}
#define RISCV_IOMMU_FSC_BARE 0
++++ +++/*
++++ +++ * This function sends IOTINVAL commands as required by the RISC-V
++++ +++ * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version)
++++ +++ * after modifying DDT or PDT entries
++++ +++ */
++++ +++static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu,
++++ +++ bool inval_pdt, unsigned long iohgatp,
++++ +++ struct riscv_iommu_dc *dc,
++++ +++ struct riscv_iommu_pc *pc)
++++ +++{
++++ +++ struct riscv_iommu_command cmd;
++++ ++
++++ +++ riscv_iommu_cmd_inval_vma(&cmd);
+
++++ +++ if (FIELD_GET(RISCV_IOMMU_DC_IOHGATP_MODE, iohgatp) ==
++++ +++ RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
++++ +++ if (inval_pdt) {
++++ +++ /*
++++ +++ * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
++++ +++ * PSCID=PC.PSCID
++++ +++ */
++++ +++ riscv_iommu_cmd_inval_set_pscid(&cmd,
++++ +++ FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
++++ +++ } else {
++++ +++ if (!FIELD_GET(RISCV_IOMMU_DC_TC_PDTV, dc->tc) &&
++++ +++ FIELD_GET(RISCV_IOMMU_DC_FSC_MODE, dc->fsc) !=
++++ +++ RISCV_IOMMU_DC_FSC_MODE_BARE) {
++++ +++ /*
++++ +++ * DC.tc.PDTV == 0 && DC.fsc.MODE != Bare
++++ +++ * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
++++ +++ * PSCID=DC.ta.PSCID
++++ +++ */
++++ +++ riscv_iommu_cmd_inval_set_pscid(&cmd,
++++ +++ FIELD_GET(RISCV_IOMMU_DC_TA_PSCID, dc->ta));
++++ +++ }
++++ +++ /* else: IOTINVAL.VMA with GV=AV=PSCV=0 */
++++ +++ }
++++ +++ } else {
++++ +++ riscv_iommu_cmd_inval_set_gscid(&cmd,
++++ +++ FIELD_GET(RISCV_IOMMU_DC_IOHGATP_GSCID, iohgatp));
++++ +++
++++ +++ if (inval_pdt) {
++++ +++ /*
++++ +++ * IOTINVAL.VMA with GV=1, AV=0, and PSCV=1, and
++++ +++ * GSCID=DC.iohgatp.GSCID, PSCID=PC.PSCID
++++ +++ */
++++ +++ riscv_iommu_cmd_inval_set_pscid(&cmd,
++++ +++ FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
++++ +++ }
++++ +++ /*
++++ +++ * else: IOTINVAL.VMA with GV=1,AV=PSCV=0,and
++++ +++ * GSCID=DC.iohgatp.GSCID
++++ +++ *
++++ +++ * IOTINVAL.GVMA with GV=1,AV=0,and
++++ +++ * GSCID=DC.iohgatp.GSCID
++++ +++ * TODO: For now, the Second-Stage feature have not yet been merged,
++++ +++ * also issue IOTINVAL.GVMA once second-stage support is merged.
++++ +++ */
++++ +++ }
++++ +++ riscv_iommu_cmd_send(iommu, &cmd);
++++ +++}
/*
* Update IODIR for the device.
*
* capability.NL (non-leaf) IOTINVAL command.
*/
riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
------- iommu_put_pages_list(&freelist);
---- -- }
---- --
---- -- return rc;
---- -- }
---- --
---- -- static size_t riscv_iommu_unmap_pages(struct iommu_domain *iommu_domain,
---- -- unsigned long iova, size_t pgsize,
---- -- size_t pgcount,
---- -- struct iommu_iotlb_gather *gather)
---- -- {
---- -- struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
---- -- size_t size = pgcount << __ffs(pgsize);
---- -- unsigned long *ptr, old;
---- -- size_t unmapped = 0;
---- -- size_t pte_size;
---- --
---- -- while (unmapped < size) {
---- -- ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size);
---- -- if (!ptr)
---- -- return unmapped;
---- --
---- -- /* partial unmap is not allowed, fail. */
---- -- if (iova & (pte_size - 1))
---- -- return unmapped;
---- --
---- -- old = READ_ONCE(*ptr);
---- -- if (cmpxchg_relaxed(ptr, old, 0) != old)
---- -- continue;
---- --
---- -- iommu_iotlb_gather_add_page(&domain->domain, gather, iova,
---- -- pte_size);
---- --
---- -- iova += pte_size;
---- -- unmapped += pte_size;
+++++++ iommu_put_pages_list(&gather->freelist);
}
-
- return rc;
- }
-
- static size_t riscv_iommu_unmap_pages(struct iommu_domain *iommu_domain,
- unsigned long iova, size_t pgsize,
- size_t pgcount,
- struct iommu_iotlb_gather *gather)
- {
- struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
- size_t size = pgcount << __ffs(pgsize);
- unsigned long *ptr, old;
- size_t unmapped = 0;
- size_t pte_size;
-
- while (unmapped < size) {
- ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size);
- if (!ptr)
- return unmapped;
-
- /* partial unmap is not allowed, fail. */
- if (iova & (pte_size - 1))
- return unmapped;
-
- old = READ_ONCE(*ptr);
- if (cmpxchg_relaxed(ptr, old, 0) != old)
- continue;
-
- iommu_iotlb_gather_add_page(&domain->domain, gather, iova,
- pte_size);
-
- iova += pte_size;
- unmapped += pte_size;
- }
-------
------- return unmapped;
------- }
-------
------- static phys_addr_t riscv_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
------- dma_addr_t iova)
------- {
------- struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
------- size_t pte_size;
------- unsigned long *ptr;
-------
------- ptr = riscv_iommu_pte_fetch(domain, iova, &pte_size);
------- if (!ptr)
------- return 0;
-------
------- return pfn_to_phys(__page_val_to_pfn(*ptr)) | (iova & (pte_size - 1));
}
static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)