iommu/amd: Refactor helper function for setting / clearing GCR3

Refactor GCR3 helper functions in preparation to use per device
GCR3 table.
  * Add new function update_gcr3 to update per device GCR3 table

  * Remove per domain default GCR3 setup during v2 page table allocation.
    Subsequent patch will add support to setup default gcr3 while
    attaching device to domain.

  * Remove amd_iommu_domain_update() from V2 page table path as device
    detach path will take care of updating the domain.

  * Consolidate GCR3 table related code in one place so that its easy
    to maintain.

  * Rename functions to reflect its usage.

Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Co-developed-by: Vasant Hegde <vasant.hegde@amd.com>
Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240205115615.6053-11-vasant.hegde@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 1f4bbc6bf1e536d5a773f33caea379049c5cd19b..6e03ff2aaebeb4071da80eb5a836cfb16a4a338a 100644 (file)
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -44,6 +44,11 @@ bool amd_iommu_v2_supported(void);
 int amd_iommu_pdev_enable_cap_pri(struct pci_dev *pdev);
 void amd_iommu_pdev_disable_cap_pri(struct pci_dev *pdev);
 
+/* GCR3 setup */
+int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data,
+                      ioasid_t pasid, unsigned long gcr3);
+int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid);
+
 int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid, u64 address);
 /*
  * This function flushes all internal caches of
@@ -61,9 +66,6 @@ void amd_iommu_dev_flush_pasid_all(struct iommu_dev_data *dev_data,
                                   ioasid_t pasid);
 
 int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid);
-int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
-                             unsigned long cr3);
-int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid);
 
 #ifdef CONFIG_IRQ_REMAP
 int amd_iommu_create_irq_domain(struct amd_iommu *iommu);

diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c
index 6d69ba60744f063462ae8205284004de78d95ed2..93489d2db4e8d04359d0a5e9c1fbacd3da573f8d 100644 (file)
--- a/drivers/iommu/amd/io_pgtable_v2.c
+++ b/drivers/iommu/amd/io_pgtable_v2.c
@@ -350,38 +350,26 @@ static const struct iommu_flush_ops v2_flush_ops = {
 
 static void v2_free_pgtable(struct io_pgtable *iop)
 {
-       struct protection_domain *pdom;
        struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);
 
-       pdom = container_of(pgtable, struct protection_domain, iop);
-       if (!(pdom->flags & PD_IOMMUV2_MASK))
+       if (!pgtable || !pgtable->pgd)
                return;
 
-       /* Clear gcr3 entry */
-       amd_iommu_domain_clear_gcr3(&pdom->domain, 0);
-
-       /* Make changes visible to IOMMUs */
-       amd_iommu_domain_update(pdom);
-
        /* Free page table */
        free_pgtable(pgtable->pgd, get_pgtable_level());
+       pgtable->pgd = NULL;
 }
 
 static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
 {
        struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
        struct protection_domain *pdom = (struct protection_domain *)cookie;
-       int ret;
        int ias = IOMMU_IN_ADDR_BIT_SIZE;
 
        pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC);
        if (!pgtable->pgd)
                return NULL;
 
-       ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, iommu_virt_to_phys(pgtable->pgd));
-       if (ret)
-               goto err_free_pgd;
-
        if (get_pgtable_level() == PAGE_MODE_5_LEVEL)
                ias = 57;
 
@@ -395,11 +383,6 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
        cfg->tlb           = &v2_flush_ops;
 
        return &pgtable->iop;
-
-err_free_pgd:
-       free_pgtable_page(pgtable->pgd);
-
-       return NULL;
 }
 
 struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = {

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index d8b42b0a253923295be4ffd2c0d9c95a889f217f..4fda158e043fcb3f9b4f94a1376963affd625fae 100644 (file)
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1714,10 +1714,13 @@ static int setup_gcr3_table(struct protection_domain *domain, int pasids)
        return 0;
 }
 
-static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)
+static u64 *__get_gcr3_pte(struct gcr3_tbl_info *gcr3_info,
+                          ioasid_t pasid, bool alloc)
 {
        int index;
        u64 *pte;
+       u64 *root = gcr3_info->gcr3_tbl;
+       int level = gcr3_info->glx;
 
        while (true) {
 
@@ -1746,6 +1749,56 @@ static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)
        return pte;
 }
 
+static int update_gcr3(struct iommu_dev_data *dev_data,
+                      ioasid_t pasid, unsigned long gcr3, bool set)
+{
+       struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
+       u64 *pte;
+
+       pte = __get_gcr3_pte(gcr3_info, pasid, true);
+       if (pte == NULL)
+               return -ENOMEM;
+
+       if (set)
+               *pte = (gcr3 & PAGE_MASK) | GCR3_VALID;
+       else
+               *pte = 0;
+
+       amd_iommu_dev_flush_pasid_all(dev_data, pasid);
+       return 0;
+}
+
+int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid,
+                      unsigned long gcr3)
+{
+       struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
+       int ret;
+
+       iommu_group_mutex_assert(dev_data->dev);
+
+       ret = update_gcr3(dev_data, pasid, gcr3, true);
+       if (ret)
+               return ret;
+
+       gcr3_info->pasid_cnt++;
+       return ret;
+}
+
+int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)
+{
+       struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;
+       int ret;
+
+       iommu_group_mutex_assert(dev_data->dev);
+
+       ret = update_gcr3(dev_data, pasid, 0, false);
+       if (ret)
+               return ret;
+
+       gcr3_info->pasid_cnt--;
+       return ret;
+}
+
 static void set_dte_entry(struct amd_iommu *iommu,
                          struct iommu_dev_data *dev_data)
 {
@@ -2769,66 +2822,6 @@ int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)
        return ret;
 }
 
-static int __set_gcr3(struct protection_domain *domain, u32 pasid,
-                     unsigned long cr3)
-{
-       u64 *pte;
-
-       if (domain->iop.mode != PAGE_MODE_NONE)
-               return -EINVAL;
-
-       pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
-       if (pte == NULL)
-               return -ENOMEM;
-
-       *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
-
-       return __amd_iommu_flush_tlb(domain, pasid);
-}
-
-static int __clear_gcr3(struct protection_domain *domain, u32 pasid)
-{
-       u64 *pte;
-
-       if (domain->iop.mode != PAGE_MODE_NONE)
-               return -EINVAL;
-
-       pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
-       if (pte == NULL)
-               return 0;
-
-       *pte = 0;
-
-       return __amd_iommu_flush_tlb(domain, pasid);
-}
-
-int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
-                             unsigned long cr3)
-{
-       struct protection_domain *domain = to_pdomain(dom);
-       unsigned long flags;
-       int ret;
-
-       spin_lock_irqsave(&domain->lock, flags);
-       ret = __set_gcr3(domain, pasid, cr3);
-       spin_unlock_irqrestore(&domain->lock, flags);
-
-       return ret;
-}
-
-int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)
-{
-       struct protection_domain *domain = to_pdomain(dom);
-       unsigned long flags;
-       int ret;
-
-       spin_lock_irqsave(&domain->lock, flags);
-       ret = __clear_gcr3(domain, pasid);
-       spin_unlock_irqrestore(&domain->lock, flags);
-
-       return ret;
-}
-
 int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
                           int status, int tag)
 {