iommu/amd: xarray to track protection_domain->iommu list

Use xarray to track IOMMU attached to protection domain instead of
static array of MAX_IOMMUS. Also add lockdep assertion.

Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
Reviewed-by: Joerg Roedel <jroedel@suse.de>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20241030063556.6104-5-vasant.hegde@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 90a752f574635a914a0b9dee2cf7467576dca9c9..f88f6e4e910fe56e448a442503c3cd1f68db7a46 100644 (file)
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -565,6 +565,12 @@ struct pdom_dev_data {
        struct list_head list;
 };
 
+/* Keeps track of the IOMMUs attached to protection domain */
+struct pdom_iommu_info {
+       struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
+       u32 refcnt;     /* Count of attached dev/pasid per domain/IOMMU */
+};
+
 /*
  * This structure contains generic data for  IOMMU protection domains
  * independent of their use.
@@ -578,7 +584,7 @@ struct protection_domain {
        u16 id;                 /* the domain id written to the device table */
        enum protection_domain_mode pd_mode; /* Track page table type */
        bool dirty_tracking;    /* dirty tracking is enabled in the domain */
-       unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
+       struct xarray iommu_array;      /* per-IOMMU reference count */
 
        struct mmu_notifier mn; /* mmu notifier for the SVA domain */
        struct list_head dev_data_list; /* List of pdom_dev_data */

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 7f6acbf8e8f435f62b86dfe24cd45982c245b2f6..48472f45d6b7cfcb46c4f1ac692d5ca4423a9d2b 100644 (file)
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1257,18 +1257,17 @@ out_unlock:
 
 static void domain_flush_complete(struct protection_domain *domain)
 {
-       int i;
+       struct pdom_iommu_info *pdom_iommu_info;
+       unsigned long i;
 
-       for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
-               if (domain && !domain->dev_iommu[i])
-                       continue;
+       lockdep_assert_held(&domain->lock);
 
-               /*
-                * Devices of this domain are behind this IOMMU
-                * We need to wait for completion of all commands.
-                */
-               iommu_completion_wait(amd_iommus[i]);
-       }
+       /*
+        * Devices of this domain are behind this IOMMU
+        * We need to wait for completion of all commands.
+        */
+        xa_for_each(&domain->iommu_array, i, pdom_iommu_info)
+               iommu_completion_wait(pdom_iommu_info->iommu);
 }
 
 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
@@ -1450,21 +1449,22 @@ static int domain_flush_pages_v2(struct protection_domain *pdom,
 static int domain_flush_pages_v1(struct protection_domain *pdom,
                                 u64 address, size_t size)
 {
+       struct pdom_iommu_info *pdom_iommu_info;
        struct iommu_cmd cmd;
-       int ret = 0, i;
+       int ret = 0;
+       unsigned long i;
+
+       lockdep_assert_held(&pdom->lock);
 
        build_inv_iommu_pages(&cmd, address, size,
                              pdom->id, IOMMU_NO_PASID, false);
 
-       for (i = 0; i < amd_iommu_get_num_iommus(); ++i) {
-               if (!pdom->dev_iommu[i])
-                       continue;
-
+       xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) {
                /*
                 * Devices of this domain are behind this IOMMU
                 * We need a TLB flush
                 */
-               ret |= iommu_queue_command(amd_iommus[i], &cmd);
+               ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd);
        }
 
        return ret;
@@ -1503,6 +1503,8 @@ static void __domain_flush_pages(struct protection_domain *domain,
 void amd_iommu_domain_flush_pages(struct protection_domain *domain,
                                  u64 address, size_t size)
 {
+       lockdep_assert_held(&domain->lock);
+
        if (likely(!amd_iommu_np_cache)) {
                __domain_flush_pages(domain, address, size);
 
@@ -2014,6 +2016,50 @@ static void destroy_gcr3_table(struct iommu_dev_data *dev_data,
        free_gcr3_table(gcr3_info);
 }
 
+static int pdom_attach_iommu(struct amd_iommu *iommu,
+                            struct protection_domain *pdom)
+{
+       struct pdom_iommu_info *pdom_iommu_info, *curr;
+
+       pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);
+       if (pdom_iommu_info) {
+               pdom_iommu_info->refcnt++;
+               return 0;
+       }
+
+       pdom_iommu_info = kzalloc(sizeof(*pdom_iommu_info), GFP_ATOMIC);
+       if (!pdom_iommu_info)
+               return -ENOMEM;
+
+       pdom_iommu_info->iommu = iommu;
+       pdom_iommu_info->refcnt = 1;
+
+       curr = xa_cmpxchg(&pdom->iommu_array, iommu->index,
+                         NULL, pdom_iommu_info, GFP_ATOMIC);
+       if (curr) {
+               kfree(pdom_iommu_info);
+               return -ENOSPC;
+       }
+
+       return 0;
+}
+
+static void pdom_detach_iommu(struct amd_iommu *iommu,
+                             struct protection_domain *pdom)
+{
+       struct pdom_iommu_info *pdom_iommu_info;
+
+       pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index);
+       if (!pdom_iommu_info)
+               return;
+
+       pdom_iommu_info->refcnt--;
+       if (pdom_iommu_info->refcnt == 0) {
+               xa_erase(&pdom->iommu_array, iommu->index);
+               kfree(pdom_iommu_info);
+       }
+}
+
 static int do_attach(struct iommu_dev_data *dev_data,
                     struct protection_domain *domain)
 {
@@ -2030,13 +2076,17 @@ static int do_attach(struct iommu_dev_data *dev_data,
                cfg->amd.nid = dev_to_node(dev_data->dev);
 
        /* Do reference counting */
-       domain->dev_iommu[iommu->index] += 1;
+       ret = pdom_attach_iommu(iommu, domain);
+       if (ret)
+               return ret;
 
        /* Setup GCR3 table */
        if (pdom_is_sva_capable(domain)) {
                ret = init_gcr3_table(dev_data, domain);
-               if (ret)
+               if (ret) {
+                       pdom_detach_iommu(iommu, domain);
                        return ret;
+               }
        }
 
        return ret;
@@ -2062,7 +2112,7 @@ static void do_detach(struct iommu_dev_data *dev_data)
        list_del(&dev_data->list);
 
        /* decrease reference counters - needs to happen after the flushes */
-       domain->dev_iommu[iommu->index] -= 1;
+       pdom_detach_iommu(iommu, domain);
 }
 
 /*
@@ -2258,6 +2308,7 @@ static void protection_domain_init(struct protection_domain *domain, int nid)
        spin_lock_init(&domain->lock);
        INIT_LIST_HEAD(&domain->dev_list);
        INIT_LIST_HEAD(&domain->dev_data_list);
+       xa_init(&domain->iommu_array);
        domain->iop.pgtbl.cfg.amd.nid = nid;
 }