iommu: Introduce a replace API for device pasid

Provide a high-level API to allow replacements of one domain with another
for specific pasid of a device. This is similar to
iommu_replace_group_handle() and it is expected to be used only by IOMMUFD.

Link: https://patch.msgid.link/r/20250321171940.7213-3-yi.l.liu@intel.com
Co-developed-by: Lu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Reviewed-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Yi Liu <yi.l.liu@intel.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
index c74fff25be78faa784e82d552a4af66502ce10c4..b8528d5dd1e718568b8cc0a5b38ef76df20b415a 100644 (file)
--- a/drivers/iommu/iommu-priv.h
+++ b/drivers/iommu/iommu-priv.h
@@ -56,4 +56,7 @@ static inline int iommufd_sw_msi(struct iommu_domain *domain,
 }
 #endif /* CONFIG_IOMMUFD_DRIVER_CORE && CONFIG_IRQ_MSI_IOMMU */
 
+int iommu_replace_device_pasid(struct iommu_domain *domain,
+                              struct device *dev, ioasid_t pasid,
+                              struct iommu_attach_handle *handle);
 #endif /* __LINUX_IOMMU_PRIV_H */

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 34ea7e5e7d00dbfe6440946957b72fcd6e83c43c..f24963be61700e1a43677e4abc0f3ab2e2af4477 100644 (file)
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -514,6 +514,13 @@ static void iommu_deinit_device(struct device *dev)
        dev_iommu_free(dev);
 }
 
+static struct iommu_domain *pasid_array_entry_to_domain(void *entry)
+{
+       if (xa_pointer_tag(entry) == IOMMU_PASID_ARRAY_DOMAIN)
+               return xa_untag_pointer(entry);
+       return ((struct iommu_attach_handle *)xa_untag_pointer(entry))->domain;
+}
+
 DEFINE_MUTEX(iommu_probe_device_lock);
 
 static int __iommu_probe_device(struct device *dev, struct list_head *group_list)
@@ -3324,14 +3331,15 @@ static void iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
 }
 
 static int __iommu_set_group_pasid(struct iommu_domain *domain,
-                                  struct iommu_group *group, ioasid_t pasid)
+                                  struct iommu_group *group, ioasid_t pasid,
+                                  struct iommu_domain *old)
 {
        struct group_device *device, *last_gdev;
        int ret;
 
        for_each_group_device(group, device) {
                ret = domain->ops->set_dev_pasid(domain, device->dev,
-                                                pasid, NULL);
+                                                pasid, old);
                if (ret)
                        goto err_revert;
        }
@@ -3343,7 +3351,15 @@ err_revert:
        for_each_group_device(group, device) {
                if (device == last_gdev)
                        break;
-               iommu_remove_dev_pasid(device->dev, pasid, domain);
+               /*
+                * If no old domain, undo the succeeded devices/pasid.
+                * Otherwise, rollback the succeeded devices/pasid to the old
+                * domain. And it is a driver bug to fail attaching with a
+                * previously good domain.
+                */
+               if (!old || WARN_ON(old->ops->set_dev_pasid(old, device->dev,
+                                                           pasid, domain)))
+                       iommu_remove_dev_pasid(device->dev, pasid, domain);
        }
        return ret;
 }
@@ -3412,7 +3428,7 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
        if (ret)
                goto out_unlock;
 
-       ret = __iommu_set_group_pasid(domain, group, pasid);
+       ret = __iommu_set_group_pasid(domain, group, pasid, NULL);
        if (ret) {
                xa_release(&group->pasid_array, pasid);
                goto out_unlock;
@@ -3433,6 +3449,97 @@ out_unlock:
 }
 EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
 
+/**
+ * iommu_replace_device_pasid - Replace the domain that a specific pasid
+ *                              of the device is attached to
+ * @domain: the new iommu domain
+ * @dev: the attached device.
+ * @pasid: the pasid of the device.
+ * @handle: the attach handle.
+ *
+ * This API allows the pasid to switch domains. The @pasid should have been
+ * attached. Otherwise, this fails. The pasid will keep the old configuration
+ * if replacement failed.
+ *
+ * Caller should always provide a new handle to avoid race with the paths
+ * that have lockless reference to handle if it intends to pass a valid handle.
+ *
+ * Return 0 on success, or an error.
+ */
+int iommu_replace_device_pasid(struct iommu_domain *domain,
+                              struct device *dev, ioasid_t pasid,
+                              struct iommu_attach_handle *handle)
+{
+       /* Caller must be a probed driver on dev */
+       struct iommu_group *group = dev->iommu_group;
+       struct iommu_attach_handle *entry;
+       struct iommu_domain *curr_domain;
+       void *curr;
+       int ret;
+
+       if (!group)
+               return -ENODEV;
+
+       if (!domain->ops->set_dev_pasid)
+               return -EOPNOTSUPP;
+
+       if (dev_iommu_ops(dev) != domain->owner ||
+           pasid == IOMMU_NO_PASID || !handle)
+               return -EINVAL;
+
+       mutex_lock(&group->mutex);
+       entry = iommu_make_pasid_array_entry(domain, handle);
+       curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
+                         XA_ZERO_ENTRY, GFP_KERNEL);
+       if (xa_is_err(curr)) {
+               ret = xa_err(curr);
+               goto out_unlock;
+       }
+
+       /*
+        * No domain (with or without handle) attached, hence not
+        * a replace case.
+        */
+       if (!curr) {
+               xa_release(&group->pasid_array, pasid);
+               ret = -EINVAL;
+               goto out_unlock;
+       }
+
+       /*
+        * Reusing handle is problematic as there are paths that refers
+        * the handle without lock. To avoid race, reject the callers that
+        * attempt it.
+        */
+       if (curr == entry) {
+               WARN_ON(1);
+               ret = -EINVAL;
+               goto out_unlock;
+       }
+
+       curr_domain = pasid_array_entry_to_domain(curr);
+       ret = 0;
+
+       if (curr_domain != domain) {
+               ret = __iommu_set_group_pasid(domain, group,
+                                             pasid, curr_domain);
+               if (ret)
+                       goto out_unlock;
+       }
+
+       /*
+        * The above xa_cmpxchg() reserved the memory, and the
+        * group->mutex is held, this cannot fail.
+        */
+       WARN_ON(xa_is_err(xa_store(&group->pasid_array,
+                                  pasid, entry, GFP_KERNEL)));
+
+out_unlock:
+       mutex_unlock(&group->mutex);
+       return ret;
+}
+EXPORT_SYMBOL_NS_GPL(iommu_replace_device_pasid, "IOMMUFD_INTERNAL");
+
 /*
  * iommu_detach_device_pasid() - Detach the domain from pasid of device
  * @domain: the iommu domain.