This includes :
- Add data structure to track per protection domain dev/pasid binding details
protection_domain->dev_data_list will track attached list of
dev_data/PASIDs.
- Move 'to_pdomain()' to header file
- Add iommu_sva_set_dev_pasid(). It will check whether PASID is supported
or not. Also adds PASID to SVA protection domain list as well as to
device GCR3 table.
- Add iommu_ops.remove_dev_pasid support. It will unbind PASID from
device. Also remove pasid data from protection domain device list.
- Add IOMMU_SVA as dependency to AMD_IOMMU driver
For a given PASID, iommu_set_dev_pasid() will bind all devices to same
SVA protection domain (1 PASID : 1 SVA protection domain : N devices).
This protection domain is different from device protection domain (one
that's mapped in attach_device() path). IOMMU uses domain ID for caching,
invalidation, etc. In SVA mode it will use per-device-domain-ID. Hence in
invalidation path we retrieve domain ID from gcr3_info_table structure and
use that for invalidation.
Co-developed-by: Wei Huang <wei.huang2@amd.com>
Signed-off-by: Wei Huang <wei.huang2@amd.com>
Co-developed-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20240418103400.6229-14-vasant.hegde@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
select IOMMU_API
select IOMMU_IOVA
select IOMMU_IO_PGTABLE
+ select IOMMU_SVA
select IOMMU_IOPF
select IOMMUFD_DRIVER if IOMMUFD
depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o
+obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o pasid.o
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o
extern enum io_pgtable_fmt amd_iommu_pgtable;
extern int amd_iommu_gpt_level;
+/* Protection domain ops */
+int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
+ struct device *dev, ioasid_t pasid);
+void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid);
+
+/* SVA/PASID */
bool amd_iommu_pasid_supported(void);
/* IOPF */
return iommu_get_iommu_dev(dev_data->dev, struct amd_iommu, iommu);
}
+static inline struct protection_domain *to_pdomain(struct iommu_domain *dom)
+{
+ return container_of(dom, struct protection_domain, domain);
+}
+
bool translation_pre_enabled(struct amd_iommu *iommu);
bool amd_iommu_is_attach_deferred(struct device *dev);
int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line);
#ifndef _ASM_X86_AMD_IOMMU_TYPES_H
#define _ASM_X86_AMD_IOMMU_TYPES_H
+#include <linux/iommu.h>
#include <linux/types.h>
+#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/msi.h>
#include <linux/list.h>
list_for_each_entry((iommu), &amd_iommu_list, list)
#define for_each_iommu_safe(iommu, next) \
list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
+/* Making iterating over protection_domain->dev_data_list easier */
+#define for_each_pdom_dev_data(pdom_dev_data, pdom) \
+ list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
+#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
+ list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
struct amd_iommu;
struct iommu_domain;
PD_MODE_V2,
};
+/* Track dev_data/PASID list for the protection domain */
+struct pdom_dev_data {
+ /* Points to attached device data */
+ struct iommu_dev_data *dev_data;
+ /* PASID attached to the protection domain */
+ ioasid_t pasid;
+ /* For protection_domain->dev_data_list */
+ struct list_head list;
+};
+
/*
* This structure contains generic data for IOMMU protection domains
* independent of their use.
bool dirty_tracking; /* dirty tracking is enabled in the domain */
unsigned dev_cnt; /* devices assigned to this domain */
unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
+
+ struct list_head dev_data_list; /* List of pdom_dev_data */
};
/*
return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));
}
-static struct protection_domain *to_pdomain(struct iommu_domain *dom)
-{
- return container_of(dom, struct protection_domain, domain);
-}
-
static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)
{
struct iommu_dev_data *dev_data;
spin_lock_init(&domain->lock);
INIT_LIST_HEAD(&domain->dev_list);
+ INIT_LIST_HEAD(&domain->dev_data_list);
domain->nid = NUMA_NO_NODE;
switch (type) {
.def_domain_type = amd_iommu_def_domain_type,
.dev_enable_feat = amd_iommu_dev_enable_feature,
.dev_disable_feat = amd_iommu_dev_disable_feature,
+ .remove_dev_pasid = amd_iommu_remove_dev_pasid,
.page_response = amd_iommu_page_response,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = amd_iommu_attach_device,
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2024 Advanced Micro Devices, Inc.
+ */
+
+#define pr_fmt(fmt) "AMD-Vi: " fmt
+#define dev_fmt(fmt) pr_fmt(fmt)
+
+#include <linux/iommu.h>
+#include <linux/mm_types.h>
+
+#include "amd_iommu.h"
+
+static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data)
+{
+ if (dev_data->pasid_enabled && dev_data->max_pasids &&
+ dev_data->gcr3_info.gcr3_tbl != NULL)
+ return true;
+
+ return false;
+}
+
+static inline bool is_pasid_valid(struct iommu_dev_data *dev_data,
+ ioasid_t pasid)
+{
+ if (pasid > 0 && pasid < dev_data->max_pasids)
+ return true;
+
+ return false;
+}
+
+static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data)
+{
+ /* Update GCR3 table and flush IOTLB */
+ amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid);
+
+ list_del(&pdom_dev_data->list);
+ kfree(pdom_dev_data);
+}
+
+/* Clear PASID from device GCR3 table and remove pdom_dev_data from list */
+static void remove_pdom_dev_pasid(struct protection_domain *pdom,
+ struct device *dev, ioasid_t pasid)
+{
+ struct pdom_dev_data *pdom_dev_data;
+ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+
+ lockdep_assert_held(&pdom->lock);
+
+ for_each_pdom_dev_data(pdom_dev_data, pdom) {
+ if (pdom_dev_data->dev_data == dev_data &&
+ pdom_dev_data->pasid == pasid) {
+ remove_dev_pasid(pdom_dev_data);
+ break;
+ }
+ }
+}
+
+int iommu_sva_set_dev_pasid(struct iommu_domain *domain,
+ struct device *dev, ioasid_t pasid)
+{
+ struct pdom_dev_data *pdom_dev_data;
+ struct protection_domain *sva_pdom = to_pdomain(domain);
+ struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);
+ unsigned long flags;
+ int ret = -EINVAL;
+
+ /* PASID zero is used for requests from the I/O device without PASID */
+ if (!is_pasid_valid(dev_data, pasid))
+ return ret;
+
+ /* Make sure PASID is enabled */
+ if (!is_pasid_enabled(dev_data))
+ return ret;
+
+ /* Add PASID to protection domain pasid list */
+ pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL);
+ if (pdom_dev_data == NULL)
+ return ret;
+
+ pdom_dev_data->pasid = pasid;
+ pdom_dev_data->dev_data = dev_data;
+
+ spin_lock_irqsave(&sva_pdom->lock, flags);
+
+ /* Setup GCR3 table */
+ ret = amd_iommu_set_gcr3(dev_data, pasid,
+ iommu_virt_to_phys(domain->mm->pgd));
+ if (ret) {
+ kfree(pdom_dev_data);
+ goto out_unlock;
+ }
+
+ list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list);
+
+out_unlock:
+ spin_unlock_irqrestore(&sva_pdom->lock, flags);
+ return ret;
+}
+
+void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
+{
+ struct protection_domain *sva_pdom;
+ struct iommu_domain *domain;
+ unsigned long flags;
+
+ if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid))
+ return;
+
+ /* Get protection domain */
+ domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
+ if (!domain)
+ return;
+ sva_pdom = to_pdomain(domain);
+
+ spin_lock_irqsave(&sva_pdom->lock, flags);
+
+ /* Remove PASID from dev_data_list */
+ remove_pdom_dev_pasid(sva_pdom, dev, pasid);
+
+ spin_unlock_irqrestore(&sva_pdom->lock, flags);
+}
projects / thirdparty / kernel / linux.git / commitdiff
