iommupt: Directly call iommupt's unmap_range()

The common algorithm in iommupt does not require the iommu_pgsize()
calculations, it can directly unmap any arbitrary range. Add a new function
pointer to directly call an iommupt unmap_range op and make
__iommu_unmap() call it directly.

Gives about a 5% gain on single page unmappings.

The function pointer is run through pt_iommu_ops instead of
iommu_domain_ops to discourage using it outside iommupt. All drivers with
their own page tables should continue to use the simplified
map/unmap_pages() style interfaces.

Reviewed-by: Samiullah Khawaja <skhawaja@google.com>
Reviewed-by: Kevin Tian <kevin.tian@intel.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index 9c08bb594e41731c94ddd110ec8c62386b6c024f..a627c26fa62dbdb380bd243d7b8611f6a6e2dcd7 100644 (file)
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -1031,34 +1031,12 @@ start_oa:
        return ret;
 }
 
-/**
- * unmap_pages() - Make a range of IOVA empty/not present
- * @domain: Domain to manipulate
- * @iova: IO virtual address to start
- * @pgsize: Length of each page
- * @pgcount: Length of the range in pgsize units starting from @iova
- * @iotlb_gather: Gather struct that must be flushed on return
- *
- * unmap_pages() will remove a translation created by map_pages(). It cannot
- * subdivide a mapping created by map_pages(), so it should be called with IOVA
- * ranges that match those passed to map_pages(). The IOVA range can aggregate
- * contiguous map_pages() calls so long as no individual range is split.
- *
- * Context: The caller must hold a write range lock that includes
- * the whole range.
- *
- * Returns: Number of bytes of VA unmapped. iova + res will be the point
- * unmapping stopped.
- */
-size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova,
-                             size_t pgsize, size_t pgcount,
+static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+                             dma_addr_t len,
                              struct iommu_iotlb_gather *iotlb_gather)
 {
-       struct pt_iommu *iommu_table =
-               container_of(domain, struct pt_iommu, domain);
        struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT(
                                               unmap.free_list) };
-       pt_vaddr_t len = pgsize * pgcount;
        struct pt_range range;
        int ret;
 
@@ -1073,7 +1051,6 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova,
 
        return unmap.unmapped;
 }
-EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(unmap_pages), "GENERIC_PT_IOMMU");
 
 static void NS(get_info)(struct pt_iommu *iommu_table,
                         struct pt_iommu_info *info)
@@ -1121,6 +1098,7 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
 }
 
 static const struct pt_iommu_ops NS(ops) = {
+       .unmap_range = NS(unmap_range),
 #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \
        IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty)
        .set_dirty = NS(set_dirty),
@@ -1183,6 +1161,7 @@ static int pt_iommu_init_domain(struct pt_iommu *iommu_table,
 
        domain->type = __IOMMU_DOMAIN_PAGING;
        domain->pgsize_bitmap = info.pgsize_bitmap;
+       domain->is_iommupt = true;
 
        if (pt_feature(common, PT_FEAT_DYNAMIC_TOP))
                range = _pt_top_range(common,

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 35db5178095404fec87cd0f18e44ea97cf354e78..f68269707101a326c37d1e7173a1e8324459c99f 100644 (file)
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -34,6 +34,7 @@
 #include <linux/sched/mm.h>
 #include <linux/msi.h>
 #include <uapi/linux/iommufd.h>
+#include <linux/generic_pt/iommu.h>
 
 #include "dma-iommu.h"
 #include "iommu-priv.h"
@@ -2666,13 +2667,12 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 EXPORT_SYMBOL_GPL(iommu_map);
 
-static size_t __iommu_unmap(struct iommu_domain *domain,
-                           unsigned long iova, size_t size,
-                           struct iommu_iotlb_gather *iotlb_gather)
+static size_t
+__iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova,
+                          size_t size, struct iommu_iotlb_gather *iotlb_gather)
 {
        const struct iommu_domain_ops *ops = domain->ops;
        size_t unmapped_page, unmapped = 0;
-       unsigned long orig_iova = iova;
        unsigned int min_pagesz;
 
        if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING)))
@@ -2718,8 +2718,23 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
                unmapped += unmapped_page;
        }
 
-       trace_unmap(orig_iova, size, unmapped);
-       iommu_debug_unmap_end(domain, orig_iova, size, unmapped);
+       return unmapped;
+}
+
+static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova,
+                           size_t size,
+                           struct iommu_iotlb_gather *iotlb_gather)
+{
+       struct pt_iommu *pt = iommupt_from_domain(domain);
+       size_t unmapped;
+
+       if (pt)
+               unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather);
+       else
+               unmapped = __iommu_unmap_domain_pgtbl(domain, iova, size,
+                                                     iotlb_gather);
+       trace_unmap(iova, size, unmapped);
+       iommu_debug_unmap_end(domain, iova, size, unmapped);
        return unmapped;
 }
 

diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h
index 49d9addb98c52c8e42da571089ff104cdb00528d..0da971134a37f5af4e1be95a9f78b61963c2d6c7 100644 (file)
--- a/include/linux/generic_pt/iommu.h
+++ b/include/linux/generic_pt/iommu.h
@@ -66,6 +66,13 @@ struct pt_iommu {
        struct device *iommu_device;
 };
 
+static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain)
+{
+       if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt)
+               return NULL;
+       return container_of(domain, struct pt_iommu, domain);
+}
+
 /**
  * struct pt_iommu_info - Details about the IOMMU page table
  *
@@ -80,6 +87,29 @@ struct pt_iommu_info {
 };
 
 struct pt_iommu_ops {
+       /**
+        * @unmap_range: Make a range of IOVA empty/not present
+        * @iommu_table: Table to manipulate
+        * @iova: IO virtual address to start
+        * @len: Length of the range starting from @iova
+        * @iotlb_gather: Gather struct that must be flushed on return
+        *
+        * unmap_range() will remove a translation created by map_range(). It
+        * cannot subdivide a mapping created by map_range(), so it should be
+        * called with IOVA ranges that match those passed to map_pages. The
+        * IOVA range can aggregate contiguous map_range() calls so long as no
+        * individual range is split.
+        *
+        * Context: The caller must hold a write range lock that includes
+        * the whole range.
+        *
+        * Returns: Number of bytes of VA unmapped. iova + res will be the
+        * point unmapping stopped.
+        */
+       size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
+                             dma_addr_t len,
+                             struct iommu_iotlb_gather *iotlb_gather);
+
        /**
         * @set_dirty: Make the iova write dirty
         * @iommu_table: Table to manipulate
@@ -198,10 +228,6 @@ struct pt_iommu_cfg {
                                       unsigned long iova, phys_addr_t paddr,  \
                                       size_t pgsize, size_t pgcount,          \
                                       int prot, gfp_t gfp, size_t *mapped);   \
-       size_t pt_iommu_##fmt##_unmap_pages(                                   \
-               struct iommu_domain *domain, unsigned long iova,               \
-               size_t pgsize, size_t pgcount,                                 \
-               struct iommu_iotlb_gather *iotlb_gather);                      \
        int pt_iommu_##fmt##_read_and_clear_dirty(                             \
                struct iommu_domain *domain, unsigned long iova, size_t size,  \
                unsigned long flags, struct iommu_dirty_bitmap *dirty);        \
@@ -223,8 +249,7 @@ struct pt_iommu_cfg {
  */
 #define IOMMU_PT_DOMAIN_OPS(fmt)                        \
        .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \
-       .map_pages = &pt_iommu_##fmt##_map_pages,       \
-       .unmap_pages = &pt_iommu_##fmt##_unmap_pages
+       .map_pages = &pt_iommu_##fmt##_map_pages
 #define IOMMU_PT_DIRTY_OPS(fmt) \
        .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty
 

diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 54b8b48c762e88153eb7e4d3039fdeb5afa61472..7ca648c0133634ac1072256ed6440479919457b8 100644 (file)
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -223,6 +223,7 @@ enum iommu_domain_cookie_type {
 struct iommu_domain {
        unsigned type;
        enum iommu_domain_cookie_type cookie_type;
+       bool is_iommupt;
        const struct iommu_domain_ops *ops;
        const struct iommu_dirty_ops *dirty_ops;
        const struct iommu_ops *owner; /* Whose domain_alloc we came from */