iommu/riscv: Enable PT_FEAT_DETAILED_GATHER and pass gather to iotlb_inval

RISC-V can use the information from PT_FEAT_DETAILED_GATHER to
compute the best stride to generate the single TLB invalidations.

Pass the gather down to the lower functions and create a full-range
gather for the flush-all callback.

Reviewed-by: Tomasz Jeznach <tjeznach@rivosinc.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Andrew Jones <andrew.jones@oss.qualcomm.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

diff --git a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
index cbf60fffa9bf7d7c816962c5045385ea584cfc5d..b18fc4d109f538ff92da3832e2f33a008efc0259 100644 (file)
--- a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
+++ b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
@@ -6,6 +6,8 @@
 #define PT_FMT_VARIANT 64
 #define PT_SUPPORTED_FEATURES                                  \
        (BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \
-        BIT(PT_FEAT_RISCV_SVNAPOT_64K))
+        BIT(PT_FEAT_RISCV_SVNAPOT_64K) |                      \
+        BIT(PT_FEAT_DETAILED_GATHER))
+#define PT_FORCE_ENABLED_FEATURES BIT(PT_FEAT_DETAILED_GATHER)
 
 #include "iommu_template.h"

diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index 6c324f9fdc53470f1b0f37b2bdd1e84f7084d4a5..8ea085263f44e99e8ff4e4e8c7528a382528d705 100644 (file)
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -929,12 +929,28 @@ static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain,
 #define RISCV_IOMMU_IOTLB_INVAL_LIMIT  (2 << 20)
 
 static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
-                                   unsigned long start, unsigned long end)
+                                   struct iommu_iotlb_gather *gather)
 {
+       unsigned long start;
+       unsigned long end;
        struct riscv_iommu_bond *bond;
        struct riscv_iommu_device *iommu, *prev;
        struct riscv_iommu_command cmd;
 
+       /*
+        * When non-leaf page table entries were changed, the base spec
+        * requires a full PSCID invalidation (AV=0) since there is no
+        * way to do targeted non-leaf invalidation without the NL
+        * extension. Force global invalidation to preserve correctness.
+        */
+       if (gather->pt.table_levels_bitmap) {
+               start = 0;
+               end = ULONG_MAX;
+       } else {
+               start = gather->start;
+               end = gather->end;
+       }
+
        /*
         * For each IOMMU linked with this protection domain (via bonds->dev),
         * an IOTLB invaliation command will be submitted and executed.
@@ -1145,8 +1161,14 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 static void riscv_iommu_iotlb_flush_all(struct iommu_domain *iommu_domain)
 {
        struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
+       struct iommu_iotlb_gather gather = {
+               .start = 0,
+               .end = ULONG_MAX,
+               .pt.leaf_levels_bitmap = 0xFF,
+               .pt.table_levels_bitmap = 0xFE,
+       };
 
-       riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
+       riscv_iommu_iotlb_inval(domain, &gather);
 }
 
 static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
@@ -1154,19 +1176,8 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
 {
        struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
 
-       if (iommu_pages_list_empty(&gather->freelist)) {
-               riscv_iommu_iotlb_inval(domain, gather->start, gather->end);
-       } else {
-               /*
-                * In 1.0 spec version, the smallest scope we can use to
-                * invalidate all levels of page table (i.e. leaf and non-leaf)
-                * is an invalidate-all-PSCID IOTINVAL.VMA with AV=0.
-                * This will be updated with hardware support for
-                * capability.NL (non-leaf) IOTINVAL command.
-                */
-               riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
-               iommu_put_pages_list(&gather->freelist);
-       }
+       riscv_iommu_iotlb_inval(domain, gather);
+       iommu_put_pages_list(&gather->freelist);
 }
 
 static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
@@ -1267,7 +1278,8 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
         */
        cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) |
                              BIT(PT_FEAT_FLUSH_RANGE) |
-                             BIT(PT_FEAT_RISCV_SVNAPOT_64K);
+                             BIT(PT_FEAT_RISCV_SVNAPOT_64K) |
+                             BIT(PT_FEAT_DETAILED_GATHER);
        if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT)
                cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT);
        domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);