iommu/riscv: Add RISCV_IOMMU_CAPABILITIES_NL

Non-leaf invalidation allows the single invalidate command to also
clear the walk cache. If NL is available, set the NL bit if the
gather indicates tables have been changed. The stride is already
calculated properly.

Reviewed-by: Tomasz Jeznach <tjeznach@rivosinc.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Andrew Jones <andrew.jones@oss.qualcomm.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
index 29a0040b1c32ea7409127ec6ae9580e9cfb106b5..f01b49ac8155861502c89cf6e625141a140acb00 100644 (file)
--- a/drivers/iommu/riscv/iommu-bits.h
+++ b/drivers/iommu/riscv/iommu-bits.h
@@ -63,6 +63,7 @@
 #define RISCV_IOMMU_CAPABILITIES_PD8           BIT_ULL(38)
 #define RISCV_IOMMU_CAPABILITIES_PD17          BIT_ULL(39)
 #define RISCV_IOMMU_CAPABILITIES_PD20          BIT_ULL(40)
+#define RISCV_IOMMU_CAPABILITIES_NL            BIT_ULL(42)
 
 /**
  * enum riscv_iommu_igs_settings - Interrupt Generation Support Settings
@@ -473,6 +474,7 @@ struct riscv_iommu_command {
 #define RISCV_IOMMU_CMD_IOTINVAL_PSCV          BIT_ULL(32)
 #define RISCV_IOMMU_CMD_IOTINVAL_GV            BIT_ULL(33)
 #define RISCV_IOMMU_CMD_IOTINVAL_GSCID         GENMASK_ULL(59, 44)
+#define RISCV_IOMMU_CMD_IOTINVAL_NL            BIT_ULL(34)
 /* dword1[61:10] is the 4K-aligned page address */
 #define RISCV_IOMMU_CMD_IOTINVAL_ADDR          GENMASK_ULL(61, 10)
 
@@ -724,6 +726,11 @@ static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cm
        cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
 }
 
+static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd)
+{
+       cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_NL;
+}
+
 static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd,
                                                   int pscid)
 {

diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index 719d3b475d61704106adf0e0d8ca251e9d5afc89..165ced9937562b533723e4ed2bd526b6e11ecac2 100644 (file)
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -966,6 +966,8 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,
                                          int pscid,
                                          struct riscv_iommu_tlbi *tlbi)
 {
+       bool use_nl = tlbi->non_leaf &&
+                     (iommu->caps & RISCV_IOMMU_CAPABILITIES_NL);
        struct riscv_iommu_command cmd;
        unsigned long iova;
        unsigned int i;
@@ -974,17 +976,17 @@ static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,
        riscv_iommu_cmd_inval_set_pscid(&cmd, pscid);
 
        /*
-        * When non-leaf page table entries were changed, the base spec
-        * requires a full PSCID invalidation (AV=0) since there is no
-        * way to do targeted non-leaf invalidation without the NL
-        * extension. Force global invalidation to preserve correctness.
+        * If non-leaf entries were changed and the IOMMU doesn't
+        * support NL, we must fall back to global invalidation (AV=0).
         */
-       if (tlbi->single.use_global || tlbi->non_leaf)
+       if (tlbi->single.use_global || (tlbi->non_leaf && !use_nl))
                goto global;
 
        iova = tlbi->start;
        for (i = 0; i < tlbi->single.num; i++) {
                riscv_iommu_cmd_inval_set_addr(&cmd, iova);
+               if (use_nl)
+                       riscv_iommu_cmd_inval_set_nl(&cmd);
                riscv_iommu_cmd_send(iommu, &cmd);
                iova += 1ULL << tlbi->single.stride_lg2;
        }