iommu/arm-smmu-v3: Pre-allocate a per-master invalidation array

When a master is attached from an old domain to a new domain, it needs to
build an invalidation array to delete and add the array entries from/onto
the invalidation arrays of those two domains, passed via the to_merge and
to_unref arguments into arm_smmu_invs_merge/unref() respectively.

Since the master->num_streams might differ across masters, a memory would
have to be allocated when building an to_merge/to_unref array which might
fail with -ENOMEM.

On the other hand, an attachment to arm_smmu_blocked_domain must not fail
so it's the best to avoid any memory allocation in that path.

Pre-allocate a fixed size invalidation array for every master. This array
will be used as a scratch to fill dynamically when building a to_merge or
to_unref invs array. Sort fwspec->ids in an ascending order to fit to the
arm_smmu_invs_merge() function.

Co-developed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Pranjal Shrivastava <praan@google.com>
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 10050b1a6d945ab5647c4aa811e87a05aa39ed40..8e651cf6b86eee5fffe5f2f2fdbb4ade768e9493 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3789,26 +3789,57 @@ static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
        return 0;
 }
 
+static int arm_smmu_stream_id_cmp(const void *_l, const void *_r)
+{
+       const typeof_member(struct arm_smmu_stream, id) *l = _l;
+       const typeof_member(struct arm_smmu_stream, id) *r = _r;
+
+       return cmp_int(*l, *r);
+}
+
 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
                                  struct arm_smmu_master *master)
 {
        int i;
        int ret = 0;
        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
+       bool ats_supported = dev_is_pci(master->dev) &&
+                            pci_ats_supported(to_pci_dev(master->dev));
 
        master->streams = kzalloc_objs(*master->streams, fwspec->num_ids);
        if (!master->streams)
                return -ENOMEM;
        master->num_streams = fwspec->num_ids;
 
-       mutex_lock(&smmu->streams_mutex);
+       if (!ats_supported) {
+               /* Base case has 1 ASID entry or maximum 2 VMID entries */
+               master->build_invs = arm_smmu_invs_alloc(2);
+       } else {
+               /* ATS case adds num_ids of entries, on top of the base case */
+               master->build_invs = arm_smmu_invs_alloc(2 + fwspec->num_ids);
+       }
+       if (!master->build_invs) {
+               kfree(master->streams);
+               return -ENOMEM;
+       }
+
        for (i = 0; i < fwspec->num_ids; i++) {
                struct arm_smmu_stream *new_stream = &master->streams[i];
-               struct rb_node *existing;
-               u32 sid = fwspec->ids[i];
 
-               new_stream->id = sid;
+               new_stream->id = fwspec->ids[i];
                new_stream->master = master;
+       }
+
+       /* Put the ids into order for sorted to_merge/to_unref arrays */
+       sort_nonatomic(master->streams, master->num_streams,
+                      sizeof(master->streams[0]), arm_smmu_stream_id_cmp,
+                      NULL);
+
+       mutex_lock(&smmu->streams_mutex);
+       for (i = 0; i < fwspec->num_ids; i++) {
+               struct arm_smmu_stream *new_stream = &master->streams[i];
+               struct rb_node *existing;
+               u32 sid = new_stream->id;
 
                ret = arm_smmu_init_sid_strtab(smmu, sid);
                if (ret)
@@ -3838,6 +3869,7 @@ static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
                for (i--; i >= 0; i--)
                        rb_erase(&master->streams[i].node, &smmu->streams);
                kfree(master->streams);
+               kfree(master->build_invs);
        }
        mutex_unlock(&smmu->streams_mutex);
 
@@ -3859,6 +3891,7 @@ static void arm_smmu_remove_master(struct arm_smmu_master *master)
        mutex_unlock(&smmu->streams_mutex);
 
        kfree(master->streams);
+       kfree(master->build_invs);
 }
 
 static struct iommu_device *arm_smmu_probe_device(struct device *dev)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index ed8820f12ba321ce1fcd3ba5efd52880fc804b4a..5e0e5055af1ebe9a52cfae99b82c4e7f280219d4 100644 (file)
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -928,6 +928,14 @@ struct arm_smmu_master {
        struct arm_smmu_device          *smmu;
        struct device                   *dev;
        struct arm_smmu_stream          *streams;
+       /*
+        * Scratch memory for a to_merge or to_unref array to build a per-domain
+        * invalidation array. It'll be pre-allocated with enough enries for all
+        * possible build scenarios. It can be used by only one caller at a time
+        * until the arm_smmu_invs_merge/unref() finishes. Must be locked by the
+        * iommu_group mutex.
+        */
+       struct arm_smmu_invs            *build_invs;
        struct arm_smmu_vmaster         *vmaster; /* use smmu->streams_mutex */
        /* Locked by the iommu core using the group mutex */
        struct arm_smmu_ctx_desc_cfg    cd_table;