iommu/amd: Store the nid in io_pgtable_cfg instead of the domain

[ Upstream commit 47f218d108950984b24af81f66356ceda380eb74 ]

We already have memory in the union here that is being wasted in AMD's
case, use it to store the nid.

Putting the nid here further isolates the io_pgtable code from the struct
protection_domain.

Fixup protection_domain_alloc so that the NID from the device is provided,
at this point dev is never NULL for AMD so this will now allocate the
first table pointer on the correct NUMA node.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Vasant Hegde <vasant.hegde@amd.com>
Link: https://lore.kernel.org/r/8-v2-831cdc4d00f3+1a315-amd_iopgtbl_jgg@nvidia.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
Stable-dep-of: 016991606aa0 ("iommu/amd/pgtbl_v2: Take protection domain lock before invalidating TLB")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 5a050080d2e8141bb6f61386acc89f11e8f22ca1..5459f726fb29d6faacaac12a0ca70f3935d501ee 100644 (file)
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -45,7 +45,7 @@ extern enum io_pgtable_fmt amd_iommu_pgtable;
 extern int amd_iommu_gpt_level;
 
 /* Protection domain ops */
-struct protection_domain *protection_domain_alloc(unsigned int type);
+struct protection_domain *protection_domain_alloc(unsigned int type, int nid);
 void protection_domain_free(struct protection_domain *domain);
 struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
                                                struct mm_struct *mm);

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 90a2e4790bffdf5a17ece81a81dd180a05f37108..caa469abf032727a6443e78804a4b194fbc2e857 100644 (file)
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -580,7 +580,6 @@ struct protection_domain {
        struct amd_io_pgtable iop;
        spinlock_t lock;        /* mostly used to lock the page table*/
        u16 id;                 /* the domain id written to the device table */
-       int nid;                /* Node ID */
        enum protection_domain_mode pd_mode; /* Track page table type */
        bool dirty_tracking;    /* dirty tracking is enabled in the domain */
        unsigned dev_cnt;       /* devices assigned to this domain */

diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index dab1cf53b1f3fff23480fa954ee303213eb6e679..f71140416fcf6c5fecd27ef2bd16ddca10fa0c16 100644 (file)
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -141,11 +141,12 @@ static bool increase_address_space(struct protection_domain *domain,
                                   unsigned long address,
                                   gfp_t gfp)
 {
+       struct io_pgtable_cfg *cfg = &domain->iop.pgtbl.cfg;
        unsigned long flags;
        bool ret = true;
        u64 *pte;
 
-       pte = iommu_alloc_page_node(domain->nid, gfp);
+       pte = iommu_alloc_page_node(cfg->amd.nid, gfp);
        if (!pte)
                return false;
 
@@ -182,6 +183,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
                      gfp_t gfp,
                      bool *updated)
 {
+       struct io_pgtable_cfg *cfg = &domain->iop.pgtbl.cfg;
        int level, end_lvl;
        u64 *pte, *page;
 
@@ -233,7 +235,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
 
                if (!IOMMU_PTE_PRESENT(__pte) ||
                    pte_level == PAGE_MODE_NONE) {
-                       page = iommu_alloc_page_node(domain->nid, gfp);
+                       page = iommu_alloc_page_node(cfg->amd.nid, gfp);
 
                        if (!page)
                                return NULL;
@@ -560,7 +562,7 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
 {
        struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
 
-       pgtable->root = iommu_alloc_page(GFP_KERNEL);
+       pgtable->root = iommu_alloc_page_node(cfg->amd.nid, GFP_KERNEL);
        if (!pgtable->root)
                return NULL;
        pgtable->mode = PAGE_MODE_3_LEVEL;

diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c
index de60f6f4cb2f925d3fabd0a586f4c4a399c0e551..c881ec848f545992d059fd5caaef0ecbb18ce070 100644 (file)
--- a/drivers/iommu/amd/io_pgtable_v2.c
+++ b/drivers/iommu/amd/io_pgtable_v2.c
@@ -251,7 +251,7 @@ static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
 
        while (mapped_size < size) {
                map_size = get_alloc_page_size(pgsize);
-               pte = v2_alloc_pte(pdom->nid, pdom->iop.pgd,
+               pte = v2_alloc_pte(cfg->amd.nid, pdom->iop.pgd,
                                   iova, map_size, gfp, &updated);
                if (!pte) {
                        ret = -EINVAL;
@@ -359,10 +359,9 @@ static void v2_free_pgtable(struct io_pgtable *iop)
 static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
 {
        struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
-       struct protection_domain *pdom = (struct protection_domain *)cookie;
        int ias = IOMMU_IN_ADDR_BIT_SIZE;
 
-       pgtable->pgd = iommu_alloc_page_node(pdom->nid, GFP_KERNEL);
+       pgtable->pgd = iommu_alloc_page_node(cfg->amd.nid, GFP_KERNEL);
        if (!pgtable->pgd)
                return NULL;
 

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index b63f0d1bb3251c737826b1026f2c282cf2b3965c..554179f7de29eba566c8a7e41f7a20a715f8e7ec 100644 (file)
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2030,6 +2030,7 @@ static int do_attach(struct iommu_dev_data *dev_data,
                     struct protection_domain *domain)
 {
        struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);
+       struct io_pgtable_cfg *cfg = &domain->iop.pgtbl.cfg;
        int ret = 0;
 
        /* Update data structures */
@@ -2037,8 +2038,8 @@ static int do_attach(struct iommu_dev_data *dev_data,
        list_add(&dev_data->list, &domain->dev_list);
 
        /* Update NUMA Node ID */
-       if (domain->nid == NUMA_NO_NODE)
-               domain->nid = dev_to_node(dev_data->dev);
+       if (cfg->amd.nid == NUMA_NO_NODE)
+               cfg->amd.nid = dev_to_node(dev_data->dev);
 
        /* Do reference counting */
        domain->dev_iommu[iommu->index] += 1;
@@ -2273,7 +2274,7 @@ void protection_domain_free(struct protection_domain *domain)
        kfree(domain);
 }
 
-struct protection_domain *protection_domain_alloc(unsigned int type)
+struct protection_domain *protection_domain_alloc(unsigned int type, int nid)
 {
        struct io_pgtable_ops *pgtbl_ops;
        struct protection_domain *domain;
@@ -2290,7 +2291,7 @@ struct protection_domain *protection_domain_alloc(unsigned int type)
        spin_lock_init(&domain->lock);
        INIT_LIST_HEAD(&domain->dev_list);
        INIT_LIST_HEAD(&domain->dev_data_list);
-       domain->nid = NUMA_NO_NODE;
+       domain->iop.pgtbl.cfg.amd.nid = nid;
 
        switch (type) {
        /* No need to allocate io pgtable ops in passthrough mode */
@@ -2366,7 +2367,8 @@ static struct iommu_domain *do_iommu_domain_alloc(unsigned int type,
        if (dirty_tracking && !amd_iommu_hd_support(iommu))
                return ERR_PTR(-EOPNOTSUPP);
 
-       domain = protection_domain_alloc(type);
+       domain = protection_domain_alloc(type,
+                                        dev ? dev_to_node(dev) : NUMA_NO_NODE);
        if (!domain)
                return ERR_PTR(-ENOMEM);
 

diff --git a/drivers/iommu/amd/pasid.c b/drivers/iommu/amd/pasid.c
index a68215f2b3e1d6a4a422bffa9d2081ee3a3bed20..0657b9373be5475fdcd765048d3f359106fd8b2b 100644 (file)
--- a/drivers/iommu/amd/pasid.c
+++ b/drivers/iommu/amd/pasid.c
@@ -181,7 +181,7 @@ struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev,
        struct protection_domain *pdom;
        int ret;
 
-       pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA);
+       pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA, dev_to_node(dev));
        if (!pdom)
                return ERR_PTR(-ENOMEM);
 

diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index f9a81761bfceda1a3b5175c661d20e7de76b88ab..b1ecfc3cd5bcc07569b51271df91bc2610267ef5 100644 (file)
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -171,6 +171,10 @@ struct io_pgtable_cfg {
                        u64 ttbr[4];
                        u32 n_ttbrs;
                } apple_dart_cfg;
+
+               struct {
+                       int nid;
+               } amd;
        };
 };