iommu/amd: Enable support for up to 2K interrupts per function

AMD IOMMU optionally supports up to 2K interrupts per function on newer
platforms. Support for this feature is indicated through Extended
Feature 2 Register (MMIO Offset 01A0h[NumIntRemapSup]). Allocate 2K IRTEs
per device when this support is available.

Co-developed-by: Sairaj Kodilkar <sarunkod@amd.com>
Signed-off-by: Sairaj Kodilkar <sarunkod@amd.com>
Signed-off-by: Kishon Vijay Abraham I <kvijayab@amd.com>
Reviewed-by: Vasant Hegde <vasant.hegde@amd.com>
Link: https://lore.kernel.org/r/20250307095822.2274-5-sarunkod@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index 433eca602a4eef9c4b951b365f560ef5b897884b..5089b58e528a7bf1ca100cdede21c8ccd3b5152e 100644 (file)
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -112,6 +112,10 @@
 #define FEATURE_SNPAVICSUP_GAM(x) \
        (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
 
+#define FEATURE_NUM_INT_REMAP_SUP      GENMASK_ULL(9, 8)
+#define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
+       (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
+
 /* Note:
  * The current driver only support 16-bit PASID.
  * Currently, hardware only implement upto 16-bit PASID
@@ -175,6 +179,9 @@
 #define CONTROL_GAM_EN         25
 #define CONTROL_GALOG_EN       28
 #define CONTROL_GAINT_EN       29
+#define CONTROL_NUM_INT_REMAP_MODE     43
+#define CONTROL_NUM_INT_REMAP_MODE_MASK        0x03
+#define CONTROL_NUM_INT_REMAP_MODE_2K  0x01
 #define CONTROL_EPH_EN         45
 #define CONTROL_XT_EN          50
 #define CONTROL_INTCAPXT_EN    51
@@ -309,14 +316,13 @@
 #define DTE_IRQ_REMAP_INTCTL    (2ULL << 60)
 #define DTE_IRQ_REMAP_ENABLE    1ULL
 
-/*
- * AMD IOMMU hardware only support 512 IRTEs despite
- * the architectural limitation of 2048 entries.
- */
 #define DTE_INTTABLEN_MASK      (0xfULL << 1)
 #define DTE_INTTABLEN_VALUE_512 9ULL
 #define DTE_INTTABLEN_512       (DTE_INTTABLEN_VALUE_512 << 1)
 #define MAX_IRQS_PER_TABLE_512  BIT(DTE_INTTABLEN_VALUE_512)
+#define DTE_INTTABLEN_VALUE_2K 11ULL
+#define DTE_INTTABLEN_2K       (DTE_INTTABLEN_VALUE_2K << 1)
+#define MAX_IRQS_PER_TABLE_2K  BIT(DTE_INTTABLEN_VALUE_2K)
 
 #define PAGE_MODE_NONE    0x00
 #define PAGE_MODE_1_LEVEL 0x01
@@ -847,6 +853,7 @@ struct iommu_dev_data {
        struct device *dev;
        u16 devid;                        /* PCI Device ID */
 
+       unsigned int max_irqs;            /* Maximum IRQs supported by device */
        u32 max_pasids;                   /* Max supported PASIDs */
        u32 flags;                        /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
        int ats_qdep;

diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index d91c04253fd3ca4c15d2538bc191c66de5c86853..dd9e26b7b71848dd5449316df1eb2ee31eecb67f 100644 (file)
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -1060,7 +1060,8 @@ static bool __copy_device_table(struct amd_iommu *iommu)
                int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;
                if (irq_v && (int_ctl || int_tab_len)) {
                        if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
-                           (int_tab_len != DTE_INTTABLEN_512)) {
+                           (int_tab_len != DTE_INTTABLEN_512 &&
+                            int_tab_len != DTE_INTTABLEN_2K)) {
                                pr_err("Wrong old irq remapping flag: %#x\n", devid);
                                memunmap(old_devtb);
                                return false;
@@ -2736,6 +2737,17 @@ static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
                iommu->irtcachedis_enabled ? "disabled" : "enabled");
 }
 
+static void iommu_enable_2k_int(struct amd_iommu *iommu)
+{
+       if (!FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
+               return;
+
+       iommu_feature_set(iommu,
+                         CONTROL_NUM_INT_REMAP_MODE_2K,
+                         CONTROL_NUM_INT_REMAP_MODE_MASK,
+                         CONTROL_NUM_INT_REMAP_MODE);
+}
+
 static void early_enable_iommu(struct amd_iommu *iommu)
 {
        iommu_disable(iommu);
@@ -2748,6 +2760,7 @@ static void early_enable_iommu(struct amd_iommu *iommu)
        iommu_enable_ga(iommu);
        iommu_enable_xt(iommu);
        iommu_enable_irtcachedis(iommu);
+       iommu_enable_2k_int(iommu);
        iommu_enable(iommu);
        amd_iommu_flush_all_caches(iommu);
 }
@@ -2804,6 +2817,7 @@ static void early_enable_iommus(void)
                        iommu_enable_ga(iommu);
                        iommu_enable_xt(iommu);
                        iommu_enable_irtcachedis(iommu);
+                       iommu_enable_2k_int(iommu);
                        iommu_set_device_table(iommu);
                        amd_iommu_flush_all_caches(iommu);
                }

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index d4559d555fa0a9911ba0d3bcbd53a30f60f9988d..02d016b04d3f9f271f7c2038727cd860ab6323dd 100644 (file)
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2394,8 +2394,14 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev)
        }
 
 out_err:
+
        iommu_completion_wait(iommu);
 
+       if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
+               dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;
+       else
+               dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;
+
        if (dev_is_pci(dev))
                pci_prepare_ats(to_pci_dev(dev), PAGE_SHIFT);
 
@@ -3076,6 +3082,13 @@ out:
        raw_spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
+static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)
+{
+       if (dev_data && dev_data->max_irqs == MAX_IRQS_PER_TABLE_2K)
+               return DTE_INTTABLEN_2K;
+       return DTE_INTTABLEN_512;
+}
+
 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,
                              struct irq_remap_table *table)
 {
@@ -3090,7 +3103,7 @@ static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,
        new &= ~DTE_IRQ_PHYS_ADDR_MASK;
        new |= iommu_virt_to_phys(table->table);
        new |= DTE_IRQ_REMAP_INTCTL;
-       new |= DTE_INTTABLEN_512;
+       new |= iommu_get_int_tablen(dev_data);
        new |= DTE_IRQ_REMAP_ENABLE;
        WRITE_ONCE(dte->data[2], new);
 
@@ -3171,13 +3184,14 @@ static inline size_t get_irq_table_size(unsigned int max_irqs)
 }
 
 static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu,
-                                              u16 devid, struct pci_dev *pdev)
+                                              u16 devid, struct pci_dev *pdev,
+                                              unsigned int max_irqs)
 {
        struct irq_remap_table *table = NULL;
        struct irq_remap_table *new_table = NULL;
        struct amd_iommu_pci_seg *pci_seg;
        unsigned long flags;
-       int order = get_order(get_irq_table_size(MAX_IRQS_PER_TABLE));
+       int order = get_order(get_irq_table_size(max_irqs));
        int nid = iommu && iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
        u16 alias;
 
@@ -3239,13 +3253,14 @@ out_unlock:
 }
 
 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,
-                          bool align, struct pci_dev *pdev)
+                          bool align, struct pci_dev *pdev,
+                          unsigned long max_irqs)
 {
        struct irq_remap_table *table;
        int index, c, alignment = 1;
        unsigned long flags;
 
-       table = alloc_irq_table(iommu, devid, pdev);
+       table = alloc_irq_table(iommu, devid, pdev, max_irqs);
        if (!table)
                return -ENODEV;
 
@@ -3256,7 +3271,7 @@ static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,
 
        /* Scan table for free entries */
        for (index = ALIGN(table->min_index, alignment), c = 0;
-            index < MAX_IRQS_PER_TABLE;) {
+            index < max_irqs;) {
                if (!iommu->irte_ops->is_allocated(table, index)) {
                        c += 1;
                } else {
@@ -3526,6 +3541,14 @@ static void fill_msi_msg(struct msi_msg *msg, u32 index)
        msg->data = index;
        msg->address_lo = 0;
        msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
+       /*
+        * The struct msi_msg.dest_mode_logical is used to set the DM bit
+        * in MSI Message Address Register. For device w/ 2K int-remap support,
+        * this is bit must be set to 1 regardless of the actual destination
+        * mode, which is signified by the IRTE[DM].
+        */
+       if (FEATURE_NUM_INT_REMAP_SUP_2K(amd_iommu_efr2))
+               msg->arch_addr_lo.dest_mode_logical = true;
        msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
 }
 
@@ -3588,6 +3611,8 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
        struct amd_ir_data *data = NULL;
        struct amd_iommu *iommu;
        struct irq_cfg *cfg;
+       struct iommu_dev_data *dev_data;
+       unsigned long max_irqs;
        int i, ret, devid, seg, sbdf;
        int index;
 
@@ -3606,6 +3631,9 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
        if (!iommu)
                return -EINVAL;
 
+       dev_data = search_dev_data(iommu, devid);
+       max_irqs = dev_data ? dev_data->max_irqs : MAX_IRQS_PER_TABLE_512;
+
        ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
        if (ret < 0)
                return ret;
@@ -3613,7 +3641,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
        if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) {
                struct irq_remap_table *table;
 
-               table = alloc_irq_table(iommu, devid, NULL);
+               table = alloc_irq_table(iommu, devid, NULL, max_irqs);
                if (table) {
                        if (!table->min_index) {
                                /*
@@ -3634,9 +3662,11 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
                bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);
 
                index = alloc_irq_index(iommu, devid, nr_irqs, align,
-                                       msi_desc_to_pci_dev(info->desc));
+                                       msi_desc_to_pci_dev(info->desc),
+                                       max_irqs);
        } else {
-               index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);
+               index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL,
+                                       max_irqs);
        }
 
        if (index < 0) {