1 From: Suresh Siddha <suresh.b.siddha@intel.com>
2 Subject: x64, x2apic/intr-remap: Interrupt remapping infrastructure
3 References: fate #303948 and fate #303984
4 Patch-Mainline: queued for .28
5 Commit-ID: 2ae21010694e56461a63bfc80e960090ce0a5ed9
7 Signed-off-by: Thomas Renninger <trenn@suse.de>
9 Interrupt remapping (part of Intel Virtualization Tech for directed I/O)
12 Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
13 Cc: akpm@linux-foundation.org
14 Cc: arjan@linux.intel.com
15 Cc: andi@firstfloor.org
16 Cc: ebiederm@xmission.com
17 Cc: jbarnes@virtuousgeek.org
19 Signed-off-by: Ingo Molnar <mingo@elte.hu>
22 drivers/pci/dma_remapping.h | 2
23 drivers/pci/dmar.c | 16 +++++
24 drivers/pci/intel-iommu.c | 25 ++-----
25 drivers/pci/intel-iommu.h | 24 ++++++-
26 drivers/pci/intr_remapping.c | 137 +++++++++++++++++++++++++++++++++++++++++++
27 drivers/pci/intr_remapping.h | 2
28 include/linux/dmar.h | 120 ++++++++++++++++++++++++++-----------
29 7 files changed, 272 insertions(+), 54 deletions(-)
31 --- a/drivers/pci/dmar.c
32 +++ b/drivers/pci/dmar.c
33 @@ -451,6 +451,22 @@ int __init early_dmar_detect(void)
34 return (ACPI_SUCCESS(status) ? 1 : 0);
37 +void __init detect_intel_iommu(void)
41 + ret = early_dmar_detect();
45 + if (ret && !no_iommu && !iommu_detected && !swiotlb &&
53 int alloc_iommu(struct dmar_drhd_unit *drhd)
55 struct intel_iommu *iommu;
56 --- a/drivers/pci/dma_remapping.h
57 +++ b/drivers/pci/dma_remapping.h
58 @@ -145,6 +145,8 @@ struct device_domain_info {
59 extern int init_dmars(void);
60 extern void free_dmar_iommu(struct intel_iommu *iommu);
62 +extern int dmar_disabled;
64 #ifndef CONFIG_DMAR_GFX_WA
65 static inline void iommu_prepare_gfx_mapping(void)
67 --- a/drivers/pci/intel-iommu.c
68 +++ b/drivers/pci/intel-iommu.c
69 @@ -78,7 +78,7 @@ static long list_size;
71 static void domain_remove_dev_info(struct dmar_domain *domain);
73 -static int dmar_disabled;
75 static int __initdata dmar_map_gfx = 1;
76 static int dmar_forcedac;
77 static int intel_iommu_strict;
78 @@ -2259,19 +2259,6 @@ static struct dmi_system_id __initdata i
83 -void __init detect_intel_iommu(void)
85 - if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
87 - if (early_dmar_detect()) {
88 - dmi_check_system(intel_iommu_dmi_table);
95 static void __init init_no_remapping_devices(void)
97 struct dmar_drhd_unit *drhd;
98 @@ -2318,15 +2305,19 @@ int __init intel_iommu_init(void)
102 - if (no_iommu || swiotlb || dmar_disabled)
105 if (dmar_table_init())
108 if (dmar_dev_scope_init())
112 + * Check the need for DMA-remapping initialization now.
113 + * Above initialization will also be used by Interrupt-remapping.
115 + if (no_iommu || swiotlb || dmar_disabled)
118 iommu_init_mempool();
119 dmar_init_reserved_ranges();
121 --- a/drivers/pci/intel-iommu.h
122 +++ b/drivers/pci/intel-iommu.h
124 #define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */
125 #define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */
126 #define DMAR_ICS_REG 0x98 /* Invalidation complete status register */
127 +#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */
129 #define OFFSET_STRIDE (9)
131 @@ -157,16 +158,20 @@ static inline void dmar_writeq(void __io
132 #define DMA_GCMD_SRTP (((u32)1) << 30)
133 #define DMA_GCMD_SFL (((u32)1) << 29)
134 #define DMA_GCMD_EAFL (((u32)1) << 28)
135 -#define DMA_GCMD_QIE (((u32)1) << 26)
136 #define DMA_GCMD_WBF (((u32)1) << 27)
137 +#define DMA_GCMD_QIE (((u32)1) << 26)
138 +#define DMA_GCMD_SIRTP (((u32)1) << 24)
139 +#define DMA_GCMD_IRE (((u32) 1) << 25)
142 #define DMA_GSTS_TES (((u32)1) << 31)
143 #define DMA_GSTS_RTPS (((u32)1) << 30)
144 #define DMA_GSTS_FLS (((u32)1) << 29)
145 #define DMA_GSTS_AFLS (((u32)1) << 28)
146 -#define DMA_GSTS_QIES (((u32)1) << 26)
147 #define DMA_GSTS_WBFS (((u32)1) << 27)
148 +#define DMA_GSTS_QIES (((u32)1) << 26)
149 +#define DMA_GSTS_IRTPS (((u32)1) << 24)
150 +#define DMA_GSTS_IRES (((u32)1) << 25)
153 #define DMA_CCMD_ICC (((u64)1) << 63)
154 @@ -245,6 +250,16 @@ struct q_inval {
158 +#ifdef CONFIG_INTR_REMAP
159 +/* 1MB - maximum possible interrupt remapping table size */
160 +#define INTR_REMAP_PAGE_ORDER 8
161 +#define INTR_REMAP_TABLE_REG_SIZE 0xf
169 void __iomem *reg; /* Pointer to hardware regs, virtual addr */
171 @@ -266,6 +281,9 @@ struct intel_iommu {
172 struct sys_device sysdev;
174 struct q_inval *qi; /* Queued invalidation info */
175 +#ifdef CONFIG_INTR_REMAP
176 + struct ir_table *ir_table; /* Interrupt remapping info */
180 static inline void __iommu_flush_cache(
181 @@ -279,5 +297,7 @@ extern struct dmar_drhd_unit * dmar_find
183 extern int alloc_iommu(struct dmar_drhd_unit *drhd);
184 extern void free_iommu(struct intel_iommu *iommu);
185 +extern int dmar_enable_qi(struct intel_iommu *iommu);
186 +extern void qi_global_iec(struct intel_iommu *iommu);
189 --- a/drivers/pci/intr_remapping.c
190 +++ b/drivers/pci/intr_remapping.c
192 #include <linux/dmar.h>
193 +#include <linux/spinlock.h>
194 +#include <linux/jiffies.h>
195 +#include <linux/pci.h>
196 #include <asm/io_apic.h>
197 #include "intel-iommu.h"
198 #include "intr_remapping.h"
200 static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
201 static int ir_ioapic_num;
202 +int intr_remapping_enabled;
204 +static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
208 + unsigned long flags;
210 + addr = virt_to_phys((void *)iommu->ir_table->base);
212 + spin_lock_irqsave(&iommu->register_lock, flags);
214 + dmar_writeq(iommu->reg + DMAR_IRTA_REG,
215 + (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
217 + /* Set interrupt-remapping table pointer */
218 + cmd = iommu->gcmd | DMA_GCMD_SIRTP;
219 + writel(cmd, iommu->reg + DMAR_GCMD_REG);
221 + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
222 + readl, (sts & DMA_GSTS_IRTPS), sts);
223 + spin_unlock_irqrestore(&iommu->register_lock, flags);
226 + * global invalidation of interrupt entry cache before enabling
227 + * interrupt-remapping.
229 + qi_global_iec(iommu);
231 + spin_lock_irqsave(&iommu->register_lock, flags);
233 + /* Enable interrupt-remapping */
234 + cmd = iommu->gcmd | DMA_GCMD_IRE;
235 + iommu->gcmd |= DMA_GCMD_IRE;
236 + writel(cmd, iommu->reg + DMAR_GCMD_REG);
238 + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
239 + readl, (sts & DMA_GSTS_IRES), sts);
241 + spin_unlock_irqrestore(&iommu->register_lock, flags);
245 +static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
247 + struct ir_table *ir_table;
248 + struct page *pages;
250 + ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
253 + if (!iommu->ir_table)
256 + pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
259 + printk(KERN_ERR "failed to allocate pages of order %d\n",
260 + INTR_REMAP_PAGE_ORDER);
261 + kfree(iommu->ir_table);
265 + ir_table->base = page_address(pages);
267 + iommu_set_intr_remapping(iommu, mode);
271 +int __init enable_intr_remapping(int eim)
273 + struct dmar_drhd_unit *drhd;
277 + * check for the Interrupt-remapping support
279 + for_each_drhd_unit(drhd) {
280 + struct intel_iommu *iommu = drhd->iommu;
282 + if (!ecap_ir_support(iommu->ecap))
285 + if (eim && !ecap_eim_support(iommu->ecap)) {
286 + printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
287 + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
293 + * Enable queued invalidation for all the DRHD's.
295 + for_each_drhd_unit(drhd) {
297 + struct intel_iommu *iommu = drhd->iommu;
298 + ret = dmar_enable_qi(iommu);
301 + printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
302 + " invalidation, ecap %Lx, ret %d\n",
303 + drhd->reg_base_addr, iommu->ecap, ret);
309 + * Setup Interrupt-remapping for all the DRHD's now.
311 + for_each_drhd_unit(drhd) {
312 + struct intel_iommu *iommu = drhd->iommu;
314 + if (!ecap_ir_support(iommu->ecap))
317 + if (setup_intr_remapping(iommu, eim))
326 + intr_remapping_enabled = 1;
332 + * handle error condition gracefully here!
337 static int ir_parse_ioapic_scope(struct acpi_dmar_header *header,
338 struct intel_iommu *iommu)
339 --- a/drivers/pci/intr_remapping.h
340 +++ b/drivers/pci/intr_remapping.h
341 @@ -4,3 +4,5 @@ struct ioapic_scope {
342 struct intel_iommu *iommu;
346 +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
347 --- a/include/linux/dmar.h
348 +++ b/include/linux/dmar.h
350 #include <linux/types.h>
351 #include <linux/msi.h>
354 +#if defined(CONFIG_DMAR) || defined(CONFIG_INTR_REMAP)
357 +struct dmar_drhd_unit {
358 + struct list_head list; /* list of drhd units */
359 + struct acpi_dmar_header *hdr; /* ACPI header */
360 + u64 reg_base_addr; /* register base address*/
361 + struct pci_dev **devices; /* target device array */
362 + int devices_cnt; /* target device count */
363 + u8 ignored:1; /* ignore drhd */
365 + struct intel_iommu *iommu;
368 +extern struct list_head dmar_drhd_units;
370 +#define for_each_drhd_unit(drhd) \
371 + list_for_each_entry(drhd, &dmar_drhd_units, list)
373 +extern int dmar_table_init(void);
374 +extern int early_dmar_detect(void);
375 +extern int dmar_dev_scope_init(void);
377 +/* Intel IOMMU detection */
378 +extern void detect_intel_iommu(void);
381 +extern int parse_ioapics_under_ir(void);
382 +extern int alloc_iommu(struct dmar_drhd_unit *);
384 +static inline void detect_intel_iommu(void)
389 +static inline int dmar_table_init(void)
393 +#endif /* !CONFIG_DMAR && !CONFIG_INTR_REMAP */
395 +#ifdef CONFIG_INTR_REMAP
396 +extern int intr_remapping_enabled;
397 +extern int enable_intr_remapping(int);
428 +#define enable_intr_remapping(mode) (-1)
429 +#define intr_remapping_enabled (0)
433 extern const char *dmar_get_fault_reason(u8 fault_reason);
435 /* Can't use the common MSI interrupt functions
436 @@ -40,29 +116,8 @@ extern void dmar_msi_write(int irq, stru
437 extern int dmar_set_interrupt(struct intel_iommu *iommu);
438 extern int arch_setup_dmar_msi(unsigned int irq);
440 -/* Intel IOMMU detection and initialization functions */
441 -extern void detect_intel_iommu(void);
442 -extern int intel_iommu_init(void);
444 -extern int dmar_table_init(void);
445 -extern int early_dmar_detect(void);
446 -extern int dmar_dev_scope_init(void);
447 -extern int parse_ioapics_under_ir(void);
449 -extern struct list_head dmar_drhd_units;
450 +extern int iommu_detected, no_iommu;
451 extern struct list_head dmar_rmrr_units;
453 -struct dmar_drhd_unit {
454 - struct list_head list; /* list of drhd units */
455 - struct acpi_dmar_header *hdr; /* ACPI header */
456 - u64 reg_base_addr; /* register base address*/
457 - struct pci_dev **devices; /* target device array */
458 - int devices_cnt; /* target device count */
459 - u8 ignored:1; /* ignore drhd */
461 - struct intel_iommu *iommu;
464 struct dmar_rmrr_unit {
465 struct list_head list; /* list of rmrr units */
466 struct acpi_dmar_header *hdr; /* ACPI header */
467 @@ -72,24 +127,19 @@ struct dmar_rmrr_unit {
468 int devices_cnt; /* target device count */
471 -#define for_each_drhd_unit(drhd) \
472 - list_for_each_entry(drhd, &dmar_drhd_units, list)
473 #define for_each_rmrr_units(rmrr) \
474 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
476 -extern int alloc_iommu(struct dmar_drhd_unit *);
477 +/* Intel DMAR initialization functions */
478 +extern int intel_iommu_init(void);
479 +extern int dmar_disabled;
481 -static inline void detect_intel_iommu(void)
485 static inline int intel_iommu_init(void)
487 +#ifdef CONFIG_INTR_REMAP
488 + return dmar_dev_scope_init();
492 -static inline int dmar_table_init(void)
497 #endif /* !CONFIG_DMAR */
498 #endif /* __DMAR_H__ */