]>
Commit | Line | Data |
---|---|---|
2cb7cef9 BS |
1 | From: Suresh Siddha <suresh.b.siddha@intel.com> |
2 | Subject: x64, x2apic/intr-remap: Interrupt remapping infrastructure | |
3 | References: fate #303948 and fate #303984 | |
4 | Patch-Mainline: queued for .28 | |
5 | Commit-ID: 2ae21010694e56461a63bfc80e960090ce0a5ed9 | |
6 | ||
7 | Signed-off-by: Thomas Renninger <trenn@suse.de> | |
8 | ||
9 | Interrupt remapping (part of Intel Virtualization Tech for directed I/O) | |
10 | infrastructure. | |
11 | ||
12 | Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com> | |
13 | Cc: akpm@linux-foundation.org | |
14 | Cc: arjan@linux.intel.com | |
15 | Cc: andi@firstfloor.org | |
16 | Cc: ebiederm@xmission.com | |
17 | Cc: jbarnes@virtuousgeek.org | |
18 | Cc: steiner@sgi.com | |
19 | Signed-off-by: Ingo Molnar <mingo@elte.hu> | |
20 | ||
21 | --- | |
22 | drivers/pci/dma_remapping.h | 2 | |
23 | drivers/pci/dmar.c | 16 +++++ | |
24 | drivers/pci/intel-iommu.c | 25 ++----- | |
25 | drivers/pci/intel-iommu.h | 24 ++++++- | |
26 | drivers/pci/intr_remapping.c | 137 +++++++++++++++++++++++++++++++++++++++++++ | |
27 | drivers/pci/intr_remapping.h | 2 | |
28 | include/linux/dmar.h | 120 ++++++++++++++++++++++++++----------- | |
29 | 7 files changed, 272 insertions(+), 54 deletions(-) | |
30 | ||
31 | --- a/drivers/pci/dmar.c | |
32 | +++ b/drivers/pci/dmar.c | |
33 | @@ -451,6 +451,22 @@ int __init early_dmar_detect(void) | |
34 | return (ACPI_SUCCESS(status) ? 1 : 0); | |
35 | } | |
36 | ||
37 | +void __init detect_intel_iommu(void) | |
38 | +{ | |
39 | + int ret; | |
40 | + | |
41 | + ret = early_dmar_detect(); | |
42 | + | |
43 | +#ifdef CONFIG_DMAR | |
44 | + { | |
45 | + if (ret && !no_iommu && !iommu_detected && !swiotlb && | |
46 | + !dmar_disabled) | |
47 | + iommu_detected = 1; | |
48 | + } | |
49 | +#endif | |
50 | +} | |
51 | + | |
52 | + | |
53 | int alloc_iommu(struct dmar_drhd_unit *drhd) | |
54 | { | |
55 | struct intel_iommu *iommu; | |
56 | --- a/drivers/pci/dma_remapping.h | |
57 | +++ b/drivers/pci/dma_remapping.h | |
58 | @@ -145,6 +145,8 @@ struct device_domain_info { | |
59 | extern int init_dmars(void); | |
60 | extern void free_dmar_iommu(struct intel_iommu *iommu); | |
61 | ||
62 | +extern int dmar_disabled; | |
63 | + | |
64 | #ifndef CONFIG_DMAR_GFX_WA | |
65 | static inline void iommu_prepare_gfx_mapping(void) | |
66 | { | |
67 | --- a/drivers/pci/intel-iommu.c | |
68 | +++ b/drivers/pci/intel-iommu.c | |
69 | @@ -78,7 +78,7 @@ static long list_size; | |
70 | ||
71 | static void domain_remove_dev_info(struct dmar_domain *domain); | |
72 | ||
73 | -static int dmar_disabled; | |
74 | +int dmar_disabled; | |
75 | static int __initdata dmar_map_gfx = 1; | |
76 | static int dmar_forcedac; | |
77 | static int intel_iommu_strict; | |
78 | @@ -2259,19 +2259,6 @@ static struct dmi_system_id __initdata i | |
79 | { } | |
80 | }; | |
81 | ||
82 | - | |
83 | -void __init detect_intel_iommu(void) | |
84 | -{ | |
85 | - if (swiotlb || no_iommu || iommu_detected || dmar_disabled) | |
86 | - return; | |
87 | - if (early_dmar_detect()) { | |
88 | - dmi_check_system(intel_iommu_dmi_table); | |
89 | - if (dmar_disabled) | |
90 | - return; | |
91 | - iommu_detected = 1; | |
92 | - } | |
93 | -} | |
94 | - | |
95 | static void __init init_no_remapping_devices(void) | |
96 | { | |
97 | struct dmar_drhd_unit *drhd; | |
98 | @@ -2318,15 +2305,19 @@ int __init intel_iommu_init(void) | |
99 | { | |
100 | int ret = 0; | |
101 | ||
102 | - if (no_iommu || swiotlb || dmar_disabled) | |
103 | - return -ENODEV; | |
104 | - | |
105 | if (dmar_table_init()) | |
106 | return -ENODEV; | |
107 | ||
108 | if (dmar_dev_scope_init()) | |
109 | return -ENODEV; | |
110 | ||
111 | + /* | |
112 | + * Check the need for DMA-remapping initialization now. | |
113 | + * Above initialization will also be used by Interrupt-remapping. | |
114 | + */ | |
115 | + if (no_iommu || swiotlb || dmar_disabled) | |
116 | + return -ENODEV; | |
117 | + | |
118 | iommu_init_mempool(); | |
119 | dmar_init_reserved_ranges(); | |
120 | ||
121 | --- a/drivers/pci/intel-iommu.h | |
122 | +++ b/drivers/pci/intel-iommu.h | |
123 | @@ -56,6 +56,7 @@ | |
124 | #define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */ | |
125 | #define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */ | |
126 | #define DMAR_ICS_REG 0x98 /* Invalidation complete status register */ | |
127 | +#define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */ | |
128 | ||
129 | #define OFFSET_STRIDE (9) | |
130 | /* | |
131 | @@ -157,16 +158,20 @@ static inline void dmar_writeq(void __io | |
132 | #define DMA_GCMD_SRTP (((u32)1) << 30) | |
133 | #define DMA_GCMD_SFL (((u32)1) << 29) | |
134 | #define DMA_GCMD_EAFL (((u32)1) << 28) | |
135 | -#define DMA_GCMD_QIE (((u32)1) << 26) | |
136 | #define DMA_GCMD_WBF (((u32)1) << 27) | |
137 | +#define DMA_GCMD_QIE (((u32)1) << 26) | |
138 | +#define DMA_GCMD_SIRTP (((u32)1) << 24) | |
139 | +#define DMA_GCMD_IRE (((u32) 1) << 25) | |
140 | ||
141 | /* GSTS_REG */ | |
142 | #define DMA_GSTS_TES (((u32)1) << 31) | |
143 | #define DMA_GSTS_RTPS (((u32)1) << 30) | |
144 | #define DMA_GSTS_FLS (((u32)1) << 29) | |
145 | #define DMA_GSTS_AFLS (((u32)1) << 28) | |
146 | -#define DMA_GSTS_QIES (((u32)1) << 26) | |
147 | #define DMA_GSTS_WBFS (((u32)1) << 27) | |
148 | +#define DMA_GSTS_QIES (((u32)1) << 26) | |
149 | +#define DMA_GSTS_IRTPS (((u32)1) << 24) | |
150 | +#define DMA_GSTS_IRES (((u32)1) << 25) | |
151 | ||
152 | /* CCMD_REG */ | |
153 | #define DMA_CCMD_ICC (((u64)1) << 63) | |
154 | @@ -245,6 +250,16 @@ struct q_inval { | |
155 | int free_cnt; | |
156 | }; | |
157 | ||
158 | +#ifdef CONFIG_INTR_REMAP | |
159 | +/* 1MB - maximum possible interrupt remapping table size */ | |
160 | +#define INTR_REMAP_PAGE_ORDER 8 | |
161 | +#define INTR_REMAP_TABLE_REG_SIZE 0xf | |
162 | + | |
163 | +struct ir_table { | |
164 | + struct irte *base; | |
165 | +}; | |
166 | +#endif | |
167 | + | |
168 | struct intel_iommu { | |
169 | void __iomem *reg; /* Pointer to hardware regs, virtual addr */ | |
170 | u64 cap; | |
171 | @@ -266,6 +281,9 @@ struct intel_iommu { | |
172 | struct sys_device sysdev; | |
173 | #endif | |
174 | struct q_inval *qi; /* Queued invalidation info */ | |
175 | +#ifdef CONFIG_INTR_REMAP | |
176 | + struct ir_table *ir_table; /* Interrupt remapping info */ | |
177 | +#endif | |
178 | }; | |
179 | ||
180 | static inline void __iommu_flush_cache( | |
181 | @@ -279,5 +297,7 @@ extern struct dmar_drhd_unit * dmar_find | |
182 | ||
183 | extern int alloc_iommu(struct dmar_drhd_unit *drhd); | |
184 | extern void free_iommu(struct intel_iommu *iommu); | |
185 | +extern int dmar_enable_qi(struct intel_iommu *iommu); | |
186 | +extern void qi_global_iec(struct intel_iommu *iommu); | |
187 | ||
188 | #endif | |
189 | --- a/drivers/pci/intr_remapping.c | |
190 | +++ b/drivers/pci/intr_remapping.c | |
191 | @@ -1,10 +1,147 @@ | |
192 | #include <linux/dmar.h> | |
193 | +#include <linux/spinlock.h> | |
194 | +#include <linux/jiffies.h> | |
195 | +#include <linux/pci.h> | |
196 | #include <asm/io_apic.h> | |
197 | #include "intel-iommu.h" | |
198 | #include "intr_remapping.h" | |
199 | ||
200 | static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; | |
201 | static int ir_ioapic_num; | |
202 | +int intr_remapping_enabled; | |
203 | + | |
204 | +static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode) | |
205 | +{ | |
206 | + u64 addr; | |
207 | + u32 cmd, sts; | |
208 | + unsigned long flags; | |
209 | + | |
210 | + addr = virt_to_phys((void *)iommu->ir_table->base); | |
211 | + | |
212 | + spin_lock_irqsave(&iommu->register_lock, flags); | |
213 | + | |
214 | + dmar_writeq(iommu->reg + DMAR_IRTA_REG, | |
215 | + (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); | |
216 | + | |
217 | + /* Set interrupt-remapping table pointer */ | |
218 | + cmd = iommu->gcmd | DMA_GCMD_SIRTP; | |
219 | + writel(cmd, iommu->reg + DMAR_GCMD_REG); | |
220 | + | |
221 | + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, | |
222 | + readl, (sts & DMA_GSTS_IRTPS), sts); | |
223 | + spin_unlock_irqrestore(&iommu->register_lock, flags); | |
224 | + | |
225 | + /* | |
226 | + * global invalidation of interrupt entry cache before enabling | |
227 | + * interrupt-remapping. | |
228 | + */ | |
229 | + qi_global_iec(iommu); | |
230 | + | |
231 | + spin_lock_irqsave(&iommu->register_lock, flags); | |
232 | + | |
233 | + /* Enable interrupt-remapping */ | |
234 | + cmd = iommu->gcmd | DMA_GCMD_IRE; | |
235 | + iommu->gcmd |= DMA_GCMD_IRE; | |
236 | + writel(cmd, iommu->reg + DMAR_GCMD_REG); | |
237 | + | |
238 | + IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, | |
239 | + readl, (sts & DMA_GSTS_IRES), sts); | |
240 | + | |
241 | + spin_unlock_irqrestore(&iommu->register_lock, flags); | |
242 | +} | |
243 | + | |
244 | + | |
245 | +static int setup_intr_remapping(struct intel_iommu *iommu, int mode) | |
246 | +{ | |
247 | + struct ir_table *ir_table; | |
248 | + struct page *pages; | |
249 | + | |
250 | + ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table), | |
251 | + GFP_KERNEL); | |
252 | + | |
253 | + if (!iommu->ir_table) | |
254 | + return -ENOMEM; | |
255 | + | |
256 | + pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER); | |
257 | + | |
258 | + if (!pages) { | |
259 | + printk(KERN_ERR "failed to allocate pages of order %d\n", | |
260 | + INTR_REMAP_PAGE_ORDER); | |
261 | + kfree(iommu->ir_table); | |
262 | + return -ENOMEM; | |
263 | + } | |
264 | + | |
265 | + ir_table->base = page_address(pages); | |
266 | + | |
267 | + iommu_set_intr_remapping(iommu, mode); | |
268 | + return 0; | |
269 | +} | |
270 | + | |
271 | +int __init enable_intr_remapping(int eim) | |
272 | +{ | |
273 | + struct dmar_drhd_unit *drhd; | |
274 | + int setup = 0; | |
275 | + | |
276 | + /* | |
277 | + * check for the Interrupt-remapping support | |
278 | + */ | |
279 | + for_each_drhd_unit(drhd) { | |
280 | + struct intel_iommu *iommu = drhd->iommu; | |
281 | + | |
282 | + if (!ecap_ir_support(iommu->ecap)) | |
283 | + continue; | |
284 | + | |
285 | + if (eim && !ecap_eim_support(iommu->ecap)) { | |
286 | + printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " | |
287 | + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); | |
288 | + return -1; | |
289 | + } | |
290 | + } | |
291 | + | |
292 | + /* | |
293 | + * Enable queued invalidation for all the DRHD's. | |
294 | + */ | |
295 | + for_each_drhd_unit(drhd) { | |
296 | + int ret; | |
297 | + struct intel_iommu *iommu = drhd->iommu; | |
298 | + ret = dmar_enable_qi(iommu); | |
299 | + | |
300 | + if (ret) { | |
301 | + printk(KERN_ERR "DRHD %Lx: failed to enable queued, " | |
302 | + " invalidation, ecap %Lx, ret %d\n", | |
303 | + drhd->reg_base_addr, iommu->ecap, ret); | |
304 | + return -1; | |
305 | + } | |
306 | + } | |
307 | + | |
308 | + /* | |
309 | + * Setup Interrupt-remapping for all the DRHD's now. | |
310 | + */ | |
311 | + for_each_drhd_unit(drhd) { | |
312 | + struct intel_iommu *iommu = drhd->iommu; | |
313 | + | |
314 | + if (!ecap_ir_support(iommu->ecap)) | |
315 | + continue; | |
316 | + | |
317 | + if (setup_intr_remapping(iommu, eim)) | |
318 | + goto error; | |
319 | + | |
320 | + setup = 1; | |
321 | + } | |
322 | + | |
323 | + if (!setup) | |
324 | + goto error; | |
325 | + | |
326 | + intr_remapping_enabled = 1; | |
327 | + | |
328 | + return 0; | |
329 | + | |
330 | +error: | |
331 | + /* | |
332 | + * handle error condition gracefully here! | |
333 | + */ | |
334 | + return -1; | |
335 | +} | |
336 | ||
337 | static int ir_parse_ioapic_scope(struct acpi_dmar_header *header, | |
338 | struct intel_iommu *iommu) | |
339 | --- a/drivers/pci/intr_remapping.h | |
340 | +++ b/drivers/pci/intr_remapping.h | |
341 | @@ -4,3 +4,5 @@ struct ioapic_scope { | |
342 | struct intel_iommu *iommu; | |
343 | unsigned int id; | |
344 | }; | |
345 | + | |
346 | +#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) | |
347 | --- a/include/linux/dmar.h | |
348 | +++ b/include/linux/dmar.h | |
349 | @@ -25,9 +25,85 @@ | |
350 | #include <linux/types.h> | |
351 | #include <linux/msi.h> | |
352 | ||
353 | -#ifdef CONFIG_DMAR | |
354 | +#if defined(CONFIG_DMAR) || defined(CONFIG_INTR_REMAP) | |
355 | struct intel_iommu; | |
356 | ||
357 | +struct dmar_drhd_unit { | |
358 | + struct list_head list; /* list of drhd units */ | |
359 | + struct acpi_dmar_header *hdr; /* ACPI header */ | |
360 | + u64 reg_base_addr; /* register base address*/ | |
361 | + struct pci_dev **devices; /* target device array */ | |
362 | + int devices_cnt; /* target device count */ | |
363 | + u8 ignored:1; /* ignore drhd */ | |
364 | + u8 include_all:1; | |
365 | + struct intel_iommu *iommu; | |
366 | +}; | |
367 | + | |
368 | +extern struct list_head dmar_drhd_units; | |
369 | + | |
370 | +#define for_each_drhd_unit(drhd) \ | |
371 | + list_for_each_entry(drhd, &dmar_drhd_units, list) | |
372 | + | |
373 | +extern int dmar_table_init(void); | |
374 | +extern int early_dmar_detect(void); | |
375 | +extern int dmar_dev_scope_init(void); | |
376 | + | |
377 | +/* Intel IOMMU detection */ | |
378 | +extern void detect_intel_iommu(void); | |
379 | + | |
380 | + | |
381 | +extern int parse_ioapics_under_ir(void); | |
382 | +extern int alloc_iommu(struct dmar_drhd_unit *); | |
383 | +#else | |
384 | +static inline void detect_intel_iommu(void) | |
385 | +{ | |
386 | + return; | |
387 | +} | |
388 | + | |
389 | +static inline int dmar_table_init(void) | |
390 | +{ | |
391 | + return -ENODEV; | |
392 | +} | |
393 | +#endif /* !CONFIG_DMAR && !CONFIG_INTR_REMAP */ | |
394 | + | |
395 | +#ifdef CONFIG_INTR_REMAP | |
396 | +extern int intr_remapping_enabled; | |
397 | +extern int enable_intr_remapping(int); | |
398 | + | |
399 | +struct irte { | |
400 | + union { | |
401 | + struct { | |
402 | + __u64 present : 1, | |
403 | + fpd : 1, | |
404 | + dst_mode : 1, | |
405 | + redir_hint : 1, | |
406 | + trigger_mode : 1, | |
407 | + dlvry_mode : 3, | |
408 | + avail : 4, | |
409 | + __reserved_1 : 4, | |
410 | + vector : 8, | |
411 | + __reserved_2 : 8, | |
412 | + dest_id : 32; | |
413 | + }; | |
414 | + __u64 low; | |
415 | + }; | |
416 | + | |
417 | + union { | |
418 | + struct { | |
419 | + __u64 sid : 16, | |
420 | + sq : 2, | |
421 | + svt : 2, | |
422 | + __reserved_3 : 44; | |
423 | + }; | |
424 | + __u64 high; | |
425 | + }; | |
426 | +}; | |
427 | +#else | |
428 | +#define enable_intr_remapping(mode) (-1) | |
429 | +#define intr_remapping_enabled (0) | |
430 | +#endif | |
431 | + | |
432 | +#ifdef CONFIG_DMAR | |
433 | extern const char *dmar_get_fault_reason(u8 fault_reason); | |
434 | ||
435 | /* Can't use the common MSI interrupt functions | |
436 | @@ -40,29 +116,8 @@ extern void dmar_msi_write(int irq, stru | |
437 | extern int dmar_set_interrupt(struct intel_iommu *iommu); | |
438 | extern int arch_setup_dmar_msi(unsigned int irq); | |
439 | ||
440 | -/* Intel IOMMU detection and initialization functions */ | |
441 | -extern void detect_intel_iommu(void); | |
442 | -extern int intel_iommu_init(void); | |
443 | - | |
444 | -extern int dmar_table_init(void); | |
445 | -extern int early_dmar_detect(void); | |
446 | -extern int dmar_dev_scope_init(void); | |
447 | -extern int parse_ioapics_under_ir(void); | |
448 | - | |
449 | -extern struct list_head dmar_drhd_units; | |
450 | +extern int iommu_detected, no_iommu; | |
451 | extern struct list_head dmar_rmrr_units; | |
452 | - | |
453 | -struct dmar_drhd_unit { | |
454 | - struct list_head list; /* list of drhd units */ | |
455 | - struct acpi_dmar_header *hdr; /* ACPI header */ | |
456 | - u64 reg_base_addr; /* register base address*/ | |
457 | - struct pci_dev **devices; /* target device array */ | |
458 | - int devices_cnt; /* target device count */ | |
459 | - u8 ignored:1; /* ignore drhd */ | |
460 | - u8 include_all:1; | |
461 | - struct intel_iommu *iommu; | |
462 | -}; | |
463 | - | |
464 | struct dmar_rmrr_unit { | |
465 | struct list_head list; /* list of rmrr units */ | |
466 | struct acpi_dmar_header *hdr; /* ACPI header */ | |
467 | @@ -72,24 +127,19 @@ struct dmar_rmrr_unit { | |
468 | int devices_cnt; /* target device count */ | |
469 | }; | |
470 | ||
471 | -#define for_each_drhd_unit(drhd) \ | |
472 | - list_for_each_entry(drhd, &dmar_drhd_units, list) | |
473 | #define for_each_rmrr_units(rmrr) \ | |
474 | list_for_each_entry(rmrr, &dmar_rmrr_units, list) | |
475 | - | |
476 | -extern int alloc_iommu(struct dmar_drhd_unit *); | |
477 | +/* Intel DMAR initialization functions */ | |
478 | +extern int intel_iommu_init(void); | |
479 | +extern int dmar_disabled; | |
480 | #else | |
481 | -static inline void detect_intel_iommu(void) | |
482 | -{ | |
483 | - return; | |
484 | -} | |
485 | static inline int intel_iommu_init(void) | |
486 | { | |
487 | +#ifdef CONFIG_INTR_REMAP | |
488 | + return dmar_dev_scope_init(); | |
489 | +#else | |
490 | return -ENODEV; | |
491 | -} | |
492 | -static inline int dmar_table_init(void) | |
493 | -{ | |
494 | - return -ENODEV; | |
495 | +#endif | |
496 | } | |
497 | #endif /* !CONFIG_DMAR */ | |
498 | #endif /* __DMAR_H__ */ |