]> git.ipfire.org Git - people/arne_f/kernel.git/blob - drivers/iommu/intel-iommu.c
projects / people / arne_f / kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iommu/vt-d: Check capability before disabling protected memory
[people/arne_f/kernel.git] / drivers / iommu / intel-iommu.c
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
83
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
86 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
87
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
92 /*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
110 static inline int agaw_to_level(int agaw)
111 {
112 return agaw + 2;
113 }
114
115 static inline int agaw_to_width(int agaw)
116 {
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
118 }
119
120 static inline int width_to_agaw(int width)
121 {
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
123 }
124
125 static inline unsigned int level_to_offset_bits(int level)
126 {
127 return (level - 1) * LEVEL_STRIDE;
128 }
129
130 static inline int pfn_level_offset(unsigned long pfn, int level)
131 {
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133 }
134
135 static inline unsigned long level_mask(int level)
136 {
137 return -1UL << level_to_offset_bits(level);
138 }
139
140 static inline unsigned long level_size(int level)
141 {
142 return 1UL << level_to_offset_bits(level);
143 }
144
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 {
147 return (pfn + level_size(level) - 1) & level_mask(level);
148 }
149
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 {
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
153 }
154
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 {
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160 }
161
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 {
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 }
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 {
168 return mm_to_dma_pfn(page_to_pfn(pg));
169 }
170 static inline unsigned long virt_to_dma_pfn(void *p)
171 {
172 return page_to_dma_pfn(virt_to_page(p));
173 }
174
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
177
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
180
181 /*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185 static int force_on = 0;
186
187 /*
188 * 0: Present
189 * 1-11: Reserved
190 * 12-63: Context Ptr (12 - (haw-1))
191 * 64-127: Reserved
192 */
193 struct root_entry {
194 u64 lo;
195 u64 hi;
196 };
197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
198
199 /*
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
201 * if marked present.
202 */
203 static phys_addr_t root_entry_lctp(struct root_entry *re)
204 {
205 if (!(re->lo & 1))
206 return 0;
207
208 return re->lo & VTD_PAGE_MASK;
209 }
210
211 /*
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
213 * if marked present.
214 */
215 static phys_addr_t root_entry_uctp(struct root_entry *re)
216 {
217 if (!(re->hi & 1))
218 return 0;
219
220 return re->hi & VTD_PAGE_MASK;
221 }
222 /*
223 * low 64 bits:
224 * 0: present
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
228 * high 64 bits:
229 * 0-2: address width
230 * 3-6: aval
231 * 8-23: domain id
232 */
233 struct context_entry {
234 u64 lo;
235 u64 hi;
236 };
237
238 static inline void context_clear_pasid_enable(struct context_entry *context)
239 {
240 context->lo &= ~(1ULL << 11);
241 }
242
243 static inline bool context_pasid_enabled(struct context_entry *context)
244 {
245 return !!(context->lo & (1ULL << 11));
246 }
247
248 static inline void context_set_copied(struct context_entry *context)
249 {
250 context->hi |= (1ull << 3);
251 }
252
253 static inline bool context_copied(struct context_entry *context)
254 {
255 return !!(context->hi & (1ULL << 3));
256 }
257
258 static inline bool __context_present(struct context_entry *context)
259 {
260 return (context->lo & 1);
261 }
262
263 static inline bool context_present(struct context_entry *context)
264 {
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
268 }
269
270 static inline void context_set_present(struct context_entry *context)
271 {
272 context->lo |= 1;
273 }
274
275 static inline void context_set_fault_enable(struct context_entry *context)
276 {
277 context->lo &= (((u64)-1) << 2) | 1;
278 }
279
280 static inline void context_set_translation_type(struct context_entry *context,
281 unsigned long value)
282 {
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
285 }
286
287 static inline void context_set_address_root(struct context_entry *context,
288 unsigned long value)
289 {
290 context->lo &= ~VTD_PAGE_MASK;
291 context->lo |= value & VTD_PAGE_MASK;
292 }
293
294 static inline void context_set_address_width(struct context_entry *context,
295 unsigned long value)
296 {
297 context->hi |= value & 7;
298 }
299
300 static inline void context_set_domain_id(struct context_entry *context,
301 unsigned long value)
302 {
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
304 }
305
306 static inline int context_domain_id(struct context_entry *c)
307 {
308 return((c->hi >> 8) & 0xffff);
309 }
310
311 static inline void context_clear_entry(struct context_entry *context)
312 {
313 context->lo = 0;
314 context->hi = 0;
315 }
316
317 /*
318 * 0: readable
319 * 1: writable
320 * 2-6: reserved
321 * 7: super page
322 * 8-10: available
323 * 11: snoop behavior
324 * 12-63: Host physcial address
325 */
326 struct dma_pte {
327 u64 val;
328 };
329
330 static inline void dma_clear_pte(struct dma_pte *pte)
331 {
332 pte->val = 0;
333 }
334
335 static inline u64 dma_pte_addr(struct dma_pte *pte)
336 {
337 #ifdef CONFIG_64BIT
338 return pte->val & VTD_PAGE_MASK;
339 #else
340 /* Must have a full atomic 64-bit read */
341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
342 #endif
343 }
344
345 static inline bool dma_pte_present(struct dma_pte *pte)
346 {
347 return (pte->val & 3) != 0;
348 }
349
350 static inline bool dma_pte_superpage(struct dma_pte *pte)
351 {
352 return (pte->val & DMA_PTE_LARGE_PAGE);
353 }
354
355 static inline int first_pte_in_page(struct dma_pte *pte)
356 {
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
358 }
359
360 /*
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
365 */
366 static struct dmar_domain *si_domain;
367 static int hw_pass_through = 1;
368
369 /*
370 * Domain represents a virtual machine, more than one devices
371 * across iommus may be owned in one domain, e.g. kvm guest.
372 */
373 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
374
375 /* si_domain contains mulitple devices */
376 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
377
378 #define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
381
382 struct dmar_domain {
383 int nid; /* node id */
384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
388
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
393
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
404 int flags; /* flags to find out type of domain */
405
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
416 };
417
418 /* PCI domain-device relationship */
419 struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u16 pfsid; /* SRIOV physical function source ID */
425 u8 pasid_supported:3;
426 u8 pasid_enabled:1;
427 u8 pri_supported:1;
428 u8 pri_enabled:1;
429 u8 ats_supported:1;
430 u8 ats_enabled:1;
431 u8 ats_qdep;
432 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
433 struct intel_iommu *iommu; /* IOMMU used by this device */
434 struct dmar_domain *domain; /* pointer to domain */
435 };
436
437 struct dmar_rmrr_unit {
438 struct list_head list; /* list of rmrr units */
439 struct acpi_dmar_header *hdr; /* ACPI header */
440 u64 base_address; /* reserved base address*/
441 u64 end_address; /* reserved end address */
442 struct dmar_dev_scope *devices; /* target devices */
443 int devices_cnt; /* target device count */
444 };
445
446 struct dmar_atsr_unit {
447 struct list_head list; /* list of ATSR units */
448 struct acpi_dmar_header *hdr; /* ACPI header */
449 struct dmar_dev_scope *devices; /* target devices */
450 int devices_cnt; /* target device count */
451 u8 include_all:1; /* include all ports */
452 };
453
454 static LIST_HEAD(dmar_atsr_units);
455 static LIST_HEAD(dmar_rmrr_units);
456
457 #define for_each_rmrr_units(rmrr) \
458 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
459
460 static void flush_unmaps_timeout(unsigned long data);
461
462 struct deferred_flush_entry {
463 unsigned long iova_pfn;
464 unsigned long nrpages;
465 struct dmar_domain *domain;
466 struct page *freelist;
467 };
468
469 #define HIGH_WATER_MARK 250
470 struct deferred_flush_table {
471 int next;
472 struct deferred_flush_entry entries[HIGH_WATER_MARK];
473 };
474
475 struct deferred_flush_data {
476 spinlock_t lock;
477 int timer_on;
478 struct timer_list timer;
479 long size;
480 struct deferred_flush_table *tables;
481 };
482
483 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
484
485 /* bitmap for indexing intel_iommus */
486 static int g_num_of_iommus;
487
488 static void domain_exit(struct dmar_domain *domain);
489 static void domain_remove_dev_info(struct dmar_domain *domain);
490 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
491 struct device *dev);
492 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
493 static void domain_context_clear(struct intel_iommu *iommu,
494 struct device *dev);
495 static int domain_detach_iommu(struct dmar_domain *domain,
496 struct intel_iommu *iommu);
497
498 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
499 int dmar_disabled = 0;
500 #else
501 int dmar_disabled = 1;
502 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
503
504 int intel_iommu_enabled = 0;
505 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
506
507 static int dmar_map_gfx = 1;
508 static int dmar_forcedac;
509 static int intel_iommu_strict;
510 static int intel_iommu_superpage = 1;
511 static int intel_iommu_ecs = 1;
512 static int intel_iommu_pasid28;
513 static int iommu_identity_mapping;
514
515 #define IDENTMAP_ALL 1
516 #define IDENTMAP_GFX 2
517 #define IDENTMAP_AZALIA 4
518
519 /* Broadwell and Skylake have broken ECS support — normal so-called "second
520 * level" translation of DMA requests-without-PASID doesn't actually happen
521 * unless you also set the NESTE bit in an extended context-entry. Which of
522 * course means that SVM doesn't work because it's trying to do nested
523 * translation of the physical addresses it finds in the process page tables,
524 * through the IOVA->phys mapping found in the "second level" page tables.
525 *
526 * The VT-d specification was retroactively changed to change the definition
527 * of the capability bits and pretend that Broadwell/Skylake never happened...
528 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
529 * for some reason it was the PASID capability bit which was redefined (from
530 * bit 28 on BDW/SKL to bit 40 in future).
531 *
532 * So our test for ECS needs to eschew those implementations which set the old
533 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
534 * Unless we are working around the 'pasid28' limitations, that is, by putting
535 * the device into passthrough mode for normal DMA and thus masking the bug.
536 */
537 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
538 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
539 /* PASID support is thus enabled if ECS is enabled and *either* of the old
540 * or new capability bits are set. */
541 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
542 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
543
544 int intel_iommu_gfx_mapped;
545 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
546
547 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
548 static DEFINE_SPINLOCK(device_domain_lock);
549 static LIST_HEAD(device_domain_list);
550
551 static const struct iommu_ops intel_iommu_ops;
552
553 static bool translation_pre_enabled(struct intel_iommu *iommu)
554 {
555 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
556 }
557
558 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
559 {
560 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
561 }
562
563 static void init_translation_status(struct intel_iommu *iommu)
564 {
565 u32 gsts;
566
567 gsts = readl(iommu->reg + DMAR_GSTS_REG);
568 if (gsts & DMA_GSTS_TES)
569 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
570 }
571
572 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
573 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
574 {
575 return container_of(dom, struct dmar_domain, domain);
576 }
577
578 static int __init intel_iommu_setup(char *str)
579 {
580 if (!str)
581 return -EINVAL;
582 while (*str) {
583 if (!strncmp(str, "on", 2)) {
584 dmar_disabled = 0;
585 pr_info("IOMMU enabled\n");
586 } else if (!strncmp(str, "off", 3)) {
587 dmar_disabled = 1;
588 pr_info("IOMMU disabled\n");
589 } else if (!strncmp(str, "igfx_off", 8)) {
590 dmar_map_gfx = 0;
591 pr_info("Disable GFX device mapping\n");
592 } else if (!strncmp(str, "forcedac", 8)) {
593 pr_info("Forcing DAC for PCI devices\n");
594 dmar_forcedac = 1;
595 } else if (!strncmp(str, "strict", 6)) {
596 pr_info("Disable batched IOTLB flush\n");
597 intel_iommu_strict = 1;
598 } else if (!strncmp(str, "sp_off", 6)) {
599 pr_info("Disable supported super page\n");
600 intel_iommu_superpage = 0;
601 } else if (!strncmp(str, "ecs_off", 7)) {
602 printk(KERN_INFO
603 "Intel-IOMMU: disable extended context table support\n");
604 intel_iommu_ecs = 0;
605 } else if (!strncmp(str, "pasid28", 7)) {
606 printk(KERN_INFO
607 "Intel-IOMMU: enable pre-production PASID support\n");
608 intel_iommu_pasid28 = 1;
609 iommu_identity_mapping |= IDENTMAP_GFX;
610 }
611
612 str += strcspn(str, ",");
613 while (*str == ',')
614 str++;
615 }
616 return 0;
617 }
618 __setup("intel_iommu=", intel_iommu_setup);
619
620 static struct kmem_cache *iommu_domain_cache;
621 static struct kmem_cache *iommu_devinfo_cache;
622
623 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
624 {
625 struct dmar_domain **domains;
626 int idx = did >> 8;
627
628 domains = iommu->domains[idx];
629 if (!domains)
630 return NULL;
631
632 return domains[did & 0xff];
633 }
634
635 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
636 struct dmar_domain *domain)
637 {
638 struct dmar_domain **domains;
639 int idx = did >> 8;
640
641 if (!iommu->domains[idx]) {
642 size_t size = 256 * sizeof(struct dmar_domain *);
643 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
644 }
645
646 domains = iommu->domains[idx];
647 if (WARN_ON(!domains))
648 return;
649 else
650 domains[did & 0xff] = domain;
651 }
652
653 static inline void *alloc_pgtable_page(int node)
654 {
655 struct page *page;
656 void *vaddr = NULL;
657
658 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
659 if (page)
660 vaddr = page_address(page);
661 return vaddr;
662 }
663
664 static inline void free_pgtable_page(void *vaddr)
665 {
666 free_page((unsigned long)vaddr);
667 }
668
669 static inline void *alloc_domain_mem(void)
670 {
671 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
672 }
673
674 static void free_domain_mem(void *vaddr)
675 {
676 kmem_cache_free(iommu_domain_cache, vaddr);
677 }
678
679 static inline void * alloc_devinfo_mem(void)
680 {
681 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
682 }
683
684 static inline void free_devinfo_mem(void *vaddr)
685 {
686 kmem_cache_free(iommu_devinfo_cache, vaddr);
687 }
688
689 static inline int domain_type_is_vm(struct dmar_domain *domain)
690 {
691 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
692 }
693
694 static inline int domain_type_is_si(struct dmar_domain *domain)
695 {
696 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
697 }
698
699 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
700 {
701 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
702 DOMAIN_FLAG_STATIC_IDENTITY);
703 }
704
705 static inline int domain_pfn_supported(struct dmar_domain *domain,
706 unsigned long pfn)
707 {
708 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
709
710 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
711 }
712
713 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
714 {
715 unsigned long sagaw;
716 int agaw = -1;
717
718 sagaw = cap_sagaw(iommu->cap);
719 for (agaw = width_to_agaw(max_gaw);
720 agaw >= 0; agaw--) {
721 if (test_bit(agaw, &sagaw))
722 break;
723 }
724
725 return agaw;
726 }
727
728 /*
729 * Calculate max SAGAW for each iommu.
730 */
731 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
732 {
733 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
734 }
735
736 /*
737 * calculate agaw for each iommu.
738 * "SAGAW" may be different across iommus, use a default agaw, and
739 * get a supported less agaw for iommus that don't support the default agaw.
740 */
741 int iommu_calculate_agaw(struct intel_iommu *iommu)
742 {
743 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
744 }
745
746 /* This functionin only returns single iommu in a domain */
747 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
748 {
749 int iommu_id;
750
751 /* si_domain and vm domain should not get here. */
752 BUG_ON(domain_type_is_vm_or_si(domain));
753 for_each_domain_iommu(iommu_id, domain)
754 break;
755
756 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
757 return NULL;
758
759 return g_iommus[iommu_id];
760 }
761
762 static void domain_update_iommu_coherency(struct dmar_domain *domain)
763 {
764 struct dmar_drhd_unit *drhd;
765 struct intel_iommu *iommu;
766 bool found = false;
767 int i;
768
769 domain->iommu_coherency = 1;
770
771 for_each_domain_iommu(i, domain) {
772 found = true;
773 if (!ecap_coherent(g_iommus[i]->ecap)) {
774 domain->iommu_coherency = 0;
775 break;
776 }
777 }
778 if (found)
779 return;
780
781 /* No hardware attached; use lowest common denominator */
782 rcu_read_lock();
783 for_each_active_iommu(iommu, drhd) {
784 if (!ecap_coherent(iommu->ecap)) {
785 domain->iommu_coherency = 0;
786 break;
787 }
788 }
789 rcu_read_unlock();
790 }
791
792 static int domain_update_iommu_snooping(struct intel_iommu *skip)
793 {
794 struct dmar_drhd_unit *drhd;
795 struct intel_iommu *iommu;
796 int ret = 1;
797
798 rcu_read_lock();
799 for_each_active_iommu(iommu, drhd) {
800 if (iommu != skip) {
801 if (!ecap_sc_support(iommu->ecap)) {
802 ret = 0;
803 break;
804 }
805 }
806 }
807 rcu_read_unlock();
808
809 return ret;
810 }
811
812 static int domain_update_iommu_superpage(struct intel_iommu *skip)
813 {
814 struct dmar_drhd_unit *drhd;
815 struct intel_iommu *iommu;
816 int mask = 0xf;
817
818 if (!intel_iommu_superpage) {
819 return 0;
820 }
821
822 /* set iommu_superpage to the smallest common denominator */
823 rcu_read_lock();
824 for_each_active_iommu(iommu, drhd) {
825 if (iommu != skip) {
826 mask &= cap_super_page_val(iommu->cap);
827 if (!mask)
828 break;
829 }
830 }
831 rcu_read_unlock();
832
833 return fls(mask);
834 }
835
836 /* Some capabilities may be different across iommus */
837 static void domain_update_iommu_cap(struct dmar_domain *domain)
838 {
839 domain_update_iommu_coherency(domain);
840 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
841 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
842 }
843
844 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
845 u8 bus, u8 devfn, int alloc)
846 {
847 struct root_entry *root = &iommu->root_entry[bus];
848 struct context_entry *context;
849 u64 *entry;
850
851 entry = &root->lo;
852 if (ecs_enabled(iommu)) {
853 if (devfn >= 0x80) {
854 devfn -= 0x80;
855 entry = &root->hi;
856 }
857 devfn *= 2;
858 }
859 if (*entry & 1)
860 context = phys_to_virt(*entry & VTD_PAGE_MASK);
861 else {
862 unsigned long phy_addr;
863 if (!alloc)
864 return NULL;
865
866 context = alloc_pgtable_page(iommu->node);
867 if (!context)
868 return NULL;
869
870 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
871 phy_addr = virt_to_phys((void *)context);
872 *entry = phy_addr | 1;
873 __iommu_flush_cache(iommu, entry, sizeof(*entry));
874 }
875 return &context[devfn];
876 }
877
878 static int iommu_dummy(struct device *dev)
879 {
880 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
881 }
882
883 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
884 {
885 struct dmar_drhd_unit *drhd = NULL;
886 struct intel_iommu *iommu;
887 struct device *tmp;
888 struct pci_dev *ptmp, *pdev = NULL;
889 u16 segment = 0;
890 int i;
891
892 if (iommu_dummy(dev))
893 return NULL;
894
895 if (dev_is_pci(dev)) {
896 struct pci_dev *pf_pdev;
897
898 pdev = to_pci_dev(dev);
899 /* VFs aren't listed in scope tables; we need to look up
900 * the PF instead to find the IOMMU. */
901 pf_pdev = pci_physfn(pdev);
902 dev = &pf_pdev->dev;
903 segment = pci_domain_nr(pdev->bus);
904 } else if (has_acpi_companion(dev))
905 dev = &ACPI_COMPANION(dev)->dev;
906
907 rcu_read_lock();
908 for_each_active_iommu(iommu, drhd) {
909 if (pdev && segment != drhd->segment)
910 continue;
911
912 for_each_active_dev_scope(drhd->devices,
913 drhd->devices_cnt, i, tmp) {
914 if (tmp == dev) {
915 /* For a VF use its original BDF# not that of the PF
916 * which we used for the IOMMU lookup. Strictly speaking
917 * we could do this for all PCI devices; we only need to
918 * get the BDF# from the scope table for ACPI matches. */
919 if (pdev && pdev->is_virtfn)
920 goto got_pdev;
921
922 *bus = drhd->devices[i].bus;
923 *devfn = drhd->devices[i].devfn;
924 goto out;
925 }
926
927 if (!pdev || !dev_is_pci(tmp))
928 continue;
929
930 ptmp = to_pci_dev(tmp);
931 if (ptmp->subordinate &&
932 ptmp->subordinate->number <= pdev->bus->number &&
933 ptmp->subordinate->busn_res.end >= pdev->bus->number)
934 goto got_pdev;
935 }
936
937 if (pdev && drhd->include_all) {
938 got_pdev:
939 *bus = pdev->bus->number;
940 *devfn = pdev->devfn;
941 goto out;
942 }
943 }
944 iommu = NULL;
945 out:
946 rcu_read_unlock();
947
948 return iommu;
949 }
950
951 static void domain_flush_cache(struct dmar_domain *domain,
952 void *addr, int size)
953 {
954 if (!domain->iommu_coherency)
955 clflush_cache_range(addr, size);
956 }
957
958 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
959 {
960 struct context_entry *context;
961 int ret = 0;
962 unsigned long flags;
963
964 spin_lock_irqsave(&iommu->lock, flags);
965 context = iommu_context_addr(iommu, bus, devfn, 0);
966 if (context)
967 ret = context_present(context);
968 spin_unlock_irqrestore(&iommu->lock, flags);
969 return ret;
970 }
971
972 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
973 {
974 struct context_entry *context;
975 unsigned long flags;
976
977 spin_lock_irqsave(&iommu->lock, flags);
978 context = iommu_context_addr(iommu, bus, devfn, 0);
979 if (context) {
980 context_clear_entry(context);
981 __iommu_flush_cache(iommu, context, sizeof(*context));
982 }
983 spin_unlock_irqrestore(&iommu->lock, flags);
984 }
985
986 static void free_context_table(struct intel_iommu *iommu)
987 {
988 int i;
989 unsigned long flags;
990 struct context_entry *context;
991
992 spin_lock_irqsave(&iommu->lock, flags);
993 if (!iommu->root_entry) {
994 goto out;
995 }
996 for (i = 0; i < ROOT_ENTRY_NR; i++) {
997 context = iommu_context_addr(iommu, i, 0, 0);
998 if (context)
999 free_pgtable_page(context);
1000
1001 if (!ecs_enabled(iommu))
1002 continue;
1003
1004 context = iommu_context_addr(iommu, i, 0x80, 0);
1005 if (context)
1006 free_pgtable_page(context);
1007
1008 }
1009 free_pgtable_page(iommu->root_entry);
1010 iommu->root_entry = NULL;
1011 out:
1012 spin_unlock_irqrestore(&iommu->lock, flags);
1013 }
1014
1015 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1016 unsigned long pfn, int *target_level)
1017 {
1018 struct dma_pte *parent, *pte = NULL;
1019 int level = agaw_to_level(domain->agaw);
1020 int offset;
1021
1022 BUG_ON(!domain->pgd);
1023
1024 if (!domain_pfn_supported(domain, pfn))
1025 /* Address beyond IOMMU's addressing capabilities. */
1026 return NULL;
1027
1028 parent = domain->pgd;
1029
1030 while (1) {
1031 void *tmp_page;
1032
1033 offset = pfn_level_offset(pfn, level);
1034 pte = &parent[offset];
1035 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1036 break;
1037 if (level == *target_level)
1038 break;
1039
1040 if (!dma_pte_present(pte)) {
1041 uint64_t pteval;
1042
1043 tmp_page = alloc_pgtable_page(domain->nid);
1044
1045 if (!tmp_page)
1046 return NULL;
1047
1048 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1049 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1050 if (cmpxchg64(&pte->val, 0ULL, pteval))
1051 /* Someone else set it while we were thinking; use theirs. */
1052 free_pgtable_page(tmp_page);
1053 else
1054 domain_flush_cache(domain, pte, sizeof(*pte));
1055 }
1056 if (level == 1)
1057 break;
1058
1059 parent = phys_to_virt(dma_pte_addr(pte));
1060 level--;
1061 }
1062
1063 if (!*target_level)
1064 *target_level = level;
1065
1066 return pte;
1067 }
1068
1069
1070 /* return address's pte at specific level */
1071 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1072 unsigned long pfn,
1073 int level, int *large_page)
1074 {
1075 struct dma_pte *parent, *pte = NULL;
1076 int total = agaw_to_level(domain->agaw);
1077 int offset;
1078
1079 parent = domain->pgd;
1080 while (level <= total) {
1081 offset = pfn_level_offset(pfn, total);
1082 pte = &parent[offset];
1083 if (level == total)
1084 return pte;
1085
1086 if (!dma_pte_present(pte)) {
1087 *large_page = total;
1088 break;
1089 }
1090
1091 if (dma_pte_superpage(pte)) {
1092 *large_page = total;
1093 return pte;
1094 }
1095
1096 parent = phys_to_virt(dma_pte_addr(pte));
1097 total--;
1098 }
1099 return NULL;
1100 }
1101
1102 /* clear last level pte, a tlb flush should be followed */
1103 static void dma_pte_clear_range(struct dmar_domain *domain,
1104 unsigned long start_pfn,
1105 unsigned long last_pfn)
1106 {
1107 unsigned int large_page = 1;
1108 struct dma_pte *first_pte, *pte;
1109
1110 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1111 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1112 BUG_ON(start_pfn > last_pfn);
1113
1114 /* we don't need lock here; nobody else touches the iova range */
1115 do {
1116 large_page = 1;
1117 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1118 if (!pte) {
1119 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1120 continue;
1121 }
1122 do {
1123 dma_clear_pte(pte);
1124 start_pfn += lvl_to_nr_pages(large_page);
1125 pte++;
1126 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1127
1128 domain_flush_cache(domain, first_pte,
1129 (void *)pte - (void *)first_pte);
1130
1131 } while (start_pfn && start_pfn <= last_pfn);
1132 }
1133
1134 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1135 struct dma_pte *pte, unsigned long pfn,
1136 unsigned long start_pfn, unsigned long last_pfn)
1137 {
1138 pfn = max(start_pfn, pfn);
1139 pte = &pte[pfn_level_offset(pfn, level)];
1140
1141 do {
1142 unsigned long level_pfn;
1143 struct dma_pte *level_pte;
1144
1145 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1146 goto next;
1147
1148 level_pfn = pfn & level_mask(level);
1149 level_pte = phys_to_virt(dma_pte_addr(pte));
1150
1151 if (level > 2)
1152 dma_pte_free_level(domain, level - 1, level_pte,
1153 level_pfn, start_pfn, last_pfn);
1154
1155 /* If range covers entire pagetable, free it */
1156 if (!(start_pfn > level_pfn ||
1157 last_pfn < level_pfn + level_size(level) - 1)) {
1158 dma_clear_pte(pte);
1159 domain_flush_cache(domain, pte, sizeof(*pte));
1160 free_pgtable_page(level_pte);
1161 }
1162 next:
1163 pfn += level_size(level);
1164 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1165 }
1166
1167 /* clear last level (leaf) ptes and free page table pages. */
1168 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1169 unsigned long start_pfn,
1170 unsigned long last_pfn)
1171 {
1172 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1173 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1174 BUG_ON(start_pfn > last_pfn);
1175
1176 dma_pte_clear_range(domain, start_pfn, last_pfn);
1177
1178 /* We don't need lock here; nobody else touches the iova range */
1179 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1180 domain->pgd, 0, start_pfn, last_pfn);
1181
1182 /* free pgd */
1183 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1184 free_pgtable_page(domain->pgd);
1185 domain->pgd = NULL;
1186 }
1187 }
1188
1189 /* When a page at a given level is being unlinked from its parent, we don't
1190 need to *modify* it at all. All we need to do is make a list of all the
1191 pages which can be freed just as soon as we've flushed the IOTLB and we
1192 know the hardware page-walk will no longer touch them.
1193 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1194 be freed. */
1195 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1196 int level, struct dma_pte *pte,
1197 struct page *freelist)
1198 {
1199 struct page *pg;
1200
1201 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1202 pg->freelist = freelist;
1203 freelist = pg;
1204
1205 if (level == 1)
1206 return freelist;
1207
1208 pte = page_address(pg);
1209 do {
1210 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1211 freelist = dma_pte_list_pagetables(domain, level - 1,
1212 pte, freelist);
1213 pte++;
1214 } while (!first_pte_in_page(pte));
1215
1216 return freelist;
1217 }
1218
1219 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1220 struct dma_pte *pte, unsigned long pfn,
1221 unsigned long start_pfn,
1222 unsigned long last_pfn,
1223 struct page *freelist)
1224 {
1225 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1226
1227 pfn = max(start_pfn, pfn);
1228 pte = &pte[pfn_level_offset(pfn, level)];
1229
1230 do {
1231 unsigned long level_pfn;
1232
1233 if (!dma_pte_present(pte))
1234 goto next;
1235
1236 level_pfn = pfn & level_mask(level);
1237
1238 /* If range covers entire pagetable, free it */
1239 if (start_pfn <= level_pfn &&
1240 last_pfn >= level_pfn + level_size(level) - 1) {
1241 /* These suborbinate page tables are going away entirely. Don't
1242 bother to clear them; we're just going to *free* them. */
1243 if (level > 1 && !dma_pte_superpage(pte))
1244 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1245
1246 dma_clear_pte(pte);
1247 if (!first_pte)
1248 first_pte = pte;
1249 last_pte = pte;
1250 } else if (level > 1) {
1251 /* Recurse down into a level that isn't *entirely* obsolete */
1252 freelist = dma_pte_clear_level(domain, level - 1,
1253 phys_to_virt(dma_pte_addr(pte)),
1254 level_pfn, start_pfn, last_pfn,
1255 freelist);
1256 }
1257 next:
1258 pfn += level_size(level);
1259 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1260
1261 if (first_pte)
1262 domain_flush_cache(domain, first_pte,
1263 (void *)++last_pte - (void *)first_pte);
1264
1265 return freelist;
1266 }
1267
1268 /* We can't just free the pages because the IOMMU may still be walking
1269 the page tables, and may have cached the intermediate levels. The
1270 pages can only be freed after the IOTLB flush has been done. */
1271 static struct page *domain_unmap(struct dmar_domain *domain,
1272 unsigned long start_pfn,
1273 unsigned long last_pfn)
1274 {
1275 struct page *freelist = NULL;
1276
1277 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1278 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1279 BUG_ON(start_pfn > last_pfn);
1280
1281 /* we don't need lock here; nobody else touches the iova range */
1282 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1283 domain->pgd, 0, start_pfn, last_pfn, NULL);
1284
1285 /* free pgd */
1286 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1287 struct page *pgd_page = virt_to_page(domain->pgd);
1288 pgd_page->freelist = freelist;
1289 freelist = pgd_page;
1290
1291 domain->pgd = NULL;
1292 }
1293
1294 return freelist;
1295 }
1296
1297 static void dma_free_pagelist(struct page *freelist)
1298 {
1299 struct page *pg;
1300
1301 while ((pg = freelist)) {
1302 freelist = pg->freelist;
1303 free_pgtable_page(page_address(pg));
1304 }
1305 }
1306
1307 /* iommu handling */
1308 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1309 {
1310 struct root_entry *root;
1311 unsigned long flags;
1312
1313 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1314 if (!root) {
1315 pr_err("Allocating root entry for %s failed\n",
1316 iommu->name);
1317 return -ENOMEM;
1318 }
1319
1320 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1321
1322 spin_lock_irqsave(&iommu->lock, flags);
1323 iommu->root_entry = root;
1324 spin_unlock_irqrestore(&iommu->lock, flags);
1325
1326 return 0;
1327 }
1328
1329 static void iommu_set_root_entry(struct intel_iommu *iommu)
1330 {
1331 u64 addr;
1332 u32 sts;
1333 unsigned long flag;
1334
1335 addr = virt_to_phys(iommu->root_entry);
1336 if (ecs_enabled(iommu))
1337 addr |= DMA_RTADDR_RTT;
1338
1339 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1340 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1341
1342 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1343
1344 /* Make sure hardware complete it */
1345 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1346 readl, (sts & DMA_GSTS_RTPS), sts);
1347
1348 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1349 }
1350
1351 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1352 {
1353 u32 val;
1354 unsigned long flag;
1355
1356 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1357 return;
1358
1359 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1360 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1361
1362 /* Make sure hardware complete it */
1363 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1364 readl, (!(val & DMA_GSTS_WBFS)), val);
1365
1366 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1367 }
1368
1369 /* return value determine if we need a write buffer flush */
1370 static void __iommu_flush_context(struct intel_iommu *iommu,
1371 u16 did, u16 source_id, u8 function_mask,
1372 u64 type)
1373 {
1374 u64 val = 0;
1375 unsigned long flag;
1376
1377 switch (type) {
1378 case DMA_CCMD_GLOBAL_INVL:
1379 val = DMA_CCMD_GLOBAL_INVL;
1380 break;
1381 case DMA_CCMD_DOMAIN_INVL:
1382 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1383 break;
1384 case DMA_CCMD_DEVICE_INVL:
1385 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1386 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1387 break;
1388 default:
1389 BUG();
1390 }
1391 val |= DMA_CCMD_ICC;
1392
1393 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1394 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1395
1396 /* Make sure hardware complete it */
1397 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1398 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1399
1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1401 }
1402
1403 /* return value determine if we need a write buffer flush */
1404 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1405 u64 addr, unsigned int size_order, u64 type)
1406 {
1407 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1408 u64 val = 0, val_iva = 0;
1409 unsigned long flag;
1410
1411 switch (type) {
1412 case DMA_TLB_GLOBAL_FLUSH:
1413 /* global flush doesn't need set IVA_REG */
1414 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1415 break;
1416 case DMA_TLB_DSI_FLUSH:
1417 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1418 break;
1419 case DMA_TLB_PSI_FLUSH:
1420 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1421 /* IH bit is passed in as part of address */
1422 val_iva = size_order | addr;
1423 break;
1424 default:
1425 BUG();
1426 }
1427 /* Note: set drain read/write */
1428 #if 0
1429 /*
1430 * This is probably to be super secure.. Looks like we can
1431 * ignore it without any impact.
1432 */
1433 if (cap_read_drain(iommu->cap))
1434 val |= DMA_TLB_READ_DRAIN;
1435 #endif
1436 if (cap_write_drain(iommu->cap))
1437 val |= DMA_TLB_WRITE_DRAIN;
1438
1439 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1440 /* Note: Only uses first TLB reg currently */
1441 if (val_iva)
1442 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1443 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1444
1445 /* Make sure hardware complete it */
1446 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1447 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1448
1449 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1450
1451 /* check IOTLB invalidation granularity */
1452 if (DMA_TLB_IAIG(val) == 0)
1453 pr_err("Flush IOTLB failed\n");
1454 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1455 pr_debug("TLB flush request %Lx, actual %Lx\n",
1456 (unsigned long long)DMA_TLB_IIRG(type),
1457 (unsigned long long)DMA_TLB_IAIG(val));
1458 }
1459
1460 static struct device_domain_info *
1461 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1462 u8 bus, u8 devfn)
1463 {
1464 struct device_domain_info *info;
1465
1466 assert_spin_locked(&device_domain_lock);
1467
1468 if (!iommu->qi)
1469 return NULL;
1470
1471 list_for_each_entry(info, &domain->devices, link)
1472 if (info->iommu == iommu && info->bus == bus &&
1473 info->devfn == devfn) {
1474 if (info->ats_supported && info->dev)
1475 return info;
1476 break;
1477 }
1478
1479 return NULL;
1480 }
1481
1482 static void domain_update_iotlb(struct dmar_domain *domain)
1483 {
1484 struct device_domain_info *info;
1485 bool has_iotlb_device = false;
1486
1487 assert_spin_locked(&device_domain_lock);
1488
1489 list_for_each_entry(info, &domain->devices, link) {
1490 struct pci_dev *pdev;
1491
1492 if (!info->dev || !dev_is_pci(info->dev))
1493 continue;
1494
1495 pdev = to_pci_dev(info->dev);
1496 if (pdev->ats_enabled) {
1497 has_iotlb_device = true;
1498 break;
1499 }
1500 }
1501
1502 domain->has_iotlb_device = has_iotlb_device;
1503 }
1504
1505 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1506 {
1507 struct pci_dev *pdev;
1508
1509 assert_spin_locked(&device_domain_lock);
1510
1511 if (!info || !dev_is_pci(info->dev))
1512 return;
1513
1514 pdev = to_pci_dev(info->dev);
1515 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1516 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1517 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1518 * reserved, which should be set to 0.
1519 */
1520 if (!ecap_dit(info->iommu->ecap))
1521 info->pfsid = 0;
1522 else {
1523 struct pci_dev *pf_pdev;
1524
1525 /* pdev will be returned if device is not a vf */
1526 pf_pdev = pci_physfn(pdev);
1527 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1528 }
1529
1530 #ifdef CONFIG_INTEL_IOMMU_SVM
1531 /* The PCIe spec, in its wisdom, declares that the behaviour of
1532 the device if you enable PASID support after ATS support is
1533 undefined. So always enable PASID support on devices which
1534 have it, even if we can't yet know if we're ever going to
1535 use it. */
1536 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1537 info->pasid_enabled = 1;
1538
1539 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1540 info->pri_enabled = 1;
1541 #endif
1542 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1543 info->ats_enabled = 1;
1544 domain_update_iotlb(info->domain);
1545 info->ats_qdep = pci_ats_queue_depth(pdev);
1546 }
1547 }
1548
1549 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1550 {
1551 struct pci_dev *pdev;
1552
1553 assert_spin_locked(&device_domain_lock);
1554
1555 if (!dev_is_pci(info->dev))
1556 return;
1557
1558 pdev = to_pci_dev(info->dev);
1559
1560 if (info->ats_enabled) {
1561 pci_disable_ats(pdev);
1562 info->ats_enabled = 0;
1563 domain_update_iotlb(info->domain);
1564 }
1565 #ifdef CONFIG_INTEL_IOMMU_SVM
1566 if (info->pri_enabled) {
1567 pci_disable_pri(pdev);
1568 info->pri_enabled = 0;
1569 }
1570 if (info->pasid_enabled) {
1571 pci_disable_pasid(pdev);
1572 info->pasid_enabled = 0;
1573 }
1574 #endif
1575 }
1576
1577 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1578 u64 addr, unsigned mask)
1579 {
1580 u16 sid, qdep;
1581 unsigned long flags;
1582 struct device_domain_info *info;
1583
1584 if (!domain->has_iotlb_device)
1585 return;
1586
1587 spin_lock_irqsave(&device_domain_lock, flags);
1588 list_for_each_entry(info, &domain->devices, link) {
1589 if (!info->ats_enabled)
1590 continue;
1591
1592 sid = info->bus << 8 | info->devfn;
1593 qdep = info->ats_qdep;
1594 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1595 qdep, addr, mask);
1596 }
1597 spin_unlock_irqrestore(&device_domain_lock, flags);
1598 }
1599
1600 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1601 struct dmar_domain *domain,
1602 unsigned long pfn, unsigned int pages,
1603 int ih, int map)
1604 {
1605 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1606 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1607 u16 did = domain->iommu_did[iommu->seq_id];
1608
1609 BUG_ON(pages == 0);
1610
1611 if (ih)
1612 ih = 1 << 6;
1613 /*
1614 * Fallback to domain selective flush if no PSI support or the size is
1615 * too big.
1616 * PSI requires page size to be 2 ^ x, and the base address is naturally
1617 * aligned to the size
1618 */
1619 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1620 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1621 DMA_TLB_DSI_FLUSH);
1622 else
1623 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1624 DMA_TLB_PSI_FLUSH);
1625
1626 /*
1627 * In caching mode, changes of pages from non-present to present require
1628 * flush. However, device IOTLB doesn't need to be flushed in this case.
1629 */
1630 if (!cap_caching_mode(iommu->cap) || !map)
1631 iommu_flush_dev_iotlb(domain, addr, mask);
1632 }
1633
1634 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1635 {
1636 u32 pmen;
1637 unsigned long flags;
1638
1639 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1640 return;
1641
1642 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1643 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1644 pmen &= ~DMA_PMEN_EPM;
1645 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1646
1647 /* wait for the protected region status bit to clear */
1648 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1649 readl, !(pmen & DMA_PMEN_PRS), pmen);
1650
1651 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1652 }
1653
1654 static void iommu_enable_translation(struct intel_iommu *iommu)
1655 {
1656 u32 sts;
1657 unsigned long flags;
1658
1659 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1660 iommu->gcmd |= DMA_GCMD_TE;
1661 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1662
1663 /* Make sure hardware complete it */
1664 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1665 readl, (sts & DMA_GSTS_TES), sts);
1666
1667 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1668 }
1669
1670 static void iommu_disable_translation(struct intel_iommu *iommu)
1671 {
1672 u32 sts;
1673 unsigned long flag;
1674
1675 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1676 iommu->gcmd &= ~DMA_GCMD_TE;
1677 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1678
1679 /* Make sure hardware complete it */
1680 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1681 readl, (!(sts & DMA_GSTS_TES)), sts);
1682
1683 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1684 }
1685
1686
1687 static int iommu_init_domains(struct intel_iommu *iommu)
1688 {
1689 u32 ndomains, nlongs;
1690 size_t size;
1691
1692 ndomains = cap_ndoms(iommu->cap);
1693 pr_debug("%s: Number of Domains supported <%d>\n",
1694 iommu->name, ndomains);
1695 nlongs = BITS_TO_LONGS(ndomains);
1696
1697 spin_lock_init(&iommu->lock);
1698
1699 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1700 if (!iommu->domain_ids) {
1701 pr_err("%s: Allocating domain id array failed\n",
1702 iommu->name);
1703 return -ENOMEM;
1704 }
1705
1706 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1707 iommu->domains = kzalloc(size, GFP_KERNEL);
1708
1709 if (iommu->domains) {
1710 size = 256 * sizeof(struct dmar_domain *);
1711 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1712 }
1713
1714 if (!iommu->domains || !iommu->domains[0]) {
1715 pr_err("%s: Allocating domain array failed\n",
1716 iommu->name);
1717 kfree(iommu->domain_ids);
1718 kfree(iommu->domains);
1719 iommu->domain_ids = NULL;
1720 iommu->domains = NULL;
1721 return -ENOMEM;
1722 }
1723
1724
1725
1726 /*
1727 * If Caching mode is set, then invalid translations are tagged
1728 * with domain-id 0, hence we need to pre-allocate it. We also
1729 * use domain-id 0 as a marker for non-allocated domain-id, so
1730 * make sure it is not used for a real domain.
1731 */
1732 set_bit(0, iommu->domain_ids);
1733
1734 return 0;
1735 }
1736
1737 static void disable_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739 struct device_domain_info *info, *tmp;
1740 unsigned long flags;
1741
1742 if (!iommu->domains || !iommu->domain_ids)
1743 return;
1744
1745 again:
1746 spin_lock_irqsave(&device_domain_lock, flags);
1747 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1748 struct dmar_domain *domain;
1749
1750 if (info->iommu != iommu)
1751 continue;
1752
1753 if (!info->dev || !info->domain)
1754 continue;
1755
1756 domain = info->domain;
1757
1758 __dmar_remove_one_dev_info(info);
1759
1760 if (!domain_type_is_vm_or_si(domain)) {
1761 /*
1762 * The domain_exit() function can't be called under
1763 * device_domain_lock, as it takes this lock itself.
1764 * So release the lock here and re-run the loop
1765 * afterwards.
1766 */
1767 spin_unlock_irqrestore(&device_domain_lock, flags);
1768 domain_exit(domain);
1769 goto again;
1770 }
1771 }
1772 spin_unlock_irqrestore(&device_domain_lock, flags);
1773
1774 if (iommu->gcmd & DMA_GCMD_TE)
1775 iommu_disable_translation(iommu);
1776 }
1777
1778 static void free_dmar_iommu(struct intel_iommu *iommu)
1779 {
1780 if ((iommu->domains) && (iommu->domain_ids)) {
1781 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1782 int i;
1783
1784 for (i = 0; i < elems; i++)
1785 kfree(iommu->domains[i]);
1786 kfree(iommu->domains);
1787 kfree(iommu->domain_ids);
1788 iommu->domains = NULL;
1789 iommu->domain_ids = NULL;
1790 }
1791
1792 g_iommus[iommu->seq_id] = NULL;
1793
1794 /* free context mapping */
1795 free_context_table(iommu);
1796
1797 #ifdef CONFIG_INTEL_IOMMU_SVM
1798 if (pasid_enabled(iommu)) {
1799 if (ecap_prs(iommu->ecap))
1800 intel_svm_finish_prq(iommu);
1801 intel_svm_free_pasid_tables(iommu);
1802 }
1803 #endif
1804 }
1805
1806 static struct dmar_domain *alloc_domain(int flags)
1807 {
1808 struct dmar_domain *domain;
1809
1810 domain = alloc_domain_mem();
1811 if (!domain)
1812 return NULL;
1813
1814 memset(domain, 0, sizeof(*domain));
1815 domain->nid = -1;
1816 domain->flags = flags;
1817 domain->has_iotlb_device = false;
1818 INIT_LIST_HEAD(&domain->devices);
1819
1820 return domain;
1821 }
1822
1823 /* Must be called with iommu->lock */
1824 static int domain_attach_iommu(struct dmar_domain *domain,
1825 struct intel_iommu *iommu)
1826 {
1827 unsigned long ndomains;
1828 int num;
1829
1830 assert_spin_locked(&device_domain_lock);
1831 assert_spin_locked(&iommu->lock);
1832
1833 domain->iommu_refcnt[iommu->seq_id] += 1;
1834 domain->iommu_count += 1;
1835 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1836 ndomains = cap_ndoms(iommu->cap);
1837 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1838
1839 if (num >= ndomains) {
1840 pr_err("%s: No free domain ids\n", iommu->name);
1841 domain->iommu_refcnt[iommu->seq_id] -= 1;
1842 domain->iommu_count -= 1;
1843 return -ENOSPC;
1844 }
1845
1846 set_bit(num, iommu->domain_ids);
1847 set_iommu_domain(iommu, num, domain);
1848
1849 domain->iommu_did[iommu->seq_id] = num;
1850 domain->nid = iommu->node;
1851
1852 domain_update_iommu_cap(domain);
1853 }
1854
1855 return 0;
1856 }
1857
1858 static int domain_detach_iommu(struct dmar_domain *domain,
1859 struct intel_iommu *iommu)
1860 {
1861 int num, count = INT_MAX;
1862
1863 assert_spin_locked(&device_domain_lock);
1864 assert_spin_locked(&iommu->lock);
1865
1866 domain->iommu_refcnt[iommu->seq_id] -= 1;
1867 count = --domain->iommu_count;
1868 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1869 num = domain->iommu_did[iommu->seq_id];
1870 clear_bit(num, iommu->domain_ids);
1871 set_iommu_domain(iommu, num, NULL);
1872
1873 domain_update_iommu_cap(domain);
1874 domain->iommu_did[iommu->seq_id] = 0;
1875 }
1876
1877 return count;
1878 }
1879
1880 static struct iova_domain reserved_iova_list;
1881 static struct lock_class_key reserved_rbtree_key;
1882
1883 static int dmar_init_reserved_ranges(void)
1884 {
1885 struct pci_dev *pdev = NULL;
1886 struct iova *iova;
1887 int i;
1888
1889 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1890 DMA_32BIT_PFN);
1891
1892 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1893 &reserved_rbtree_key);
1894
1895 /* IOAPIC ranges shouldn't be accessed by DMA */
1896 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1897 IOVA_PFN(IOAPIC_RANGE_END));
1898 if (!iova) {
1899 pr_err("Reserve IOAPIC range failed\n");
1900 return -ENODEV;
1901 }
1902
1903 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1904 for_each_pci_dev(pdev) {
1905 struct resource *r;
1906
1907 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1908 r = &pdev->resource[i];
1909 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1910 continue;
1911 iova = reserve_iova(&reserved_iova_list,
1912 IOVA_PFN(r->start),
1913 IOVA_PFN(r->end));
1914 if (!iova) {
1915 pr_err("Reserve iova failed\n");
1916 return -ENODEV;
1917 }
1918 }
1919 }
1920 return 0;
1921 }
1922
1923 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1924 {
1925 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1926 }
1927
1928 static inline int guestwidth_to_adjustwidth(int gaw)
1929 {
1930 int agaw;
1931 int r = (gaw - 12) % 9;
1932
1933 if (r == 0)
1934 agaw = gaw;
1935 else
1936 agaw = gaw + 9 - r;
1937 if (agaw > 64)
1938 agaw = 64;
1939 return agaw;
1940 }
1941
1942 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1943 int guest_width)
1944 {
1945 int adjust_width, agaw;
1946 unsigned long sagaw;
1947
1948 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1949 DMA_32BIT_PFN);
1950 domain_reserve_special_ranges(domain);
1951
1952 /* calculate AGAW */
1953 if (guest_width > cap_mgaw(iommu->cap))
1954 guest_width = cap_mgaw(iommu->cap);
1955 domain->gaw = guest_width;
1956 adjust_width = guestwidth_to_adjustwidth(guest_width);
1957 agaw = width_to_agaw(adjust_width);
1958 sagaw = cap_sagaw(iommu->cap);
1959 if (!test_bit(agaw, &sagaw)) {
1960 /* hardware doesn't support it, choose a bigger one */
1961 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1962 agaw = find_next_bit(&sagaw, 5, agaw);
1963 if (agaw >= 5)
1964 return -ENODEV;
1965 }
1966 domain->agaw = agaw;
1967
1968 if (ecap_coherent(iommu->ecap))
1969 domain->iommu_coherency = 1;
1970 else
1971 domain->iommu_coherency = 0;
1972
1973 if (ecap_sc_support(iommu->ecap))
1974 domain->iommu_snooping = 1;
1975 else
1976 domain->iommu_snooping = 0;
1977
1978 if (intel_iommu_superpage)
1979 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1980 else
1981 domain->iommu_superpage = 0;
1982
1983 domain->nid = iommu->node;
1984
1985 /* always allocate the top pgd */
1986 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1987 if (!domain->pgd)
1988 return -ENOMEM;
1989 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1990 return 0;
1991 }
1992
1993 static void domain_exit(struct dmar_domain *domain)
1994 {
1995 struct page *freelist = NULL;
1996
1997 /* Domain 0 is reserved, so dont process it */
1998 if (!domain)
1999 return;
2000
2001 /* Flush any lazy unmaps that may reference this domain */
2002 if (!intel_iommu_strict) {
2003 int cpu;
2004
2005 for_each_possible_cpu(cpu)
2006 flush_unmaps_timeout(cpu);
2007 }
2008
2009 /* Remove associated devices and clear attached or cached domains */
2010 rcu_read_lock();
2011 domain_remove_dev_info(domain);
2012 rcu_read_unlock();
2013
2014 /* destroy iovas */
2015 put_iova_domain(&domain->iovad);
2016
2017 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2018
2019 dma_free_pagelist(freelist);
2020
2021 free_domain_mem(domain);
2022 }
2023
2024 static int domain_context_mapping_one(struct dmar_domain *domain,
2025 struct intel_iommu *iommu,
2026 u8 bus, u8 devfn)
2027 {
2028 u16 did = domain->iommu_did[iommu->seq_id];
2029 int translation = CONTEXT_TT_MULTI_LEVEL;
2030 struct device_domain_info *info = NULL;
2031 struct context_entry *context;
2032 unsigned long flags;
2033 struct dma_pte *pgd;
2034 int ret, agaw;
2035
2036 WARN_ON(did == 0);
2037
2038 if (hw_pass_through && domain_type_is_si(domain))
2039 translation = CONTEXT_TT_PASS_THROUGH;
2040
2041 pr_debug("Set context mapping for %02x:%02x.%d\n",
2042 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2043
2044 BUG_ON(!domain->pgd);
2045
2046 spin_lock_irqsave(&device_domain_lock, flags);
2047 spin_lock(&iommu->lock);
2048
2049 ret = -ENOMEM;
2050 context = iommu_context_addr(iommu, bus, devfn, 1);
2051 if (!context)
2052 goto out_unlock;
2053
2054 ret = 0;
2055 if (context_present(context))
2056 goto out_unlock;
2057
2058 /*
2059 * For kdump cases, old valid entries may be cached due to the
2060 * in-flight DMA and copied pgtable, but there is no unmapping
2061 * behaviour for them, thus we need an explicit cache flush for
2062 * the newly-mapped device. For kdump, at this point, the device
2063 * is supposed to finish reset at its driver probe stage, so no
2064 * in-flight DMA will exist, and we don't need to worry anymore
2065 * hereafter.
2066 */
2067 if (context_copied(context)) {
2068 u16 did_old = context_domain_id(context);
2069
2070 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) {
2071 iommu->flush.flush_context(iommu, did_old,
2072 (((u16)bus) << 8) | devfn,
2073 DMA_CCMD_MASK_NOBIT,
2074 DMA_CCMD_DEVICE_INVL);
2075 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2076 DMA_TLB_DSI_FLUSH);
2077 }
2078 }
2079
2080 pgd = domain->pgd;
2081
2082 context_clear_entry(context);
2083 context_set_domain_id(context, did);
2084
2085 /*
2086 * Skip top levels of page tables for iommu which has less agaw
2087 * than default. Unnecessary for PT mode.
2088 */
2089 if (translation != CONTEXT_TT_PASS_THROUGH) {
2090 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2091 ret = -ENOMEM;
2092 pgd = phys_to_virt(dma_pte_addr(pgd));
2093 if (!dma_pte_present(pgd))
2094 goto out_unlock;
2095 }
2096
2097 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2098 if (info && info->ats_supported)
2099 translation = CONTEXT_TT_DEV_IOTLB;
2100 else
2101 translation = CONTEXT_TT_MULTI_LEVEL;
2102
2103 context_set_address_root(context, virt_to_phys(pgd));
2104 context_set_address_width(context, agaw);
2105 } else {
2106 /*
2107 * In pass through mode, AW must be programmed to
2108 * indicate the largest AGAW value supported by
2109 * hardware. And ASR is ignored by hardware.
2110 */
2111 context_set_address_width(context, iommu->msagaw);
2112 }
2113
2114 context_set_translation_type(context, translation);
2115 context_set_fault_enable(context);
2116 context_set_present(context);
2117 domain_flush_cache(domain, context, sizeof(*context));
2118
2119 /*
2120 * It's a non-present to present mapping. If hardware doesn't cache
2121 * non-present entry we only need to flush the write-buffer. If the
2122 * _does_ cache non-present entries, then it does so in the special
2123 * domain #0, which we have to flush:
2124 */
2125 if (cap_caching_mode(iommu->cap)) {
2126 iommu->flush.flush_context(iommu, 0,
2127 (((u16)bus) << 8) | devfn,
2128 DMA_CCMD_MASK_NOBIT,
2129 DMA_CCMD_DEVICE_INVL);
2130 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2131 } else {
2132 iommu_flush_write_buffer(iommu);
2133 }
2134 iommu_enable_dev_iotlb(info);
2135
2136 ret = 0;
2137
2138 out_unlock:
2139 spin_unlock(&iommu->lock);
2140 spin_unlock_irqrestore(&device_domain_lock, flags);
2141
2142 return ret;
2143 }
2144
2145 struct domain_context_mapping_data {
2146 struct dmar_domain *domain;
2147 struct intel_iommu *iommu;
2148 };
2149
2150 static int domain_context_mapping_cb(struct pci_dev *pdev,
2151 u16 alias, void *opaque)
2152 {
2153 struct domain_context_mapping_data *data = opaque;
2154
2155 return domain_context_mapping_one(data->domain, data->iommu,
2156 PCI_BUS_NUM(alias), alias & 0xff);
2157 }
2158
2159 static int
2160 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2161 {
2162 struct intel_iommu *iommu;
2163 u8 bus, devfn;
2164 struct domain_context_mapping_data data;
2165
2166 iommu = device_to_iommu(dev, &bus, &devfn);
2167 if (!iommu)
2168 return -ENODEV;
2169
2170 if (!dev_is_pci(dev))
2171 return domain_context_mapping_one(domain, iommu, bus, devfn);
2172
2173 data.domain = domain;
2174 data.iommu = iommu;
2175
2176 return pci_for_each_dma_alias(to_pci_dev(dev),
2177 &domain_context_mapping_cb, &data);
2178 }
2179
2180 static int domain_context_mapped_cb(struct pci_dev *pdev,
2181 u16 alias, void *opaque)
2182 {
2183 struct intel_iommu *iommu = opaque;
2184
2185 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2186 }
2187
2188 static int domain_context_mapped(struct device *dev)
2189 {
2190 struct intel_iommu *iommu;
2191 u8 bus, devfn;
2192
2193 iommu = device_to_iommu(dev, &bus, &devfn);
2194 if (!iommu)
2195 return -ENODEV;
2196
2197 if (!dev_is_pci(dev))
2198 return device_context_mapped(iommu, bus, devfn);
2199
2200 return !pci_for_each_dma_alias(to_pci_dev(dev),
2201 domain_context_mapped_cb, iommu);
2202 }
2203
2204 /* Returns a number of VTD pages, but aligned to MM page size */
2205 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2206 size_t size)
2207 {
2208 host_addr &= ~PAGE_MASK;
2209 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2210 }
2211
2212 /* Return largest possible superpage level for a given mapping */
2213 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2214 unsigned long iov_pfn,
2215 unsigned long phy_pfn,
2216 unsigned long pages)
2217 {
2218 int support, level = 1;
2219 unsigned long pfnmerge;
2220
2221 support = domain->iommu_superpage;
2222
2223 /* To use a large page, the virtual *and* physical addresses
2224 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2225 of them will mean we have to use smaller pages. So just
2226 merge them and check both at once. */
2227 pfnmerge = iov_pfn | phy_pfn;
2228
2229 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2230 pages >>= VTD_STRIDE_SHIFT;
2231 if (!pages)
2232 break;
2233 pfnmerge >>= VTD_STRIDE_SHIFT;
2234 level++;
2235 support--;
2236 }
2237 return level;
2238 }
2239
2240 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2241 struct scatterlist *sg, unsigned long phys_pfn,
2242 unsigned long nr_pages, int prot)
2243 {
2244 struct dma_pte *first_pte = NULL, *pte = NULL;
2245 phys_addr_t uninitialized_var(pteval);
2246 unsigned long sg_res = 0;
2247 unsigned int largepage_lvl = 0;
2248 unsigned long lvl_pages = 0;
2249
2250 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2251
2252 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2253 return -EINVAL;
2254
2255 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2256
2257 if (!sg) {
2258 sg_res = nr_pages;
2259 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2260 }
2261
2262 while (nr_pages > 0) {
2263 uint64_t tmp;
2264
2265 if (!sg_res) {
2266 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2267
2268 sg_res = aligned_nrpages(sg->offset, sg->length);
2269 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2270 sg->dma_length = sg->length;
2271 pteval = (sg_phys(sg) - pgoff) | prot;
2272 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2273 }
2274
2275 if (!pte) {
2276 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2277
2278 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2279 if (!pte)
2280 return -ENOMEM;
2281 /* It is large page*/
2282 if (largepage_lvl > 1) {
2283 unsigned long nr_superpages, end_pfn;
2284
2285 pteval |= DMA_PTE_LARGE_PAGE;
2286 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2287
2288 nr_superpages = sg_res / lvl_pages;
2289 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2290
2291 /*
2292 * Ensure that old small page tables are
2293 * removed to make room for superpage(s).
2294 */
2295 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2296 } else {
2297 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2298 }
2299
2300 }
2301 /* We don't need lock here, nobody else
2302 * touches the iova range
2303 */
2304 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2305 if (tmp) {
2306 static int dumps = 5;
2307 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2308 iov_pfn, tmp, (unsigned long long)pteval);
2309 if (dumps) {
2310 dumps--;
2311 debug_dma_dump_mappings(NULL);
2312 }
2313 WARN_ON(1);
2314 }
2315
2316 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2317
2318 BUG_ON(nr_pages < lvl_pages);
2319 BUG_ON(sg_res < lvl_pages);
2320
2321 nr_pages -= lvl_pages;
2322 iov_pfn += lvl_pages;
2323 phys_pfn += lvl_pages;
2324 pteval += lvl_pages * VTD_PAGE_SIZE;
2325 sg_res -= lvl_pages;
2326
2327 /* If the next PTE would be the first in a new page, then we
2328 need to flush the cache on the entries we've just written.
2329 And then we'll need to recalculate 'pte', so clear it and
2330 let it get set again in the if (!pte) block above.
2331
2332 If we're done (!nr_pages) we need to flush the cache too.
2333
2334 Also if we've been setting superpages, we may need to
2335 recalculate 'pte' and switch back to smaller pages for the
2336 end of the mapping, if the trailing size is not enough to
2337 use another superpage (i.e. sg_res < lvl_pages). */
2338 pte++;
2339 if (!nr_pages || first_pte_in_page(pte) ||
2340 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2341 domain_flush_cache(domain, first_pte,
2342 (void *)pte - (void *)first_pte);
2343 pte = NULL;
2344 }
2345
2346 if (!sg_res && nr_pages)
2347 sg = sg_next(sg);
2348 }
2349 return 0;
2350 }
2351
2352 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2353 struct scatterlist *sg, unsigned long nr_pages,
2354 int prot)
2355 {
2356 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2357 }
2358
2359 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2360 unsigned long phys_pfn, unsigned long nr_pages,
2361 int prot)
2362 {
2363 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2364 }
2365
2366 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2367 {
2368 if (!iommu)
2369 return;
2370
2371 clear_context_table(iommu, bus, devfn);
2372 iommu->flush.flush_context(iommu, 0, 0, 0,
2373 DMA_CCMD_GLOBAL_INVL);
2374 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2375 }
2376
2377 static inline void unlink_domain_info(struct device_domain_info *info)
2378 {
2379 assert_spin_locked(&device_domain_lock);
2380 list_del(&info->link);
2381 list_del(&info->global);
2382 if (info->dev)
2383 info->dev->archdata.iommu = NULL;
2384 }
2385
2386 static void domain_remove_dev_info(struct dmar_domain *domain)
2387 {
2388 struct device_domain_info *info, *tmp;
2389 unsigned long flags;
2390
2391 spin_lock_irqsave(&device_domain_lock, flags);
2392 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2393 __dmar_remove_one_dev_info(info);
2394 spin_unlock_irqrestore(&device_domain_lock, flags);
2395 }
2396
2397 /*
2398 * find_domain
2399 * Note: we use struct device->archdata.iommu stores the info
2400 */
2401 static struct dmar_domain *find_domain(struct device *dev)
2402 {
2403 struct device_domain_info *info;
2404
2405 /* No lock here, assumes no domain exit in normal case */
2406 info = dev->archdata.iommu;
2407 if (info)
2408 return info->domain;
2409 return NULL;
2410 }
2411
2412 static inline struct device_domain_info *
2413 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2414 {
2415 struct device_domain_info *info;
2416
2417 list_for_each_entry(info, &device_domain_list, global)
2418 if (info->iommu->segment == segment && info->bus == bus &&
2419 info->devfn == devfn)
2420 return info;
2421
2422 return NULL;
2423 }
2424
2425 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2426 int bus, int devfn,
2427 struct device *dev,
2428 struct dmar_domain *domain)
2429 {
2430 struct dmar_domain *found = NULL;
2431 struct device_domain_info *info;
2432 unsigned long flags;
2433 int ret;
2434
2435 info = alloc_devinfo_mem();
2436 if (!info)
2437 return NULL;
2438
2439 info->bus = bus;
2440 info->devfn = devfn;
2441 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2442 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2443 info->ats_qdep = 0;
2444 info->dev = dev;
2445 info->domain = domain;
2446 info->iommu = iommu;
2447
2448 if (dev && dev_is_pci(dev)) {
2449 struct pci_dev *pdev = to_pci_dev(info->dev);
2450
2451 if (ecap_dev_iotlb_support(iommu->ecap) &&
2452 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2453 dmar_find_matched_atsr_unit(pdev))
2454 info->ats_supported = 1;
2455
2456 if (ecs_enabled(iommu)) {
2457 if (pasid_enabled(iommu)) {
2458 int features = pci_pasid_features(pdev);
2459 if (features >= 0)
2460 info->pasid_supported = features | 1;
2461 }
2462
2463 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2464 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2465 info->pri_supported = 1;
2466 }
2467 }
2468
2469 spin_lock_irqsave(&device_domain_lock, flags);
2470 if (dev)
2471 found = find_domain(dev);
2472
2473 if (!found) {
2474 struct device_domain_info *info2;
2475 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2476 if (info2) {
2477 found = info2->domain;
2478 info2->dev = dev;
2479 }
2480 }
2481
2482 if (found) {
2483 spin_unlock_irqrestore(&device_domain_lock, flags);
2484 free_devinfo_mem(info);
2485 /* Caller must free the original domain */
2486 return found;
2487 }
2488
2489 spin_lock(&iommu->lock);
2490 ret = domain_attach_iommu(domain, iommu);
2491 spin_unlock(&iommu->lock);
2492
2493 if (ret) {
2494 spin_unlock_irqrestore(&device_domain_lock, flags);
2495 free_devinfo_mem(info);
2496 return NULL;
2497 }
2498
2499 list_add(&info->link, &domain->devices);
2500 list_add(&info->global, &device_domain_list);
2501 if (dev)
2502 dev->archdata.iommu = info;
2503 spin_unlock_irqrestore(&device_domain_lock, flags);
2504
2505 if (dev && domain_context_mapping(domain, dev)) {
2506 pr_err("Domain context map for %s failed\n", dev_name(dev));
2507 dmar_remove_one_dev_info(domain, dev);
2508 return NULL;
2509 }
2510
2511 return domain;
2512 }
2513
2514 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2515 {
2516 *(u16 *)opaque = alias;
2517 return 0;
2518 }
2519
2520 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2521 {
2522 struct device_domain_info *info = NULL;
2523 struct dmar_domain *domain = NULL;
2524 struct intel_iommu *iommu;
2525 u16 req_id, dma_alias;
2526 unsigned long flags;
2527 u8 bus, devfn;
2528
2529 iommu = device_to_iommu(dev, &bus, &devfn);
2530 if (!iommu)
2531 return NULL;
2532
2533 req_id = ((u16)bus << 8) | devfn;
2534
2535 if (dev_is_pci(dev)) {
2536 struct pci_dev *pdev = to_pci_dev(dev);
2537
2538 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2539
2540 spin_lock_irqsave(&device_domain_lock, flags);
2541 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2542 PCI_BUS_NUM(dma_alias),
2543 dma_alias & 0xff);
2544 if (info) {
2545 iommu = info->iommu;
2546 domain = info->domain;
2547 }
2548 spin_unlock_irqrestore(&device_domain_lock, flags);
2549
2550 /* DMA alias already has a domain, use it */
2551 if (info)
2552 goto out;
2553 }
2554
2555 /* Allocate and initialize new domain for the device */
2556 domain = alloc_domain(0);
2557 if (!domain)
2558 return NULL;
2559 if (domain_init(domain, iommu, gaw)) {
2560 domain_exit(domain);
2561 return NULL;
2562 }
2563
2564 out:
2565
2566 return domain;
2567 }
2568
2569 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2570 struct dmar_domain *domain)
2571 {
2572 struct intel_iommu *iommu;
2573 struct dmar_domain *tmp;
2574 u16 req_id, dma_alias;
2575 u8 bus, devfn;
2576
2577 iommu = device_to_iommu(dev, &bus, &devfn);
2578 if (!iommu)
2579 return NULL;
2580
2581 req_id = ((u16)bus << 8) | devfn;
2582
2583 if (dev_is_pci(dev)) {
2584 struct pci_dev *pdev = to_pci_dev(dev);
2585
2586 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2587
2588 /* register PCI DMA alias device */
2589 if (req_id != dma_alias) {
2590 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2591 dma_alias & 0xff, NULL, domain);
2592
2593 if (!tmp || tmp != domain)
2594 return tmp;
2595 }
2596 }
2597
2598 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2599 if (!tmp || tmp != domain)
2600 return tmp;
2601
2602 return domain;
2603 }
2604
2605 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2606 {
2607 struct dmar_domain *domain, *tmp;
2608
2609 domain = find_domain(dev);
2610 if (domain)
2611 goto out;
2612
2613 domain = find_or_alloc_domain(dev, gaw);
2614 if (!domain)
2615 goto out;
2616
2617 tmp = set_domain_for_dev(dev, domain);
2618 if (!tmp || domain != tmp) {
2619 domain_exit(domain);
2620 domain = tmp;
2621 }
2622
2623 out:
2624
2625 return domain;
2626 }
2627
2628 static int iommu_domain_identity_map(struct dmar_domain *domain,
2629 unsigned long long start,
2630 unsigned long long end)
2631 {
2632 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2633 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2634
2635 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2636 dma_to_mm_pfn(last_vpfn))) {
2637 pr_err("Reserving iova failed\n");
2638 return -ENOMEM;
2639 }
2640
2641 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2642 /*
2643 * RMRR range might have overlap with physical memory range,
2644 * clear it first
2645 */
2646 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2647
2648 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2649 last_vpfn - first_vpfn + 1,
2650 DMA_PTE_READ|DMA_PTE_WRITE);
2651 }
2652
2653 static int domain_prepare_identity_map(struct device *dev,
2654 struct dmar_domain *domain,
2655 unsigned long long start,
2656 unsigned long long end)
2657 {
2658 /* For _hardware_ passthrough, don't bother. But for software
2659 passthrough, we do it anyway -- it may indicate a memory
2660 range which is reserved in E820, so which didn't get set
2661 up to start with in si_domain */
2662 if (domain == si_domain && hw_pass_through) {
2663 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2664 dev_name(dev), start, end);
2665 return 0;
2666 }
2667
2668 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2669 dev_name(dev), start, end);
2670
2671 if (end < start) {
2672 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2673 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2674 dmi_get_system_info(DMI_BIOS_VENDOR),
2675 dmi_get_system_info(DMI_BIOS_VERSION),
2676 dmi_get_system_info(DMI_PRODUCT_VERSION));
2677 return -EIO;
2678 }
2679
2680 if (end >> agaw_to_width(domain->agaw)) {
2681 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2682 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2683 agaw_to_width(domain->agaw),
2684 dmi_get_system_info(DMI_BIOS_VENDOR),
2685 dmi_get_system_info(DMI_BIOS_VERSION),
2686 dmi_get_system_info(DMI_PRODUCT_VERSION));
2687 return -EIO;
2688 }
2689
2690 return iommu_domain_identity_map(domain, start, end);
2691 }
2692
2693 static int iommu_prepare_identity_map(struct device *dev,
2694 unsigned long long start,
2695 unsigned long long end)
2696 {
2697 struct dmar_domain *domain;
2698 int ret;
2699
2700 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2701 if (!domain)
2702 return -ENOMEM;
2703
2704 ret = domain_prepare_identity_map(dev, domain, start, end);
2705 if (ret)
2706 domain_exit(domain);
2707
2708 return ret;
2709 }
2710
2711 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2712 struct device *dev)
2713 {
2714 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2715 return 0;
2716 return iommu_prepare_identity_map(dev, rmrr->base_address,
2717 rmrr->end_address);
2718 }
2719
2720 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2721 static inline void iommu_prepare_isa(void)
2722 {
2723 struct pci_dev *pdev;
2724 int ret;
2725
2726 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2727 if (!pdev)
2728 return;
2729
2730 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2731 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2732
2733 if (ret)
2734 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2735
2736 pci_dev_put(pdev);
2737 }
2738 #else
2739 static inline void iommu_prepare_isa(void)
2740 {
2741 return;
2742 }
2743 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2744
2745 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2746
2747 static int __init si_domain_init(int hw)
2748 {
2749 int nid, ret = 0;
2750
2751 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2752 if (!si_domain)
2753 return -EFAULT;
2754
2755 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2756 domain_exit(si_domain);
2757 return -EFAULT;
2758 }
2759
2760 pr_debug("Identity mapping domain allocated\n");
2761
2762 if (hw)
2763 return 0;
2764
2765 for_each_online_node(nid) {
2766 unsigned long start_pfn, end_pfn;
2767 int i;
2768
2769 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2770 ret = iommu_domain_identity_map(si_domain,
2771 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2772 if (ret)
2773 return ret;
2774 }
2775 }
2776
2777 return 0;
2778 }
2779
2780 static int identity_mapping(struct device *dev)
2781 {
2782 struct device_domain_info *info;
2783
2784 if (likely(!iommu_identity_mapping))
2785 return 0;
2786
2787 info = dev->archdata.iommu;
2788 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2789 return (info->domain == si_domain);
2790
2791 return 0;
2792 }
2793
2794 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2795 {
2796 struct dmar_domain *ndomain;
2797 struct intel_iommu *iommu;
2798 u8 bus, devfn;
2799
2800 iommu = device_to_iommu(dev, &bus, &devfn);
2801 if (!iommu)
2802 return -ENODEV;
2803
2804 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2805 if (ndomain != domain)
2806 return -EBUSY;
2807
2808 return 0;
2809 }
2810
2811 static bool device_has_rmrr(struct device *dev)
2812 {
2813 struct dmar_rmrr_unit *rmrr;
2814 struct device *tmp;
2815 int i;
2816
2817 rcu_read_lock();
2818 for_each_rmrr_units(rmrr) {
2819 /*
2820 * Return TRUE if this RMRR contains the device that
2821 * is passed in.
2822 */
2823 for_each_active_dev_scope(rmrr->devices,
2824 rmrr->devices_cnt, i, tmp)
2825 if (tmp == dev) {
2826 rcu_read_unlock();
2827 return true;
2828 }
2829 }
2830 rcu_read_unlock();
2831 return false;
2832 }
2833
2834 /*
2835 * There are a couple cases where we need to restrict the functionality of
2836 * devices associated with RMRRs. The first is when evaluating a device for
2837 * identity mapping because problems exist when devices are moved in and out
2838 * of domains and their respective RMRR information is lost. This means that
2839 * a device with associated RMRRs will never be in a "passthrough" domain.
2840 * The second is use of the device through the IOMMU API. This interface
2841 * expects to have full control of the IOVA space for the device. We cannot
2842 * satisfy both the requirement that RMRR access is maintained and have an
2843 * unencumbered IOVA space. We also have no ability to quiesce the device's
2844 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2845 * We therefore prevent devices associated with an RMRR from participating in
2846 * the IOMMU API, which eliminates them from device assignment.
2847 *
2848 * In both cases we assume that PCI USB devices with RMRRs have them largely
2849 * for historical reasons and that the RMRR space is not actively used post
2850 * boot. This exclusion may change if vendors begin to abuse it.
2851 *
2852 * The same exception is made for graphics devices, with the requirement that
2853 * any use of the RMRR regions will be torn down before assigning the device
2854 * to a guest.
2855 */
2856 static bool device_is_rmrr_locked(struct device *dev)
2857 {
2858 if (!device_has_rmrr(dev))
2859 return false;
2860
2861 if (dev_is_pci(dev)) {
2862 struct pci_dev *pdev = to_pci_dev(dev);
2863
2864 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2865 return false;
2866 }
2867
2868 return true;
2869 }
2870
2871 static int iommu_should_identity_map(struct device *dev, int startup)
2872 {
2873
2874 if (dev_is_pci(dev)) {
2875 struct pci_dev *pdev = to_pci_dev(dev);
2876
2877 if (device_is_rmrr_locked(dev))
2878 return 0;
2879
2880 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2881 return 1;
2882
2883 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2884 return 1;
2885
2886 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2887 return 0;
2888
2889 /*
2890 * We want to start off with all devices in the 1:1 domain, and
2891 * take them out later if we find they can't access all of memory.
2892 *
2893 * However, we can't do this for PCI devices behind bridges,
2894 * because all PCI devices behind the same bridge will end up
2895 * with the same source-id on their transactions.
2896 *
2897 * Practically speaking, we can't change things around for these
2898 * devices at run-time, because we can't be sure there'll be no
2899 * DMA transactions in flight for any of their siblings.
2900 *
2901 * So PCI devices (unless they're on the root bus) as well as
2902 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2903 * the 1:1 domain, just in _case_ one of their siblings turns out
2904 * not to be able to map all of memory.
2905 */
2906 if (!pci_is_pcie(pdev)) {
2907 if (!pci_is_root_bus(pdev->bus))
2908 return 0;
2909 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2910 return 0;
2911 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2912 return 0;
2913 } else {
2914 if (device_has_rmrr(dev))
2915 return 0;
2916 }
2917
2918 /*
2919 * At boot time, we don't yet know if devices will be 64-bit capable.
2920 * Assume that they will — if they turn out not to be, then we can
2921 * take them out of the 1:1 domain later.
2922 */
2923 if (!startup) {
2924 /*
2925 * If the device's dma_mask is less than the system's memory
2926 * size then this is not a candidate for identity mapping.
2927 */
2928 u64 dma_mask = *dev->dma_mask;
2929
2930 if (dev->coherent_dma_mask &&
2931 dev->coherent_dma_mask < dma_mask)
2932 dma_mask = dev->coherent_dma_mask;
2933
2934 return dma_mask >= dma_get_required_mask(dev);
2935 }
2936
2937 return 1;
2938 }
2939
2940 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2941 {
2942 int ret;
2943
2944 if (!iommu_should_identity_map(dev, 1))
2945 return 0;
2946
2947 ret = domain_add_dev_info(si_domain, dev);
2948 if (!ret)
2949 pr_info("%s identity mapping for device %s\n",
2950 hw ? "Hardware" : "Software", dev_name(dev));
2951 else if (ret == -ENODEV)
2952 /* device not associated with an iommu */
2953 ret = 0;
2954
2955 return ret;
2956 }
2957
2958
2959 static int __init iommu_prepare_static_identity_mapping(int hw)
2960 {
2961 struct pci_dev *pdev = NULL;
2962 struct dmar_drhd_unit *drhd;
2963 struct intel_iommu *iommu;
2964 struct device *dev;
2965 int i;
2966 int ret = 0;
2967
2968 for_each_pci_dev(pdev) {
2969 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2970 if (ret)
2971 return ret;
2972 }
2973
2974 for_each_active_iommu(iommu, drhd)
2975 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2976 struct acpi_device_physical_node *pn;
2977 struct acpi_device *adev;
2978
2979 if (dev->bus != &acpi_bus_type)
2980 continue;
2981
2982 adev= to_acpi_device(dev);
2983 mutex_lock(&adev->physical_node_lock);
2984 list_for_each_entry(pn, &adev->physical_node_list, node) {
2985 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2986 if (ret)
2987 break;
2988 }
2989 mutex_unlock(&adev->physical_node_lock);
2990 if (ret)
2991 return ret;
2992 }
2993
2994 return 0;
2995 }
2996
2997 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2998 {
2999 /*
3000 * Start from the sane iommu hardware state.
3001 * If the queued invalidation is already initialized by us
3002 * (for example, while enabling interrupt-remapping) then
3003 * we got the things already rolling from a sane state.
3004 */
3005 if (!iommu->qi) {
3006 /*
3007 * Clear any previous faults.
3008 */
3009 dmar_fault(-1, iommu);
3010 /*
3011 * Disable queued invalidation if supported and already enabled
3012 * before OS handover.
3013 */
3014 dmar_disable_qi(iommu);
3015 }
3016
3017 if (dmar_enable_qi(iommu)) {
3018 /*
3019 * Queued Invalidate not enabled, use Register Based Invalidate
3020 */
3021 iommu->flush.flush_context = __iommu_flush_context;
3022 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3023 pr_info("%s: Using Register based invalidation\n",
3024 iommu->name);
3025 } else {
3026 iommu->flush.flush_context = qi_flush_context;
3027 iommu->flush.flush_iotlb = qi_flush_iotlb;
3028 pr_info("%s: Using Queued invalidation\n", iommu->name);
3029 }
3030 }
3031
3032 static int copy_context_table(struct intel_iommu *iommu,
3033 struct root_entry *old_re,
3034 struct context_entry **tbl,
3035 int bus, bool ext)
3036 {
3037 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3038 struct context_entry *new_ce = NULL, ce;
3039 struct context_entry *old_ce = NULL;
3040 struct root_entry re;
3041 phys_addr_t old_ce_phys;
3042
3043 tbl_idx = ext ? bus * 2 : bus;
3044 memcpy(&re, old_re, sizeof(re));
3045
3046 for (devfn = 0; devfn < 256; devfn++) {
3047 /* First calculate the correct index */
3048 idx = (ext ? devfn * 2 : devfn) % 256;
3049
3050 if (idx == 0) {
3051 /* First save what we may have and clean up */
3052 if (new_ce) {
3053 tbl[tbl_idx] = new_ce;
3054 __iommu_flush_cache(iommu, new_ce,
3055 VTD_PAGE_SIZE);
3056 pos = 1;
3057 }
3058
3059 if (old_ce)
3060 memunmap(old_ce);
3061
3062 ret = 0;
3063 if (devfn < 0x80)
3064 old_ce_phys = root_entry_lctp(&re);
3065 else
3066 old_ce_phys = root_entry_uctp(&re);
3067
3068 if (!old_ce_phys) {
3069 if (ext && devfn == 0) {
3070 /* No LCTP, try UCTP */
3071 devfn = 0x7f;
3072 continue;
3073 } else {
3074 goto out;
3075 }
3076 }
3077
3078 ret = -ENOMEM;
3079 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3080 MEMREMAP_WB);
3081 if (!old_ce)
3082 goto out;
3083
3084 new_ce = alloc_pgtable_page(iommu->node);
3085 if (!new_ce)
3086 goto out_unmap;
3087
3088 ret = 0;
3089 }
3090
3091 /* Now copy the context entry */
3092 memcpy(&ce, old_ce + idx, sizeof(ce));
3093
3094 if (!__context_present(&ce))
3095 continue;
3096
3097 did = context_domain_id(&ce);
3098 if (did >= 0 && did < cap_ndoms(iommu->cap))
3099 set_bit(did, iommu->domain_ids);
3100
3101 /*
3102 * We need a marker for copied context entries. This
3103 * marker needs to work for the old format as well as
3104 * for extended context entries.
3105 *
3106 * Bit 67 of the context entry is used. In the old
3107 * format this bit is available to software, in the
3108 * extended format it is the PGE bit, but PGE is ignored
3109 * by HW if PASIDs are disabled (and thus still
3110 * available).
3111 *
3112 * So disable PASIDs first and then mark the entry
3113 * copied. This means that we don't copy PASID
3114 * translations from the old kernel, but this is fine as
3115 * faults there are not fatal.
3116 */
3117 context_clear_pasid_enable(&ce);
3118 context_set_copied(&ce);
3119
3120 new_ce[idx] = ce;
3121 }
3122
3123 tbl[tbl_idx + pos] = new_ce;
3124
3125 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3126
3127 out_unmap:
3128 memunmap(old_ce);
3129
3130 out:
3131 return ret;
3132 }
3133
3134 static int copy_translation_tables(struct intel_iommu *iommu)
3135 {
3136 struct context_entry **ctxt_tbls;
3137 struct root_entry *old_rt;
3138 phys_addr_t old_rt_phys;
3139 int ctxt_table_entries;
3140 unsigned long flags;
3141 u64 rtaddr_reg;
3142 int bus, ret;
3143 bool new_ext, ext;
3144
3145 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3146 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3147 new_ext = !!ecap_ecs(iommu->ecap);
3148
3149 /*
3150 * The RTT bit can only be changed when translation is disabled,
3151 * but disabling translation means to open a window for data
3152 * corruption. So bail out and don't copy anything if we would
3153 * have to change the bit.
3154 */
3155 if (new_ext != ext)
3156 return -EINVAL;
3157
3158 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3159 if (!old_rt_phys)
3160 return -EINVAL;
3161
3162 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3163 if (!old_rt)
3164 return -ENOMEM;
3165
3166 /* This is too big for the stack - allocate it from slab */
3167 ctxt_table_entries = ext ? 512 : 256;
3168 ret = -ENOMEM;
3169 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3170 if (!ctxt_tbls)
3171 goto out_unmap;
3172
3173 for (bus = 0; bus < 256; bus++) {
3174 ret = copy_context_table(iommu, &old_rt[bus],
3175 ctxt_tbls, bus, ext);
3176 if (ret) {
3177 pr_err("%s: Failed to copy context table for bus %d\n",
3178 iommu->name, bus);
3179 continue;
3180 }
3181 }
3182
3183 spin_lock_irqsave(&iommu->lock, flags);
3184
3185 /* Context tables are copied, now write them to the root_entry table */
3186 for (bus = 0; bus < 256; bus++) {
3187 int idx = ext ? bus * 2 : bus;
3188 u64 val;
3189
3190 if (ctxt_tbls[idx]) {
3191 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3192 iommu->root_entry[bus].lo = val;
3193 }
3194
3195 if (!ext || !ctxt_tbls[idx + 1])
3196 continue;
3197
3198 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3199 iommu->root_entry[bus].hi = val;
3200 }
3201
3202 spin_unlock_irqrestore(&iommu->lock, flags);
3203
3204 kfree(ctxt_tbls);
3205
3206 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3207
3208 ret = 0;
3209
3210 out_unmap:
3211 memunmap(old_rt);
3212
3213 return ret;
3214 }
3215
3216 static int __init init_dmars(void)
3217 {
3218 struct dmar_drhd_unit *drhd;
3219 struct dmar_rmrr_unit *rmrr;
3220 bool copied_tables = false;
3221 struct device *dev;
3222 struct intel_iommu *iommu;
3223 int i, ret, cpu;
3224
3225 /*
3226 * for each drhd
3227 * allocate root
3228 * initialize and program root entry to not present
3229 * endfor
3230 */
3231 for_each_drhd_unit(drhd) {
3232 /*
3233 * lock not needed as this is only incremented in the single
3234 * threaded kernel __init code path all other access are read
3235 * only
3236 */
3237 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3238 g_num_of_iommus++;
3239 continue;
3240 }
3241 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3242 }
3243
3244 /* Preallocate enough resources for IOMMU hot-addition */
3245 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3246 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3247
3248 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3249 GFP_KERNEL);
3250 if (!g_iommus) {
3251 pr_err("Allocating global iommu array failed\n");
3252 ret = -ENOMEM;
3253 goto error;
3254 }
3255
3256 for_each_possible_cpu(cpu) {
3257 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3258 cpu);
3259
3260 dfd->tables = kzalloc(g_num_of_iommus *
3261 sizeof(struct deferred_flush_table),
3262 GFP_KERNEL);
3263 if (!dfd->tables) {
3264 ret = -ENOMEM;
3265 goto free_g_iommus;
3266 }
3267
3268 spin_lock_init(&dfd->lock);
3269 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3270 }
3271
3272 for_each_active_iommu(iommu, drhd) {
3273 g_iommus[iommu->seq_id] = iommu;
3274
3275 intel_iommu_init_qi(iommu);
3276
3277 ret = iommu_init_domains(iommu);
3278 if (ret)
3279 goto free_iommu;
3280
3281 init_translation_status(iommu);
3282
3283 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3284 iommu_disable_translation(iommu);
3285 clear_translation_pre_enabled(iommu);
3286 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3287 iommu->name);
3288 }
3289
3290 /*
3291 * TBD:
3292 * we could share the same root & context tables
3293 * among all IOMMU's. Need to Split it later.
3294 */
3295 ret = iommu_alloc_root_entry(iommu);
3296 if (ret)
3297 goto free_iommu;
3298
3299 if (translation_pre_enabled(iommu)) {
3300 pr_info("Translation already enabled - trying to copy translation structures\n");
3301
3302 ret = copy_translation_tables(iommu);
3303 if (ret) {
3304 /*
3305 * We found the IOMMU with translation
3306 * enabled - but failed to copy over the
3307 * old root-entry table. Try to proceed
3308 * by disabling translation now and
3309 * allocating a clean root-entry table.
3310 * This might cause DMAR faults, but
3311 * probably the dump will still succeed.
3312 */
3313 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3314 iommu->name);
3315 iommu_disable_translation(iommu);
3316 clear_translation_pre_enabled(iommu);
3317 } else {
3318 pr_info("Copied translation tables from previous kernel for %s\n",
3319 iommu->name);
3320 copied_tables = true;
3321 }
3322 }
3323
3324 if (!ecap_pass_through(iommu->ecap))
3325 hw_pass_through = 0;
3326 #ifdef CONFIG_INTEL_IOMMU_SVM
3327 if (pasid_enabled(iommu))
3328 intel_svm_alloc_pasid_tables(iommu);
3329 #endif
3330 }
3331
3332 /*
3333 * Now that qi is enabled on all iommus, set the root entry and flush
3334 * caches. This is required on some Intel X58 chipsets, otherwise the
3335 * flush_context function will loop forever and the boot hangs.
3336 */
3337 for_each_active_iommu(iommu, drhd) {
3338 iommu_flush_write_buffer(iommu);
3339 iommu_set_root_entry(iommu);
3340 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3341 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3342 }
3343
3344 if (iommu_pass_through)
3345 iommu_identity_mapping |= IDENTMAP_ALL;
3346
3347 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3348 iommu_identity_mapping |= IDENTMAP_GFX;
3349 #endif
3350
3351 check_tylersburg_isoch();
3352
3353 if (iommu_identity_mapping) {
3354 ret = si_domain_init(hw_pass_through);
3355 if (ret)
3356 goto free_iommu;
3357 }
3358
3359
3360 /*
3361 * If we copied translations from a previous kernel in the kdump
3362 * case, we can not assign the devices to domains now, as that
3363 * would eliminate the old mappings. So skip this part and defer
3364 * the assignment to device driver initialization time.
3365 */
3366 if (copied_tables)
3367 goto domains_done;
3368
3369 /*
3370 * If pass through is not set or not enabled, setup context entries for
3371 * identity mappings for rmrr, gfx, and isa and may fall back to static
3372 * identity mapping if iommu_identity_mapping is set.
3373 */
3374 if (iommu_identity_mapping) {
3375 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3376 if (ret) {
3377 pr_crit("Failed to setup IOMMU pass-through\n");
3378 goto free_iommu;
3379 }
3380 }
3381 /*
3382 * For each rmrr
3383 * for each dev attached to rmrr
3384 * do
3385 * locate drhd for dev, alloc domain for dev
3386 * allocate free domain
3387 * allocate page table entries for rmrr
3388 * if context not allocated for bus
3389 * allocate and init context
3390 * set present in root table for this bus
3391 * init context with domain, translation etc
3392 * endfor
3393 * endfor
3394 */
3395 pr_info("Setting RMRR:\n");
3396 for_each_rmrr_units(rmrr) {
3397 /* some BIOS lists non-exist devices in DMAR table. */
3398 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3399 i, dev) {
3400 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3401 if (ret)
3402 pr_err("Mapping reserved region failed\n");
3403 }
3404 }
3405
3406 iommu_prepare_isa();
3407
3408 domains_done:
3409
3410 /*
3411 * for each drhd
3412 * enable fault log
3413 * global invalidate context cache
3414 * global invalidate iotlb
3415 * enable translation
3416 */
3417 for_each_iommu(iommu, drhd) {
3418 if (drhd->ignored) {
3419 /*
3420 * we always have to disable PMRs or DMA may fail on
3421 * this device
3422 */
3423 if (force_on)
3424 iommu_disable_protect_mem_regions(iommu);
3425 continue;
3426 }
3427
3428 iommu_flush_write_buffer(iommu);
3429
3430 #ifdef CONFIG_INTEL_IOMMU_SVM
3431 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3432 ret = intel_svm_enable_prq(iommu);
3433 if (ret)
3434 goto free_iommu;
3435 }
3436 #endif
3437 ret = dmar_set_interrupt(iommu);
3438 if (ret)
3439 goto free_iommu;
3440
3441 if (!translation_pre_enabled(iommu))
3442 iommu_enable_translation(iommu);
3443
3444 iommu_disable_protect_mem_regions(iommu);
3445 }
3446
3447 return 0;
3448
3449 free_iommu:
3450 for_each_active_iommu(iommu, drhd) {
3451 disable_dmar_iommu(iommu);
3452 free_dmar_iommu(iommu);
3453 }
3454 free_g_iommus:
3455 for_each_possible_cpu(cpu)
3456 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3457 kfree(g_iommus);
3458 error:
3459 return ret;
3460 }
3461
3462 /* This takes a number of _MM_ pages, not VTD pages */
3463 static unsigned long intel_alloc_iova(struct device *dev,
3464 struct dmar_domain *domain,
3465 unsigned long nrpages, uint64_t dma_mask)
3466 {
3467 unsigned long iova_pfn = 0;
3468
3469 /* Restrict dma_mask to the width that the iommu can handle */
3470 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3471 /* Ensure we reserve the whole size-aligned region */
3472 nrpages = __roundup_pow_of_two(nrpages);
3473
3474 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3475 /*
3476 * First try to allocate an io virtual address in
3477 * DMA_BIT_MASK(32) and if that fails then try allocating
3478 * from higher range
3479 */
3480 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3481 IOVA_PFN(DMA_BIT_MASK(32)));
3482 if (iova_pfn)
3483 return iova_pfn;
3484 }
3485 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3486 if (unlikely(!iova_pfn)) {
3487 pr_err("Allocating %ld-page iova for %s failed",
3488 nrpages, dev_name(dev));
3489 return 0;
3490 }
3491
3492 return iova_pfn;
3493 }
3494
3495 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3496 {
3497 struct dmar_domain *domain, *tmp;
3498 struct dmar_rmrr_unit *rmrr;
3499 struct device *i_dev;
3500 int i, ret;
3501
3502 domain = find_domain(dev);
3503 if (domain)
3504 goto out;
3505
3506 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3507 if (!domain)
3508 goto out;
3509
3510 /* We have a new domain - setup possible RMRRs for the device */
3511 rcu_read_lock();
3512 for_each_rmrr_units(rmrr) {
3513 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3514 i, i_dev) {
3515 if (i_dev != dev)
3516 continue;
3517
3518 ret = domain_prepare_identity_map(dev, domain,
3519 rmrr->base_address,
3520 rmrr->end_address);
3521 if (ret)
3522 dev_err(dev, "Mapping reserved region failed\n");
3523 }
3524 }
3525 rcu_read_unlock();
3526
3527 tmp = set_domain_for_dev(dev, domain);
3528 if (!tmp || domain != tmp) {
3529 domain_exit(domain);
3530 domain = tmp;
3531 }
3532
3533 out:
3534
3535 if (!domain)
3536 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3537
3538
3539 return domain;
3540 }
3541
3542 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3543 {
3544 struct device_domain_info *info;
3545
3546 /* No lock here, assumes no domain exit in normal case */
3547 info = dev->archdata.iommu;
3548 if (likely(info))
3549 return info->domain;
3550
3551 return __get_valid_domain_for_dev(dev);
3552 }
3553
3554 /* Check if the dev needs to go through non-identity map and unmap process.*/
3555 static int iommu_no_mapping(struct device *dev)
3556 {
3557 int found;
3558
3559 if (iommu_dummy(dev))
3560 return 1;
3561
3562 if (!iommu_identity_mapping)
3563 return 0;
3564
3565 found = identity_mapping(dev);
3566 if (found) {
3567 if (iommu_should_identity_map(dev, 0))
3568 return 1;
3569 else {
3570 /*
3571 * 32 bit DMA is removed from si_domain and fall back
3572 * to non-identity mapping.
3573 */
3574 dmar_remove_one_dev_info(si_domain, dev);
3575 pr_info("32bit %s uses non-identity mapping\n",
3576 dev_name(dev));
3577 return 0;
3578 }
3579 } else {
3580 /*
3581 * In case of a detached 64 bit DMA device from vm, the device
3582 * is put into si_domain for identity mapping.
3583 */
3584 if (iommu_should_identity_map(dev, 0)) {
3585 int ret;
3586 ret = domain_add_dev_info(si_domain, dev);
3587 if (!ret) {
3588 pr_info("64bit %s uses identity mapping\n",
3589 dev_name(dev));
3590 return 1;
3591 }
3592 }
3593 }
3594
3595 return 0;
3596 }
3597
3598 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3599 size_t size, int dir, u64 dma_mask)
3600 {
3601 struct dmar_domain *domain;
3602 phys_addr_t start_paddr;
3603 unsigned long iova_pfn;
3604 int prot = 0;
3605 int ret;
3606 struct intel_iommu *iommu;
3607 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3608
3609 BUG_ON(dir == DMA_NONE);
3610
3611 if (iommu_no_mapping(dev))
3612 return paddr;
3613
3614 domain = get_valid_domain_for_dev(dev);
3615 if (!domain)
3616 return 0;
3617
3618 iommu = domain_get_iommu(domain);
3619 size = aligned_nrpages(paddr, size);
3620
3621 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3622 if (!iova_pfn)
3623 goto error;
3624
3625 /*
3626 * Check if DMAR supports zero-length reads on write only
3627 * mappings..
3628 */
3629 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3630 !cap_zlr(iommu->cap))
3631 prot |= DMA_PTE_READ;
3632 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3633 prot |= DMA_PTE_WRITE;
3634 /*
3635 * paddr - (paddr + size) might be partial page, we should map the whole
3636 * page. Note: if two part of one page are separately mapped, we
3637 * might have two guest_addr mapping to the same host paddr, but this
3638 * is not a big problem
3639 */
3640 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3641 mm_to_dma_pfn(paddr_pfn), size, prot);
3642 if (ret)
3643 goto error;
3644
3645 /* it's a non-present to present mapping. Only flush if caching mode */
3646 if (cap_caching_mode(iommu->cap))
3647 iommu_flush_iotlb_psi(iommu, domain,
3648 mm_to_dma_pfn(iova_pfn),
3649 size, 0, 1);
3650 else
3651 iommu_flush_write_buffer(iommu);
3652
3653 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3654 start_paddr += paddr & ~PAGE_MASK;
3655 return start_paddr;
3656
3657 error:
3658 if (iova_pfn)
3659 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3660 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3661 dev_name(dev), size, (unsigned long long)paddr, dir);
3662 return 0;
3663 }
3664
3665 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3666 unsigned long offset, size_t size,
3667 enum dma_data_direction dir,
3668 unsigned long attrs)
3669 {
3670 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3671 dir, *dev->dma_mask);
3672 }
3673
3674 static void flush_unmaps(struct deferred_flush_data *flush_data)
3675 {
3676 int i, j;
3677
3678 flush_data->timer_on = 0;
3679
3680 /* just flush them all */
3681 for (i = 0; i < g_num_of_iommus; i++) {
3682 struct intel_iommu *iommu = g_iommus[i];
3683 struct deferred_flush_table *flush_table =
3684 &flush_data->tables[i];
3685 if (!iommu)
3686 continue;
3687
3688 if (!flush_table->next)
3689 continue;
3690
3691 /* In caching mode, global flushes turn emulation expensive */
3692 if (!cap_caching_mode(iommu->cap))
3693 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3694 DMA_TLB_GLOBAL_FLUSH);
3695 for (j = 0; j < flush_table->next; j++) {
3696 unsigned long mask;
3697 struct deferred_flush_entry *entry =
3698 &flush_table->entries[j];
3699 unsigned long iova_pfn = entry->iova_pfn;
3700 unsigned long nrpages = entry->nrpages;
3701 struct dmar_domain *domain = entry->domain;
3702 struct page *freelist = entry->freelist;
3703
3704 /* On real hardware multiple invalidations are expensive */
3705 if (cap_caching_mode(iommu->cap))
3706 iommu_flush_iotlb_psi(iommu, domain,
3707 mm_to_dma_pfn(iova_pfn),
3708 nrpages, !freelist, 0);
3709 else {
3710 mask = ilog2(nrpages);
3711 iommu_flush_dev_iotlb(domain,
3712 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3713 }
3714 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3715 if (freelist)
3716 dma_free_pagelist(freelist);
3717 }
3718 flush_table->next = 0;
3719 }
3720
3721 flush_data->size = 0;
3722 }
3723
3724 static void flush_unmaps_timeout(unsigned long cpuid)
3725 {
3726 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3727 unsigned long flags;
3728
3729 spin_lock_irqsave(&flush_data->lock, flags);
3730 flush_unmaps(flush_data);
3731 spin_unlock_irqrestore(&flush_data->lock, flags);
3732 }
3733
3734 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3735 unsigned long nrpages, struct page *freelist)
3736 {
3737 unsigned long flags;
3738 int entry_id, iommu_id;
3739 struct intel_iommu *iommu;
3740 struct deferred_flush_entry *entry;
3741 struct deferred_flush_data *flush_data;
3742 unsigned int cpuid;
3743
3744 cpuid = get_cpu();
3745 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3746
3747 /* Flush all CPUs' entries to avoid deferring too much. If
3748 * this becomes a bottleneck, can just flush us, and rely on
3749 * flush timer for the rest.
3750 */
3751 if (flush_data->size == HIGH_WATER_MARK) {
3752 int cpu;
3753
3754 for_each_online_cpu(cpu)
3755 flush_unmaps_timeout(cpu);
3756 }
3757
3758 spin_lock_irqsave(&flush_data->lock, flags);
3759
3760 iommu = domain_get_iommu(dom);
3761 iommu_id = iommu->seq_id;
3762
3763 entry_id = flush_data->tables[iommu_id].next;
3764 ++(flush_data->tables[iommu_id].next);
3765
3766 entry = &flush_data->tables[iommu_id].entries[entry_id];
3767 entry->domain = dom;
3768 entry->iova_pfn = iova_pfn;
3769 entry->nrpages = nrpages;
3770 entry->freelist = freelist;
3771
3772 if (!flush_data->timer_on) {
3773 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3774 flush_data->timer_on = 1;
3775 }
3776 flush_data->size++;
3777 spin_unlock_irqrestore(&flush_data->lock, flags);
3778
3779 put_cpu();
3780 }
3781
3782 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3783 {
3784 struct dmar_domain *domain;
3785 unsigned long start_pfn, last_pfn;
3786 unsigned long nrpages;
3787 unsigned long iova_pfn;
3788 struct intel_iommu *iommu;
3789 struct page *freelist;
3790
3791 if (iommu_no_mapping(dev))
3792 return;
3793
3794 domain = find_domain(dev);
3795 BUG_ON(!domain);
3796
3797 iommu = domain_get_iommu(domain);
3798
3799 iova_pfn = IOVA_PFN(dev_addr);
3800
3801 nrpages = aligned_nrpages(dev_addr, size);
3802 start_pfn = mm_to_dma_pfn(iova_pfn);
3803 last_pfn = start_pfn + nrpages - 1;
3804
3805 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3806 dev_name(dev), start_pfn, last_pfn);
3807
3808 freelist = domain_unmap(domain, start_pfn, last_pfn);
3809
3810 if (intel_iommu_strict) {
3811 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3812 nrpages, !freelist, 0);
3813 /* free iova */
3814 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3815 dma_free_pagelist(freelist);
3816 } else {
3817 add_unmap(domain, iova_pfn, nrpages, freelist);
3818 /*
3819 * queue up the release of the unmap to save the 1/6th of the
3820 * cpu used up by the iotlb flush operation...
3821 */
3822 }
3823 }
3824
3825 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3826 size_t size, enum dma_data_direction dir,
3827 unsigned long attrs)
3828 {
3829 intel_unmap(dev, dev_addr, size);
3830 }
3831
3832 static void *intel_alloc_coherent(struct device *dev, size_t size,
3833 dma_addr_t *dma_handle, gfp_t flags,
3834 unsigned long attrs)
3835 {
3836 struct page *page = NULL;
3837 int order;
3838
3839 size = PAGE_ALIGN(size);
3840 order = get_order(size);
3841
3842 if (!iommu_no_mapping(dev))
3843 flags &= ~(GFP_DMA | GFP_DMA32);
3844 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3845 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3846 flags |= GFP_DMA;
3847 else
3848 flags |= GFP_DMA32;
3849 }
3850
3851 if (gfpflags_allow_blocking(flags)) {
3852 unsigned int count = size >> PAGE_SHIFT;
3853
3854 page = dma_alloc_from_contiguous(dev, count, order);
3855 if (page && iommu_no_mapping(dev) &&
3856 page_to_phys(page) + size > dev->coherent_dma_mask) {
3857 dma_release_from_contiguous(dev, page, count);
3858 page = NULL;
3859 }
3860 }
3861
3862 if (!page)
3863 page = alloc_pages(flags, order);
3864 if (!page)
3865 return NULL;
3866 memset(page_address(page), 0, size);
3867
3868 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3869 DMA_BIDIRECTIONAL,
3870 dev->coherent_dma_mask);
3871 if (*dma_handle)
3872 return page_address(page);
3873 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3874 __free_pages(page, order);
3875
3876 return NULL;
3877 }
3878
3879 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3880 dma_addr_t dma_handle, unsigned long attrs)
3881 {
3882 int order;
3883 struct page *page = virt_to_page(vaddr);
3884
3885 size = PAGE_ALIGN(size);
3886 order = get_order(size);
3887
3888 intel_unmap(dev, dma_handle, size);
3889 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3890 __free_pages(page, order);
3891 }
3892
3893 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3894 int nelems, enum dma_data_direction dir,
3895 unsigned long attrs)
3896 {
3897 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3898 unsigned long nrpages = 0;
3899 struct scatterlist *sg;
3900 int i;
3901
3902 for_each_sg(sglist, sg, nelems, i) {
3903 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3904 }
3905
3906 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3907 }
3908
3909 static int intel_nontranslate_map_sg(struct device *hddev,
3910 struct scatterlist *sglist, int nelems, int dir)
3911 {
3912 int i;
3913 struct scatterlist *sg;
3914
3915 for_each_sg(sglist, sg, nelems, i) {
3916 BUG_ON(!sg_page(sg));
3917 sg->dma_address = sg_phys(sg);
3918 sg->dma_length = sg->length;
3919 }
3920 return nelems;
3921 }
3922
3923 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3924 enum dma_data_direction dir, unsigned long attrs)
3925 {
3926 int i;
3927 struct dmar_domain *domain;
3928 size_t size = 0;
3929 int prot = 0;
3930 unsigned long iova_pfn;
3931 int ret;
3932 struct scatterlist *sg;
3933 unsigned long start_vpfn;
3934 struct intel_iommu *iommu;
3935
3936 BUG_ON(dir == DMA_NONE);
3937 if (iommu_no_mapping(dev))
3938 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3939
3940 domain = get_valid_domain_for_dev(dev);
3941 if (!domain)
3942 return 0;
3943
3944 iommu = domain_get_iommu(domain);
3945
3946 for_each_sg(sglist, sg, nelems, i)
3947 size += aligned_nrpages(sg->offset, sg->length);
3948
3949 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3950 *dev->dma_mask);
3951 if (!iova_pfn) {
3952 sglist->dma_length = 0;
3953 return 0;
3954 }
3955
3956 /*
3957 * Check if DMAR supports zero-length reads on write only
3958 * mappings..
3959 */
3960 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3961 !cap_zlr(iommu->cap))
3962 prot |= DMA_PTE_READ;
3963 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3964 prot |= DMA_PTE_WRITE;
3965
3966 start_vpfn = mm_to_dma_pfn(iova_pfn);
3967
3968 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3969 if (unlikely(ret)) {
3970 dma_pte_free_pagetable(domain, start_vpfn,
3971 start_vpfn + size - 1);
3972 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3973 return 0;
3974 }
3975
3976 /* it's a non-present to present mapping. Only flush if caching mode */
3977 if (cap_caching_mode(iommu->cap))
3978 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3979 else
3980 iommu_flush_write_buffer(iommu);
3981
3982 return nelems;
3983 }
3984
3985 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3986 {
3987 return !dma_addr;
3988 }
3989
3990 struct dma_map_ops intel_dma_ops = {
3991 .alloc = intel_alloc_coherent,
3992 .free = intel_free_coherent,
3993 .map_sg = intel_map_sg,
3994 .unmap_sg = intel_unmap_sg,
3995 .map_page = intel_map_page,
3996 .unmap_page = intel_unmap_page,
3997 .mapping_error = intel_mapping_error,
3998 };
3999
4000 static inline int iommu_domain_cache_init(void)
4001 {
4002 int ret = 0;
4003
4004 iommu_domain_cache = kmem_cache_create("iommu_domain",
4005 sizeof(struct dmar_domain),
4006 0,
4007 SLAB_HWCACHE_ALIGN,
4008
4009 NULL);
4010 if (!iommu_domain_cache) {
4011 pr_err("Couldn't create iommu_domain cache\n");
4012 ret = -ENOMEM;
4013 }
4014
4015 return ret;
4016 }
4017
4018 static inline int iommu_devinfo_cache_init(void)
4019 {
4020 int ret = 0;
4021
4022 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
4023 sizeof(struct device_domain_info),
4024 0,
4025 SLAB_HWCACHE_ALIGN,
4026 NULL);
4027 if (!iommu_devinfo_cache) {
4028 pr_err("Couldn't create devinfo cache\n");
4029 ret = -ENOMEM;
4030 }
4031
4032 return ret;
4033 }
4034
4035 static int __init iommu_init_mempool(void)
4036 {
4037 int ret;
4038 ret = iova_cache_get();
4039 if (ret)
4040 return ret;
4041
4042 ret = iommu_domain_cache_init();
4043 if (ret)
4044 goto domain_error;
4045
4046 ret = iommu_devinfo_cache_init();
4047 if (!ret)
4048 return ret;
4049
4050 kmem_cache_destroy(iommu_domain_cache);
4051 domain_error:
4052 iova_cache_put();
4053
4054 return -ENOMEM;
4055 }
4056
4057 static void __init iommu_exit_mempool(void)
4058 {
4059 kmem_cache_destroy(iommu_devinfo_cache);
4060 kmem_cache_destroy(iommu_domain_cache);
4061 iova_cache_put();
4062 }
4063
4064 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4065 {
4066 struct dmar_drhd_unit *drhd;
4067 u32 vtbar;
4068 int rc;
4069
4070 /* We know that this device on this chipset has its own IOMMU.
4071 * If we find it under a different IOMMU, then the BIOS is lying
4072 * to us. Hope that the IOMMU for this device is actually
4073 * disabled, and it needs no translation...
4074 */
4075 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4076 if (rc) {
4077 /* "can't" happen */
4078 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4079 return;
4080 }
4081 vtbar &= 0xffff0000;
4082
4083 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4084 drhd = dmar_find_matched_drhd_unit(pdev);
4085 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4086 TAINT_FIRMWARE_WORKAROUND,
4087 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4088 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4089 }
4090 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4091
4092 static void __init init_no_remapping_devices(void)
4093 {
4094 struct dmar_drhd_unit *drhd;
4095 struct device *dev;
4096 int i;
4097
4098 for_each_drhd_unit(drhd) {
4099 if (!drhd->include_all) {
4100 for_each_active_dev_scope(drhd->devices,
4101 drhd->devices_cnt, i, dev)
4102 break;
4103 /* ignore DMAR unit if no devices exist */
4104 if (i == drhd->devices_cnt)
4105 drhd->ignored = 1;
4106 }
4107 }
4108
4109 for_each_active_drhd_unit(drhd) {
4110 if (drhd->include_all)
4111 continue;
4112
4113 for_each_active_dev_scope(drhd->devices,
4114 drhd->devices_cnt, i, dev)
4115 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4116 break;
4117 if (i < drhd->devices_cnt)
4118 continue;
4119
4120 /* This IOMMU has *only* gfx devices. Either bypass it or
4121 set the gfx_mapped flag, as appropriate */
4122 if (dmar_map_gfx) {
4123 intel_iommu_gfx_mapped = 1;
4124 } else {
4125 drhd->ignored = 1;
4126 for_each_active_dev_scope(drhd->devices,
4127 drhd->devices_cnt, i, dev)
4128 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4129 }
4130 }
4131 }
4132
4133 #ifdef CONFIG_SUSPEND
4134 static int init_iommu_hw(void)
4135 {
4136 struct dmar_drhd_unit *drhd;
4137 struct intel_iommu *iommu = NULL;
4138
4139 for_each_active_iommu(iommu, drhd)
4140 if (iommu->qi)
4141 dmar_reenable_qi(iommu);
4142
4143 for_each_iommu(iommu, drhd) {
4144 if (drhd->ignored) {
4145 /*
4146 * we always have to disable PMRs or DMA may fail on
4147 * this device
4148 */
4149 if (force_on)
4150 iommu_disable_protect_mem_regions(iommu);
4151 continue;
4152 }
4153
4154 iommu_flush_write_buffer(iommu);
4155
4156 iommu_set_root_entry(iommu);
4157
4158 iommu->flush.flush_context(iommu, 0, 0, 0,
4159 DMA_CCMD_GLOBAL_INVL);
4160 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4161 iommu_enable_translation(iommu);
4162 iommu_disable_protect_mem_regions(iommu);
4163 }
4164
4165 return 0;
4166 }
4167
4168 static void iommu_flush_all(void)
4169 {
4170 struct dmar_drhd_unit *drhd;
4171 struct intel_iommu *iommu;
4172
4173 for_each_active_iommu(iommu, drhd) {
4174 iommu->flush.flush_context(iommu, 0, 0, 0,
4175 DMA_CCMD_GLOBAL_INVL);
4176 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4177 DMA_TLB_GLOBAL_FLUSH);
4178 }
4179 }
4180
4181 static int iommu_suspend(void)
4182 {
4183 struct dmar_drhd_unit *drhd;
4184 struct intel_iommu *iommu = NULL;
4185 unsigned long flag;
4186
4187 for_each_active_iommu(iommu, drhd) {
4188 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4189 GFP_ATOMIC);
4190 if (!iommu->iommu_state)
4191 goto nomem;
4192 }
4193
4194 iommu_flush_all();
4195
4196 for_each_active_iommu(iommu, drhd) {
4197 iommu_disable_translation(iommu);
4198
4199 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4200
4201 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4202 readl(iommu->reg + DMAR_FECTL_REG);
4203 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4204 readl(iommu->reg + DMAR_FEDATA_REG);
4205 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4206 readl(iommu->reg + DMAR_FEADDR_REG);
4207 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4208 readl(iommu->reg + DMAR_FEUADDR_REG);
4209
4210 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4211 }
4212 return 0;
4213
4214 nomem:
4215 for_each_active_iommu(iommu, drhd)
4216 kfree(iommu->iommu_state);
4217
4218 return -ENOMEM;
4219 }
4220
4221 static void iommu_resume(void)
4222 {
4223 struct dmar_drhd_unit *drhd;
4224 struct intel_iommu *iommu = NULL;
4225 unsigned long flag;
4226
4227 if (init_iommu_hw()) {
4228 if (force_on)
4229 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4230 else
4231 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4232 return;
4233 }
4234
4235 for_each_active_iommu(iommu, drhd) {
4236
4237 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4238
4239 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4240 iommu->reg + DMAR_FECTL_REG);
4241 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4242 iommu->reg + DMAR_FEDATA_REG);
4243 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4244 iommu->reg + DMAR_FEADDR_REG);
4245 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4246 iommu->reg + DMAR_FEUADDR_REG);
4247
4248 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4249 }
4250
4251 for_each_active_iommu(iommu, drhd)
4252 kfree(iommu->iommu_state);
4253 }
4254
4255 static struct syscore_ops iommu_syscore_ops = {
4256 .resume = iommu_resume,
4257 .suspend = iommu_suspend,
4258 };
4259
4260 static void __init init_iommu_pm_ops(void)
4261 {
4262 register_syscore_ops(&iommu_syscore_ops);
4263 }
4264
4265 #else
4266 static inline void init_iommu_pm_ops(void) {}
4267 #endif /* CONFIG_PM */
4268
4269
4270 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4271 {
4272 struct acpi_dmar_reserved_memory *rmrr;
4273 struct dmar_rmrr_unit *rmrru;
4274
4275 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4276 if (!rmrru)
4277 return -ENOMEM;
4278
4279 rmrru->hdr = header;
4280 rmrr = (struct acpi_dmar_reserved_memory *)header;
4281 rmrru->base_address = rmrr->base_address;
4282 rmrru->end_address = rmrr->end_address;
4283 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4284 ((void *)rmrr) + rmrr->header.length,
4285 &rmrru->devices_cnt);
4286 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4287 kfree(rmrru);
4288 return -ENOMEM;
4289 }
4290
4291 list_add(&rmrru->list, &dmar_rmrr_units);
4292
4293 return 0;
4294 }
4295
4296 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4297 {
4298 struct dmar_atsr_unit *atsru;
4299 struct acpi_dmar_atsr *tmp;
4300
4301 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4302 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4303 if (atsr->segment != tmp->segment)
4304 continue;
4305 if (atsr->header.length != tmp->header.length)
4306 continue;
4307 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4308 return atsru;
4309 }
4310
4311 return NULL;
4312 }
4313
4314 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4315 {
4316 struct acpi_dmar_atsr *atsr;
4317 struct dmar_atsr_unit *atsru;
4318
4319 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4320 return 0;
4321
4322 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4323 atsru = dmar_find_atsr(atsr);
4324 if (atsru)
4325 return 0;
4326
4327 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4328 if (!atsru)
4329 return -ENOMEM;
4330
4331 /*
4332 * If memory is allocated from slab by ACPI _DSM method, we need to
4333 * copy the memory content because the memory buffer will be freed
4334 * on return.
4335 */
4336 atsru->hdr = (void *)(atsru + 1);
4337 memcpy(atsru->hdr, hdr, hdr->length);
4338 atsru->include_all = atsr->flags & 0x1;
4339 if (!atsru->include_all) {
4340 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4341 (void *)atsr + atsr->header.length,
4342 &atsru->devices_cnt);
4343 if (atsru->devices_cnt && atsru->devices == NULL) {
4344 kfree(atsru);
4345 return -ENOMEM;
4346 }
4347 }
4348
4349 list_add_rcu(&atsru->list, &dmar_atsr_units);
4350
4351 return 0;
4352 }
4353
4354 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4355 {
4356 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4357 kfree(atsru);
4358 }
4359
4360 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4361 {
4362 struct acpi_dmar_atsr *atsr;
4363 struct dmar_atsr_unit *atsru;
4364
4365 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4366 atsru = dmar_find_atsr(atsr);
4367 if (atsru) {
4368 list_del_rcu(&atsru->list);
4369 synchronize_rcu();
4370 intel_iommu_free_atsr(atsru);
4371 }
4372
4373 return 0;
4374 }
4375
4376 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4377 {
4378 int i;
4379 struct device *dev;
4380 struct acpi_dmar_atsr *atsr;
4381 struct dmar_atsr_unit *atsru;
4382
4383 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4384 atsru = dmar_find_atsr(atsr);
4385 if (!atsru)
4386 return 0;
4387
4388 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4389 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4390 i, dev)
4391 return -EBUSY;
4392 }
4393
4394 return 0;
4395 }
4396
4397 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4398 {
4399 int sp, ret = 0;
4400 struct intel_iommu *iommu = dmaru->iommu;
4401
4402 if (g_iommus[iommu->seq_id])
4403 return 0;
4404
4405 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4406 pr_warn("%s: Doesn't support hardware pass through.\n",
4407 iommu->name);
4408 return -ENXIO;
4409 }
4410 if (!ecap_sc_support(iommu->ecap) &&
4411 domain_update_iommu_snooping(iommu)) {
4412 pr_warn("%s: Doesn't support snooping.\n",
4413 iommu->name);
4414 return -ENXIO;
4415 }
4416 sp = domain_update_iommu_superpage(iommu) - 1;
4417 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4418 pr_warn("%s: Doesn't support large page.\n",
4419 iommu->name);
4420 return -ENXIO;
4421 }
4422
4423 /*
4424 * Disable translation if already enabled prior to OS handover.
4425 */
4426 if (iommu->gcmd & DMA_GCMD_TE)
4427 iommu_disable_translation(iommu);
4428
4429 g_iommus[iommu->seq_id] = iommu;
4430 ret = iommu_init_domains(iommu);
4431 if (ret == 0)
4432 ret = iommu_alloc_root_entry(iommu);
4433 if (ret)
4434 goto out;
4435
4436 #ifdef CONFIG_INTEL_IOMMU_SVM
4437 if (pasid_enabled(iommu))
4438 intel_svm_alloc_pasid_tables(iommu);
4439 #endif
4440
4441 if (dmaru->ignored) {
4442 /*
4443 * we always have to disable PMRs or DMA may fail on this device
4444 */
4445 if (force_on)
4446 iommu_disable_protect_mem_regions(iommu);
4447 return 0;
4448 }
4449
4450 intel_iommu_init_qi(iommu);
4451 iommu_flush_write_buffer(iommu);
4452
4453 #ifdef CONFIG_INTEL_IOMMU_SVM
4454 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4455 ret = intel_svm_enable_prq(iommu);
4456 if (ret)
4457 goto disable_iommu;
4458 }
4459 #endif
4460 ret = dmar_set_interrupt(iommu);
4461 if (ret)
4462 goto disable_iommu;
4463
4464 iommu_set_root_entry(iommu);
4465 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4466 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4467 iommu_enable_translation(iommu);
4468
4469 iommu_disable_protect_mem_regions(iommu);
4470 return 0;
4471
4472 disable_iommu:
4473 disable_dmar_iommu(iommu);
4474 out:
4475 free_dmar_iommu(iommu);
4476 return ret;
4477 }
4478
4479 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4480 {
4481 int ret = 0;
4482 struct intel_iommu *iommu = dmaru->iommu;
4483
4484 if (!intel_iommu_enabled)
4485 return 0;
4486 if (iommu == NULL)
4487 return -EINVAL;
4488
4489 if (insert) {
4490 ret = intel_iommu_add(dmaru);
4491 } else {
4492 disable_dmar_iommu(iommu);
4493 free_dmar_iommu(iommu);
4494 }
4495
4496 return ret;
4497 }
4498
4499 static void intel_iommu_free_dmars(void)
4500 {
4501 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4502 struct dmar_atsr_unit *atsru, *atsr_n;
4503
4504 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4505 list_del(&rmrru->list);
4506 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4507 kfree(rmrru);
4508 }
4509
4510 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4511 list_del(&atsru->list);
4512 intel_iommu_free_atsr(atsru);
4513 }
4514 }
4515
4516 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4517 {
4518 int i, ret = 1;
4519 struct pci_bus *bus;
4520 struct pci_dev *bridge = NULL;
4521 struct device *tmp;
4522 struct acpi_dmar_atsr *atsr;
4523 struct dmar_atsr_unit *atsru;
4524
4525 dev = pci_physfn(dev);
4526 for (bus = dev->bus; bus; bus = bus->parent) {
4527 bridge = bus->self;
4528 /* If it's an integrated device, allow ATS */
4529 if (!bridge)
4530 return 1;
4531 /* Connected via non-PCIe: no ATS */
4532 if (!pci_is_pcie(bridge) ||
4533 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4534 return 0;
4535 /* If we found the root port, look it up in the ATSR */
4536 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4537 break;
4538 }
4539
4540 rcu_read_lock();
4541 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4542 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4543 if (atsr->segment != pci_domain_nr(dev->bus))
4544 continue;
4545
4546 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4547 if (tmp == &bridge->dev)
4548 goto out;
4549
4550 if (atsru->include_all)
4551 goto out;
4552 }
4553 ret = 0;
4554 out:
4555 rcu_read_unlock();
4556
4557 return ret;
4558 }
4559
4560 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4561 {
4562 int ret = 0;
4563 struct dmar_rmrr_unit *rmrru;
4564 struct dmar_atsr_unit *atsru;
4565 struct acpi_dmar_atsr *atsr;
4566 struct acpi_dmar_reserved_memory *rmrr;
4567
4568 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4569 return 0;
4570
4571 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4572 rmrr = container_of(rmrru->hdr,
4573 struct acpi_dmar_reserved_memory, header);
4574 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4575 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4576 ((void *)rmrr) + rmrr->header.length,
4577 rmrr->segment, rmrru->devices,
4578 rmrru->devices_cnt);
4579 if(ret < 0)
4580 return ret;
4581 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4582 dmar_remove_dev_scope(info, rmrr->segment,
4583 rmrru->devices, rmrru->devices_cnt);
4584 }
4585 }
4586
4587 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4588 if (atsru->include_all)
4589 continue;
4590
4591 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4592 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4593 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4594 (void *)atsr + atsr->header.length,
4595 atsr->segment, atsru->devices,
4596 atsru->devices_cnt);
4597 if (ret > 0)
4598 break;
4599 else if(ret < 0)
4600 return ret;
4601 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4602 if (dmar_remove_dev_scope(info, atsr->segment,
4603 atsru->devices, atsru->devices_cnt))
4604 break;
4605 }
4606 }
4607
4608 return 0;
4609 }
4610
4611 /*
4612 * Here we only respond to action of unbound device from driver.
4613 *
4614 * Added device is not attached to its DMAR domain here yet. That will happen
4615 * when mapping the device to iova.
4616 */
4617 static int device_notifier(struct notifier_block *nb,
4618 unsigned long action, void *data)
4619 {
4620 struct device *dev = data;
4621 struct dmar_domain *domain;
4622
4623 if (iommu_dummy(dev))
4624 return 0;
4625
4626 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4627 return 0;
4628
4629 domain = find_domain(dev);
4630 if (!domain)
4631 return 0;
4632
4633 dmar_remove_one_dev_info(domain, dev);
4634 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4635 domain_exit(domain);
4636
4637 return 0;
4638 }
4639
4640 static struct notifier_block device_nb = {
4641 .notifier_call = device_notifier,
4642 };
4643
4644 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4645 unsigned long val, void *v)
4646 {
4647 struct memory_notify *mhp = v;
4648 unsigned long long start, end;
4649 unsigned long start_vpfn, last_vpfn;
4650
4651 switch (val) {
4652 case MEM_GOING_ONLINE:
4653 start = mhp->start_pfn << PAGE_SHIFT;
4654 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4655 if (iommu_domain_identity_map(si_domain, start, end)) {
4656 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4657 start, end);
4658 return NOTIFY_BAD;
4659 }
4660 break;
4661
4662 case MEM_OFFLINE:
4663 case MEM_CANCEL_ONLINE:
4664 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4665 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4666 while (start_vpfn <= last_vpfn) {
4667 struct iova *iova;
4668 struct dmar_drhd_unit *drhd;
4669 struct intel_iommu *iommu;
4670 struct page *freelist;
4671
4672 iova = find_iova(&si_domain->iovad, start_vpfn);
4673 if (iova == NULL) {
4674 pr_debug("Failed get IOVA for PFN %lx\n",
4675 start_vpfn);
4676 break;
4677 }
4678
4679 iova = split_and_remove_iova(&si_domain->iovad, iova,
4680 start_vpfn, last_vpfn);
4681 if (iova == NULL) {
4682 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4683 start_vpfn, last_vpfn);
4684 return NOTIFY_BAD;
4685 }
4686
4687 freelist = domain_unmap(si_domain, iova->pfn_lo,
4688 iova->pfn_hi);
4689
4690 rcu_read_lock();
4691 for_each_active_iommu(iommu, drhd)
4692 iommu_flush_iotlb_psi(iommu, si_domain,
4693 iova->pfn_lo, iova_size(iova),
4694 !freelist, 0);
4695 rcu_read_unlock();
4696 dma_free_pagelist(freelist);
4697
4698 start_vpfn = iova->pfn_hi + 1;
4699 free_iova_mem(iova);
4700 }
4701 break;
4702 }
4703
4704 return NOTIFY_OK;
4705 }
4706
4707 static struct notifier_block intel_iommu_memory_nb = {
4708 .notifier_call = intel_iommu_memory_notifier,
4709 .priority = 0
4710 };
4711
4712 static void free_all_cpu_cached_iovas(unsigned int cpu)
4713 {
4714 int i;
4715
4716 for (i = 0; i < g_num_of_iommus; i++) {
4717 struct intel_iommu *iommu = g_iommus[i];
4718 struct dmar_domain *domain;
4719 int did;
4720
4721 if (!iommu)
4722 continue;
4723
4724 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4725 domain = get_iommu_domain(iommu, (u16)did);
4726
4727 if (!domain)
4728 continue;
4729 free_cpu_cached_iovas(cpu, &domain->iovad);
4730 }
4731 }
4732 }
4733
4734 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4735 unsigned long action, void *v)
4736 {
4737 unsigned int cpu = (unsigned long)v;
4738
4739 switch (action) {
4740 case CPU_DEAD:
4741 case CPU_DEAD_FROZEN:
4742 free_all_cpu_cached_iovas(cpu);
4743 flush_unmaps_timeout(cpu);
4744 break;
4745 }
4746 return NOTIFY_OK;
4747 }
4748
4749 static struct notifier_block intel_iommu_cpu_nb = {
4750 .notifier_call = intel_iommu_cpu_notifier,
4751 };
4752
4753 static ssize_t intel_iommu_show_version(struct device *dev,
4754 struct device_attribute *attr,
4755 char *buf)
4756 {
4757 struct intel_iommu *iommu = dev_get_drvdata(dev);
4758 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4759 return sprintf(buf, "%d:%d\n",
4760 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4761 }
4762 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4763
4764 static ssize_t intel_iommu_show_address(struct device *dev,
4765 struct device_attribute *attr,
4766 char *buf)
4767 {
4768 struct intel_iommu *iommu = dev_get_drvdata(dev);
4769 return sprintf(buf, "%llx\n", iommu->reg_phys);
4770 }
4771 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4772
4773 static ssize_t intel_iommu_show_cap(struct device *dev,
4774 struct device_attribute *attr,
4775 char *buf)
4776 {
4777 struct intel_iommu *iommu = dev_get_drvdata(dev);
4778 return sprintf(buf, "%llx\n", iommu->cap);
4779 }
4780 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4781
4782 static ssize_t intel_iommu_show_ecap(struct device *dev,
4783 struct device_attribute *attr,
4784 char *buf)
4785 {
4786 struct intel_iommu *iommu = dev_get_drvdata(dev);
4787 return sprintf(buf, "%llx\n", iommu->ecap);
4788 }
4789 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4790
4791 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4792 struct device_attribute *attr,
4793 char *buf)
4794 {
4795 struct intel_iommu *iommu = dev_get_drvdata(dev);
4796 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4797 }
4798 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4799
4800 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4801 struct device_attribute *attr,
4802 char *buf)
4803 {
4804 struct intel_iommu *iommu = dev_get_drvdata(dev);
4805 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4806 cap_ndoms(iommu->cap)));
4807 }
4808 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4809
4810 static struct attribute *intel_iommu_attrs[] = {
4811 &dev_attr_version.attr,
4812 &dev_attr_address.attr,
4813 &dev_attr_cap.attr,
4814 &dev_attr_ecap.attr,
4815 &dev_attr_domains_supported.attr,
4816 &dev_attr_domains_used.attr,
4817 NULL,
4818 };
4819
4820 static struct attribute_group intel_iommu_group = {
4821 .name = "intel-iommu",
4822 .attrs = intel_iommu_attrs,
4823 };
4824
4825 const struct attribute_group *intel_iommu_groups[] = {
4826 &intel_iommu_group,
4827 NULL,
4828 };
4829
4830 int __init intel_iommu_init(void)
4831 {
4832 int ret = -ENODEV;
4833 struct dmar_drhd_unit *drhd;
4834 struct intel_iommu *iommu;
4835
4836 /* VT-d is required for a TXT/tboot launch, so enforce that */
4837 force_on = tboot_force_iommu();
4838
4839 if (iommu_init_mempool()) {
4840 if (force_on)
4841 panic("tboot: Failed to initialize iommu memory\n");
4842 return -ENOMEM;
4843 }
4844
4845 down_write(&dmar_global_lock);
4846 if (dmar_table_init()) {
4847 if (force_on)
4848 panic("tboot: Failed to initialize DMAR table\n");
4849 goto out_free_dmar;
4850 }
4851
4852 if (dmar_dev_scope_init() < 0) {
4853 if (force_on)
4854 panic("tboot: Failed to initialize DMAR device scope\n");
4855 goto out_free_dmar;
4856 }
4857
4858 if (no_iommu || dmar_disabled)
4859 goto out_free_dmar;
4860
4861 if (list_empty(&dmar_rmrr_units))
4862 pr_info("No RMRR found\n");
4863
4864 if (list_empty(&dmar_atsr_units))
4865 pr_info("No ATSR found\n");
4866
4867 if (dmar_init_reserved_ranges()) {
4868 if (force_on)
4869 panic("tboot: Failed to reserve iommu ranges\n");
4870 goto out_free_reserved_range;
4871 }
4872
4873 init_no_remapping_devices();
4874
4875 ret = init_dmars();
4876 if (ret) {
4877 if (force_on)
4878 panic("tboot: Failed to initialize DMARs\n");
4879 pr_err("Initialization failed\n");
4880 goto out_free_reserved_range;
4881 }
4882 up_write(&dmar_global_lock);
4883 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4884
4885 #ifdef CONFIG_SWIOTLB
4886 swiotlb = 0;
4887 #endif
4888 dma_ops = &intel_dma_ops;
4889
4890 init_iommu_pm_ops();
4891
4892 for_each_active_iommu(iommu, drhd)
4893 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4894 intel_iommu_groups,
4895 "%s", iommu->name);
4896
4897 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4898 bus_register_notifier(&pci_bus_type, &device_nb);
4899 if (si_domain && !hw_pass_through)
4900 register_memory_notifier(&intel_iommu_memory_nb);
4901 register_hotcpu_notifier(&intel_iommu_cpu_nb);
4902
4903 intel_iommu_enabled = 1;
4904
4905 return 0;
4906
4907 out_free_reserved_range:
4908 put_iova_domain(&reserved_iova_list);
4909 out_free_dmar:
4910 intel_iommu_free_dmars();
4911 up_write(&dmar_global_lock);
4912 iommu_exit_mempool();
4913 return ret;
4914 }
4915
4916 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4917 {
4918 struct intel_iommu *iommu = opaque;
4919
4920 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4921 return 0;
4922 }
4923
4924 /*
4925 * NB - intel-iommu lacks any sort of reference counting for the users of
4926 * dependent devices. If multiple endpoints have intersecting dependent
4927 * devices, unbinding the driver from any one of them will possibly leave
4928 * the others unable to operate.
4929 */
4930 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4931 {
4932 if (!iommu || !dev || !dev_is_pci(dev))
4933 return;
4934
4935 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4936 }
4937
4938 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4939 {
4940 struct intel_iommu *iommu;
4941 unsigned long flags;
4942
4943 assert_spin_locked(&device_domain_lock);
4944
4945 if (WARN_ON(!info))
4946 return;
4947
4948 iommu = info->iommu;
4949
4950 if (info->dev) {
4951 iommu_disable_dev_iotlb(info);
4952 domain_context_clear(iommu, info->dev);
4953 }
4954
4955 unlink_domain_info(info);
4956
4957 spin_lock_irqsave(&iommu->lock, flags);
4958 domain_detach_iommu(info->domain, iommu);
4959 spin_unlock_irqrestore(&iommu->lock, flags);
4960
4961 free_devinfo_mem(info);
4962 }
4963
4964 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4965 struct device *dev)
4966 {
4967 struct device_domain_info *info;
4968 unsigned long flags;
4969
4970 spin_lock_irqsave(&device_domain_lock, flags);
4971 info = dev->archdata.iommu;
4972 __dmar_remove_one_dev_info(info);
4973 spin_unlock_irqrestore(&device_domain_lock, flags);
4974 }
4975
4976 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4977 {
4978 int adjust_width;
4979
4980 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4981 DMA_32BIT_PFN);
4982 domain_reserve_special_ranges(domain);
4983
4984 /* calculate AGAW */
4985 domain->gaw = guest_width;
4986 adjust_width = guestwidth_to_adjustwidth(guest_width);
4987 domain->agaw = width_to_agaw(adjust_width);
4988
4989 domain->iommu_coherency = 0;
4990 domain->iommu_snooping = 0;
4991 domain->iommu_superpage = 0;
4992 domain->max_addr = 0;
4993
4994 /* always allocate the top pgd */
4995 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4996 if (!domain->pgd)
4997 return -ENOMEM;
4998 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4999 return 0;
5000 }
5001
5002 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5003 {
5004 struct dmar_domain *dmar_domain;
5005 struct iommu_domain *domain;
5006
5007 if (type != IOMMU_DOMAIN_UNMANAGED)
5008 return NULL;
5009
5010 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5011 if (!dmar_domain) {
5012 pr_err("Can't allocate dmar_domain\n");
5013 return NULL;
5014 }
5015 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5016 pr_err("Domain initialization failed\n");
5017 domain_exit(dmar_domain);
5018 return NULL;
5019 }
5020 domain_update_iommu_cap(dmar_domain);
5021
5022 domain = &dmar_domain->domain;
5023 domain->geometry.aperture_start = 0;
5024 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5025 domain->geometry.force_aperture = true;
5026
5027 return domain;
5028 }
5029
5030 static void intel_iommu_domain_free(struct iommu_domain *domain)
5031 {
5032 domain_exit(to_dmar_domain(domain));
5033 }
5034
5035 static int intel_iommu_attach_device(struct iommu_domain *domain,
5036 struct device *dev)
5037 {
5038 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5039 struct intel_iommu *iommu;
5040 int addr_width;
5041 u8 bus, devfn;
5042
5043 if (device_is_rmrr_locked(dev)) {
5044 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5045 return -EPERM;
5046 }
5047
5048 /* normally dev is not mapped */
5049 if (unlikely(domain_context_mapped(dev))) {
5050 struct dmar_domain *old_domain;
5051
5052 old_domain = find_domain(dev);
5053 if (old_domain) {
5054 rcu_read_lock();
5055 dmar_remove_one_dev_info(old_domain, dev);
5056 rcu_read_unlock();
5057
5058 if (!domain_type_is_vm_or_si(old_domain) &&
5059 list_empty(&old_domain->devices))
5060 domain_exit(old_domain);
5061 }
5062 }
5063
5064 iommu = device_to_iommu(dev, &bus, &devfn);
5065 if (!iommu)
5066 return -ENODEV;
5067
5068 /* check if this iommu agaw is sufficient for max mapped address */
5069 addr_width = agaw_to_width(iommu->agaw);
5070 if (addr_width > cap_mgaw(iommu->cap))
5071 addr_width = cap_mgaw(iommu->cap);
5072
5073 if (dmar_domain->max_addr > (1LL << addr_width)) {
5074 pr_err("%s: iommu width (%d) is not "
5075 "sufficient for the mapped address (%llx)\n",
5076 __func__, addr_width, dmar_domain->max_addr);
5077 return -EFAULT;
5078 }
5079 dmar_domain->gaw = addr_width;
5080
5081 /*
5082 * Knock out extra levels of page tables if necessary
5083 */
5084 while (iommu->agaw < dmar_domain->agaw) {
5085 struct dma_pte *pte;
5086
5087 pte = dmar_domain->pgd;
5088 if (dma_pte_present(pte)) {
5089 dmar_domain->pgd = (struct dma_pte *)
5090 phys_to_virt(dma_pte_addr(pte));
5091 free_pgtable_page(pte);
5092 }
5093 dmar_domain->agaw--;
5094 }
5095
5096 return domain_add_dev_info(dmar_domain, dev);
5097 }
5098
5099 static void intel_iommu_detach_device(struct iommu_domain *domain,
5100 struct device *dev)
5101 {
5102 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5103 }
5104
5105 static int intel_iommu_map(struct iommu_domain *domain,
5106 unsigned long iova, phys_addr_t hpa,
5107 size_t size, int iommu_prot)
5108 {
5109 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5110 u64 max_addr;
5111 int prot = 0;
5112 int ret;
5113
5114 if (iommu_prot & IOMMU_READ)
5115 prot |= DMA_PTE_READ;
5116 if (iommu_prot & IOMMU_WRITE)
5117 prot |= DMA_PTE_WRITE;
5118 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5119 prot |= DMA_PTE_SNP;
5120
5121 max_addr = iova + size;
5122 if (dmar_domain->max_addr < max_addr) {
5123 u64 end;
5124
5125 /* check if minimum agaw is sufficient for mapped address */
5126 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5127 if (end < max_addr) {
5128 pr_err("%s: iommu width (%d) is not "
5129 "sufficient for the mapped address (%llx)\n",
5130 __func__, dmar_domain->gaw, max_addr);
5131 return -EFAULT;
5132 }
5133 dmar_domain->max_addr = max_addr;
5134 }
5135 /* Round up size to next multiple of PAGE_SIZE, if it and
5136 the low bits of hpa would take us onto the next page */
5137 size = aligned_nrpages(hpa, size);
5138 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5139 hpa >> VTD_PAGE_SHIFT, size, prot);
5140 return ret;
5141 }
5142
5143 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5144 unsigned long iova, size_t size)
5145 {
5146 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5147 struct page *freelist = NULL;
5148 struct intel_iommu *iommu;
5149 unsigned long start_pfn, last_pfn;
5150 unsigned int npages;
5151 int iommu_id, level = 0;
5152
5153 /* Cope with horrid API which requires us to unmap more than the
5154 size argument if it happens to be a large-page mapping. */
5155 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5156
5157 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5158 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5159
5160 start_pfn = iova >> VTD_PAGE_SHIFT;
5161 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5162
5163 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5164
5165 npages = last_pfn - start_pfn + 1;
5166
5167 for_each_domain_iommu(iommu_id, dmar_domain) {
5168 iommu = g_iommus[iommu_id];
5169
5170 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5171 start_pfn, npages, !freelist, 0);
5172 }
5173
5174 dma_free_pagelist(freelist);
5175
5176 if (dmar_domain->max_addr == iova + size)
5177 dmar_domain->max_addr = iova;
5178
5179 return size;
5180 }
5181
5182 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5183 dma_addr_t iova)
5184 {
5185 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5186 struct dma_pte *pte;
5187 int level = 0;
5188 u64 phys = 0;
5189
5190 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5191 if (pte)
5192 phys = dma_pte_addr(pte);
5193
5194 return phys;
5195 }
5196
5197 static bool intel_iommu_capable(enum iommu_cap cap)
5198 {
5199 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5200 return domain_update_iommu_snooping(NULL) == 1;
5201 if (cap == IOMMU_CAP_INTR_REMAP)
5202 return irq_remapping_enabled == 1;
5203
5204 return false;
5205 }
5206
5207 static int intel_iommu_add_device(struct device *dev)
5208 {
5209 struct intel_iommu *iommu;
5210 struct iommu_group *group;
5211 u8 bus, devfn;
5212
5213 iommu = device_to_iommu(dev, &bus, &devfn);
5214 if (!iommu)
5215 return -ENODEV;
5216
5217 iommu_device_link(iommu->iommu_dev, dev);
5218
5219 group = iommu_group_get_for_dev(dev);
5220
5221 if (IS_ERR(group))
5222 return PTR_ERR(group);
5223
5224 iommu_group_put(group);
5225 return 0;
5226 }
5227
5228 static void intel_iommu_remove_device(struct device *dev)
5229 {
5230 struct intel_iommu *iommu;
5231 u8 bus, devfn;
5232
5233 iommu = device_to_iommu(dev, &bus, &devfn);
5234 if (!iommu)
5235 return;
5236
5237 iommu_group_remove_device(dev);
5238
5239 iommu_device_unlink(iommu->iommu_dev, dev);
5240 }
5241
5242 #ifdef CONFIG_INTEL_IOMMU_SVM
5243 #define MAX_NR_PASID_BITS (20)
5244 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5245 {
5246 /*
5247 * Convert ecap_pss to extend context entry pts encoding, also
5248 * respect the soft pasid_max value set by the iommu.
5249 * - number of PASID bits = ecap_pss + 1
5250 * - number of PASID table entries = 2^(pts + 5)
5251 * Therefore, pts = ecap_pss - 4
5252 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5253 */
5254 if (ecap_pss(iommu->ecap) < 5)
5255 return 0;
5256
5257 /* pasid_max is encoded as actual number of entries not the bits */
5258 return find_first_bit((unsigned long *)&iommu->pasid_max,
5259 MAX_NR_PASID_BITS) - 5;
5260 }
5261
5262 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5263 {
5264 struct device_domain_info *info;
5265 struct context_entry *context;
5266 struct dmar_domain *domain;
5267 unsigned long flags;
5268 u64 ctx_lo;
5269 int ret;
5270
5271 domain = get_valid_domain_for_dev(sdev->dev);
5272 if (!domain)
5273 return -EINVAL;
5274
5275 spin_lock_irqsave(&device_domain_lock, flags);
5276 spin_lock(&iommu->lock);
5277
5278 ret = -EINVAL;
5279 info = sdev->dev->archdata.iommu;
5280 if (!info || !info->pasid_supported)
5281 goto out;
5282
5283 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5284 if (WARN_ON(!context))
5285 goto out;
5286
5287 ctx_lo = context[0].lo;
5288
5289 sdev->did = domain->iommu_did[iommu->seq_id];
5290 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5291
5292 if (!(ctx_lo & CONTEXT_PASIDE)) {
5293 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5294 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5295 intel_iommu_get_pts(iommu);
5296
5297 wmb();
5298 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5299 * extended to permit requests-with-PASID if the PASIDE bit
5300 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5301 * however, the PASIDE bit is ignored and requests-with-PASID
5302 * are unconditionally blocked. Which makes less sense.
5303 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5304 * "guest mode" translation types depending on whether ATS
5305 * is available or not. Annoyingly, we can't use the new
5306 * modes *unless* PASIDE is set. */
5307 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5308 ctx_lo &= ~CONTEXT_TT_MASK;
5309 if (info->ats_supported)
5310 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5311 else
5312 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5313 }
5314 ctx_lo |= CONTEXT_PASIDE;
5315 if (iommu->pasid_state_table)
5316 ctx_lo |= CONTEXT_DINVE;
5317 if (info->pri_supported)
5318 ctx_lo |= CONTEXT_PRS;
5319 context[0].lo = ctx_lo;
5320 wmb();
5321 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5322 DMA_CCMD_MASK_NOBIT,
5323 DMA_CCMD_DEVICE_INVL);
5324 }
5325
5326 /* Enable PASID support in the device, if it wasn't already */
5327 if (!info->pasid_enabled)
5328 iommu_enable_dev_iotlb(info);
5329
5330 if (info->ats_enabled) {
5331 sdev->dev_iotlb = 1;
5332 sdev->qdep = info->ats_qdep;
5333 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5334 sdev->qdep = 0;
5335 }
5336 ret = 0;
5337
5338 out:
5339 spin_unlock(&iommu->lock);
5340 spin_unlock_irqrestore(&device_domain_lock, flags);
5341
5342 return ret;
5343 }
5344
5345 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5346 {
5347 struct intel_iommu *iommu;
5348 u8 bus, devfn;
5349
5350 if (iommu_dummy(dev)) {
5351 dev_warn(dev,
5352 "No IOMMU translation for device; cannot enable SVM\n");
5353 return NULL;
5354 }
5355
5356 iommu = device_to_iommu(dev, &bus, &devfn);
5357 if ((!iommu)) {
5358 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5359 return NULL;
5360 }
5361
5362 if (!iommu->pasid_table) {
5363 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5364 return NULL;
5365 }
5366
5367 return iommu;
5368 }
5369 #endif /* CONFIG_INTEL_IOMMU_SVM */
5370
5371 static const struct iommu_ops intel_iommu_ops = {
5372 .capable = intel_iommu_capable,
5373 .domain_alloc = intel_iommu_domain_alloc,
5374 .domain_free = intel_iommu_domain_free,
5375 .attach_dev = intel_iommu_attach_device,
5376 .detach_dev = intel_iommu_detach_device,
5377 .map = intel_iommu_map,
5378 .unmap = intel_iommu_unmap,
5379 .map_sg = default_iommu_map_sg,
5380 .iova_to_phys = intel_iommu_iova_to_phys,
5381 .add_device = intel_iommu_add_device,
5382 .remove_device = intel_iommu_remove_device,
5383 .device_group = pci_device_group,
5384 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5385 };
5386
5387 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5388 {
5389 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5390 pr_info("Disabling IOMMU for graphics on this chipset\n");
5391 dmar_map_gfx = 0;
5392 }
5393
5394 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5395 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5396 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5397 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5398 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5399 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5400 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5401
5402 static void quirk_iommu_rwbf(struct pci_dev *dev)
5403 {
5404 /*
5405 * Mobile 4 Series Chipset neglects to set RWBF capability,
5406 * but needs it. Same seems to hold for the desktop versions.
5407 */
5408 pr_info("Forcing write-buffer flush capability\n");
5409 rwbf_quirk = 1;
5410 }
5411
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5414 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5415 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5416 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5417 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5418 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5419
5420 #define GGC 0x52
5421 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5422 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5423 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5424 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5425 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5426 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5427 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5428 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5429
5430 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5431 {
5432 unsigned short ggc;
5433
5434 if (pci_read_config_word(dev, GGC, &ggc))
5435 return;
5436
5437 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5438 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5439 dmar_map_gfx = 0;
5440 } else if (dmar_map_gfx) {
5441 /* we have to ensure the gfx device is idle before we flush */
5442 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5443 intel_iommu_strict = 1;
5444 }
5445 }
5446 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5447 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5448 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5449 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5450
5451 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5452 ISOCH DMAR unit for the Azalia sound device, but not give it any
5453 TLB entries, which causes it to deadlock. Check for that. We do
5454 this in a function called from init_dmars(), instead of in a PCI
5455 quirk, because we don't want to print the obnoxious "BIOS broken"
5456 message if VT-d is actually disabled.
5457 */
5458 static void __init check_tylersburg_isoch(void)
5459 {
5460 struct pci_dev *pdev;
5461 uint32_t vtisochctrl;
5462
5463 /* If there's no Azalia in the system anyway, forget it. */
5464 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5465 if (!pdev)
5466 return;
5467 pci_dev_put(pdev);
5468
5469 /* System Management Registers. Might be hidden, in which case
5470 we can't do the sanity check. But that's OK, because the
5471 known-broken BIOSes _don't_ actually hide it, so far. */
5472 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5473 if (!pdev)
5474 return;
5475
5476 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5477 pci_dev_put(pdev);
5478 return;
5479 }
5480
5481 pci_dev_put(pdev);
5482
5483 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5484 if (vtisochctrl & 1)
5485 return;
5486
5487 /* Drop all bits other than the number of TLB entries */
5488 vtisochctrl &= 0x1c;
5489
5490 /* If we have the recommended number of TLB entries (16), fine. */
5491 if (vtisochctrl == 0x10)
5492 return;
5493
5494 /* Zero TLB entries? You get to ride the short bus to school. */
5495 if (!vtisochctrl) {
5496 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5497 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5498 dmi_get_system_info(DMI_BIOS_VENDOR),
5499 dmi_get_system_info(DMI_BIOS_VERSION),
5500 dmi_get_system_info(DMI_PRODUCT_VERSION));
5501 iommu_identity_mapping |= IDENTMAP_AZALIA;
5502 return;
5503 }
5504
5505 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5506 vtisochctrl);
5507 }
Arne's tree of linux kernel.