]> git.ipfire.org Git - people/arne_f/kernel.git/blame - drivers/pci/intel-iommu.c
projects / people / arne_f / kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel-iommu: Only unlink device domains from iommu
[people/arne_f/kernel.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
134fac3f 39#include <linux/syscore_ops.h>
69575d38 40#include <linux/tboot.h>
adb2fe02 41#include <linux/dmi.h>
ba395927 42#include <asm/cacheflush.h>
46a7fa27 43#include <asm/iommu.h>
ba395927
KA		 44#include "pci.h"
45
5b6985ce
FY		 46#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
ba395927
KA		 49#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 51#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 52
53#define IOAPIC_RANGE_START (0xfee00000)
54#define IOAPIC_RANGE_END (0xfeefffff)
55#define IOVA_START_ADDR (0x1000)
56
57#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
4ed0d3e6
FY		 59#define MAX_AGAW_WIDTH 64
60
2ebe3151
DW		 61#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 69
f27be03b 70#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 71#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 72#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 73
df08cdc7
AM		 74/* page table handling */
75#define LEVEL_STRIDE (9)
76#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
77
78static inline int agaw_to_level(int agaw)
79{
80 return agaw + 2;
81}
82
83static inline int agaw_to_width(int agaw)
84{
85 return 30 + agaw * LEVEL_STRIDE;
86}
87
88static inline int width_to_agaw(int width)
89{
90 return (width - 30) / LEVEL_STRIDE;
91}
92
93static inline unsigned int level_to_offset_bits(int level)
94{
95 return (level - 1) * LEVEL_STRIDE;
96}
97
98static inline int pfn_level_offset(unsigned long pfn, int level)
99{
100 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
101}
102
103static inline unsigned long level_mask(int level)
104{
105 return -1UL << level_to_offset_bits(level);
106}
107
108static inline unsigned long level_size(int level)
109{
110 return 1UL << level_to_offset_bits(level);
111}
112
113static inline unsigned long align_to_level(unsigned long pfn, int level)
114{
115 return (pfn + level_size(level) - 1) & level_mask(level);
116}
fd18de50 117
6dd9a7c7
YS		 118static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
119{
120 return 1 << ((lvl - 1) * LEVEL_STRIDE);
121}
122
dd4e8319
DW		 123/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
124 are never going to work. */
125static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
126{
127 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
128}
129
130static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
131{
132 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
133}
134static inline unsigned long page_to_dma_pfn(struct page *pg)
135{
136 return mm_to_dma_pfn(page_to_pfn(pg));
137}
138static inline unsigned long virt_to_dma_pfn(void *p)
139{
140 return page_to_dma_pfn(virt_to_page(p));
141}
142
d9630fe9
WH		 143/* global iommu list, set NULL for ignored DMAR units */
144static struct intel_iommu **g_iommus;
145
e0fc7e0b 146static void __init check_tylersburg_isoch(void);
9af88143
DW		 147static int rwbf_quirk;
148
b779260b
JC		 149/*
150 * set to 1 to panic kernel if can't successfully enable VT-d
151 * (used when kernel is launched w/ TXT)
152 */
153static int force_on = 0;
154
46b08e1a
MM		 155/*
156 * 0: Present
157 * 1-11: Reserved
158 * 12-63: Context Ptr (12 - (haw-1))
159 * 64-127: Reserved
160 */
161struct root_entry {
162 u64 val;
163 u64 rsvd1;
164};
165#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
166static inline bool root_present(struct root_entry *root)
167{
168 return (root->val & 1);
169}
170static inline void set_root_present(struct root_entry *root)
171{
172 root->val |= 1;
173}
174static inline void set_root_value(struct root_entry *root, unsigned long value)
175{
176 root->val |= value & VTD_PAGE_MASK;
177}
178
179static inline struct context_entry *
180get_context_addr_from_root(struct root_entry *root)
181{
182 return (struct context_entry *)
183 (root_present(root)?phys_to_virt(
184 root->val & VTD_PAGE_MASK) :
185 NULL);
186}
187
7a8fc25e
MM		 188/*
189 * low 64 bits:
190 * 0: present
191 * 1: fault processing disable
192 * 2-3: translation type
193 * 12-63: address space root
194 * high 64 bits:
195 * 0-2: address width
196 * 3-6: aval
197 * 8-23: domain id
198 */
199struct context_entry {
200 u64 lo;
201 u64 hi;
202};
c07e7d21
MM		 203
204static inline bool context_present(struct context_entry *context)
205{
206 return (context->lo & 1);
207}
208static inline void context_set_present(struct context_entry *context)
209{
210 context->lo |= 1;
211}
212
213static inline void context_set_fault_enable(struct context_entry *context)
214{
215 context->lo &= (((u64)-1) << 2) | 1;
216}
217
c07e7d21
MM		 218static inline void context_set_translation_type(struct context_entry *context,
219 unsigned long value)
220{
221 context->lo &= (((u64)-1) << 4) | 3;
222 context->lo |= (value & 3) << 2;
223}
224
225static inline void context_set_address_root(struct context_entry *context,
226 unsigned long value)
227{
228 context->lo |= value & VTD_PAGE_MASK;
229}
230
231static inline void context_set_address_width(struct context_entry *context,
232 unsigned long value)
233{
234 context->hi |= value & 7;
235}
236
237static inline void context_set_domain_id(struct context_entry *context,
238 unsigned long value)
239{
240 context->hi |= (value & ((1 << 16) - 1)) << 8;
241}
242
243static inline void context_clear_entry(struct context_entry *context)
244{
245 context->lo = 0;
246 context->hi = 0;
247}
7a8fc25e 248
622ba12a
MM		 249/*
250 * 0: readable
251 * 1: writable
252 * 2-6: reserved
253 * 7: super page
9cf06697
SY		 254 * 8-10: available
255 * 11: snoop behavior
622ba12a
MM		 256 * 12-63: Host physcial address
257 */
258struct dma_pte {
259 u64 val;
260};
622ba12a 261
19c239ce
MM		 262static inline void dma_clear_pte(struct dma_pte *pte)
263{
264 pte->val = 0;
265}
266
267static inline void dma_set_pte_readable(struct dma_pte *pte)
268{
269 pte->val |= DMA_PTE_READ;
270}
271
272static inline void dma_set_pte_writable(struct dma_pte *pte)
273{
274 pte->val |= DMA_PTE_WRITE;
275}
276
9cf06697
SY		 277static inline void dma_set_pte_snp(struct dma_pte *pte)
278{
279 pte->val |= DMA_PTE_SNP;
280}
281
19c239ce
MM		 282static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
283{
284 pte->val = (pte->val & ~3) | (prot & 3);
285}
286
287static inline u64 dma_pte_addr(struct dma_pte *pte)
288{
c85994e4
DW		 289#ifdef CONFIG_64BIT
290 return pte->val & VTD_PAGE_MASK;
291#else
292 /* Must have a full atomic 64-bit read */
1a8bd481 293 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 294#endif
19c239ce
MM		 295}
296
dd4e8319 297static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 298{
dd4e8319 299 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 300}
301
302static inline bool dma_pte_present(struct dma_pte *pte)
303{
304 return (pte->val & 3) != 0;
305}
622ba12a 306
75e6bf96
DW		 307static inline int first_pte_in_page(struct dma_pte *pte)
308{
309 return !((unsigned long)pte & ~VTD_PAGE_MASK);
310}
311
2c2e2c38
FY		 312/*
313 * This domain is a statically identity mapping domain.
314 * 1. This domain creats a static 1:1 mapping to all usable memory.
315 * 2. It maps to each iommu if successful.
316 * 3. Each iommu mapps to this domain if successful.
317 */
19943b0e
DW		 318static struct dmar_domain *si_domain;
319static int hw_pass_through = 1;
2c2e2c38 320
3b5410e7 321/* devices under the same p2p bridge are owned in one domain */
cdc7b837 322#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 323
1ce28feb
WH		 324/* domain represents a virtual machine, more than one devices
325 * across iommus may be owned in one domain, e.g. kvm guest.
326 */
327#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
328
2c2e2c38
FY		 329/* si_domain contains mulitple devices */
330#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
331
99126f7c
MM		 332struct dmar_domain {
333 int id; /* domain id */
4c923d47 334 int nid; /* node id */
8c11e798 335 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 336
337 struct list_head devices; /* all devices' list */
338 struct iova_domain iovad; /* iova's that belong to this domain */
339
340 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 341 int gaw; /* max guest address width */
342
343 /* adjusted guest address width, 0 is level 2 30-bit */
344 int agaw;
345
3b5410e7 346 int flags; /* flags to find out type of domain */
8e604097
WH		 347
348 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 349 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 350 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 351 int iommu_superpage;/* Level of superpages supported:
352 0 == 4KiB (no superpages), 1 == 2MiB,
353 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 354 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 355 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 356};
357
a647dacb
MM		 358/* PCI domain-device relationship */
359struct device_domain_info {
360 struct list_head link; /* link to domain siblings */
361 struct list_head global; /* link to global list */
276dbf99
DW		 362 int segment; /* PCI domain */
363 u8 bus; /* PCI bus number */
a647dacb 364 u8 devfn; /* PCI devfn number */
45e829ea 365 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 366 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 367 struct dmar_domain *domain; /* pointer to domain */
368};
369
5e0d2a6f 370static void flush_unmaps_timeout(unsigned long data);
371
372DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
373
80b20dd8 374#define HIGH_WATER_MARK 250
375struct deferred_flush_tables {
376 int next;
377 struct iova *iova[HIGH_WATER_MARK];
378 struct dmar_domain *domain[HIGH_WATER_MARK];
379};
380
381static struct deferred_flush_tables *deferred_flush;
382
5e0d2a6f 383/* bitmap for indexing intel_iommus */
5e0d2a6f 384static int g_num_of_iommus;
385
386static DEFINE_SPINLOCK(async_umap_flush_lock);
387static LIST_HEAD(unmaps_to_do);
388
389static int timer_on;
390static long list_size;
5e0d2a6f 391
ba395927
KA		 392static void domain_remove_dev_info(struct dmar_domain *domain);
393
0cd5c3c8
KM		 394#ifdef CONFIG_DMAR_DEFAULT_ON
395int dmar_disabled = 0;
396#else
397int dmar_disabled = 1;
398#endif /*CONFIG_DMAR_DEFAULT_ON*/
399
2d9e667e 400static int dmar_map_gfx = 1;
7d3b03ce 401static int dmar_forcedac;
5e0d2a6f 402static int intel_iommu_strict;
6dd9a7c7 403static int intel_iommu_superpage = 1;
ba395927
KA		 404
405#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
406static DEFINE_SPINLOCK(device_domain_lock);
407static LIST_HEAD(device_domain_list);
408
a8bcbb0d
JR		 409static struct iommu_ops intel_iommu_ops;
410
ba395927
KA		 411static int __init intel_iommu_setup(char *str)
412{
413 if (!str)
414 return -EINVAL;
415 while (*str) {
0cd5c3c8
KM		 416 if (!strncmp(str, "on", 2)) {
417 dmar_disabled = 0;
418 printk(KERN_INFO "Intel-IOMMU: enabled\n");
419 } else if (!strncmp(str, "off", 3)) {
ba395927 420 dmar_disabled = 1;
0cd5c3c8 421 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 422 } else if (!strncmp(str, "igfx_off", 8)) {
423 dmar_map_gfx = 0;
424 printk(KERN_INFO
425 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 426 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 427 printk(KERN_INFO
7d3b03ce
KA		 428 "Intel-IOMMU: Forcing DAC for PCI devices\n");
429 dmar_forcedac = 1;
5e0d2a6f 430 } else if (!strncmp(str, "strict", 6)) {
431 printk(KERN_INFO
432 "Intel-IOMMU: disable batched IOTLB flush\n");
433 intel_iommu_strict = 1;
6dd9a7c7
YS		 434 } else if (!strncmp(str, "sp_off", 6)) {
435 printk(KERN_INFO
436 "Intel-IOMMU: disable supported super page\n");
437 intel_iommu_superpage = 0;
ba395927
KA		 438 }
439
440 str += strcspn(str, ",");
441 while (*str == ',')
442 str++;
443 }
444 return 0;
445}
446__setup("intel_iommu=", intel_iommu_setup);
447
448static struct kmem_cache *iommu_domain_cache;
449static struct kmem_cache *iommu_devinfo_cache;
450static struct kmem_cache *iommu_iova_cache;
451
4c923d47 452static inline void *alloc_pgtable_page(int node)
eb3fa7cb 453{
4c923d47
SS		 454 struct page *page;
455 void *vaddr = NULL;
eb3fa7cb 456
4c923d47
SS		 457 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
458 if (page)
459 vaddr = page_address(page);
eb3fa7cb 460 return vaddr;
ba395927
KA		 461}
462
463static inline void free_pgtable_page(void *vaddr)
464{
465 free_page((unsigned long)vaddr);
466}
467
468static inline void *alloc_domain_mem(void)
469{
354bb65e 470 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 471}
472
38717946 473static void free_domain_mem(void *vaddr)
ba395927
KA		 474{
475 kmem_cache_free(iommu_domain_cache, vaddr);
476}
477
478static inline void * alloc_devinfo_mem(void)
479{
354bb65e 480 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 481}
482
483static inline void free_devinfo_mem(void *vaddr)
484{
485 kmem_cache_free(iommu_devinfo_cache, vaddr);
486}
487
488struct iova *alloc_iova_mem(void)
489{
354bb65e 490 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 491}
492
493void free_iova_mem(struct iova *iova)
494{
495 kmem_cache_free(iommu_iova_cache, iova);
496}
497
1b573683 498
4ed0d3e6 499static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 500{
501 unsigned long sagaw;
502 int agaw = -1;
503
504 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 505 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 506 agaw >= 0; agaw--) {
507 if (test_bit(agaw, &sagaw))
508 break;
509 }
510
511 return agaw;
512}
513
4ed0d3e6
FY		 514/*
515 * Calculate max SAGAW for each iommu.
516 */
517int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
518{
519 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
520}
521
522/*
523 * calculate agaw for each iommu.
524 * "SAGAW" may be different across iommus, use a default agaw, and
525 * get a supported less agaw for iommus that don't support the default agaw.
526 */
527int iommu_calculate_agaw(struct intel_iommu *iommu)
528{
529 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
530}
531
2c2e2c38 532/* This functionin only returns single iommu in a domain */
8c11e798
WH		 533static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
534{
535 int iommu_id;
536
2c2e2c38 537 /* si_domain and vm domain should not get here. */
1ce28feb 538 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 539 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 540
8c11e798
WH		 541 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
542 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
543 return NULL;
544
545 return g_iommus[iommu_id];
546}
547
8e604097
WH		 548static void domain_update_iommu_coherency(struct dmar_domain *domain)
549{
550 int i;
551
552 domain->iommu_coherency = 1;
553
a45946ab 554 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 555 if (!ecap_coherent(g_iommus[i]->ecap)) {
556 domain->iommu_coherency = 0;
557 break;
558 }
8e604097
WH		 559 }
560}
561
58c610bd
SY		 562static void domain_update_iommu_snooping(struct dmar_domain *domain)
563{
564 int i;
565
566 domain->iommu_snooping = 1;
567
a45946ab 568 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 569 if (!ecap_sc_support(g_iommus[i]->ecap)) {
570 domain->iommu_snooping = 0;
571 break;
572 }
58c610bd
SY		 573 }
574}
575
6dd9a7c7
YS		 576static void domain_update_iommu_superpage(struct dmar_domain *domain)
577{
578 int i, mask = 0xf;
579
580 if (!intel_iommu_superpage) {
581 domain->iommu_superpage = 0;
582 return;
583 }
584
585 domain->iommu_superpage = 4; /* 1TiB */
586
587 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
588 mask |= cap_super_page_val(g_iommus[i]->cap);
589 if (!mask) {
590 break;
591 }
592 }
593 domain->iommu_superpage = fls(mask);
594}
595
58c610bd
SY		 596/* Some capabilities may be different across iommus */
597static void domain_update_iommu_cap(struct dmar_domain *domain)
598{
599 domain_update_iommu_coherency(domain);
600 domain_update_iommu_snooping(domain);
6dd9a7c7 601 domain_update_iommu_superpage(domain);
58c610bd
SY		 602}
603
276dbf99 604static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 605{
606 struct dmar_drhd_unit *drhd = NULL;
607 int i;
608
609 for_each_drhd_unit(drhd) {
610 if (drhd->ignored)
611 continue;
276dbf99
DW		 612 if (segment != drhd->segment)
613 continue;
c7151a8d 614
924b6231 615 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 616 if (drhd->devices[i] &&
617 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 618 drhd->devices[i]->devfn == devfn)
619 return drhd->iommu;
4958c5dc
DW		 620 if (drhd->devices[i] &&
621 drhd->devices[i]->subordinate &&
924b6231
DW		 622 drhd->devices[i]->subordinate->number <= bus &&
623 drhd->devices[i]->subordinate->subordinate >= bus)
624 return drhd->iommu;
625 }
c7151a8d
WH		 626
627 if (drhd->include_all)
628 return drhd->iommu;
629 }
630
631 return NULL;
632}
633
5331fe6f
WH		 634static void domain_flush_cache(struct dmar_domain *domain,
635 void *addr, int size)
636{
637 if (!domain->iommu_coherency)
638 clflush_cache_range(addr, size);
639}
640
ba395927
KA		 641/* Gets context entry for a given bus and devfn */
642static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
643 u8 bus, u8 devfn)
644{
645 struct root_entry *root;
646 struct context_entry *context;
647 unsigned long phy_addr;
648 unsigned long flags;
649
650 spin_lock_irqsave(&iommu->lock, flags);
651 root = &iommu->root_entry[bus];
652 context = get_context_addr_from_root(root);
653 if (!context) {
4c923d47
SS		 654 context = (struct context_entry *)
655 alloc_pgtable_page(iommu->node);
ba395927
KA		 656 if (!context) {
657 spin_unlock_irqrestore(&iommu->lock, flags);
658 return NULL;
659 }
5b6985ce 660 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 661 phy_addr = virt_to_phys((void *)context);
662 set_root_value(root, phy_addr);
663 set_root_present(root);
664 __iommu_flush_cache(iommu, root, sizeof(*root));
665 }
666 spin_unlock_irqrestore(&iommu->lock, flags);
667 return &context[devfn];
668}
669
670static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
671{
672 struct root_entry *root;
673 struct context_entry *context;
674 int ret;
675 unsigned long flags;
676
677 spin_lock_irqsave(&iommu->lock, flags);
678 root = &iommu->root_entry[bus];
679 context = get_context_addr_from_root(root);
680 if (!context) {
681 ret = 0;
682 goto out;
683 }
c07e7d21 684 ret = context_present(&context[devfn]);
ba395927
KA		 685out:
686 spin_unlock_irqrestore(&iommu->lock, flags);
687 return ret;
688}
689
690static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
691{
692 struct root_entry *root;
693 struct context_entry *context;
694 unsigned long flags;
695
696 spin_lock_irqsave(&iommu->lock, flags);
697 root = &iommu->root_entry[bus];
698 context = get_context_addr_from_root(root);
699 if (context) {
c07e7d21 700 context_clear_entry(&context[devfn]);
ba395927
KA		 701 __iommu_flush_cache(iommu, &context[devfn], \
702 sizeof(*context));
703 }
704 spin_unlock_irqrestore(&iommu->lock, flags);
705}
706
707static void free_context_table(struct intel_iommu *iommu)
708{
709 struct root_entry *root;
710 int i;
711 unsigned long flags;
712 struct context_entry *context;
713
714 spin_lock_irqsave(&iommu->lock, flags);
715 if (!iommu->root_entry) {
716 goto out;
717 }
718 for (i = 0; i < ROOT_ENTRY_NR; i++) {
719 root = &iommu->root_entry[i];
720 context = get_context_addr_from_root(root);
721 if (context)
722 free_pgtable_page(context);
723 }
724 free_pgtable_page(iommu->root_entry);
725 iommu->root_entry = NULL;
726out:
727 spin_unlock_irqrestore(&iommu->lock, flags);
728}
729
b026fd28 730static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
6dd9a7c7 731 unsigned long pfn, int large_level)
ba395927 732{
b026fd28 733 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 734 struct dma_pte *parent, *pte = NULL;
735 int level = agaw_to_level(domain->agaw);
6dd9a7c7 736 int offset, target_level;
ba395927
KA		 737
738 BUG_ON(!domain->pgd);
b026fd28 739 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 740 parent = domain->pgd;
741
6dd9a7c7
YS		 742 /* Search pte */
743 if (!large_level)
744 target_level = 1;
745 else
746 target_level = large_level;
747
ba395927
KA		 748 while (level > 0) {
749 void *tmp_page;
750
b026fd28 751 offset = pfn_level_offset(pfn, level);
ba395927 752 pte = &parent[offset];
6dd9a7c7
YS		 753 if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
754 break;
755 if (level == target_level)
ba395927
KA		 756 break;
757
19c239ce 758 if (!dma_pte_present(pte)) {
c85994e4
DW		 759 uint64_t pteval;
760
4c923d47 761 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 762
206a73c1 763 if (!tmp_page)
ba395927 764 return NULL;
206a73c1 765
c85994e4 766 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 767 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 768 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
769 /* Someone else set it while we were thinking; use theirs. */
770 free_pgtable_page(tmp_page);
771 } else {
772 dma_pte_addr(pte);
773 domain_flush_cache(domain, pte, sizeof(*pte));
774 }
ba395927 775 }
19c239ce 776 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 777 level--;
778 }
779
ba395927
KA		 780 return pte;
781}
782
6dd9a7c7 783
ba395927 784/* return address's pte at specific level */
90dcfb5e
DW		 785static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
786 unsigned long pfn,
6dd9a7c7 787 int level, int *large_page)
ba395927
KA		 788{
789 struct dma_pte *parent, *pte = NULL;
790 int total = agaw_to_level(domain->agaw);
791 int offset;
792
793 parent = domain->pgd;
794 while (level <= total) {
90dcfb5e 795 offset = pfn_level_offset(pfn, total);
ba395927
KA		 796 pte = &parent[offset];
797 if (level == total)
798 return pte;
799
6dd9a7c7
YS		 800 if (!dma_pte_present(pte)) {
801 *large_page = total;
ba395927 802 break;
6dd9a7c7
YS		 803 }
804
805 if (pte->val & DMA_PTE_LARGE_PAGE) {
806 *large_page = total;
807 return pte;
808 }
809
19c239ce 810 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 811 total--;
812 }
813 return NULL;
814}
815
ba395927 816/* clear last level pte, a tlb flush should be followed */
595badf5
DW		 817static void dma_pte_clear_range(struct dmar_domain *domain,
818 unsigned long start_pfn,
819 unsigned long last_pfn)
ba395927 820{
04b18e65 821 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 822 unsigned int large_page = 1;
310a5ab9 823 struct dma_pte *first_pte, *pte;
66eae846 824
04b18e65 825 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 826 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 827 BUG_ON(start_pfn > last_pfn);
ba395927 828
04b18e65 829 /* we don't need lock here; nobody else touches the iova range */
59c36286 830 do {
6dd9a7c7
YS		 831 large_page = 1;
832 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 833 if (!pte) {
6dd9a7c7 834 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 835 continue;
836 }
6dd9a7c7 837 do {
310a5ab9 838 dma_clear_pte(pte);
6dd9a7c7 839 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 840 pte++;
75e6bf96
DW		 841 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
842
310a5ab9
DW		 843 domain_flush_cache(domain, first_pte,
844 (void *)pte - (void *)first_pte);
59c36286
DW		 845
846 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 847}
848
849/* free page table pages. last level pte should already be cleared */
850static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 851 unsigned long start_pfn,
852 unsigned long last_pfn)
ba395927 853{
6660c63a 854 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
f3a0a52f 855 struct dma_pte *first_pte, *pte;
ba395927
KA		 856 int total = agaw_to_level(domain->agaw);
857 int level;
6660c63a 858 unsigned long tmp;
6dd9a7c7 859 int large_page = 2;
ba395927 860
6660c63a
DW		 861 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
862 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 863 BUG_ON(start_pfn > last_pfn);
ba395927 864
f3a0a52f 865 /* We don't need lock here; nobody else touches the iova range */
ba395927
KA		 866 level = 2;
867 while (level <= total) {
6660c63a
DW		 868 tmp = align_to_level(start_pfn, level);
869
f3a0a52f 870 /* If we can't even clear one PTE at this level, we're done */
6660c63a 871 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 872 return;
873
59c36286 874 do {
6dd9a7c7
YS		 875 large_page = level;
876 first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
877 if (large_page > level)
878 level = large_page + 1;
f3a0a52f
DW		 879 if (!pte) {
880 tmp = align_to_level(tmp + 1, level + 1);
881 continue;
882 }
75e6bf96 883 do {
6a43e574
DW		 884 if (dma_pte_present(pte)) {
885 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
886 dma_clear_pte(pte);
887 }
f3a0a52f
DW		 888 pte++;
889 tmp += level_size(level);
75e6bf96
DW		 890 } while (!first_pte_in_page(pte) &&
891 tmp + level_size(level) - 1 <= last_pfn);
892
f3a0a52f
DW		 893 domain_flush_cache(domain, first_pte,
894 (void *)pte - (void *)first_pte);
895
59c36286 896 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
ba395927
KA		 897 level++;
898 }
899 /* free pgd */
d794dc9b 900 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 901 free_pgtable_page(domain->pgd);
902 domain->pgd = NULL;
903 }
904}
905
906/* iommu handling */
907static int iommu_alloc_root_entry(struct intel_iommu *iommu)
908{
909 struct root_entry *root;
910 unsigned long flags;
911
4c923d47 912 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 913 if (!root)
914 return -ENOMEM;
915
5b6985ce 916 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 917
918 spin_lock_irqsave(&iommu->lock, flags);
919 iommu->root_entry = root;
920 spin_unlock_irqrestore(&iommu->lock, flags);
921
922 return 0;
923}
924
ba395927
KA		 925static void iommu_set_root_entry(struct intel_iommu *iommu)
926{
927 void *addr;
c416daa9 928 u32 sts;
ba395927
KA		 929 unsigned long flag;
930
931 addr = iommu->root_entry;
932
933 spin_lock_irqsave(&iommu->register_lock, flag);
934 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
935
c416daa9 936 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 937
938 /* Make sure hardware complete it */
939 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 940 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 941
942 spin_unlock_irqrestore(&iommu->register_lock, flag);
943}
944
945static void iommu_flush_write_buffer(struct intel_iommu *iommu)
946{
947 u32 val;
948 unsigned long flag;
949
9af88143 950 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 951 return;
ba395927
KA		 952
953 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 954 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 955
956 /* Make sure hardware complete it */
957 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 958 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 959
960 spin_unlock_irqrestore(&iommu->register_lock, flag);
961}
962
963/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 964static void __iommu_flush_context(struct intel_iommu *iommu,
965 u16 did, u16 source_id, u8 function_mask,
966 u64 type)
ba395927
KA		 967{
968 u64 val = 0;
969 unsigned long flag;
970
ba395927
KA		 971 switch (type) {
972 case DMA_CCMD_GLOBAL_INVL:
973 val = DMA_CCMD_GLOBAL_INVL;
974 break;
975 case DMA_CCMD_DOMAIN_INVL:
976 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
977 break;
978 case DMA_CCMD_DEVICE_INVL:
979 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
980 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
981 break;
982 default:
983 BUG();
984 }
985 val |= DMA_CCMD_ICC;
986
987 spin_lock_irqsave(&iommu->register_lock, flag);
988 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
989
990 /* Make sure hardware complete it */
991 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
992 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
993
994 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 995}
996
ba395927 997/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 998static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
999 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1000{
1001 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1002 u64 val = 0, val_iva = 0;
1003 unsigned long flag;
1004
ba395927
KA		 1005 switch (type) {
1006 case DMA_TLB_GLOBAL_FLUSH:
1007 /* global flush doesn't need set IVA_REG */
1008 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1009 break;
1010 case DMA_TLB_DSI_FLUSH:
1011 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1012 break;
1013 case DMA_TLB_PSI_FLUSH:
1014 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1015 /* Note: always flush non-leaf currently */
1016 val_iva = size_order | addr;
1017 break;
1018 default:
1019 BUG();
1020 }
1021 /* Note: set drain read/write */
1022#if 0
1023 /*
1024 * This is probably to be super secure.. Looks like we can
1025 * ignore it without any impact.
1026 */
1027 if (cap_read_drain(iommu->cap))
1028 val |= DMA_TLB_READ_DRAIN;
1029#endif
1030 if (cap_write_drain(iommu->cap))
1031 val |= DMA_TLB_WRITE_DRAIN;
1032
1033 spin_lock_irqsave(&iommu->register_lock, flag);
1034 /* Note: Only uses first TLB reg currently */
1035 if (val_iva)
1036 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1037 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1038
1039 /* Make sure hardware complete it */
1040 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1041 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1042
1043 spin_unlock_irqrestore(&iommu->register_lock, flag);
1044
1045 /* check IOTLB invalidation granularity */
1046 if (DMA_TLB_IAIG(val) == 0)
1047 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1048 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1049 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1050 (unsigned long long)DMA_TLB_IIRG(type),
1051 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1052}
1053
93a23a72
YZ		 1054static struct device_domain_info *iommu_support_dev_iotlb(
1055 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1056{
1057 int found = 0;
1058 unsigned long flags;
1059 struct device_domain_info *info;
1060 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1061
1062 if (!ecap_dev_iotlb_support(iommu->ecap))
1063 return NULL;
1064
1065 if (!iommu->qi)
1066 return NULL;
1067
1068 spin_lock_irqsave(&device_domain_lock, flags);
1069 list_for_each_entry(info, &domain->devices, link)
1070 if (info->bus == bus && info->devfn == devfn) {
1071 found = 1;
1072 break;
1073 }
1074 spin_unlock_irqrestore(&device_domain_lock, flags);
1075
1076 if (!found || !info->dev)
1077 return NULL;
1078
1079 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1080 return NULL;
1081
1082 if (!dmar_find_matched_atsr_unit(info->dev))
1083 return NULL;
1084
1085 info->iommu = iommu;
1086
1087 return info;
1088}
1089
1090static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1091{
93a23a72
YZ		 1092 if (!info)
1093 return;
1094
1095 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1096}
1097
1098static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1099{
1100 if (!info->dev || !pci_ats_enabled(info->dev))
1101 return;
1102
1103 pci_disable_ats(info->dev);
1104}
1105
1106static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1107 u64 addr, unsigned mask)
1108{
1109 u16 sid, qdep;
1110 unsigned long flags;
1111 struct device_domain_info *info;
1112
1113 spin_lock_irqsave(&device_domain_lock, flags);
1114 list_for_each_entry(info, &domain->devices, link) {
1115 if (!info->dev || !pci_ats_enabled(info->dev))
1116 continue;
1117
1118 sid = info->bus << 8 | info->devfn;
1119 qdep = pci_ats_queue_depth(info->dev);
1120 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1121 }
1122 spin_unlock_irqrestore(&device_domain_lock, flags);
1123}
1124
1f0ef2aa 1125static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1126 unsigned long pfn, unsigned int pages, int map)
ba395927 1127{
9dd2fe89 1128 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1129 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1130
ba395927
KA		 1131 BUG_ON(pages == 0);
1132
ba395927 1133 /*
9dd2fe89
YZ		 1134 * Fallback to domain selective flush if no PSI support or the size is
1135 * too big.
ba395927
KA		 1136 * PSI requires page size to be 2 ^ x, and the base address is naturally
1137 * aligned to the size
1138 */
9dd2fe89
YZ		 1139 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1140 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1141 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1142 else
1143 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1144 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1145
1146 /*
82653633
NA		 1147 * In caching mode, changes of pages from non-present to present require
1148 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1149 */
82653633 1150 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1151 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1152}
1153
f8bab735 1154static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1155{
1156 u32 pmen;
1157 unsigned long flags;
1158
1159 spin_lock_irqsave(&iommu->register_lock, flags);
1160 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1161 pmen &= ~DMA_PMEN_EPM;
1162 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1163
1164 /* wait for the protected region status bit to clear */
1165 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1166 readl, !(pmen & DMA_PMEN_PRS), pmen);
1167
1168 spin_unlock_irqrestore(&iommu->register_lock, flags);
1169}
1170
ba395927
KA		 1171static int iommu_enable_translation(struct intel_iommu *iommu)
1172{
1173 u32 sts;
1174 unsigned long flags;
1175
1176 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1177 iommu->gcmd |= DMA_GCMD_TE;
1178 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1179
1180 /* Make sure hardware complete it */
1181 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1182 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1183
ba395927
KA		 1184 spin_unlock_irqrestore(&iommu->register_lock, flags);
1185 return 0;
1186}
1187
1188static int iommu_disable_translation(struct intel_iommu *iommu)
1189{
1190 u32 sts;
1191 unsigned long flag;
1192
1193 spin_lock_irqsave(&iommu->register_lock, flag);
1194 iommu->gcmd &= ~DMA_GCMD_TE;
1195 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1196
1197 /* Make sure hardware complete it */
1198 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1199 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1200
1201 spin_unlock_irqrestore(&iommu->register_lock, flag);
1202 return 0;
1203}
1204
3460a6d9 1205
ba395927
KA		 1206static int iommu_init_domains(struct intel_iommu *iommu)
1207{
1208 unsigned long ndomains;
1209 unsigned long nlongs;
1210
1211 ndomains = cap_ndoms(iommu->cap);
680a7524
YL		 1212 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1213 ndomains);
ba395927
KA		 1214 nlongs = BITS_TO_LONGS(ndomains);
1215
94a91b50
DD		 1216 spin_lock_init(&iommu->lock);
1217
ba395927
KA		 1218 /* TBD: there might be 64K domains,
1219 * consider other allocation for future chip
1220 */
1221 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1222 if (!iommu->domain_ids) {
1223 printk(KERN_ERR "Allocating domain id array failed\n");
1224 return -ENOMEM;
1225 }
1226 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1227 GFP_KERNEL);
1228 if (!iommu->domains) {
1229 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1230 return -ENOMEM;
1231 }
1232
1233 /*
1234 * if Caching mode is set, then invalid translations are tagged
1235 * with domainid 0. Hence we need to pre-allocate it.
1236 */
1237 if (cap_caching_mode(iommu->cap))
1238 set_bit(0, iommu->domain_ids);
1239 return 0;
1240}
ba395927 1241
ba395927
KA		 1242
1243static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1244static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1245
1246void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1247{
1248 struct dmar_domain *domain;
1249 int i;
c7151a8d 1250 unsigned long flags;
ba395927 1251
94a91b50 1252 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1253 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1254 domain = iommu->domains[i];
1255 clear_bit(i, iommu->domain_ids);
1256
1257 spin_lock_irqsave(&domain->iommu_lock, flags);
1258 if (--domain->iommu_count == 0) {
1259 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1260 vm_domain_exit(domain);
1261 else
1262 domain_exit(domain);
1263 }
1264 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1265 }
ba395927
KA		 1266 }
1267
1268 if (iommu->gcmd & DMA_GCMD_TE)
1269 iommu_disable_translation(iommu);
1270
1271 if (iommu->irq) {
dced35ae 1272 irq_set_handler_data(iommu->irq, NULL);
ba395927
KA		 1273 /* This will mask the irq */
1274 free_irq(iommu->irq, iommu);
1275 destroy_irq(iommu->irq);
1276 }
1277
1278 kfree(iommu->domains);
1279 kfree(iommu->domain_ids);
1280
d9630fe9
WH		 1281 g_iommus[iommu->seq_id] = NULL;
1282
1283 /* if all iommus are freed, free g_iommus */
1284 for (i = 0; i < g_num_of_iommus; i++) {
1285 if (g_iommus[i])
1286 break;
1287 }
1288
1289 if (i == g_num_of_iommus)
1290 kfree(g_iommus);
1291
ba395927
KA		 1292 /* free context mapping */
1293 free_context_table(iommu);
ba395927
KA		 1294}
1295
2c2e2c38 1296static struct dmar_domain *alloc_domain(void)
ba395927 1297{
ba395927 1298 struct dmar_domain *domain;
ba395927
KA		 1299
1300 domain = alloc_domain_mem();
1301 if (!domain)
1302 return NULL;
1303
4c923d47 1304 domain->nid = -1;
2c2e2c38
FY		 1305 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1306 domain->flags = 0;
1307
1308 return domain;
1309}
1310
1311static int iommu_attach_domain(struct dmar_domain *domain,
1312 struct intel_iommu *iommu)
1313{
1314 int num;
1315 unsigned long ndomains;
1316 unsigned long flags;
1317
ba395927
KA		 1318 ndomains = cap_ndoms(iommu->cap);
1319
1320 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1321
ba395927
KA		 1322 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1323 if (num >= ndomains) {
1324 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1325 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1326 return -ENOMEM;
ba395927
KA		 1327 }
1328
ba395927 1329 domain->id = num;
2c2e2c38 1330 set_bit(num, iommu->domain_ids);
8c11e798 1331 set_bit(iommu->seq_id, &domain->iommu_bmp);
ba395927
KA		 1332 iommu->domains[num] = domain;
1333 spin_unlock_irqrestore(&iommu->lock, flags);
1334
2c2e2c38 1335 return 0;
ba395927
KA		 1336}
1337
2c2e2c38
FY		 1338static void iommu_detach_domain(struct dmar_domain *domain,
1339 struct intel_iommu *iommu)
ba395927
KA		 1340{
1341 unsigned long flags;
2c2e2c38
FY		 1342 int num, ndomains;
1343 int found = 0;
ba395927 1344
8c11e798 1345 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1346 ndomains = cap_ndoms(iommu->cap);
a45946ab 1347 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1348 if (iommu->domains[num] == domain) {
1349 found = 1;
1350 break;
1351 }
2c2e2c38
FY		 1352 }
1353
1354 if (found) {
1355 clear_bit(num, iommu->domain_ids);
1356 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1357 iommu->domains[num] = NULL;
1358 }
8c11e798 1359 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1360}
1361
1362static struct iova_domain reserved_iova_list;
8a443df4 1363static struct lock_class_key reserved_rbtree_key;
ba395927 1364
51a63e67 1365static int dmar_init_reserved_ranges(void)
ba395927
KA		 1366{
1367 struct pci_dev *pdev = NULL;
1368 struct iova *iova;
1369 int i;
ba395927 1370
f661197e 1371 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1372
8a443df4
MG		 1373 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1374 &reserved_rbtree_key);
1375
ba395927
KA		 1376 /* IOAPIC ranges shouldn't be accessed by DMA */
1377 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1378 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1379 if (!iova) {
ba395927 1380 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1381 return -ENODEV;
1382 }
ba395927
KA		 1383
1384 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1385 for_each_pci_dev(pdev) {
1386 struct resource *r;
1387
1388 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1389 r = &pdev->resource[i];
1390 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1391 continue;
1a4a4551
DW		 1392 iova = reserve_iova(&reserved_iova_list,
1393 IOVA_PFN(r->start),
1394 IOVA_PFN(r->end));
51a63e67 1395 if (!iova) {
ba395927 1396 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1397 return -ENODEV;
1398 }
ba395927
KA		 1399 }
1400 }
51a63e67 1401 return 0;
ba395927
KA		 1402}
1403
1404static void domain_reserve_special_ranges(struct dmar_domain *domain)
1405{
1406 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1407}
1408
1409static inline int guestwidth_to_adjustwidth(int gaw)
1410{
1411 int agaw;
1412 int r = (gaw - 12) % 9;
1413
1414 if (r == 0)
1415 agaw = gaw;
1416 else
1417 agaw = gaw + 9 - r;
1418 if (agaw > 64)
1419 agaw = 64;
1420 return agaw;
1421}
1422
1423static int domain_init(struct dmar_domain *domain, int guest_width)
1424{
1425 struct intel_iommu *iommu;
1426 int adjust_width, agaw;
1427 unsigned long sagaw;
1428
f661197e 1429 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1430 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1431
1432 domain_reserve_special_ranges(domain);
1433
1434 /* calculate AGAW */
8c11e798 1435 iommu = domain_get_iommu(domain);
ba395927
KA		 1436 if (guest_width > cap_mgaw(iommu->cap))
1437 guest_width = cap_mgaw(iommu->cap);
1438 domain->gaw = guest_width;
1439 adjust_width = guestwidth_to_adjustwidth(guest_width);
1440 agaw = width_to_agaw(adjust_width);
1441 sagaw = cap_sagaw(iommu->cap);
1442 if (!test_bit(agaw, &sagaw)) {
1443 /* hardware doesn't support it, choose a bigger one */
1444 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1445 agaw = find_next_bit(&sagaw, 5, agaw);
1446 if (agaw >= 5)
1447 return -ENODEV;
1448 }
1449 domain->agaw = agaw;
1450 INIT_LIST_HEAD(&domain->devices);
1451
8e604097
WH		 1452 if (ecap_coherent(iommu->ecap))
1453 domain->iommu_coherency = 1;
1454 else
1455 domain->iommu_coherency = 0;
1456
58c610bd
SY		 1457 if (ecap_sc_support(iommu->ecap))
1458 domain->iommu_snooping = 1;
1459 else
1460 domain->iommu_snooping = 0;
1461
6dd9a7c7 1462 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1463 domain->iommu_count = 1;
4c923d47 1464 domain->nid = iommu->node;
c7151a8d 1465
ba395927 1466 /* always allocate the top pgd */
4c923d47 1467 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1468 if (!domain->pgd)
1469 return -ENOMEM;
5b6985ce 1470 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1471 return 0;
1472}
1473
1474static void domain_exit(struct dmar_domain *domain)
1475{
2c2e2c38
FY		 1476 struct dmar_drhd_unit *drhd;
1477 struct intel_iommu *iommu;
ba395927
KA		 1478
1479 /* Domain 0 is reserved, so dont process it */
1480 if (!domain)
1481 return;
1482
7b668357
AW		 1483 /* Flush any lazy unmaps that may reference this domain */
1484 if (!intel_iommu_strict)
1485 flush_unmaps_timeout(0);
1486
ba395927
KA		 1487 domain_remove_dev_info(domain);
1488 /* destroy iovas */
1489 put_iova_domain(&domain->iovad);
ba395927
KA		 1490
1491 /* clear ptes */
595badf5 1492 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1493
1494 /* free page tables */
d794dc9b 1495 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1496
2c2e2c38
FY		 1497 for_each_active_iommu(iommu, drhd)
1498 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1499 iommu_detach_domain(domain, iommu);
1500
ba395927
KA		 1501 free_domain_mem(domain);
1502}
1503
4ed0d3e6
FY		 1504static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1505 u8 bus, u8 devfn, int translation)
ba395927
KA		 1506{
1507 struct context_entry *context;
ba395927 1508 unsigned long flags;
5331fe6f 1509 struct intel_iommu *iommu;
ea6606b0
WH		 1510 struct dma_pte *pgd;
1511 unsigned long num;
1512 unsigned long ndomains;
1513 int id;
1514 int agaw;
93a23a72 1515 struct device_domain_info *info = NULL;
ba395927
KA		 1516
1517 pr_debug("Set context mapping for %02x:%02x.%d\n",
1518 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1519
ba395927 1520 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1521 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1522 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1523
276dbf99 1524 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1525 if (!iommu)
1526 return -ENODEV;
1527
ba395927
KA		 1528 context = device_to_context_entry(iommu, bus, devfn);
1529 if (!context)
1530 return -ENOMEM;
1531 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1532 if (context_present(context)) {
ba395927
KA		 1533 spin_unlock_irqrestore(&iommu->lock, flags);
1534 return 0;
1535 }
1536
ea6606b0
WH		 1537 id = domain->id;
1538 pgd = domain->pgd;
1539
2c2e2c38
FY		 1540 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1541 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1542 int found = 0;
1543
1544 /* find an available domain id for this device in iommu */
1545 ndomains = cap_ndoms(iommu->cap);
a45946ab 1546 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1547 if (iommu->domains[num] == domain) {
1548 id = num;
1549 found = 1;
1550 break;
1551 }
ea6606b0
WH		 1552 }
1553
1554 if (found == 0) {
1555 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1556 if (num >= ndomains) {
1557 spin_unlock_irqrestore(&iommu->lock, flags);
1558 printk(KERN_ERR "IOMMU: no free domain ids\n");
1559 return -EFAULT;
1560 }
1561
1562 set_bit(num, iommu->domain_ids);
1563 iommu->domains[num] = domain;
1564 id = num;
1565 }
1566
1567 /* Skip top levels of page tables for
1568 * iommu which has less agaw than default.
1672af11 1569 * Unnecessary for PT mode.
ea6606b0 1570 */
1672af11
CW		 1571 if (translation != CONTEXT_TT_PASS_THROUGH) {
1572 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1573 pgd = phys_to_virt(dma_pte_addr(pgd));
1574 if (!dma_pte_present(pgd)) {
1575 spin_unlock_irqrestore(&iommu->lock, flags);
1576 return -ENOMEM;
1577 }
ea6606b0
WH		 1578 }
1579 }
1580 }
1581
1582 context_set_domain_id(context, id);
4ed0d3e6 1583
93a23a72
YZ		 1584 if (translation != CONTEXT_TT_PASS_THROUGH) {
1585 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1586 translation = info ? CONTEXT_TT_DEV_IOTLB :
1587 CONTEXT_TT_MULTI_LEVEL;
1588 }
4ed0d3e6
FY		 1589 /*
1590 * In pass through mode, AW must be programmed to indicate the largest
1591 * AGAW value supported by hardware. And ASR is ignored by hardware.
1592 */
93a23a72 1593 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1594 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1595 else {
1596 context_set_address_root(context, virt_to_phys(pgd));
1597 context_set_address_width(context, iommu->agaw);
1598 }
4ed0d3e6
FY		 1599
1600 context_set_translation_type(context, translation);
c07e7d21
MM		 1601 context_set_fault_enable(context);
1602 context_set_present(context);
5331fe6f 1603 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1604
4c25a2c1
DW		 1605 /*
1606 * It's a non-present to present mapping. If hardware doesn't cache
1607 * non-present entry we only need to flush the write-buffer. If the
1608 * _does_ cache non-present entries, then it does so in the special
1609 * domain #0, which we have to flush:
1610 */
1611 if (cap_caching_mode(iommu->cap)) {
1612 iommu->flush.flush_context(iommu, 0,
1613 (((u16)bus) << 8) | devfn,
1614 DMA_CCMD_MASK_NOBIT,
1615 DMA_CCMD_DEVICE_INVL);
82653633 1616 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1617 } else {
ba395927 1618 iommu_flush_write_buffer(iommu);
4c25a2c1 1619 }
93a23a72 1620 iommu_enable_dev_iotlb(info);
ba395927 1621 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1622
1623 spin_lock_irqsave(&domain->iommu_lock, flags);
1624 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1625 domain->iommu_count++;
4c923d47
SS		 1626 if (domain->iommu_count == 1)
1627 domain->nid = iommu->node;
58c610bd 1628 domain_update_iommu_cap(domain);
c7151a8d
WH		 1629 }
1630 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1631 return 0;
1632}
1633
1634static int
4ed0d3e6
FY		 1635domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1636 int translation)
ba395927
KA		 1637{
1638 int ret;
1639 struct pci_dev *tmp, *parent;
1640
276dbf99 1641 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1642 pdev->bus->number, pdev->devfn,
1643 translation);
ba395927
KA		 1644 if (ret)
1645 return ret;
1646
1647 /* dependent device mapping */
1648 tmp = pci_find_upstream_pcie_bridge(pdev);
1649 if (!tmp)
1650 return 0;
1651 /* Secondary interface's bus number and devfn 0 */
1652 parent = pdev->bus->self;
1653 while (parent != tmp) {
276dbf99
DW		 1654 ret = domain_context_mapping_one(domain,
1655 pci_domain_nr(parent->bus),
1656 parent->bus->number,
4ed0d3e6 1657 parent->devfn, translation);
ba395927
KA		 1658 if (ret)
1659 return ret;
1660 parent = parent->bus->self;
1661 }
45e829ea 1662 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1663 return domain_context_mapping_one(domain,
276dbf99 1664 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1665 tmp->subordinate->number, 0,
1666 translation);
ba395927
KA		 1667 else /* this is a legacy PCI bridge */
1668 return domain_context_mapping_one(domain,
276dbf99
DW		 1669 pci_domain_nr(tmp->bus),
1670 tmp->bus->number,
4ed0d3e6
FY		 1671 tmp->devfn,
1672 translation);
ba395927
KA		 1673}
1674
5331fe6f 1675static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1676{
1677 int ret;
1678 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1679 struct intel_iommu *iommu;
1680
276dbf99
DW		 1681 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1682 pdev->devfn);
5331fe6f
WH		 1683 if (!iommu)
1684 return -ENODEV;
ba395927 1685
276dbf99 1686 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1687 if (!ret)
1688 return ret;
1689 /* dependent device mapping */
1690 tmp = pci_find_upstream_pcie_bridge(pdev);
1691 if (!tmp)
1692 return ret;
1693 /* Secondary interface's bus number and devfn 0 */
1694 parent = pdev->bus->self;
1695 while (parent != tmp) {
8c11e798 1696 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1697 parent->devfn);
ba395927
KA		 1698 if (!ret)
1699 return ret;
1700 parent = parent->bus->self;
1701 }
5f4d91a1 1702 if (pci_is_pcie(tmp))
276dbf99
DW		 1703 return device_context_mapped(iommu, tmp->subordinate->number,
1704 0);
ba395927 1705 else
276dbf99
DW		 1706 return device_context_mapped(iommu, tmp->bus->number,
1707 tmp->devfn);
ba395927
KA		 1708}
1709
f532959b
FY		 1710/* Returns a number of VTD pages, but aligned to MM page size */
1711static inline unsigned long aligned_nrpages(unsigned long host_addr,
1712 size_t size)
1713{
1714 host_addr &= ~PAGE_MASK;
1715 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1716}
1717
6dd9a7c7
YS		 1718/* Return largest possible superpage level for a given mapping */
1719static inline int hardware_largepage_caps(struct dmar_domain *domain,
1720 unsigned long iov_pfn,
1721 unsigned long phy_pfn,
1722 unsigned long pages)
1723{
1724 int support, level = 1;
1725 unsigned long pfnmerge;
1726
1727 support = domain->iommu_superpage;
1728
1729 /* To use a large page, the virtual *and* physical addresses
1730 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1731 of them will mean we have to use smaller pages. So just
1732 merge them and check both at once. */
1733 pfnmerge = iov_pfn | phy_pfn;
1734
1735 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1736 pages >>= VTD_STRIDE_SHIFT;
1737 if (!pages)
1738 break;
1739 pfnmerge >>= VTD_STRIDE_SHIFT;
1740 level++;
1741 support--;
1742 }
1743 return level;
1744}
1745
9051aa02
DW		 1746static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1747 struct scatterlist *sg, unsigned long phys_pfn,
1748 unsigned long nr_pages, int prot)
e1605495
DW		 1749{
1750 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1751 phys_addr_t uninitialized_var(pteval);
e1605495 1752 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1753 unsigned long sg_res;
6dd9a7c7
YS		 1754 unsigned int largepage_lvl = 0;
1755 unsigned long lvl_pages = 0;
e1605495
DW		 1756
1757 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1758
1759 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1760 return -EINVAL;
1761
1762 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1763
9051aa02
DW		 1764 if (sg)
1765 sg_res = 0;
1766 else {
1767 sg_res = nr_pages + 1;
1768 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1769 }
1770
6dd9a7c7 1771 while (nr_pages > 0) {
c85994e4
DW		 1772 uint64_t tmp;
1773
e1605495 1774 if (!sg_res) {
f532959b 1775 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1776 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1777 sg->dma_length = sg->length;
1778 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1779 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1780 }
6dd9a7c7 1781
e1605495 1782 if (!pte) {
6dd9a7c7
YS		 1783 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1784
1785 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1786 if (!pte)
1787 return -ENOMEM;
6dd9a7c7
YS		 1788 /* It is large page*/
1789 if (largepage_lvl > 1)
1790 pteval |= DMA_PTE_LARGE_PAGE;
1791 else
1792 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1793
e1605495
DW		 1794 }
1795 /* We don't need lock here, nobody else
1796 * touches the iova range
1797 */
7766a3fb 1798 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1799 if (tmp) {
1bf20f0d 1800 static int dumps = 5;
c85994e4
DW		 1801 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1802 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1803 if (dumps) {
1804 dumps--;
1805 debug_dma_dump_mappings(NULL);
1806 }
1807 WARN_ON(1);
1808 }
6dd9a7c7
YS		 1809
1810 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1811
1812 BUG_ON(nr_pages < lvl_pages);
1813 BUG_ON(sg_res < lvl_pages);
1814
1815 nr_pages -= lvl_pages;
1816 iov_pfn += lvl_pages;
1817 phys_pfn += lvl_pages;
1818 pteval += lvl_pages * VTD_PAGE_SIZE;
1819 sg_res -= lvl_pages;
1820
1821 /* If the next PTE would be the first in a new page, then we
1822 need to flush the cache on the entries we've just written.
1823 And then we'll need to recalculate 'pte', so clear it and
1824 let it get set again in the if (!pte) block above.
1825
1826 If we're done (!nr_pages) we need to flush the cache too.
1827
1828 Also if we've been setting superpages, we may need to
1829 recalculate 'pte' and switch back to smaller pages for the
1830 end of the mapping, if the trailing size is not enough to
1831 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1832 pte++;
6dd9a7c7
YS		 1833 if (!nr_pages || first_pte_in_page(pte) ||
1834 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1835 domain_flush_cache(domain, first_pte,
1836 (void *)pte - (void *)first_pte);
1837 pte = NULL;
1838 }
6dd9a7c7
YS		 1839
1840 if (!sg_res && nr_pages)
e1605495
DW		 1841 sg = sg_next(sg);
1842 }
1843 return 0;
1844}
1845
9051aa02
DW		 1846static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1847 struct scatterlist *sg, unsigned long nr_pages,
1848 int prot)
ba395927 1849{
9051aa02
DW		 1850 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1851}
6f6a00e4 1852
9051aa02
DW		 1853static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1854 unsigned long phys_pfn, unsigned long nr_pages,
1855 int prot)
1856{
1857 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1858}
1859
c7151a8d 1860static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1861{
c7151a8d
WH		 1862 if (!iommu)
1863 return;
8c11e798
WH		 1864
1865 clear_context_table(iommu, bus, devfn);
1866 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1867 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1868 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1869}
1870
1871static void domain_remove_dev_info(struct dmar_domain *domain)
1872{
1873 struct device_domain_info *info;
1874 unsigned long flags;
c7151a8d 1875 struct intel_iommu *iommu;
ba395927
KA		 1876
1877 spin_lock_irqsave(&device_domain_lock, flags);
1878 while (!list_empty(&domain->devices)) {
1879 info = list_entry(domain->devices.next,
1880 struct device_domain_info, link);
1881 list_del(&info->link);
1882 list_del(&info->global);
1883 if (info->dev)
358dd8ac 1884 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1885 spin_unlock_irqrestore(&device_domain_lock, flags);
1886
93a23a72 1887 iommu_disable_dev_iotlb(info);
276dbf99 1888 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1889 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1890 free_devinfo_mem(info);
1891
1892 spin_lock_irqsave(&device_domain_lock, flags);
1893 }
1894 spin_unlock_irqrestore(&device_domain_lock, flags);
1895}
1896
1897/*
1898 * find_domain
358dd8ac 1899 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1900 */
38717946 1901static struct dmar_domain *
ba395927
KA		 1902find_domain(struct pci_dev *pdev)
1903{
1904 struct device_domain_info *info;
1905
1906 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1907 info = pdev->dev.archdata.iommu;
ba395927
KA		 1908 if (info)
1909 return info->domain;
1910 return NULL;
1911}
1912
ba395927
KA		 1913/* domain is initialized */
1914static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1915{
1916 struct dmar_domain *domain, *found = NULL;
1917 struct intel_iommu *iommu;
1918 struct dmar_drhd_unit *drhd;
1919 struct device_domain_info *info, *tmp;
1920 struct pci_dev *dev_tmp;
1921 unsigned long flags;
1922 int bus = 0, devfn = 0;
276dbf99 1923 int segment;
2c2e2c38 1924 int ret;
ba395927
KA		 1925
1926 domain = find_domain(pdev);
1927 if (domain)
1928 return domain;
1929
276dbf99
DW		 1930 segment = pci_domain_nr(pdev->bus);
1931
ba395927
KA		 1932 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1933 if (dev_tmp) {
5f4d91a1 1934 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1935 bus = dev_tmp->subordinate->number;
1936 devfn = 0;
1937 } else {
1938 bus = dev_tmp->bus->number;
1939 devfn = dev_tmp->devfn;
1940 }
1941 spin_lock_irqsave(&device_domain_lock, flags);
1942 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1943 if (info->segment == segment &&
1944 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1945 found = info->domain;
1946 break;
1947 }
1948 }
1949 spin_unlock_irqrestore(&device_domain_lock, flags);
1950 /* pcie-pci bridge already has a domain, uses it */
1951 if (found) {
1952 domain = found;
1953 goto found_domain;
1954 }
1955 }
1956
2c2e2c38
FY		 1957 domain = alloc_domain();
1958 if (!domain)
1959 goto error;
1960
ba395927
KA		 1961 /* Allocate new domain for the device */
1962 drhd = dmar_find_matched_drhd_unit(pdev);
1963 if (!drhd) {
1964 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1965 pci_name(pdev));
1966 return NULL;
1967 }
1968 iommu = drhd->iommu;
1969
2c2e2c38
FY		 1970 ret = iommu_attach_domain(domain, iommu);
1971 if (ret) {
2fe9723d 1972 free_domain_mem(domain);
ba395927 1973 goto error;
2c2e2c38 1974 }
ba395927
KA		 1975
1976 if (domain_init(domain, gaw)) {
1977 domain_exit(domain);
1978 goto error;
1979 }
1980
1981 /* register pcie-to-pci device */
1982 if (dev_tmp) {
1983 info = alloc_devinfo_mem();
1984 if (!info) {
1985 domain_exit(domain);
1986 goto error;
1987 }
276dbf99 1988 info->segment = segment;
ba395927
KA		 1989 info->bus = bus;
1990 info->devfn = devfn;
1991 info->dev = NULL;
1992 info->domain = domain;
1993 /* This domain is shared by devices under p2p bridge */
3b5410e7 1994 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1995
1996 /* pcie-to-pci bridge already has a domain, uses it */
1997 found = NULL;
1998 spin_lock_irqsave(&device_domain_lock, flags);
1999 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2000 if (tmp->segment == segment &&
2001 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2002 found = tmp->domain;
2003 break;
2004 }
2005 }
2006 if (found) {
00dfff77 2007 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2008 free_devinfo_mem(info);
2009 domain_exit(domain);
2010 domain = found;
2011 } else {
2012 list_add(&info->link, &domain->devices);
2013 list_add(&info->global, &device_domain_list);
00dfff77 2014 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2015 }
ba395927
KA		 2016 }
2017
2018found_domain:
2019 info = alloc_devinfo_mem();
2020 if (!info)
2021 goto error;
276dbf99 2022 info->segment = segment;
ba395927
KA		 2023 info->bus = pdev->bus->number;
2024 info->devfn = pdev->devfn;
2025 info->dev = pdev;
2026 info->domain = domain;
2027 spin_lock_irqsave(&device_domain_lock, flags);
2028 /* somebody is fast */
2029 found = find_domain(pdev);
2030 if (found != NULL) {
2031 spin_unlock_irqrestore(&device_domain_lock, flags);
2032 if (found != domain) {
2033 domain_exit(domain);
2034 domain = found;
2035 }
2036 free_devinfo_mem(info);
2037 return domain;
2038 }
2039 list_add(&info->link, &domain->devices);
2040 list_add(&info->global, &device_domain_list);
358dd8ac 2041 pdev->dev.archdata.iommu = info;
ba395927
KA		 2042 spin_unlock_irqrestore(&device_domain_lock, flags);
2043 return domain;
2044error:
2045 /* recheck it here, maybe others set it */
2046 return find_domain(pdev);
2047}
2048
2c2e2c38 2049static int iommu_identity_mapping;
e0fc7e0b
DW		 2050#define IDENTMAP_ALL 1
2051#define IDENTMAP_GFX 2
2052#define IDENTMAP_AZALIA 4
2c2e2c38 2053
b213203e
DW		 2054static int iommu_domain_identity_map(struct dmar_domain *domain,
2055 unsigned long long start,
2056 unsigned long long end)
ba395927 2057{
c5395d5c
DW		 2058 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2059 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2060
2061 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2062 dma_to_mm_pfn(last_vpfn))) {
ba395927 2063 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2064 return -ENOMEM;
ba395927
KA		 2065 }
2066
c5395d5c
DW		 2067 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2068 start, end, domain->id);
ba395927
KA		 2069 /*
2070 * RMRR range might have overlap with physical memory range,
2071 * clear it first
2072 */
c5395d5c 2073 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2074
c5395d5c
DW		 2075 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2076 last_vpfn - first_vpfn + 1,
61df7443 2077 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2078}
2079
2080static int iommu_prepare_identity_map(struct pci_dev *pdev,
2081 unsigned long long start,
2082 unsigned long long end)
2083{
2084 struct dmar_domain *domain;
2085 int ret;
2086
c7ab48d2 2087 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2088 if (!domain)
2089 return -ENOMEM;
2090
19943b0e
DW		 2091 /* For _hardware_ passthrough, don't bother. But for software
2092 passthrough, we do it anyway -- it may indicate a memory
2093 range which is reserved in E820, so which didn't get set
2094 up to start with in si_domain */
2095 if (domain == si_domain && hw_pass_through) {
2096 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2097 pci_name(pdev), start, end);
2098 return 0;
2099 }
2100
2101 printk(KERN_INFO
2102 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2103 pci_name(pdev), start, end);
2ff729f5 2104
5595b528
DW		 2105 if (end < start) {
2106 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2107 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2108 dmi_get_system_info(DMI_BIOS_VENDOR),
2109 dmi_get_system_info(DMI_BIOS_VERSION),
2110 dmi_get_system_info(DMI_PRODUCT_VERSION));
2111 ret = -EIO;
2112 goto error;
2113 }
2114
2ff729f5
DW		 2115 if (end >> agaw_to_width(domain->agaw)) {
2116 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2117 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2118 agaw_to_width(domain->agaw),
2119 dmi_get_system_info(DMI_BIOS_VENDOR),
2120 dmi_get_system_info(DMI_BIOS_VERSION),
2121 dmi_get_system_info(DMI_PRODUCT_VERSION));
2122 ret = -EIO;
2123 goto error;
2124 }
19943b0e 2125
b213203e 2126 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2127 if (ret)
2128 goto error;
2129
2130 /* context entry init */
4ed0d3e6 2131 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2132 if (ret)
2133 goto error;
2134
2135 return 0;
2136
2137 error:
ba395927
KA		 2138 domain_exit(domain);
2139 return ret;
ba395927
KA		 2140}
2141
2142static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2143 struct pci_dev *pdev)
2144{
358dd8ac 2145 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2146 return 0;
2147 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2148 rmrr->end_address + 1);
2149}
2150
49a0429e
KA		 2151#ifdef CONFIG_DMAR_FLOPPY_WA
2152static inline void iommu_prepare_isa(void)
2153{
2154 struct pci_dev *pdev;
2155 int ret;
2156
2157 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2158 if (!pdev)
2159 return;
2160
c7ab48d2 2161 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
49a0429e
KA		 2162 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2163
2164 if (ret)
c7ab48d2
DW		 2165 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2166 "floppy might not work\n");
49a0429e
KA		 2167
2168}
2169#else
2170static inline void iommu_prepare_isa(void)
2171{
2172 return;
2173}
2174#endif /* !CONFIG_DMAR_FLPY_WA */
2175
2c2e2c38 2176static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2
DW		 2177
2178static int __init si_domain_work_fn(unsigned long start_pfn,
2179 unsigned long end_pfn, void *datax)
2180{
2181 int *ret = datax;
2182
2183 *ret = iommu_domain_identity_map(si_domain,
2184 (uint64_t)start_pfn << PAGE_SHIFT,
2185 (uint64_t)end_pfn << PAGE_SHIFT);
2186 return *ret;
2187
2188}
2189
071e1374 2190static int __init si_domain_init(int hw)
2c2e2c38
FY		 2191{
2192 struct dmar_drhd_unit *drhd;
2193 struct intel_iommu *iommu;
c7ab48d2 2194 int nid, ret = 0;
2c2e2c38
FY		 2195
2196 si_domain = alloc_domain();
2197 if (!si_domain)
2198 return -EFAULT;
2199
c7ab48d2 2200 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2201
2202 for_each_active_iommu(iommu, drhd) {
2203 ret = iommu_attach_domain(si_domain, iommu);
2204 if (ret) {
2205 domain_exit(si_domain);
2206 return -EFAULT;
2207 }
2208 }
2209
2210 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2211 domain_exit(si_domain);
2212 return -EFAULT;
2213 }
2214
2215 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2216
19943b0e
DW		 2217 if (hw)
2218 return 0;
2219
c7ab48d2
DW		 2220 for_each_online_node(nid) {
2221 work_with_active_regions(nid, si_domain_work_fn, &ret);
2222 if (ret)
2223 return ret;
2224 }
2225
2c2e2c38
FY		 2226 return 0;
2227}
2228
2229static void domain_remove_one_dev_info(struct dmar_domain *domain,
2230 struct pci_dev *pdev);
2231static int identity_mapping(struct pci_dev *pdev)
2232{
2233 struct device_domain_info *info;
2234
2235 if (likely(!iommu_identity_mapping))
2236 return 0;
2237
2238
2239 list_for_each_entry(info, &si_domain->devices, link)
2240 if (info->dev == pdev)
2241 return 1;
2242 return 0;
2243}
2244
2245static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2246 struct pci_dev *pdev,
2247 int translation)
2c2e2c38
FY		 2248{
2249 struct device_domain_info *info;
2250 unsigned long flags;
5fe60f4e 2251 int ret;
2c2e2c38
FY		 2252
2253 info = alloc_devinfo_mem();
2254 if (!info)
2255 return -ENOMEM;
2256
5fe60f4e
DW		 2257 ret = domain_context_mapping(domain, pdev, translation);
2258 if (ret) {
2259 free_devinfo_mem(info);
2260 return ret;
2261 }
2262
2c2e2c38
FY		 2263 info->segment = pci_domain_nr(pdev->bus);
2264 info->bus = pdev->bus->number;
2265 info->devfn = pdev->devfn;
2266 info->dev = pdev;
2267 info->domain = domain;
2268
2269 spin_lock_irqsave(&device_domain_lock, flags);
2270 list_add(&info->link, &domain->devices);
2271 list_add(&info->global, &device_domain_list);
2272 pdev->dev.archdata.iommu = info;
2273 spin_unlock_irqrestore(&device_domain_lock, flags);
2274
2275 return 0;
2276}
2277
6941af28
DW		 2278static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2279{
e0fc7e0b
DW		 2280 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2281 return 1;
2282
2283 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2284 return 1;
2285
2286 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2287 return 0;
6941af28 2288
3dfc813d
DW		 2289 /*
2290 * We want to start off with all devices in the 1:1 domain, and
2291 * take them out later if we find they can't access all of memory.
2292 *
2293 * However, we can't do this for PCI devices behind bridges,
2294 * because all PCI devices behind the same bridge will end up
2295 * with the same source-id on their transactions.
2296 *
2297 * Practically speaking, we can't change things around for these
2298 * devices at run-time, because we can't be sure there'll be no
2299 * DMA transactions in flight for any of their siblings.
2300 *
2301 * So PCI devices (unless they're on the root bus) as well as
2302 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2303 * the 1:1 domain, just in _case_ one of their siblings turns out
2304 * not to be able to map all of memory.
2305 */
5f4d91a1 2306 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2307 if (!pci_is_root_bus(pdev->bus))
2308 return 0;
2309 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2310 return 0;
2311 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2312 return 0;
2313
2314 /*
2315 * At boot time, we don't yet know if devices will be 64-bit capable.
2316 * Assume that they will -- if they turn out not to be, then we can
2317 * take them out of the 1:1 domain later.
2318 */
6941af28
DW		 2319 if (!startup)
2320 return pdev->dma_mask > DMA_BIT_MASK(32);
2321
2322 return 1;
2323}
2324
071e1374 2325static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2326{
2c2e2c38
FY		 2327 struct pci_dev *pdev = NULL;
2328 int ret;
2329
19943b0e 2330 ret = si_domain_init(hw);
2c2e2c38
FY		 2331 if (ret)
2332 return -EFAULT;
2333
2c2e2c38 2334 for_each_pci_dev(pdev) {
6941af28 2335 if (iommu_should_identity_map(pdev, 1)) {
19943b0e
DW		 2336 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2337 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2338
5fe60f4e 2339 ret = domain_add_dev_info(si_domain, pdev,
19943b0e 2340 hw ? CONTEXT_TT_PASS_THROUGH :
62edf5dc
DW		 2341 CONTEXT_TT_MULTI_LEVEL);
2342 if (ret)
2343 return ret;
62edf5dc 2344 }
2c2e2c38
FY		 2345 }
2346
2347 return 0;
2348}
2349
b779260b 2350static int __init init_dmars(void)
ba395927
KA		 2351{
2352 struct dmar_drhd_unit *drhd;
2353 struct dmar_rmrr_unit *rmrr;
2354 struct pci_dev *pdev;
2355 struct intel_iommu *iommu;
9d783ba0 2356 int i, ret;
2c2e2c38 2357
ba395927
KA		 2358 /*
2359 * for each drhd
2360 * allocate root
2361 * initialize and program root entry to not present
2362 * endfor
2363 */
2364 for_each_drhd_unit(drhd) {
5e0d2a6f 2365 g_num_of_iommus++;
2366 /*
2367 * lock not needed as this is only incremented in the single
2368 * threaded kernel __init code path all other access are read
2369 * only
2370 */
2371 }
2372
d9630fe9
WH		 2373 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2374 GFP_KERNEL);
2375 if (!g_iommus) {
2376 printk(KERN_ERR "Allocating global iommu array failed\n");
2377 ret = -ENOMEM;
2378 goto error;
2379 }
2380
80b20dd8 2381 deferred_flush = kzalloc(g_num_of_iommus *
2382 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2383 if (!deferred_flush) {
5e0d2a6f 2384 ret = -ENOMEM;
2385 goto error;
2386 }
2387
5e0d2a6f 2388 for_each_drhd_unit(drhd) {
2389 if (drhd->ignored)
2390 continue;
1886e8a9
SS		 2391
2392 iommu = drhd->iommu;
d9630fe9 2393 g_iommus[iommu->seq_id] = iommu;
ba395927 2394
e61d98d8
SS		 2395 ret = iommu_init_domains(iommu);
2396 if (ret)
2397 goto error;
2398
ba395927
KA		 2399 /*
2400 * TBD:
2401 * we could share the same root & context tables
25985edc 2402 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2403 */
2404 ret = iommu_alloc_root_entry(iommu);
2405 if (ret) {
2406 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2407 goto error;
2408 }
4ed0d3e6 2409 if (!ecap_pass_through(iommu->ecap))
19943b0e 2410 hw_pass_through = 0;
ba395927
KA		 2411 }
2412
1531a6a6
SS		 2413 /*
2414 * Start from the sane iommu hardware state.
2415 */
a77b67d4
YS		 2416 for_each_drhd_unit(drhd) {
2417 if (drhd->ignored)
2418 continue;
2419
2420 iommu = drhd->iommu;
1531a6a6
SS		 2421
2422 /*
2423 * If the queued invalidation is already initialized by us
2424 * (for example, while enabling interrupt-remapping) then
2425 * we got the things already rolling from a sane state.
2426 */
2427 if (iommu->qi)
2428 continue;
2429
2430 /*
2431 * Clear any previous faults.
2432 */
2433 dmar_fault(-1, iommu);
2434 /*
2435 * Disable queued invalidation if supported and already enabled
2436 * before OS handover.
2437 */
2438 dmar_disable_qi(iommu);
2439 }
2440
2441 for_each_drhd_unit(drhd) {
2442 if (drhd->ignored)
2443 continue;
2444
2445 iommu = drhd->iommu;
2446
a77b67d4
YS		 2447 if (dmar_enable_qi(iommu)) {
2448 /*
2449 * Queued Invalidate not enabled, use Register Based
2450 * Invalidate
2451 */
2452 iommu->flush.flush_context = __iommu_flush_context;
2453 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2454 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2455 "invalidation\n",
680a7524 2456 iommu->seq_id,
b4e0f9eb 2457 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2458 } else {
2459 iommu->flush.flush_context = qi_flush_context;
2460 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2461 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2462 "invalidation\n",
680a7524 2463 iommu->seq_id,
b4e0f9eb 2464 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2465 }
2466 }
2467
19943b0e 2468 if (iommu_pass_through)
e0fc7e0b
DW		 2469 iommu_identity_mapping |= IDENTMAP_ALL;
2470
19943b0e 2471#ifdef CONFIG_DMAR_BROKEN_GFX_WA
e0fc7e0b 2472 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2473#endif
e0fc7e0b
DW		 2474
2475 check_tylersburg_isoch();
2476
ba395927 2477 /*
19943b0e
DW		 2478 * If pass through is not set or not enabled, setup context entries for
2479 * identity mappings for rmrr, gfx, and isa and may fall back to static
2480 * identity mapping if iommu_identity_mapping is set.
ba395927 2481 */
19943b0e
DW		 2482 if (iommu_identity_mapping) {
2483 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2484 if (ret) {
19943b0e
DW		 2485 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2486 goto error;
ba395927
KA		 2487 }
2488 }
ba395927 2489 /*
19943b0e
DW		 2490 * For each rmrr
2491 * for each dev attached to rmrr
2492 * do
2493 * locate drhd for dev, alloc domain for dev
2494 * allocate free domain
2495 * allocate page table entries for rmrr
2496 * if context not allocated for bus
2497 * allocate and init context
2498 * set present in root table for this bus
2499 * init context with domain, translation etc
2500 * endfor
2501 * endfor
ba395927 2502 */
19943b0e
DW		 2503 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2504 for_each_rmrr_units(rmrr) {
2505 for (i = 0; i < rmrr->devices_cnt; i++) {
2506 pdev = rmrr->devices[i];
2507 /*
2508 * some BIOS lists non-exist devices in DMAR
2509 * table.
2510 */
2511 if (!pdev)
2512 continue;
2513 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2514 if (ret)
2515 printk(KERN_ERR
2516 "IOMMU: mapping reserved region failed\n");
ba395927 2517 }
4ed0d3e6 2518 }
49a0429e 2519
19943b0e
DW		 2520 iommu_prepare_isa();
2521
ba395927
KA		 2522 /*
2523 * for each drhd
2524 * enable fault log
2525 * global invalidate context cache
2526 * global invalidate iotlb
2527 * enable translation
2528 */
2529 for_each_drhd_unit(drhd) {
51a63e67
JC		 2530 if (drhd->ignored) {
2531 /*
2532 * we always have to disable PMRs or DMA may fail on
2533 * this device
2534 */
2535 if (force_on)
2536 iommu_disable_protect_mem_regions(drhd->iommu);
ba395927 2537 continue;
51a63e67 2538 }
ba395927 2539 iommu = drhd->iommu;
ba395927
KA		 2540
2541 iommu_flush_write_buffer(iommu);
2542
3460a6d9
KA		 2543 ret = dmar_set_interrupt(iommu);
2544 if (ret)
2545 goto error;
2546
ba395927
KA		 2547 iommu_set_root_entry(iommu);
2548
4c25a2c1 2549 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2550 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2551
ba395927
KA		 2552 ret = iommu_enable_translation(iommu);
2553 if (ret)
2554 goto error;
b94996c9
DW		 2555
2556 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2557 }
2558
2559 return 0;
2560error:
2561 for_each_drhd_unit(drhd) {
2562 if (drhd->ignored)
2563 continue;
2564 iommu = drhd->iommu;
2565 free_iommu(iommu);
2566 }
d9630fe9 2567 kfree(g_iommus);
ba395927
KA		 2568 return ret;
2569}
2570
5a5e02a6 2571/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2572static struct iova *intel_alloc_iova(struct device *dev,
2573 struct dmar_domain *domain,
2574 unsigned long nrpages, uint64_t dma_mask)
ba395927 2575{
ba395927 2576 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2577 struct iova *iova = NULL;
ba395927 2578
875764de
DW		 2579 /* Restrict dma_mask to the width that the iommu can handle */
2580 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2581
2582 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2583 /*
2584 * First try to allocate an io virtual address in
284901a9 2585 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2586 * from higher range
ba395927 2587 */
875764de
DW		 2588 iova = alloc_iova(&domain->iovad, nrpages,
2589 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2590 if (iova)
2591 return iova;
2592 }
2593 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2594 if (unlikely(!iova)) {
2595 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2596 nrpages, pci_name(pdev));
f76aec76
KA		 2597 return NULL;
2598 }
2599
2600 return iova;
2601}
2602
147202aa 2603static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2604{
2605 struct dmar_domain *domain;
2606 int ret;
2607
2608 domain = get_domain_for_dev(pdev,
2609 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2610 if (!domain) {
2611 printk(KERN_ERR
2612 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2613 return NULL;
ba395927
KA		 2614 }
2615
2616 /* make sure context mapping is ok */
5331fe6f 2617 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2618 ret = domain_context_mapping(domain, pdev,
2619 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2620 if (ret) {
2621 printk(KERN_ERR
2622 "Domain context map for %s failed",
2623 pci_name(pdev));
4fe05bbc 2624 return NULL;
f76aec76 2625 }
ba395927
KA		 2626 }
2627
f76aec76
KA		 2628 return domain;
2629}
2630
147202aa
DW		 2631static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2632{
2633 struct device_domain_info *info;
2634
2635 /* No lock here, assumes no domain exit in normal case */
2636 info = dev->dev.archdata.iommu;
2637 if (likely(info))
2638 return info->domain;
2639
2640 return __get_valid_domain_for_dev(dev);
2641}
2642
2c2e2c38
FY		 2643static int iommu_dummy(struct pci_dev *pdev)
2644{
2645 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2646}
2647
2648/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2649static int iommu_no_mapping(struct device *dev)
2c2e2c38 2650{
73676832 2651 struct pci_dev *pdev;
2c2e2c38
FY		 2652 int found;
2653
73676832
DW		 2654 if (unlikely(dev->bus != &pci_bus_type))
2655 return 1;
2656
2657 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2658 if (iommu_dummy(pdev))
2659 return 1;
2660
2c2e2c38 2661 if (!iommu_identity_mapping)
1e4c64c4 2662 return 0;
2c2e2c38
FY		 2663
2664 found = identity_mapping(pdev);
2665 if (found) {
6941af28 2666 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2667 return 1;
2668 else {
2669 /*
2670 * 32 bit DMA is removed from si_domain and fall back
2671 * to non-identity mapping.
2672 */
2673 domain_remove_one_dev_info(si_domain, pdev);
2674 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2675 pci_name(pdev));
2676 return 0;
2677 }
2678 } else {
2679 /*
2680 * In case of a detached 64 bit DMA device from vm, the device
2681 * is put into si_domain for identity mapping.
2682 */
6941af28 2683 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2684 int ret;
5fe60f4e
DW		 2685 ret = domain_add_dev_info(si_domain, pdev,
2686 hw_pass_through ?
2687 CONTEXT_TT_PASS_THROUGH :
2688 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2689 if (!ret) {
2690 printk(KERN_INFO "64bit %s uses identity mapping\n",
2691 pci_name(pdev));
2692 return 1;
2693 }
2694 }
2695 }
2696
1e4c64c4 2697 return 0;
2c2e2c38
FY		 2698}
2699
bb9e6d65
FT		 2700static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2701 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2702{
2703 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2704 struct dmar_domain *domain;
5b6985ce 2705 phys_addr_t start_paddr;
f76aec76
KA		 2706 struct iova *iova;
2707 int prot = 0;
6865f0d1 2708 int ret;
8c11e798 2709 struct intel_iommu *iommu;
33041ec0 2710 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2711
2712 BUG_ON(dir == DMA_NONE);
2c2e2c38 2713
73676832 2714 if (iommu_no_mapping(hwdev))
6865f0d1 2715 return paddr;
f76aec76
KA		 2716
2717 domain = get_valid_domain_for_dev(pdev);
2718 if (!domain)
2719 return 0;
2720
8c11e798 2721 iommu = domain_get_iommu(domain);
88cb6a74 2722 size = aligned_nrpages(paddr, size);
f76aec76 2723
5a5e02a6
DW		 2724 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2725 pdev->dma_mask);
f76aec76
KA		 2726 if (!iova)
2727 goto error;
2728
ba395927
KA		 2729 /*
2730 * Check if DMAR supports zero-length reads on write only
2731 * mappings..
2732 */
2733 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2734 !cap_zlr(iommu->cap))
ba395927
KA		 2735 prot |= DMA_PTE_READ;
2736 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2737 prot |= DMA_PTE_WRITE;
2738 /*
6865f0d1 2739 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2740 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2741 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2742 * is not a big problem
2743 */
0ab36de2 2744 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2745 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2746 if (ret)
2747 goto error;
2748
1f0ef2aa
DW		 2749 /* it's a non-present to present mapping. Only flush if caching mode */
2750 if (cap_caching_mode(iommu->cap))
82653633 2751 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2752 else
8c11e798 2753 iommu_flush_write_buffer(iommu);
f76aec76 2754
03d6a246
DW		 2755 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2756 start_paddr += paddr & ~PAGE_MASK;
2757 return start_paddr;
ba395927 2758
ba395927 2759error:
f76aec76
KA		 2760 if (iova)
2761 __free_iova(&domain->iovad, iova);
4cf2e75d 2762 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2763 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2764 return 0;
2765}
2766
ffbbef5c
FT		 2767static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2768 unsigned long offset, size_t size,
2769 enum dma_data_direction dir,
2770 struct dma_attrs *attrs)
bb9e6d65 2771{
ffbbef5c
FT		 2772 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2773 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2774}
2775
5e0d2a6f 2776static void flush_unmaps(void)
2777{
80b20dd8 2778 int i, j;
5e0d2a6f 2779
5e0d2a6f 2780 timer_on = 0;
2781
2782 /* just flush them all */
2783 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2784 struct intel_iommu *iommu = g_iommus[i];
2785 if (!iommu)
2786 continue;
c42d9f32 2787
9dd2fe89
YZ		 2788 if (!deferred_flush[i].next)
2789 continue;
2790
78d5f0f5
NA		 2791 /* In caching mode, global flushes turn emulation expensive */
2792 if (!cap_caching_mode(iommu->cap))
2793 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2794 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2795 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2796 unsigned long mask;
2797 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2798 struct dmar_domain *domain = deferred_flush[i].domain[j];
2799
2800 /* On real hardware multiple invalidations are expensive */
2801 if (cap_caching_mode(iommu->cap))
2802 iommu_flush_iotlb_psi(iommu, domain->id,
2803 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2804 else {
2805 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2806 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2807 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2808 }
93a23a72 2809 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2810 }
9dd2fe89 2811 deferred_flush[i].next = 0;
5e0d2a6f 2812 }
2813
5e0d2a6f 2814 list_size = 0;
5e0d2a6f 2815}
2816
2817static void flush_unmaps_timeout(unsigned long data)
2818{
80b20dd8 2819 unsigned long flags;
2820
2821 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2822 flush_unmaps();
80b20dd8 2823 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2824}
2825
2826static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2827{
2828 unsigned long flags;
80b20dd8 2829 int next, iommu_id;
8c11e798 2830 struct intel_iommu *iommu;
5e0d2a6f 2831
2832 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2833 if (list_size == HIGH_WATER_MARK)
2834 flush_unmaps();
2835
8c11e798
WH		 2836 iommu = domain_get_iommu(dom);
2837 iommu_id = iommu->seq_id;
c42d9f32 2838
80b20dd8 2839 next = deferred_flush[iommu_id].next;
2840 deferred_flush[iommu_id].domain[next] = dom;
2841 deferred_flush[iommu_id].iova[next] = iova;
2842 deferred_flush[iommu_id].next++;
5e0d2a6f 2843
2844 if (!timer_on) {
2845 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2846 timer_on = 1;
2847 }
2848 list_size++;
2849 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2850}
2851
ffbbef5c
FT		 2852static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2853 size_t size, enum dma_data_direction dir,
2854 struct dma_attrs *attrs)
ba395927 2855{
ba395927 2856 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2857 struct dmar_domain *domain;
d794dc9b 2858 unsigned long start_pfn, last_pfn;
ba395927 2859 struct iova *iova;
8c11e798 2860 struct intel_iommu *iommu;
ba395927 2861
73676832 2862 if (iommu_no_mapping(dev))
f76aec76 2863 return;
2c2e2c38 2864
ba395927
KA		 2865 domain = find_domain(pdev);
2866 BUG_ON(!domain);
2867
8c11e798
WH		 2868 iommu = domain_get_iommu(domain);
2869
ba395927 2870 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2871 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2872 (unsigned long long)dev_addr))
ba395927 2873 return;
ba395927 2874
d794dc9b
DW		 2875 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2876 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2877
d794dc9b
DW		 2878 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2879 pci_name(pdev), start_pfn, last_pfn);
ba395927 2880
f76aec76 2881 /* clear the whole page */
d794dc9b
DW		 2882 dma_pte_clear_range(domain, start_pfn, last_pfn);
2883
f76aec76 2884 /* free page tables */
d794dc9b
DW		 2885 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2886
5e0d2a6f 2887 if (intel_iommu_strict) {
03d6a246 2888 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2889 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2890 /* free iova */
2891 __free_iova(&domain->iovad, iova);
2892 } else {
2893 add_unmap(domain, iova);
2894 /*
2895 * queue up the release of the unmap to save the 1/6th of the
2896 * cpu used up by the iotlb flush operation...
2897 */
5e0d2a6f 2898 }
ba395927
KA		 2899}
2900
d7ab5c46
FT		 2901static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2902 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2903{
2904 void *vaddr;
2905 int order;
2906
5b6985ce 2907 size = PAGE_ALIGN(size);
ba395927 2908 order = get_order(size);
e8bb910d
AW		 2909
2910 if (!iommu_no_mapping(hwdev))
2911 flags &= ~(GFP_DMA | GFP_DMA32);
2912 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2913 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2914 flags |= GFP_DMA;
2915 else
2916 flags |= GFP_DMA32;
2917 }
ba395927
KA		 2918
2919 vaddr = (void *)__get_free_pages(flags, order);
2920 if (!vaddr)
2921 return NULL;
2922 memset(vaddr, 0, size);
2923
bb9e6d65
FT		 2924 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2925 DMA_BIDIRECTIONAL,
2926 hwdev->coherent_dma_mask);
ba395927
KA		 2927 if (*dma_handle)
2928 return vaddr;
2929 free_pages((unsigned long)vaddr, order);
2930 return NULL;
2931}
2932
d7ab5c46
FT		 2933static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2934 dma_addr_t dma_handle)
ba395927
KA		 2935{
2936 int order;
2937
5b6985ce 2938 size = PAGE_ALIGN(size);
ba395927
KA		 2939 order = get_order(size);
2940
0db9b7ae 2941 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 2942 free_pages((unsigned long)vaddr, order);
2943}
2944
d7ab5c46
FT		 2945static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2946 int nelems, enum dma_data_direction dir,
2947 struct dma_attrs *attrs)
ba395927 2948{
ba395927
KA		 2949 struct pci_dev *pdev = to_pci_dev(hwdev);
2950 struct dmar_domain *domain;
d794dc9b 2951 unsigned long start_pfn, last_pfn;
f76aec76 2952 struct iova *iova;
8c11e798 2953 struct intel_iommu *iommu;
ba395927 2954
73676832 2955 if (iommu_no_mapping(hwdev))
ba395927
KA		 2956 return;
2957
2958 domain = find_domain(pdev);
8c11e798
WH		 2959 BUG_ON(!domain);
2960
2961 iommu = domain_get_iommu(domain);
ba395927 2962
c03ab37c 2963 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 2964 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2965 (unsigned long long)sglist[0].dma_address))
f76aec76 2966 return;
f76aec76 2967
d794dc9b
DW		 2968 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2969 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 2970
2971 /* clear the whole page */
d794dc9b
DW		 2972 dma_pte_clear_range(domain, start_pfn, last_pfn);
2973
f76aec76 2974 /* free page tables */
d794dc9b 2975 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 2976
acea0018
DW		 2977 if (intel_iommu_strict) {
2978 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2979 last_pfn - start_pfn + 1, 0);
acea0018
DW		 2980 /* free iova */
2981 __free_iova(&domain->iovad, iova);
2982 } else {
2983 add_unmap(domain, iova);
2984 /*
2985 * queue up the release of the unmap to save the 1/6th of the
2986 * cpu used up by the iotlb flush operation...
2987 */
2988 }
ba395927
KA		 2989}
2990
ba395927 2991static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2992 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2993{
2994 int i;
c03ab37c 2995 struct scatterlist *sg;
ba395927 2996
c03ab37c 2997 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2998 BUG_ON(!sg_page(sg));
4cf2e75d 2999 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3000 sg->dma_length = sg->length;
ba395927
KA		 3001 }
3002 return nelems;
3003}
3004
d7ab5c46
FT		 3005static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3006 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3007{
ba395927 3008 int i;
ba395927
KA		 3009 struct pci_dev *pdev = to_pci_dev(hwdev);
3010 struct dmar_domain *domain;
f76aec76
KA		 3011 size_t size = 0;
3012 int prot = 0;
f76aec76
KA		 3013 struct iova *iova = NULL;
3014 int ret;
c03ab37c 3015 struct scatterlist *sg;
b536d24d 3016 unsigned long start_vpfn;
8c11e798 3017 struct intel_iommu *iommu;
ba395927
KA		 3018
3019 BUG_ON(dir == DMA_NONE);
73676832 3020 if (iommu_no_mapping(hwdev))
c03ab37c 3021 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3022
f76aec76
KA		 3023 domain = get_valid_domain_for_dev(pdev);
3024 if (!domain)
3025 return 0;
3026
8c11e798
WH		 3027 iommu = domain_get_iommu(domain);
3028
b536d24d 3029 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3030 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3031
5a5e02a6
DW		 3032 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3033 pdev->dma_mask);
f76aec76 3034 if (!iova) {
c03ab37c 3035 sglist->dma_length = 0;
f76aec76
KA		 3036 return 0;
3037 }
3038
3039 /*
3040 * Check if DMAR supports zero-length reads on write only
3041 * mappings..
3042 */
3043 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3044 !cap_zlr(iommu->cap))
f76aec76
KA		 3045 prot |= DMA_PTE_READ;
3046 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3047 prot |= DMA_PTE_WRITE;
3048
b536d24d 3049 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3050
f532959b 3051 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3052 if (unlikely(ret)) {
3053 /* clear the page */
3054 dma_pte_clear_range(domain, start_vpfn,
3055 start_vpfn + size - 1);
3056 /* free page tables */
3057 dma_pte_free_pagetable(domain, start_vpfn,
3058 start_vpfn + size - 1);
3059 /* free iova */
3060 __free_iova(&domain->iovad, iova);
3061 return 0;
ba395927
KA		 3062 }
3063
1f0ef2aa
DW		 3064 /* it's a non-present to present mapping. Only flush if caching mode */
3065 if (cap_caching_mode(iommu->cap))
82653633 3066 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3067 else
8c11e798 3068 iommu_flush_write_buffer(iommu);
1f0ef2aa 3069
ba395927
KA		 3070 return nelems;
3071}
3072
dfb805e8
FT		 3073static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3074{
3075 return !dma_addr;
3076}
3077
160c1d8e 3078struct dma_map_ops intel_dma_ops = {
ba395927
KA		 3079 .alloc_coherent = intel_alloc_coherent,
3080 .free_coherent = intel_free_coherent,
ba395927
KA		 3081 .map_sg = intel_map_sg,
3082 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3083 .map_page = intel_map_page,
3084 .unmap_page = intel_unmap_page,
dfb805e8 3085 .mapping_error = intel_mapping_error,
ba395927
KA		 3086};
3087
3088static inline int iommu_domain_cache_init(void)
3089{
3090 int ret = 0;
3091
3092 iommu_domain_cache = kmem_cache_create("iommu_domain",
3093 sizeof(struct dmar_domain),
3094 0,
3095 SLAB_HWCACHE_ALIGN,
3096
3097 NULL);
3098 if (!iommu_domain_cache) {
3099 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3100 ret = -ENOMEM;
3101 }
3102
3103 return ret;
3104}
3105
3106static inline int iommu_devinfo_cache_init(void)
3107{
3108 int ret = 0;
3109
3110 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3111 sizeof(struct device_domain_info),
3112 0,
3113 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3114 NULL);
3115 if (!iommu_devinfo_cache) {
3116 printk(KERN_ERR "Couldn't create devinfo cache\n");
3117 ret = -ENOMEM;
3118 }
3119
3120 return ret;
3121}
3122
3123static inline int iommu_iova_cache_init(void)
3124{
3125 int ret = 0;
3126
3127 iommu_iova_cache = kmem_cache_create("iommu_iova",
3128 sizeof(struct iova),
3129 0,
3130 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3131 NULL);
3132 if (!iommu_iova_cache) {
3133 printk(KERN_ERR "Couldn't create iova cache\n");
3134 ret = -ENOMEM;
3135 }
3136
3137 return ret;
3138}
3139
3140static int __init iommu_init_mempool(void)
3141{
3142 int ret;
3143 ret = iommu_iova_cache_init();
3144 if (ret)
3145 return ret;
3146
3147 ret = iommu_domain_cache_init();
3148 if (ret)
3149 goto domain_error;
3150
3151 ret = iommu_devinfo_cache_init();
3152 if (!ret)
3153 return ret;
3154
3155 kmem_cache_destroy(iommu_domain_cache);
3156domain_error:
3157 kmem_cache_destroy(iommu_iova_cache);
3158
3159 return -ENOMEM;
3160}
3161
3162static void __init iommu_exit_mempool(void)
3163{
3164 kmem_cache_destroy(iommu_devinfo_cache);
3165 kmem_cache_destroy(iommu_domain_cache);
3166 kmem_cache_destroy(iommu_iova_cache);
3167
3168}
3169
556ab45f
DW		 3170static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3171{
3172 struct dmar_drhd_unit *drhd;
3173 u32 vtbar;
3174 int rc;
3175
3176 /* We know that this device on this chipset has its own IOMMU.
3177 * If we find it under a different IOMMU, then the BIOS is lying
3178 * to us. Hope that the IOMMU for this device is actually
3179 * disabled, and it needs no translation...
3180 */
3181 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3182 if (rc) {
3183 /* "can't" happen */
3184 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3185 return;
3186 }
3187 vtbar &= 0xffff0000;
3188
3189 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3190 drhd = dmar_find_matched_drhd_unit(pdev);
3191 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3192 TAINT_FIRMWARE_WORKAROUND,
3193 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3194 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3195}
3196DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3197
ba395927
KA		 3198static void __init init_no_remapping_devices(void)
3199{
3200 struct dmar_drhd_unit *drhd;
3201
3202 for_each_drhd_unit(drhd) {
3203 if (!drhd->include_all) {
3204 int i;
3205 for (i = 0; i < drhd->devices_cnt; i++)
3206 if (drhd->devices[i] != NULL)
3207 break;
3208 /* ignore DMAR unit if no pci devices exist */
3209 if (i == drhd->devices_cnt)
3210 drhd->ignored = 1;
3211 }
3212 }
3213
3214 if (dmar_map_gfx)
3215 return;
3216
3217 for_each_drhd_unit(drhd) {
3218 int i;
3219 if (drhd->ignored || drhd->include_all)
3220 continue;
3221
3222 for (i = 0; i < drhd->devices_cnt; i++)
3223 if (drhd->devices[i] &&
3224 !IS_GFX_DEVICE(drhd->devices[i]))
3225 break;
3226
3227 if (i < drhd->devices_cnt)
3228 continue;
3229
3230 /* bypass IOMMU if it is just for gfx devices */
3231 drhd->ignored = 1;
3232 for (i = 0; i < drhd->devices_cnt; i++) {
3233 if (!drhd->devices[i])
3234 continue;
358dd8ac 3235 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3236 }
3237 }
3238}
3239
f59c7b69
FY		 3240#ifdef CONFIG_SUSPEND
3241static int init_iommu_hw(void)
3242{
3243 struct dmar_drhd_unit *drhd;
3244 struct intel_iommu *iommu = NULL;
3245
3246 for_each_active_iommu(iommu, drhd)
3247 if (iommu->qi)
3248 dmar_reenable_qi(iommu);
3249
b779260b
JC		 3250 for_each_iommu(iommu, drhd) {
3251 if (drhd->ignored) {
3252 /*
3253 * we always have to disable PMRs or DMA may fail on
3254 * this device
3255 */
3256 if (force_on)
3257 iommu_disable_protect_mem_regions(iommu);
3258 continue;
3259 }
3260
f59c7b69
FY		 3261 iommu_flush_write_buffer(iommu);
3262
3263 iommu_set_root_entry(iommu);
3264
3265 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3266 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3267 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3268 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3269 if (iommu_enable_translation(iommu))
3270 return 1;
b94996c9 3271 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3272 }
3273
3274 return 0;
3275}
3276
3277static void iommu_flush_all(void)
3278{
3279 struct dmar_drhd_unit *drhd;
3280 struct intel_iommu *iommu;
3281
3282 for_each_active_iommu(iommu, drhd) {
3283 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3284 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3285 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3286 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3287 }
3288}
3289
134fac3f 3290static int iommu_suspend(void)
f59c7b69
FY		 3291{
3292 struct dmar_drhd_unit *drhd;
3293 struct intel_iommu *iommu = NULL;
3294 unsigned long flag;
3295
3296 for_each_active_iommu(iommu, drhd) {
3297 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3298 GFP_ATOMIC);
3299 if (!iommu->iommu_state)
3300 goto nomem;
3301 }
3302
3303 iommu_flush_all();
3304
3305 for_each_active_iommu(iommu, drhd) {
3306 iommu_disable_translation(iommu);
3307
3308 spin_lock_irqsave(&iommu->register_lock, flag);
3309
3310 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3311 readl(iommu->reg + DMAR_FECTL_REG);
3312 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3313 readl(iommu->reg + DMAR_FEDATA_REG);
3314 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3315 readl(iommu->reg + DMAR_FEADDR_REG);
3316 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3317 readl(iommu->reg + DMAR_FEUADDR_REG);
3318
3319 spin_unlock_irqrestore(&iommu->register_lock, flag);
3320 }
3321 return 0;
3322
3323nomem:
3324 for_each_active_iommu(iommu, drhd)
3325 kfree(iommu->iommu_state);
3326
3327 return -ENOMEM;
3328}
3329
134fac3f 3330static void iommu_resume(void)
f59c7b69
FY		 3331{
3332 struct dmar_drhd_unit *drhd;
3333 struct intel_iommu *iommu = NULL;
3334 unsigned long flag;
3335
3336 if (init_iommu_hw()) {
b779260b
JC		 3337 if (force_on)
3338 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3339 else
3340 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3341 return;
f59c7b69
FY		 3342 }
3343
3344 for_each_active_iommu(iommu, drhd) {
3345
3346 spin_lock_irqsave(&iommu->register_lock, flag);
3347
3348 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3349 iommu->reg + DMAR_FECTL_REG);
3350 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3351 iommu->reg + DMAR_FEDATA_REG);
3352 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3353 iommu->reg + DMAR_FEADDR_REG);
3354 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3355 iommu->reg + DMAR_FEUADDR_REG);
3356
3357 spin_unlock_irqrestore(&iommu->register_lock, flag);
3358 }
3359
3360 for_each_active_iommu(iommu, drhd)
3361 kfree(iommu->iommu_state);
f59c7b69
FY		 3362}
3363
134fac3f 3364static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3365 .resume = iommu_resume,
3366 .suspend = iommu_suspend,
3367};
3368
134fac3f 3369static void __init init_iommu_pm_ops(void)
f59c7b69 3370{
134fac3f 3371 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3372}
3373
3374#else
134fac3f 3375static inline int init_iommu_pm_ops(void) { }
f59c7b69
FY		 3376#endif /* CONFIG_PM */
3377
99dcaded
FY		 3378/*
3379 * Here we only respond to action of unbound device from driver.
3380 *
3381 * Added device is not attached to its DMAR domain here yet. That will happen
3382 * when mapping the device to iova.
3383 */
3384static int device_notifier(struct notifier_block *nb,
3385 unsigned long action, void *data)
3386{
3387 struct device *dev = data;
3388 struct pci_dev *pdev = to_pci_dev(dev);
3389 struct dmar_domain *domain;
3390
44cd613c
DW		 3391 if (iommu_no_mapping(dev))
3392 return 0;
3393
99dcaded
FY		 3394 domain = find_domain(pdev);
3395 if (!domain)
3396 return 0;
3397
a97590e5 3398 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3399 domain_remove_one_dev_info(domain, pdev);
3400
a97590e5
AW		 3401 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3402 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3403 list_empty(&domain->devices))
3404 domain_exit(domain);
3405 }
3406
99dcaded
FY		 3407 return 0;
3408}
3409
3410static struct notifier_block device_nb = {
3411 .notifier_call = device_notifier,
3412};
3413
ba395927
KA		 3414int __init intel_iommu_init(void)
3415{
3416 int ret = 0;
3417
a59b50e9
JC		 3418 /* VT-d is required for a TXT/tboot launch, so enforce that */
3419 force_on = tboot_force_iommu();
3420
3421 if (dmar_table_init()) {
3422 if (force_on)
3423 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3424 return -ENODEV;
a59b50e9 3425 }
ba395927 3426
a59b50e9
JC		 3427 if (dmar_dev_scope_init()) {
3428 if (force_on)
3429 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3430 return -ENODEV;
a59b50e9 3431 }
1886e8a9 3432
2ae21010
SS		 3433 /*
3434 * Check the need for DMA-remapping initialization now.
3435 * Above initialization will also be used by Interrupt-remapping.
3436 */
75f1cdf1 3437 if (no_iommu || dmar_disabled)
2ae21010
SS		 3438 return -ENODEV;
3439
51a63e67
JC		 3440 if (iommu_init_mempool()) {
3441 if (force_on)
3442 panic("tboot: Failed to initialize iommu memory\n");
3443 return -ENODEV;
3444 }
3445
3446 if (dmar_init_reserved_ranges()) {
3447 if (force_on)
3448 panic("tboot: Failed to reserve iommu ranges\n");
3449 return -ENODEV;
3450 }
ba395927
KA		 3451
3452 init_no_remapping_devices();
3453
b779260b 3454 ret = init_dmars();
ba395927 3455 if (ret) {
a59b50e9
JC		 3456 if (force_on)
3457 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3458 printk(KERN_ERR "IOMMU: dmar init failed\n");
3459 put_iova_domain(&reserved_iova_list);
3460 iommu_exit_mempool();
3461 return ret;
3462 }
3463 printk(KERN_INFO
3464 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3465
5e0d2a6f 3466 init_timer(&unmap_timer);
75f1cdf1
FT		 3467#ifdef CONFIG_SWIOTLB
3468 swiotlb = 0;
3469#endif
19943b0e 3470 dma_ops = &intel_dma_ops;
4ed0d3e6 3471
134fac3f 3472 init_iommu_pm_ops();
a8bcbb0d
JR		 3473
3474 register_iommu(&intel_iommu_ops);
3475
99dcaded
FY		 3476 bus_register_notifier(&pci_bus_type, &device_nb);
3477
ba395927
KA		 3478 return 0;
3479}
e820482c 3480
3199aa6b
HW		 3481static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3482 struct pci_dev *pdev)
3483{
3484 struct pci_dev *tmp, *parent;
3485
3486 if (!iommu || !pdev)
3487 return;
3488
3489 /* dependent device detach */
3490 tmp = pci_find_upstream_pcie_bridge(pdev);
3491 /* Secondary interface's bus number and devfn 0 */
3492 if (tmp) {
3493 parent = pdev->bus->self;
3494 while (parent != tmp) {
3495 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3496 parent->devfn);
3199aa6b
HW		 3497 parent = parent->bus->self;
3498 }
45e829ea 3499 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3500 iommu_detach_dev(iommu,
3501 tmp->subordinate->number, 0);
3502 else /* this is a legacy PCI bridge */
276dbf99
DW		 3503 iommu_detach_dev(iommu, tmp->bus->number,
3504 tmp->devfn);
3199aa6b
HW		 3505 }
3506}
3507
2c2e2c38 3508static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3509 struct pci_dev *pdev)
3510{
3511 struct device_domain_info *info;
3512 struct intel_iommu *iommu;
3513 unsigned long flags;
3514 int found = 0;
3515 struct list_head *entry, *tmp;
3516
276dbf99
DW		 3517 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3518 pdev->devfn);
c7151a8d
WH		 3519 if (!iommu)
3520 return;
3521
3522 spin_lock_irqsave(&device_domain_lock, flags);
3523 list_for_each_safe(entry, tmp, &domain->devices) {
3524 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3525 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3526 if (info->bus == pdev->bus->number &&
3527 info->devfn == pdev->devfn) {
3528 list_del(&info->link);
3529 list_del(&info->global);
3530 if (info->dev)
3531 info->dev->dev.archdata.iommu = NULL;
3532 spin_unlock_irqrestore(&device_domain_lock, flags);
3533
93a23a72 3534 iommu_disable_dev_iotlb(info);
c7151a8d 3535 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3536 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3537 free_devinfo_mem(info);
3538
3539 spin_lock_irqsave(&device_domain_lock, flags);
3540
3541 if (found)
3542 break;
3543 else
3544 continue;
3545 }
3546
3547 /* if there is no other devices under the same iommu
3548 * owned by this domain, clear this iommu in iommu_bmp
3549 * update iommu count and coherency
3550 */
276dbf99
DW		 3551 if (iommu == device_to_iommu(info->segment, info->bus,
3552 info->devfn))
c7151a8d
WH		 3553 found = 1;
3554 }
3555
3556 if (found == 0) {
3557 unsigned long tmp_flags;
3558 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3559 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3560 domain->iommu_count--;
58c610bd 3561 domain_update_iommu_cap(domain);
c7151a8d 3562 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3563
9b4554b2
AW		 3564 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3565 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3566 spin_lock_irqsave(&iommu->lock, tmp_flags);
3567 clear_bit(domain->id, iommu->domain_ids);
3568 iommu->domains[domain->id] = NULL;
3569 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3570 }
c7151a8d
WH		 3571 }
3572
3573 spin_unlock_irqrestore(&device_domain_lock, flags);
3574}
3575
3576static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3577{
3578 struct device_domain_info *info;
3579 struct intel_iommu *iommu;
3580 unsigned long flags1, flags2;
3581
3582 spin_lock_irqsave(&device_domain_lock, flags1);
3583 while (!list_empty(&domain->devices)) {
3584 info = list_entry(domain->devices.next,
3585 struct device_domain_info, link);
3586 list_del(&info->link);
3587 list_del(&info->global);
3588 if (info->dev)
3589 info->dev->dev.archdata.iommu = NULL;
3590
3591 spin_unlock_irqrestore(&device_domain_lock, flags1);
3592
93a23a72 3593 iommu_disable_dev_iotlb(info);
276dbf99 3594 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3595 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3596 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3597
3598 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3599 * and capabilities
c7151a8d
WH		 3600 */
3601 spin_lock_irqsave(&domain->iommu_lock, flags2);
3602 if (test_and_clear_bit(iommu->seq_id,
3603 &domain->iommu_bmp)) {
3604 domain->iommu_count--;
58c610bd 3605 domain_update_iommu_cap(domain);
c7151a8d
WH		 3606 }
3607 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3608
3609 free_devinfo_mem(info);
3610 spin_lock_irqsave(&device_domain_lock, flags1);
3611 }
3612 spin_unlock_irqrestore(&device_domain_lock, flags1);
3613}
3614
5e98c4b1
WH		 3615/* domain id for virtual machine, it won't be set in context */
3616static unsigned long vm_domid;
3617
3618static struct dmar_domain *iommu_alloc_vm_domain(void)
3619{
3620 struct dmar_domain *domain;
3621
3622 domain = alloc_domain_mem();
3623 if (!domain)
3624 return NULL;
3625
3626 domain->id = vm_domid++;
4c923d47 3627 domain->nid = -1;
5e98c4b1
WH		 3628 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3629 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3630
3631 return domain;
3632}
3633
2c2e2c38 3634static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3635{
3636 int adjust_width;
3637
3638 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3639 spin_lock_init(&domain->iommu_lock);
3640
3641 domain_reserve_special_ranges(domain);
3642
3643 /* calculate AGAW */
3644 domain->gaw = guest_width;
3645 adjust_width = guestwidth_to_adjustwidth(guest_width);
3646 domain->agaw = width_to_agaw(adjust_width);
3647
3648 INIT_LIST_HEAD(&domain->devices);
3649
3650 domain->iommu_count = 0;
3651 domain->iommu_coherency = 0;
c5b15255 3652 domain->iommu_snooping = 0;
6dd9a7c7 3653 domain->iommu_superpage = 0;
fe40f1e0 3654 domain->max_addr = 0;
4c923d47 3655 domain->nid = -1;
5e98c4b1
WH		 3656
3657 /* always allocate the top pgd */
4c923d47 3658 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3659 if (!domain->pgd)
3660 return -ENOMEM;
3661 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3662 return 0;
3663}
3664
3665static void iommu_free_vm_domain(struct dmar_domain *domain)
3666{
3667 unsigned long flags;
3668 struct dmar_drhd_unit *drhd;
3669 struct intel_iommu *iommu;
3670 unsigned long i;
3671 unsigned long ndomains;
3672
3673 for_each_drhd_unit(drhd) {
3674 if (drhd->ignored)
3675 continue;
3676 iommu = drhd->iommu;
3677
3678 ndomains = cap_ndoms(iommu->cap);
a45946ab 3679 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3680 if (iommu->domains[i] == domain) {
3681 spin_lock_irqsave(&iommu->lock, flags);
3682 clear_bit(i, iommu->domain_ids);
3683 iommu->domains[i] = NULL;
3684 spin_unlock_irqrestore(&iommu->lock, flags);
3685 break;
3686 }
5e98c4b1
WH		 3687 }
3688 }
3689}
3690
3691static void vm_domain_exit(struct dmar_domain *domain)
3692{
5e98c4b1
WH		 3693 /* Domain 0 is reserved, so dont process it */
3694 if (!domain)
3695 return;
3696
3697 vm_domain_remove_all_dev_info(domain);
3698 /* destroy iovas */
3699 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3700
3701 /* clear ptes */
595badf5 3702 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3703
3704 /* free page tables */
d794dc9b 3705 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3706
3707 iommu_free_vm_domain(domain);
3708 free_domain_mem(domain);
3709}
3710
5d450806 3711static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3712{
5d450806 3713 struct dmar_domain *dmar_domain;
38717946 3714
5d450806
JR		 3715 dmar_domain = iommu_alloc_vm_domain();
3716 if (!dmar_domain) {
38717946 3717 printk(KERN_ERR
5d450806
JR		 3718 "intel_iommu_domain_init: dmar_domain == NULL\n");
3719 return -ENOMEM;
38717946 3720 }
2c2e2c38 3721 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3722 printk(KERN_ERR
5d450806
JR		 3723 "intel_iommu_domain_init() failed\n");
3724 vm_domain_exit(dmar_domain);
3725 return -ENOMEM;
38717946 3726 }
5d450806 3727 domain->priv = dmar_domain;
faa3d6f5 3728
5d450806 3729 return 0;
38717946 3730}
38717946 3731
5d450806 3732static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3733{
5d450806
JR		 3734 struct dmar_domain *dmar_domain = domain->priv;
3735
3736 domain->priv = NULL;
3737 vm_domain_exit(dmar_domain);
38717946 3738}
38717946 3739
4c5478c9
JR		 3740static int intel_iommu_attach_device(struct iommu_domain *domain,
3741 struct device *dev)
38717946 3742{
4c5478c9
JR		 3743 struct dmar_domain *dmar_domain = domain->priv;
3744 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3745 struct intel_iommu *iommu;
3746 int addr_width;
faa3d6f5
WH		 3747
3748 /* normally pdev is not mapped */
3749 if (unlikely(domain_context_mapped(pdev))) {
3750 struct dmar_domain *old_domain;
3751
3752 old_domain = find_domain(pdev);
3753 if (old_domain) {
2c2e2c38
FY		 3754 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3755 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3756 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3757 else
3758 domain_remove_dev_info(old_domain);
3759 }
3760 }
3761
276dbf99
DW		 3762 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3763 pdev->devfn);
fe40f1e0
WH		 3764 if (!iommu)
3765 return -ENODEV;
3766
3767 /* check if this iommu agaw is sufficient for max mapped address */
3768 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 3769 if (addr_width > cap_mgaw(iommu->cap))
3770 addr_width = cap_mgaw(iommu->cap);
3771
3772 if (dmar_domain->max_addr > (1LL << addr_width)) {
3773 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3774 "sufficient for the mapped address (%llx)\n",
a99c47a2 3775 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 3776 return -EFAULT;
3777 }
a99c47a2
TL		 3778 dmar_domain->gaw = addr_width;
3779
3780 /*
3781 * Knock out extra levels of page tables if necessary
3782 */
3783 while (iommu->agaw < dmar_domain->agaw) {
3784 struct dma_pte *pte;
3785
3786 pte = dmar_domain->pgd;
3787 if (dma_pte_present(pte)) {
25cbff16
SY		 3788 dmar_domain->pgd = (struct dma_pte *)
3789 phys_to_virt(dma_pte_addr(pte));
7a661013 3790 free_pgtable_page(pte);
a99c47a2
TL		 3791 }
3792 dmar_domain->agaw--;
3793 }
fe40f1e0 3794
5fe60f4e 3795 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 3796}
38717946 3797
4c5478c9
JR		 3798static void intel_iommu_detach_device(struct iommu_domain *domain,
3799 struct device *dev)
38717946 3800{
4c5478c9
JR		 3801 struct dmar_domain *dmar_domain = domain->priv;
3802 struct pci_dev *pdev = to_pci_dev(dev);
3803
2c2e2c38 3804 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3805}
c7151a8d 3806
b146a1c9
JR		 3807static int intel_iommu_map(struct iommu_domain *domain,
3808 unsigned long iova, phys_addr_t hpa,
3809 int gfp_order, int iommu_prot)
faa3d6f5 3810{
dde57a21 3811 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 3812 u64 max_addr;
dde57a21 3813 int prot = 0;
b146a1c9 3814 size_t size;
faa3d6f5 3815 int ret;
fe40f1e0 3816
dde57a21
JR		 3817 if (iommu_prot & IOMMU_READ)
3818 prot |= DMA_PTE_READ;
3819 if (iommu_prot & IOMMU_WRITE)
3820 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3821 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3822 prot |= DMA_PTE_SNP;
dde57a21 3823
b146a1c9 3824 size = PAGE_SIZE << gfp_order;
163cc52c 3825 max_addr = iova + size;
dde57a21 3826 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3827 u64 end;
3828
3829 /* check if minimum agaw is sufficient for mapped address */
8954da1f 3830 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 3831 if (end < max_addr) {
8954da1f 3832 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3833 "sufficient for the mapped address (%llx)\n",
8954da1f 3834 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 3835 return -EFAULT;
3836 }
dde57a21 3837 dmar_domain->max_addr = max_addr;
fe40f1e0 3838 }
ad051221
DW		 3839 /* Round up size to next multiple of PAGE_SIZE, if it and
3840 the low bits of hpa would take us onto the next page */
88cb6a74 3841 size = aligned_nrpages(hpa, size);
ad051221
DW		 3842 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3843 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 3844 return ret;
38717946 3845}
38717946 3846
b146a1c9
JR		 3847static int intel_iommu_unmap(struct iommu_domain *domain,
3848 unsigned long iova, int gfp_order)
38717946 3849{
dde57a21 3850 struct dmar_domain *dmar_domain = domain->priv;
b146a1c9 3851 size_t size = PAGE_SIZE << gfp_order;
4b99d352 3852
163cc52c
DW		 3853 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3854 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 3855
163cc52c
DW		 3856 if (dmar_domain->max_addr == iova + size)
3857 dmar_domain->max_addr = iova;
b146a1c9
JR		 3858
3859 return gfp_order;
38717946 3860}
38717946 3861
d14d6577
JR		 3862static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3863 unsigned long iova)
38717946 3864{
d14d6577 3865 struct dmar_domain *dmar_domain = domain->priv;
38717946 3866 struct dma_pte *pte;
faa3d6f5 3867 u64 phys = 0;
38717946 3868
6dd9a7c7 3869 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 3870 if (pte)
faa3d6f5 3871 phys = dma_pte_addr(pte);
38717946 3872
faa3d6f5 3873 return phys;
38717946 3874}
a8bcbb0d 3875
dbb9fd86
SY		 3876static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3877 unsigned long cap)
3878{
3879 struct dmar_domain *dmar_domain = domain->priv;
3880
3881 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3882 return dmar_domain->iommu_snooping;
323f99cb
TL		 3883 if (cap == IOMMU_CAP_INTR_REMAP)
3884 return intr_remapping_enabled;
dbb9fd86
SY		 3885
3886 return 0;
3887}
3888
a8bcbb0d
JR		 3889static struct iommu_ops intel_iommu_ops = {
3890 .domain_init = intel_iommu_domain_init,
3891 .domain_destroy = intel_iommu_domain_destroy,
3892 .attach_dev = intel_iommu_attach_device,
3893 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 3894 .map = intel_iommu_map,
3895 .unmap = intel_iommu_unmap,
a8bcbb0d 3896 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3897 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3898};
9af88143
DW		 3899
3900static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3901{
3902 /*
3903 * Mobile 4 Series Chipset neglects to set RWBF capability,
3904 * but needs it:
3905 */
3906 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3907 rwbf_quirk = 1;
2d9e667e
DW		 3908
3909 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3910 if (dev->revision == 0x07) {
3911 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3912 dmar_map_gfx = 0;
3913 }
9af88143
DW		 3914}
3915
3916DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
e0fc7e0b 3917
eecfd57f
AJ		 3918#define GGC 0x52
3919#define GGC_MEMORY_SIZE_MASK (0xf << 8)
3920#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3921#define GGC_MEMORY_SIZE_1M (0x1 << 8)
3922#define GGC_MEMORY_SIZE_2M (0x3 << 8)
3923#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3924#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3925#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3926#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3927
9eecabcb
DW		 3928static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3929{
3930 unsigned short ggc;
3931
eecfd57f 3932 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 3933 return;
3934
eecfd57f 3935 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 3936 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3937 dmar_map_gfx = 0;
3938 }
3939}
3940DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3941DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3942DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3943DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3944
e0fc7e0b
DW		 3945/* On Tylersburg chipsets, some BIOSes have been known to enable the
3946 ISOCH DMAR unit for the Azalia sound device, but not give it any
3947 TLB entries, which causes it to deadlock. Check for that. We do
3948 this in a function called from init_dmars(), instead of in a PCI
3949 quirk, because we don't want to print the obnoxious "BIOS broken"
3950 message if VT-d is actually disabled.
3951*/
3952static void __init check_tylersburg_isoch(void)
3953{
3954 struct pci_dev *pdev;
3955 uint32_t vtisochctrl;
3956
3957 /* If there's no Azalia in the system anyway, forget it. */
3958 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3959 if (!pdev)
3960 return;
3961 pci_dev_put(pdev);
3962
3963 /* System Management Registers. Might be hidden, in which case
3964 we can't do the sanity check. But that's OK, because the
3965 known-broken BIOSes _don't_ actually hide it, so far. */
3966 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3967 if (!pdev)
3968 return;
3969
3970 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3971 pci_dev_put(pdev);
3972 return;
3973 }
3974
3975 pci_dev_put(pdev);
3976
3977 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3978 if (vtisochctrl & 1)
3979 return;
3980
3981 /* Drop all bits other than the number of TLB entries */
3982 vtisochctrl &= 0x1c;
3983
3984 /* If we have the recommended number of TLB entries (16), fine. */
3985 if (vtisochctrl == 0x10)
3986 return;
3987
3988 /* Zero TLB entries? You get to ride the short bus to school. */
3989 if (!vtisochctrl) {
3990 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3991 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3992 dmi_get_system_info(DMI_BIOS_VENDOR),
3993 dmi_get_system_info(DMI_BIOS_VERSION),
3994 dmi_get_system_info(DMI_PRODUCT_VERSION));
3995 iommu_identity_mapping |= IDENTMAP_AZALIA;
3996 return;
3997 }
3998
3999 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4000 vtisochctrl);
4001}
Arne's tree of linux kernel.