1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright © 2015 Intel Corporation.
5 * Authors: David Woodhouse <dwmw2@infradead.org>
8 #include <linux/mmu_notifier.h>
9 #include <linux/sched.h>
10 #include <linux/sched/mm.h>
11 #include <linux/slab.h>
12 #include <linux/rculist.h>
13 #include <linux/pci.h>
14 #include <linux/pci-ats.h>
15 #include <linux/dmar.h>
16 #include <linux/interrupt.h>
17 #include <linux/mm_types.h>
18 #include <linux/xarray.h>
20 #include <asm/fpu/api.h>
25 #include "../iommu-sva.h"
28 static irqreturn_t
prq_event_thread(int irq
, void *d
);
29 static void intel_svm_drain_prq(struct device
*dev
, u32 pasid
);
30 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
32 static DEFINE_XARRAY_ALLOC(pasid_private_array
);
33 static int pasid_private_add(ioasid_t pasid
, void *priv
)
35 return xa_alloc(&pasid_private_array
, &pasid
, priv
,
36 XA_LIMIT(pasid
, pasid
), GFP_ATOMIC
);
39 static void pasid_private_remove(ioasid_t pasid
)
41 xa_erase(&pasid_private_array
, pasid
);
44 static void *pasid_private_find(ioasid_t pasid
)
46 return xa_load(&pasid_private_array
, pasid
);
49 static struct intel_svm_dev
*
50 svm_lookup_device_by_dev(struct intel_svm
*svm
, struct device
*dev
)
52 struct intel_svm_dev
*sdev
= NULL
, *t
;
55 list_for_each_entry_rcu(t
, &svm
->devs
, list
) {
66 int intel_svm_enable_prq(struct intel_iommu
*iommu
)
68 struct iopf_queue
*iopfq
;
72 pages
= alloc_pages(GFP_KERNEL
| __GFP_ZERO
, PRQ_ORDER
);
74 pr_warn("IOMMU: %s: Failed to allocate page request queue\n",
78 iommu
->prq
= page_address(pages
);
80 irq
= dmar_alloc_hwirq(IOMMU_IRQ_ID_OFFSET_PRQ
+ iommu
->seq_id
, iommu
->node
, iommu
);
82 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n",
89 snprintf(iommu
->iopfq_name
, sizeof(iommu
->iopfq_name
),
90 "dmar%d-iopfq", iommu
->seq_id
);
91 iopfq
= iopf_queue_alloc(iommu
->iopfq_name
);
93 pr_err("IOMMU: %s: Failed to allocate iopf queue\n", iommu
->name
);
97 iommu
->iopf_queue
= iopfq
;
99 snprintf(iommu
->prq_name
, sizeof(iommu
->prq_name
), "dmar%d-prq", iommu
->seq_id
);
101 ret
= request_threaded_irq(irq
, NULL
, prq_event_thread
, IRQF_ONESHOT
,
102 iommu
->prq_name
, iommu
);
104 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
108 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, 0ULL);
109 dmar_writeq(iommu
->reg
+ DMAR_PQT_REG
, 0ULL);
110 dmar_writeq(iommu
->reg
+ DMAR_PQA_REG
, virt_to_phys(iommu
->prq
) | PRQ_ORDER
);
112 init_completion(&iommu
->prq_complete
);
117 iopf_queue_free(iommu
->iopf_queue
);
118 iommu
->iopf_queue
= NULL
;
120 dmar_free_hwirq(irq
);
123 free_pages((unsigned long)iommu
->prq
, PRQ_ORDER
);
129 int intel_svm_finish_prq(struct intel_iommu
*iommu
)
131 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, 0ULL);
132 dmar_writeq(iommu
->reg
+ DMAR_PQT_REG
, 0ULL);
133 dmar_writeq(iommu
->reg
+ DMAR_PQA_REG
, 0ULL);
136 free_irq(iommu
->pr_irq
, iommu
);
137 dmar_free_hwirq(iommu
->pr_irq
);
141 if (iommu
->iopf_queue
) {
142 iopf_queue_free(iommu
->iopf_queue
);
143 iommu
->iopf_queue
= NULL
;
146 free_pages((unsigned long)iommu
->prq
, PRQ_ORDER
);
152 void intel_svm_check(struct intel_iommu
*iommu
)
154 if (!pasid_supported(iommu
))
157 if (cpu_feature_enabled(X86_FEATURE_GBPAGES
) &&
158 !cap_fl1gp_support(iommu
->cap
)) {
159 pr_err("%s SVM disabled, incompatible 1GB page capability\n",
164 if (cpu_feature_enabled(X86_FEATURE_LA57
) &&
165 !cap_fl5lp_support(iommu
->cap
)) {
166 pr_err("%s SVM disabled, incompatible paging mode\n",
171 iommu
->flags
|= VTD_FLAG_SVM_CAPABLE
;
174 static void __flush_svm_range_dev(struct intel_svm
*svm
,
175 struct intel_svm_dev
*sdev
,
176 unsigned long address
,
177 unsigned long pages
, int ih
)
179 struct device_domain_info
*info
= dev_iommu_priv_get(sdev
->dev
);
184 qi_flush_piotlb(sdev
->iommu
, sdev
->did
, svm
->pasid
, address
, pages
, ih
);
185 if (info
->ats_enabled
) {
186 qi_flush_dev_iotlb_pasid(sdev
->iommu
, sdev
->sid
, info
->pfsid
,
187 svm
->pasid
, sdev
->qdep
, address
,
188 order_base_2(pages
));
189 quirk_extra_dev_tlb_flush(info
, address
, order_base_2(pages
),
190 svm
->pasid
, sdev
->qdep
);
194 static void intel_flush_svm_range_dev(struct intel_svm
*svm
,
195 struct intel_svm_dev
*sdev
,
196 unsigned long address
,
197 unsigned long pages
, int ih
)
199 unsigned long shift
= ilog2(__roundup_pow_of_two(pages
));
200 unsigned long align
= (1ULL << (VTD_PAGE_SHIFT
+ shift
));
201 unsigned long start
= ALIGN_DOWN(address
, align
);
202 unsigned long end
= ALIGN(address
+ (pages
<< VTD_PAGE_SHIFT
), align
);
204 while (start
< end
) {
205 __flush_svm_range_dev(svm
, sdev
, start
, align
>> VTD_PAGE_SHIFT
, ih
);
210 static void intel_flush_svm_range(struct intel_svm
*svm
, unsigned long address
,
211 unsigned long pages
, int ih
)
213 struct intel_svm_dev
*sdev
;
216 list_for_each_entry_rcu(sdev
, &svm
->devs
, list
)
217 intel_flush_svm_range_dev(svm
, sdev
, address
, pages
, ih
);
221 /* Pages have been freed at this point */
222 static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier
*mn
,
223 struct mm_struct
*mm
,
224 unsigned long start
, unsigned long end
)
226 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
228 intel_flush_svm_range(svm
, start
,
229 (end
- start
+ PAGE_SIZE
- 1) >> VTD_PAGE_SHIFT
, 0);
232 static void intel_mm_release(struct mmu_notifier
*mn
, struct mm_struct
*mm
)
234 struct intel_svm
*svm
= container_of(mn
, struct intel_svm
, notifier
);
235 struct intel_svm_dev
*sdev
;
237 /* This might end up being called from exit_mmap(), *before* the page
238 * tables are cleared. And __mmu_notifier_release() will delete us from
239 * the list of notifiers so that our invalidate_range() callback doesn't
240 * get called when the page tables are cleared. So we need to protect
241 * against hardware accessing those page tables.
243 * We do it by clearing the entry in the PASID table and then flushing
244 * the IOTLB and the PASID table caches. This might upset hardware;
245 * perhaps we'll want to point the PASID to a dummy PGD (like the zero
246 * page) so that we end up taking a fault that the hardware really
247 * *has* to handle gracefully without affecting other processes.
250 list_for_each_entry_rcu(sdev
, &svm
->devs
, list
)
251 intel_pasid_tear_down_entry(sdev
->iommu
, sdev
->dev
,
257 static const struct mmu_notifier_ops intel_mmuops
= {
258 .release
= intel_mm_release
,
259 .arch_invalidate_secondary_tlbs
= intel_arch_invalidate_secondary_tlbs
,
262 static DEFINE_MUTEX(pasid_mutex
);
264 static int pasid_to_svm_sdev(struct device
*dev
, unsigned int pasid
,
265 struct intel_svm
**rsvm
,
266 struct intel_svm_dev
**rsdev
)
268 struct intel_svm_dev
*sdev
= NULL
;
269 struct intel_svm
*svm
;
271 /* The caller should hold the pasid_mutex lock */
272 if (WARN_ON(!mutex_is_locked(&pasid_mutex
)))
275 if (pasid
== IOMMU_PASID_INVALID
|| pasid
>= PASID_MAX
)
278 svm
= pasid_private_find(pasid
);
286 * If we found svm for the PASID, there must be at least one device
289 if (WARN_ON(list_empty(&svm
->devs
)))
291 sdev
= svm_lookup_device_by_dev(svm
, dev
);
300 static int intel_svm_bind_mm(struct intel_iommu
*iommu
, struct device
*dev
,
301 struct mm_struct
*mm
)
303 struct device_domain_info
*info
= dev_iommu_priv_get(dev
);
304 struct intel_svm_dev
*sdev
;
305 struct intel_svm
*svm
;
306 unsigned long sflags
;
309 svm
= pasid_private_find(mm
->pasid
);
311 svm
= kzalloc(sizeof(*svm
), GFP_KERNEL
);
315 svm
->pasid
= mm
->pasid
;
317 INIT_LIST_HEAD_RCU(&svm
->devs
);
319 svm
->notifier
.ops
= &intel_mmuops
;
320 ret
= mmu_notifier_register(&svm
->notifier
, mm
);
326 ret
= pasid_private_add(svm
->pasid
, svm
);
328 mmu_notifier_unregister(&svm
->notifier
, mm
);
334 sdev
= kzalloc(sizeof(*sdev
), GFP_KERNEL
);
342 sdev
->did
= FLPT_DEFAULT_DID
;
343 sdev
->sid
= PCI_DEVID(info
->bus
, info
->devfn
);
344 init_rcu_head(&sdev
->rcu
);
345 if (info
->ats_enabled
) {
346 sdev
->qdep
= info
->ats_qdep
;
347 if (sdev
->qdep
>= QI_DEV_EIOTLB_MAX_INVS
)
351 /* Setup the pasid table: */
352 sflags
= cpu_feature_enabled(X86_FEATURE_LA57
) ? PASID_FLAG_FL5LP
: 0;
353 ret
= intel_pasid_setup_first_level(iommu
, dev
, mm
->pgd
, mm
->pasid
,
354 FLPT_DEFAULT_DID
, sflags
);
358 list_add_rcu(&sdev
->list
, &svm
->devs
);
365 if (list_empty(&svm
->devs
)) {
366 mmu_notifier_unregister(&svm
->notifier
, mm
);
367 pasid_private_remove(mm
->pasid
);
374 /* Caller must hold pasid_mutex */
375 static int intel_svm_unbind_mm(struct device
*dev
, u32 pasid
)
377 struct intel_svm_dev
*sdev
;
378 struct intel_iommu
*iommu
;
379 struct intel_svm
*svm
;
380 struct mm_struct
*mm
;
383 iommu
= device_to_iommu(dev
, NULL
, NULL
);
387 ret
= pasid_to_svm_sdev(dev
, pasid
, &svm
, &sdev
);
393 list_del_rcu(&sdev
->list
);
395 * Flush the PASID cache and IOTLB for this device.
396 * Note that we do depend on the hardware *not* using
397 * the PASID any more. Just as we depend on other
398 * devices never using PASIDs that they have no right
399 * to use. We have a *shared* PASID table, because it's
400 * large and has to be physically contiguous. So it's
401 * hard to be as defensive as we might like.
403 intel_pasid_tear_down_entry(iommu
, dev
, svm
->pasid
, false);
404 intel_svm_drain_prq(dev
, svm
->pasid
);
405 kfree_rcu(sdev
, rcu
);
407 if (list_empty(&svm
->devs
)) {
408 if (svm
->notifier
.ops
)
409 mmu_notifier_unregister(&svm
->notifier
, mm
);
410 pasid_private_remove(svm
->pasid
);
412 * We mandate that no page faults may be outstanding
413 * for the PASID when intel_svm_unbind_mm() is called.
414 * If that is not obeyed, subtle errors will happen.
415 * Let's make them less subtle...
417 memset(svm
, 0x6b, sizeof(*svm
));
425 /* Page request queue descriptor */
426 struct page_req_dsc
{
431 u64 priv_data_present
:1;
454 static bool is_canonical_address(u64 addr
)
456 int shift
= 64 - (__VIRTUAL_MASK_SHIFT
+ 1);
457 long saddr
= (long) addr
;
459 return (((saddr
<< shift
) >> shift
) == saddr
);
463 * intel_svm_drain_prq - Drain page requests and responses for a pasid
464 * @dev: target device
465 * @pasid: pasid for draining
467 * Drain all pending page requests and responses related to @pasid in both
468 * software and hardware. This is supposed to be called after the device
469 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB
470 * and DevTLB have been invalidated.
472 * It waits until all pending page requests for @pasid in the page fault
473 * queue are completed by the prq handling thread. Then follow the steps
474 * described in VT-d spec CH7.10 to drain all page requests and page
475 * responses pending in the hardware.
477 static void intel_svm_drain_prq(struct device
*dev
, u32 pasid
)
479 struct device_domain_info
*info
;
480 struct dmar_domain
*domain
;
481 struct intel_iommu
*iommu
;
482 struct qi_desc desc
[3];
483 struct pci_dev
*pdev
;
488 info
= dev_iommu_priv_get(dev
);
489 if (WARN_ON(!info
|| !dev_is_pci(dev
)))
492 if (!info
->pri_enabled
)
496 domain
= info
->domain
;
497 pdev
= to_pci_dev(dev
);
498 sid
= PCI_DEVID(info
->bus
, info
->devfn
);
499 did
= domain_id_iommu(domain
, iommu
);
500 qdep
= pci_ats_queue_depth(pdev
);
503 * Check and wait until all pending page requests in the queue are
504 * handled by the prq handling thread.
507 reinit_completion(&iommu
->prq_complete
);
508 tail
= dmar_readq(iommu
->reg
+ DMAR_PQT_REG
) & PRQ_RING_MASK
;
509 head
= dmar_readq(iommu
->reg
+ DMAR_PQH_REG
) & PRQ_RING_MASK
;
510 while (head
!= tail
) {
511 struct page_req_dsc
*req
;
513 req
= &iommu
->prq
[head
/ sizeof(*req
)];
514 if (!req
->pasid_present
|| req
->pasid
!= pasid
) {
515 head
= (head
+ sizeof(*req
)) & PRQ_RING_MASK
;
519 wait_for_completion(&iommu
->prq_complete
);
524 * A work in IO page fault workqueue may try to lock pasid_mutex now.
525 * Holding pasid_mutex while waiting in iopf_queue_flush_dev() for
526 * all works in the workqueue to finish may cause deadlock.
528 * It's unnecessary to hold pasid_mutex in iopf_queue_flush_dev().
529 * Unlock it to allow the works to be handled while waiting for
532 lockdep_assert_held(&pasid_mutex
);
533 mutex_unlock(&pasid_mutex
);
534 iopf_queue_flush_dev(dev
);
535 mutex_lock(&pasid_mutex
);
538 * Perform steps described in VT-d spec CH7.10 to drain page
539 * requests and responses in hardware.
541 memset(desc
, 0, sizeof(desc
));
542 desc
[0].qw0
= QI_IWD_STATUS_DATA(QI_DONE
) |
545 desc
[1].qw0
= QI_EIOTLB_PASID(pasid
) |
547 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID
) |
549 desc
[2].qw0
= QI_DEV_EIOTLB_PASID(pasid
) |
550 QI_DEV_EIOTLB_SID(sid
) |
551 QI_DEV_EIOTLB_QDEP(qdep
) |
553 QI_DEV_IOTLB_PFSID(info
->pfsid
);
555 reinit_completion(&iommu
->prq_complete
);
556 qi_submit_sync(iommu
, desc
, 3, QI_OPT_WAIT_DRAIN
);
557 if (readl(iommu
->reg
+ DMAR_PRS_REG
) & DMA_PRS_PRO
) {
558 wait_for_completion(&iommu
->prq_complete
);
563 static int prq_to_iommu_prot(struct page_req_dsc
*req
)
568 prot
|= IOMMU_FAULT_PERM_READ
;
570 prot
|= IOMMU_FAULT_PERM_WRITE
;
572 prot
|= IOMMU_FAULT_PERM_EXEC
;
574 prot
|= IOMMU_FAULT_PERM_PRIV
;
579 static int intel_svm_prq_report(struct intel_iommu
*iommu
, struct device
*dev
,
580 struct page_req_dsc
*desc
)
582 struct iommu_fault_event event
;
584 if (!dev
|| !dev_is_pci(dev
))
587 /* Fill in event data for device specific processing */
588 memset(&event
, 0, sizeof(struct iommu_fault_event
));
589 event
.fault
.type
= IOMMU_FAULT_PAGE_REQ
;
590 event
.fault
.prm
.addr
= (u64
)desc
->addr
<< VTD_PAGE_SHIFT
;
591 event
.fault
.prm
.pasid
= desc
->pasid
;
592 event
.fault
.prm
.grpid
= desc
->prg_index
;
593 event
.fault
.prm
.perm
= prq_to_iommu_prot(desc
);
596 event
.fault
.prm
.flags
|= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE
;
597 if (desc
->pasid_present
) {
598 event
.fault
.prm
.flags
|= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID
;
599 event
.fault
.prm
.flags
|= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID
;
601 if (desc
->priv_data_present
) {
603 * Set last page in group bit if private data is present,
604 * page response is required as it does for LPIG.
605 * iommu_report_device_fault() doesn't understand this vendor
606 * specific requirement thus we set last_page as a workaround.
608 event
.fault
.prm
.flags
|= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE
;
609 event
.fault
.prm
.flags
|= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA
;
610 event
.fault
.prm
.private_data
[0] = desc
->priv_data
[0];
611 event
.fault
.prm
.private_data
[1] = desc
->priv_data
[1];
612 } else if (dmar_latency_enabled(iommu
, DMAR_LATENCY_PRQ
)) {
614 * If the private data fields are not used by hardware, use it
615 * to monitor the prq handle latency.
617 event
.fault
.prm
.private_data
[0] = ktime_to_ns(ktime_get());
620 return iommu_report_device_fault(dev
, &event
);
623 static void handle_bad_prq_event(struct intel_iommu
*iommu
,
624 struct page_req_dsc
*req
, int result
)
628 pr_err("%s: Invalid page request: %08llx %08llx\n",
629 iommu
->name
, ((unsigned long long *)req
)[0],
630 ((unsigned long long *)req
)[1]);
633 * Per VT-d spec. v3.0 ch7.7, system software must
634 * respond with page group response if private data
635 * is present (PDP) or last page in group (LPIG) bit
636 * is set. This is an additional VT-d feature beyond
639 if (!req
->lpig
&& !req
->priv_data_present
)
642 desc
.qw0
= QI_PGRP_PASID(req
->pasid
) |
643 QI_PGRP_DID(req
->rid
) |
644 QI_PGRP_PASID_P(req
->pasid_present
) |
645 QI_PGRP_PDP(req
->priv_data_present
) |
646 QI_PGRP_RESP_CODE(result
) |
648 desc
.qw1
= QI_PGRP_IDX(req
->prg_index
) |
649 QI_PGRP_LPIG(req
->lpig
);
651 if (req
->priv_data_present
) {
652 desc
.qw2
= req
->priv_data
[0];
653 desc
.qw3
= req
->priv_data
[1];
659 qi_submit_sync(iommu
, &desc
, 1, 0);
662 static irqreturn_t
prq_event_thread(int irq
, void *d
)
664 struct intel_iommu
*iommu
= d
;
665 struct page_req_dsc
*req
;
666 int head
, tail
, handled
;
667 struct pci_dev
*pdev
;
671 * Clear PPR bit before reading head/tail registers, to ensure that
672 * we get a new interrupt if needed.
674 writel(DMA_PRS_PPR
, iommu
->reg
+ DMAR_PRS_REG
);
676 tail
= dmar_readq(iommu
->reg
+ DMAR_PQT_REG
) & PRQ_RING_MASK
;
677 head
= dmar_readq(iommu
->reg
+ DMAR_PQH_REG
) & PRQ_RING_MASK
;
678 handled
= (head
!= tail
);
679 while (head
!= tail
) {
680 req
= &iommu
->prq
[head
/ sizeof(*req
)];
681 address
= (u64
)req
->addr
<< VTD_PAGE_SHIFT
;
683 if (unlikely(!req
->pasid_present
)) {
684 pr_err("IOMMU: %s: Page request without PASID\n",
687 handle_bad_prq_event(iommu
, req
, QI_RESP_INVALID
);
691 if (unlikely(!is_canonical_address(address
))) {
692 pr_err("IOMMU: %s: Address is not canonical\n",
697 if (unlikely(req
->pm_req
&& (req
->rd_req
| req
->wr_req
))) {
698 pr_err("IOMMU: %s: Page request in Privilege Mode\n",
703 if (unlikely(req
->exe_req
&& req
->rd_req
)) {
704 pr_err("IOMMU: %s: Execution request not supported\n",
709 /* Drop Stop Marker message. No need for a response. */
710 if (unlikely(req
->lpig
&& !req
->rd_req
&& !req
->wr_req
))
713 pdev
= pci_get_domain_bus_and_slot(iommu
->segment
,
714 PCI_BUS_NUM(req
->rid
),
717 * If prq is to be handled outside iommu driver via receiver of
718 * the fault notifiers, we skip the page response here.
723 if (intel_svm_prq_report(iommu
, &pdev
->dev
, req
))
724 handle_bad_prq_event(iommu
, req
, QI_RESP_INVALID
);
726 trace_prq_report(iommu
, &pdev
->dev
, req
->qw_0
, req
->qw_1
,
727 req
->priv_data
[0], req
->priv_data
[1],
728 iommu
->prq_seq_number
++);
731 head
= (head
+ sizeof(*req
)) & PRQ_RING_MASK
;
734 dmar_writeq(iommu
->reg
+ DMAR_PQH_REG
, tail
);
737 * Clear the page request overflow bit and wake up all threads that
738 * are waiting for the completion of this handling.
740 if (readl(iommu
->reg
+ DMAR_PRS_REG
) & DMA_PRS_PRO
) {
741 pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n",
743 head
= dmar_readq(iommu
->reg
+ DMAR_PQH_REG
) & PRQ_RING_MASK
;
744 tail
= dmar_readq(iommu
->reg
+ DMAR_PQT_REG
) & PRQ_RING_MASK
;
746 iopf_queue_discard_partial(iommu
->iopf_queue
);
747 writel(DMA_PRS_PRO
, iommu
->reg
+ DMAR_PRS_REG
);
748 pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared",
753 if (!completion_done(&iommu
->prq_complete
))
754 complete(&iommu
->prq_complete
);
756 return IRQ_RETVAL(handled
);
759 int intel_svm_page_response(struct device
*dev
,
760 struct iommu_fault_event
*evt
,
761 struct iommu_page_response
*msg
)
763 struct iommu_fault_page_request
*prm
;
764 struct intel_iommu
*iommu
;
765 bool private_present
;
772 if (!dev
|| !dev_is_pci(dev
))
775 iommu
= device_to_iommu(dev
, &bus
, &devfn
);
782 prm
= &evt
->fault
.prm
;
783 sid
= PCI_DEVID(bus
, devfn
);
784 pasid_present
= prm
->flags
& IOMMU_FAULT_PAGE_REQUEST_PASID_VALID
;
785 private_present
= prm
->flags
& IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA
;
786 last_page
= prm
->flags
& IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE
;
788 if (!pasid_present
) {
793 if (prm
->pasid
== 0 || prm
->pasid
>= PASID_MAX
) {
799 * Per VT-d spec. v3.0 ch7.7, system software must respond
800 * with page group response if private data is present (PDP)
801 * or last page in group (LPIG) bit is set. This is an
802 * additional VT-d requirement beyond PCI ATS spec.
804 if (last_page
|| private_present
) {
807 desc
.qw0
= QI_PGRP_PASID(prm
->pasid
) | QI_PGRP_DID(sid
) |
808 QI_PGRP_PASID_P(pasid_present
) |
809 QI_PGRP_PDP(private_present
) |
810 QI_PGRP_RESP_CODE(msg
->code
) |
812 desc
.qw1
= QI_PGRP_IDX(prm
->grpid
) | QI_PGRP_LPIG(last_page
);
816 if (private_present
) {
817 desc
.qw2
= prm
->private_data
[0];
818 desc
.qw3
= prm
->private_data
[1];
819 } else if (prm
->private_data
[0]) {
820 dmar_latency_update(iommu
, DMAR_LATENCY_PRQ
,
821 ktime_to_ns(ktime_get()) - prm
->private_data
[0]);
824 qi_submit_sync(iommu
, &desc
, 1, 0);
830 void intel_svm_remove_dev_pasid(struct device
*dev
, ioasid_t pasid
)
832 mutex_lock(&pasid_mutex
);
833 intel_svm_unbind_mm(dev
, pasid
);
834 mutex_unlock(&pasid_mutex
);
837 static int intel_svm_set_dev_pasid(struct iommu_domain
*domain
,
838 struct device
*dev
, ioasid_t pasid
)
840 struct device_domain_info
*info
= dev_iommu_priv_get(dev
);
841 struct intel_iommu
*iommu
= info
->iommu
;
842 struct mm_struct
*mm
= domain
->mm
;
845 mutex_lock(&pasid_mutex
);
846 ret
= intel_svm_bind_mm(iommu
, dev
, mm
);
847 mutex_unlock(&pasid_mutex
);
852 static void intel_svm_domain_free(struct iommu_domain
*domain
)
854 kfree(to_dmar_domain(domain
));
857 static const struct iommu_domain_ops intel_svm_domain_ops
= {
858 .set_dev_pasid
= intel_svm_set_dev_pasid
,
859 .free
= intel_svm_domain_free
862 struct iommu_domain
*intel_svm_domain_alloc(void)
864 struct dmar_domain
*domain
;
866 domain
= kzalloc(sizeof(*domain
), GFP_KERNEL
);
869 domain
->domain
.ops
= &intel_svm_domain_ops
;
871 return &domain
->domain
;