1 // SPDX-License-Identifier: GPL-2.0-only
3 * VFIO-KVM bridge pseudo device
5 * Copyright (C) 2013 Red Hat, Inc. All rights reserved.
6 * Author: Alex Williamson <alex.williamson@redhat.com>
9 #include <linux/errno.h>
10 #include <linux/file.h>
11 #include <linux/kvm_host.h>
12 #include <linux/list.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/slab.h>
16 #include <linux/uaccess.h>
17 #include <linux/vfio.h>
20 #ifdef CONFIG_SPAPR_TCE_IOMMU
21 #include <asm/kvm_ppc.h>
24 struct kvm_vfio_file
{
25 struct list_head node
;
27 #ifdef CONFIG_SPAPR_TCE_IOMMU
28 struct iommu_group
*iommu_group
;
33 struct list_head file_list
;
38 static void kvm_vfio_file_set_kvm(struct file
*file
, struct kvm
*kvm
)
40 void (*fn
)(struct file
*file
, struct kvm
*kvm
);
42 fn
= symbol_get(vfio_file_set_kvm
);
48 symbol_put(vfio_file_set_kvm
);
51 static bool kvm_vfio_file_enforced_coherent(struct file
*file
)
53 bool (*fn
)(struct file
*file
);
56 fn
= symbol_get(vfio_file_enforced_coherent
);
62 symbol_put(vfio_file_enforced_coherent
);
67 static bool kvm_vfio_file_is_valid(struct file
*file
)
69 bool (*fn
)(struct file
*file
);
72 fn
= symbol_get(vfio_file_is_valid
);
78 symbol_put(vfio_file_is_valid
);
83 #ifdef CONFIG_SPAPR_TCE_IOMMU
84 static struct iommu_group
*kvm_vfio_file_iommu_group(struct file
*file
)
86 struct iommu_group
*(*fn
)(struct file
*file
);
87 struct iommu_group
*ret
;
89 fn
= symbol_get(vfio_file_iommu_group
);
95 symbol_put(vfio_file_iommu_group
);
100 static void kvm_spapr_tce_release_vfio_group(struct kvm
*kvm
,
101 struct kvm_vfio_file
*kvf
)
103 if (WARN_ON_ONCE(!kvf
->iommu_group
))
106 kvm_spapr_tce_release_iommu_group(kvm
, kvf
->iommu_group
);
107 iommu_group_put(kvf
->iommu_group
);
108 kvf
->iommu_group
= NULL
;
113 * Groups/devices can use the same or different IOMMU domains. If the same
114 * then adding a new group/device may change the coherency of groups/devices
115 * we've previously been told about. We don't want to care about any of
116 * that so we retest each group/device and bail as soon as we find one that's
117 * noncoherent. This means we only ever [un]register_noncoherent_dma once
118 * for the whole device.
120 static void kvm_vfio_update_coherency(struct kvm_device
*dev
)
122 struct kvm_vfio
*kv
= dev
->private;
123 bool noncoherent
= false;
124 struct kvm_vfio_file
*kvf
;
126 list_for_each_entry(kvf
, &kv
->file_list
, node
) {
127 if (!kvm_vfio_file_enforced_coherent(kvf
->file
)) {
133 if (noncoherent
!= kv
->noncoherent
) {
134 kv
->noncoherent
= noncoherent
;
137 kvm_arch_register_noncoherent_dma(dev
->kvm
);
139 kvm_arch_unregister_noncoherent_dma(dev
->kvm
);
143 static int kvm_vfio_file_add(struct kvm_device
*dev
, unsigned int fd
)
145 struct kvm_vfio
*kv
= dev
->private;
146 struct kvm_vfio_file
*kvf
;
154 /* Ensure the FD is a vfio FD. */
155 if (!kvm_vfio_file_is_valid(filp
)) {
160 mutex_lock(&kv
->lock
);
162 list_for_each_entry(kvf
, &kv
->file_list
, node
) {
163 if (kvf
->file
== filp
) {
169 kvf
= kzalloc(sizeof(*kvf
), GFP_KERNEL_ACCOUNT
);
175 kvf
->file
= get_file(filp
);
176 list_add_tail(&kvf
->node
, &kv
->file_list
);
178 kvm_arch_start_assignment(dev
->kvm
);
179 kvm_vfio_file_set_kvm(kvf
->file
, dev
->kvm
);
180 kvm_vfio_update_coherency(dev
);
183 mutex_unlock(&kv
->lock
);
189 static int kvm_vfio_file_del(struct kvm_device
*dev
, unsigned int fd
)
191 struct kvm_vfio
*kv
= dev
->private;
192 struct kvm_vfio_file
*kvf
;
202 mutex_lock(&kv
->lock
);
204 list_for_each_entry(kvf
, &kv
->file_list
, node
) {
205 if (kvf
->file
!= f
.file
)
208 list_del(&kvf
->node
);
209 kvm_arch_end_assignment(dev
->kvm
);
210 #ifdef CONFIG_SPAPR_TCE_IOMMU
211 kvm_spapr_tce_release_vfio_group(dev
->kvm
, kvf
);
213 kvm_vfio_file_set_kvm(kvf
->file
, NULL
);
220 kvm_vfio_update_coherency(dev
);
222 mutex_unlock(&kv
->lock
);
229 #ifdef CONFIG_SPAPR_TCE_IOMMU
230 static int kvm_vfio_file_set_spapr_tce(struct kvm_device
*dev
,
233 struct kvm_vfio_spapr_tce param
;
234 struct kvm_vfio
*kv
= dev
->private;
235 struct kvm_vfio_file
*kvf
;
239 if (copy_from_user(¶m
, arg
, sizeof(struct kvm_vfio_spapr_tce
)))
242 f
= fdget(param
.groupfd
);
248 mutex_lock(&kv
->lock
);
250 list_for_each_entry(kvf
, &kv
->file_list
, node
) {
251 if (kvf
->file
!= f
.file
)
254 if (!kvf
->iommu_group
) {
255 kvf
->iommu_group
= kvm_vfio_file_iommu_group(kvf
->file
);
256 if (WARN_ON_ONCE(!kvf
->iommu_group
)) {
262 ret
= kvm_spapr_tce_attach_iommu_group(dev
->kvm
, param
.tablefd
,
268 mutex_unlock(&kv
->lock
);
274 static int kvm_vfio_set_file(struct kvm_device
*dev
, long attr
,
277 int32_t __user
*argp
= arg
;
281 case KVM_DEV_VFIO_FILE_ADD
:
282 if (get_user(fd
, argp
))
284 return kvm_vfio_file_add(dev
, fd
);
286 case KVM_DEV_VFIO_FILE_DEL
:
287 if (get_user(fd
, argp
))
289 return kvm_vfio_file_del(dev
, fd
);
291 #ifdef CONFIG_SPAPR_TCE_IOMMU
292 case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE
:
293 return kvm_vfio_file_set_spapr_tce(dev
, arg
);
300 static int kvm_vfio_set_attr(struct kvm_device
*dev
,
301 struct kvm_device_attr
*attr
)
303 switch (attr
->group
) {
304 case KVM_DEV_VFIO_FILE
:
305 return kvm_vfio_set_file(dev
, attr
->attr
,
306 u64_to_user_ptr(attr
->addr
));
312 static int kvm_vfio_has_attr(struct kvm_device
*dev
,
313 struct kvm_device_attr
*attr
)
315 switch (attr
->group
) {
316 case KVM_DEV_VFIO_FILE
:
317 switch (attr
->attr
) {
318 case KVM_DEV_VFIO_FILE_ADD
:
319 case KVM_DEV_VFIO_FILE_DEL
:
320 #ifdef CONFIG_SPAPR_TCE_IOMMU
321 case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE
:
332 static void kvm_vfio_release(struct kvm_device
*dev
)
334 struct kvm_vfio
*kv
= dev
->private;
335 struct kvm_vfio_file
*kvf
, *tmp
;
337 list_for_each_entry_safe(kvf
, tmp
, &kv
->file_list
, node
) {
338 #ifdef CONFIG_SPAPR_TCE_IOMMU
339 kvm_spapr_tce_release_vfio_group(dev
->kvm
, kvf
);
341 kvm_vfio_file_set_kvm(kvf
->file
, NULL
);
343 list_del(&kvf
->node
);
345 kvm_arch_end_assignment(dev
->kvm
);
348 kvm_vfio_update_coherency(dev
);
351 kfree(dev
); /* alloc by kvm_ioctl_create_device, free by .release */
354 static int kvm_vfio_create(struct kvm_device
*dev
, u32 type
);
356 static struct kvm_device_ops kvm_vfio_ops
= {
358 .create
= kvm_vfio_create
,
359 .release
= kvm_vfio_release
,
360 .set_attr
= kvm_vfio_set_attr
,
361 .has_attr
= kvm_vfio_has_attr
,
364 static int kvm_vfio_create(struct kvm_device
*dev
, u32 type
)
366 struct kvm_device
*tmp
;
369 /* Only one VFIO "device" per VM */
370 list_for_each_entry(tmp
, &dev
->kvm
->devices
, vm_node
)
371 if (tmp
->ops
== &kvm_vfio_ops
)
374 kv
= kzalloc(sizeof(*kv
), GFP_KERNEL_ACCOUNT
);
378 INIT_LIST_HEAD(&kv
->file_list
);
379 mutex_init(&kv
->lock
);
386 int kvm_vfio_ops_init(void)
388 return kvm_register_device_ops(&kvm_vfio_ops
, KVM_DEV_TYPE_VFIO
);
391 void kvm_vfio_ops_exit(void)
393 kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO
);