]> git.ipfire.org Git - thirdparty/linux.git/blob - drivers/vdpa/vdpa_user/vduse_dev.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
eventfd: simplify eventfd_signal()
[thirdparty/linux.git] / drivers / vdpa / vdpa_user / vduse_dev.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VDUSE: vDPA Device in Userspace
4 *
5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
6 *
7 * Author: Xie Yongji <xieyongji@bytedance.com>
8 *
9 */
10
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/slab.h>
17 #include <linux/wait.h>
18 #include <linux/dma-map-ops.h>
19 #include <linux/poll.h>
20 #include <linux/file.h>
21 #include <linux/uio.h>
22 #include <linux/vdpa.h>
23 #include <linux/nospec.h>
24 #include <linux/vmalloc.h>
25 #include <linux/sched/mm.h>
26 #include <uapi/linux/vduse.h>
27 #include <uapi/linux/vdpa.h>
28 #include <uapi/linux/virtio_config.h>
29 #include <uapi/linux/virtio_ids.h>
30 #include <uapi/linux/virtio_blk.h>
31 #include <linux/mod_devicetable.h>
32
33 #include "iova_domain.h"
34
35 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>"
36 #define DRV_DESC "vDPA Device in Userspace"
37 #define DRV_LICENSE "GPL v2"
38
39 #define VDUSE_DEV_MAX (1U << MINORBITS)
40 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024)
41 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024)
42 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
43 /* 128 MB reserved for virtqueue creation */
44 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024)
45 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
46
47 #define IRQ_UNBOUND -1
48
49 struct vduse_virtqueue {
50 u16 index;
51 u16 num_max;
52 u32 num;
53 u64 desc_addr;
54 u64 driver_addr;
55 u64 device_addr;
56 struct vdpa_vq_state state;
57 bool ready;
58 bool kicked;
59 spinlock_t kick_lock;
60 spinlock_t irq_lock;
61 struct eventfd_ctx *kickfd;
62 struct vdpa_callback cb;
63 struct work_struct inject;
64 struct work_struct kick;
65 int irq_effective_cpu;
66 struct cpumask irq_affinity;
67 struct kobject kobj;
68 };
69
70 struct vduse_dev;
71
72 struct vduse_vdpa {
73 struct vdpa_device vdpa;
74 struct vduse_dev *dev;
75 };
76
77 struct vduse_umem {
78 unsigned long iova;
79 unsigned long npages;
80 struct page **pages;
81 struct mm_struct *mm;
82 };
83
84 struct vduse_dev {
85 struct vduse_vdpa *vdev;
86 struct device *dev;
87 struct vduse_virtqueue **vqs;
88 struct vduse_iova_domain *domain;
89 char *name;
90 struct mutex lock;
91 spinlock_t msg_lock;
92 u64 msg_unique;
93 u32 msg_timeout;
94 wait_queue_head_t waitq;
95 struct list_head send_list;
96 struct list_head recv_list;
97 struct vdpa_callback config_cb;
98 struct work_struct inject;
99 spinlock_t irq_lock;
100 struct rw_semaphore rwsem;
101 int minor;
102 bool broken;
103 bool connected;
104 u64 api_version;
105 u64 device_features;
106 u64 driver_features;
107 u32 device_id;
108 u32 vendor_id;
109 u32 generation;
110 u32 config_size;
111 void *config;
112 u8 status;
113 u32 vq_num;
114 u32 vq_align;
115 struct vduse_umem *umem;
116 struct mutex mem_lock;
117 unsigned int bounce_size;
118 struct mutex domain_lock;
119 };
120
121 struct vduse_dev_msg {
122 struct vduse_dev_request req;
123 struct vduse_dev_response resp;
124 struct list_head list;
125 wait_queue_head_t waitq;
126 bool completed;
127 };
128
129 struct vduse_control {
130 u64 api_version;
131 };
132
133 static DEFINE_MUTEX(vduse_lock);
134 static DEFINE_IDR(vduse_idr);
135
136 static dev_t vduse_major;
137 static struct cdev vduse_ctrl_cdev;
138 static struct cdev vduse_cdev;
139 static struct workqueue_struct *vduse_irq_wq;
140 static struct workqueue_struct *vduse_irq_bound_wq;
141
142 static u32 allowed_device_id[] = {
143 VIRTIO_ID_BLOCK,
144 };
145
146 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
147 {
148 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
149
150 return vdev->dev;
151 }
152
153 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
154 {
155 struct vdpa_device *vdpa = dev_to_vdpa(dev);
156
157 return vdpa_to_vduse(vdpa);
158 }
159
160 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
161 uint32_t request_id)
162 {
163 struct vduse_dev_msg *msg;
164
165 list_for_each_entry(msg, head, list) {
166 if (msg->req.request_id == request_id) {
167 list_del(&msg->list);
168 return msg;
169 }
170 }
171
172 return NULL;
173 }
174
175 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
176 {
177 struct vduse_dev_msg *msg = NULL;
178
179 if (!list_empty(head)) {
180 msg = list_first_entry(head, struct vduse_dev_msg, list);
181 list_del(&msg->list);
182 }
183
184 return msg;
185 }
186
187 static void vduse_enqueue_msg(struct list_head *head,
188 struct vduse_dev_msg *msg)
189 {
190 list_add_tail(&msg->list, head);
191 }
192
193 static void vduse_dev_broken(struct vduse_dev *dev)
194 {
195 struct vduse_dev_msg *msg, *tmp;
196
197 if (unlikely(dev->broken))
198 return;
199
200 list_splice_init(&dev->recv_list, &dev->send_list);
201 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
202 list_del(&msg->list);
203 msg->completed = 1;
204 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
205 wake_up(&msg->waitq);
206 }
207 dev->broken = true;
208 wake_up(&dev->waitq);
209 }
210
211 static int vduse_dev_msg_sync(struct vduse_dev *dev,
212 struct vduse_dev_msg *msg)
213 {
214 int ret;
215
216 if (unlikely(dev->broken))
217 return -EIO;
218
219 init_waitqueue_head(&msg->waitq);
220 spin_lock(&dev->msg_lock);
221 if (unlikely(dev->broken)) {
222 spin_unlock(&dev->msg_lock);
223 return -EIO;
224 }
225 msg->req.request_id = dev->msg_unique++;
226 vduse_enqueue_msg(&dev->send_list, msg);
227 wake_up(&dev->waitq);
228 spin_unlock(&dev->msg_lock);
229 if (dev->msg_timeout)
230 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
231 (long)dev->msg_timeout * HZ);
232 else
233 ret = wait_event_killable(msg->waitq, msg->completed);
234
235 spin_lock(&dev->msg_lock);
236 if (!msg->completed) {
237 list_del(&msg->list);
238 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
239 /* Mark the device as malfunction when there is a timeout */
240 if (!ret)
241 vduse_dev_broken(dev);
242 }
243 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
244 spin_unlock(&dev->msg_lock);
245
246 return ret;
247 }
248
249 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
250 struct vduse_virtqueue *vq,
251 struct vdpa_vq_state_packed *packed)
252 {
253 struct vduse_dev_msg msg = { 0 };
254 int ret;
255
256 msg.req.type = VDUSE_GET_VQ_STATE;
257 msg.req.vq_state.index = vq->index;
258
259 ret = vduse_dev_msg_sync(dev, &msg);
260 if (ret)
261 return ret;
262
263 packed->last_avail_counter =
264 msg.resp.vq_state.packed.last_avail_counter & 0x0001;
265 packed->last_avail_idx =
266 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
267 packed->last_used_counter =
268 msg.resp.vq_state.packed.last_used_counter & 0x0001;
269 packed->last_used_idx =
270 msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
271
272 return 0;
273 }
274
275 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
276 struct vduse_virtqueue *vq,
277 struct vdpa_vq_state_split *split)
278 {
279 struct vduse_dev_msg msg = { 0 };
280 int ret;
281
282 msg.req.type = VDUSE_GET_VQ_STATE;
283 msg.req.vq_state.index = vq->index;
284
285 ret = vduse_dev_msg_sync(dev, &msg);
286 if (ret)
287 return ret;
288
289 split->avail_index = msg.resp.vq_state.split.avail_index;
290
291 return 0;
292 }
293
294 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
295 {
296 struct vduse_dev_msg msg = { 0 };
297
298 msg.req.type = VDUSE_SET_STATUS;
299 msg.req.s.status = status;
300
301 return vduse_dev_msg_sync(dev, &msg);
302 }
303
304 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
305 u64 start, u64 last)
306 {
307 struct vduse_dev_msg msg = { 0 };
308
309 if (last < start)
310 return -EINVAL;
311
312 msg.req.type = VDUSE_UPDATE_IOTLB;
313 msg.req.iova.start = start;
314 msg.req.iova.last = last;
315
316 return vduse_dev_msg_sync(dev, &msg);
317 }
318
319 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
320 {
321 struct file *file = iocb->ki_filp;
322 struct vduse_dev *dev = file->private_data;
323 struct vduse_dev_msg *msg;
324 int size = sizeof(struct vduse_dev_request);
325 ssize_t ret;
326
327 if (iov_iter_count(to) < size)
328 return -EINVAL;
329
330 spin_lock(&dev->msg_lock);
331 while (1) {
332 msg = vduse_dequeue_msg(&dev->send_list);
333 if (msg)
334 break;
335
336 ret = -EAGAIN;
337 if (file->f_flags & O_NONBLOCK)
338 goto unlock;
339
340 spin_unlock(&dev->msg_lock);
341 ret = wait_event_interruptible_exclusive(dev->waitq,
342 !list_empty(&dev->send_list));
343 if (ret)
344 return ret;
345
346 spin_lock(&dev->msg_lock);
347 }
348 spin_unlock(&dev->msg_lock);
349 ret = copy_to_iter(&msg->req, size, to);
350 spin_lock(&dev->msg_lock);
351 if (ret != size) {
352 ret = -EFAULT;
353 vduse_enqueue_msg(&dev->send_list, msg);
354 goto unlock;
355 }
356 vduse_enqueue_msg(&dev->recv_list, msg);
357 unlock:
358 spin_unlock(&dev->msg_lock);
359
360 return ret;
361 }
362
363 static bool is_mem_zero(const char *ptr, int size)
364 {
365 int i;
366
367 for (i = 0; i < size; i++) {
368 if (ptr[i])
369 return false;
370 }
371 return true;
372 }
373
374 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
375 {
376 struct file *file = iocb->ki_filp;
377 struct vduse_dev *dev = file->private_data;
378 struct vduse_dev_response resp;
379 struct vduse_dev_msg *msg;
380 size_t ret;
381
382 ret = copy_from_iter(&resp, sizeof(resp), from);
383 if (ret != sizeof(resp))
384 return -EINVAL;
385
386 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
387 return -EINVAL;
388
389 spin_lock(&dev->msg_lock);
390 msg = vduse_find_msg(&dev->recv_list, resp.request_id);
391 if (!msg) {
392 ret = -ENOENT;
393 goto unlock;
394 }
395
396 memcpy(&msg->resp, &resp, sizeof(resp));
397 msg->completed = 1;
398 wake_up(&msg->waitq);
399 unlock:
400 spin_unlock(&dev->msg_lock);
401
402 return ret;
403 }
404
405 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
406 {
407 struct vduse_dev *dev = file->private_data;
408 __poll_t mask = 0;
409
410 poll_wait(file, &dev->waitq, wait);
411
412 spin_lock(&dev->msg_lock);
413
414 if (unlikely(dev->broken))
415 mask |= EPOLLERR;
416 if (!list_empty(&dev->send_list))
417 mask |= EPOLLIN | EPOLLRDNORM;
418 if (!list_empty(&dev->recv_list))
419 mask |= EPOLLOUT | EPOLLWRNORM;
420
421 spin_unlock(&dev->msg_lock);
422
423 return mask;
424 }
425
426 static void vduse_dev_reset(struct vduse_dev *dev)
427 {
428 int i;
429 struct vduse_iova_domain *domain = dev->domain;
430
431 /* The coherent mappings are handled in vduse_dev_free_coherent() */
432 if (domain && domain->bounce_map)
433 vduse_domain_reset_bounce_map(domain);
434
435 down_write(&dev->rwsem);
436
437 dev->status = 0;
438 dev->driver_features = 0;
439 dev->generation++;
440 spin_lock(&dev->irq_lock);
441 dev->config_cb.callback = NULL;
442 dev->config_cb.private = NULL;
443 spin_unlock(&dev->irq_lock);
444 flush_work(&dev->inject);
445
446 for (i = 0; i < dev->vq_num; i++) {
447 struct vduse_virtqueue *vq = dev->vqs[i];
448
449 vq->ready = false;
450 vq->desc_addr = 0;
451 vq->driver_addr = 0;
452 vq->device_addr = 0;
453 vq->num = 0;
454 memset(&vq->state, 0, sizeof(vq->state));
455
456 spin_lock(&vq->kick_lock);
457 vq->kicked = false;
458 if (vq->kickfd)
459 eventfd_ctx_put(vq->kickfd);
460 vq->kickfd = NULL;
461 spin_unlock(&vq->kick_lock);
462
463 spin_lock(&vq->irq_lock);
464 vq->cb.callback = NULL;
465 vq->cb.private = NULL;
466 vq->cb.trigger = NULL;
467 spin_unlock(&vq->irq_lock);
468 flush_work(&vq->inject);
469 flush_work(&vq->kick);
470 }
471
472 up_write(&dev->rwsem);
473 }
474
475 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
476 u64 desc_area, u64 driver_area,
477 u64 device_area)
478 {
479 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
480 struct vduse_virtqueue *vq = dev->vqs[idx];
481
482 vq->desc_addr = desc_area;
483 vq->driver_addr = driver_area;
484 vq->device_addr = device_area;
485
486 return 0;
487 }
488
489 static void vduse_vq_kick(struct vduse_virtqueue *vq)
490 {
491 spin_lock(&vq->kick_lock);
492 if (!vq->ready)
493 goto unlock;
494
495 if (vq->kickfd)
496 eventfd_signal(vq->kickfd);
497 else
498 vq->kicked = true;
499 unlock:
500 spin_unlock(&vq->kick_lock);
501 }
502
503 static void vduse_vq_kick_work(struct work_struct *work)
504 {
505 struct vduse_virtqueue *vq = container_of(work,
506 struct vduse_virtqueue, kick);
507
508 vduse_vq_kick(vq);
509 }
510
511 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
512 {
513 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
514 struct vduse_virtqueue *vq = dev->vqs[idx];
515
516 if (!eventfd_signal_allowed()) {
517 schedule_work(&vq->kick);
518 return;
519 }
520 vduse_vq_kick(vq);
521 }
522
523 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
524 struct vdpa_callback *cb)
525 {
526 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
527 struct vduse_virtqueue *vq = dev->vqs[idx];
528
529 spin_lock(&vq->irq_lock);
530 vq->cb.callback = cb->callback;
531 vq->cb.private = cb->private;
532 vq->cb.trigger = cb->trigger;
533 spin_unlock(&vq->irq_lock);
534 }
535
536 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
537 {
538 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
539 struct vduse_virtqueue *vq = dev->vqs[idx];
540
541 vq->num = num;
542 }
543
544 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
545 u16 idx, bool ready)
546 {
547 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
548 struct vduse_virtqueue *vq = dev->vqs[idx];
549
550 vq->ready = ready;
551 }
552
553 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
554 {
555 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
556 struct vduse_virtqueue *vq = dev->vqs[idx];
557
558 return vq->ready;
559 }
560
561 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
562 const struct vdpa_vq_state *state)
563 {
564 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
565 struct vduse_virtqueue *vq = dev->vqs[idx];
566
567 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
568 vq->state.packed.last_avail_counter =
569 state->packed.last_avail_counter;
570 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
571 vq->state.packed.last_used_counter =
572 state->packed.last_used_counter;
573 vq->state.packed.last_used_idx = state->packed.last_used_idx;
574 } else
575 vq->state.split.avail_index = state->split.avail_index;
576
577 return 0;
578 }
579
580 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
581 struct vdpa_vq_state *state)
582 {
583 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
584 struct vduse_virtqueue *vq = dev->vqs[idx];
585
586 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
587 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
588
589 return vduse_dev_get_vq_state_split(dev, vq, &state->split);
590 }
591
592 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
593 {
594 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
595
596 return dev->vq_align;
597 }
598
599 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
600 {
601 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
602
603 return dev->device_features;
604 }
605
606 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
607 {
608 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
609
610 dev->driver_features = features;
611 return 0;
612 }
613
614 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
615 {
616 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
617
618 return dev->driver_features;
619 }
620
621 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
622 struct vdpa_callback *cb)
623 {
624 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
625
626 spin_lock(&dev->irq_lock);
627 dev->config_cb.callback = cb->callback;
628 dev->config_cb.private = cb->private;
629 spin_unlock(&dev->irq_lock);
630 }
631
632 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
633 {
634 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
635 u16 num_max = 0;
636 int i;
637
638 for (i = 0; i < dev->vq_num; i++)
639 if (num_max < dev->vqs[i]->num_max)
640 num_max = dev->vqs[i]->num_max;
641
642 return num_max;
643 }
644
645 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
646 {
647 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
648
649 return dev->device_id;
650 }
651
652 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
653 {
654 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
655
656 return dev->vendor_id;
657 }
658
659 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
660 {
661 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
662
663 return dev->status;
664 }
665
666 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
667 {
668 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
669
670 if (vduse_dev_set_status(dev, status))
671 return;
672
673 dev->status = status;
674 }
675
676 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
677 {
678 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
679
680 return dev->config_size;
681 }
682
683 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
684 void *buf, unsigned int len)
685 {
686 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
687
688 /* Initialize the buffer in case of partial copy. */
689 memset(buf, 0, len);
690
691 if (offset > dev->config_size)
692 return;
693
694 if (len > dev->config_size - offset)
695 len = dev->config_size - offset;
696
697 memcpy(buf, dev->config + offset, len);
698 }
699
700 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
701 const void *buf, unsigned int len)
702 {
703 /* Now we only support read-only configuration space */
704 }
705
706 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
707 {
708 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
709 int ret = vduse_dev_set_status(dev, 0);
710
711 vduse_dev_reset(dev);
712
713 return ret;
714 }
715
716 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
717 {
718 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
719
720 return dev->generation;
721 }
722
723 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx,
724 const struct cpumask *cpu_mask)
725 {
726 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
727
728 if (cpu_mask)
729 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask);
730 else
731 cpumask_setall(&dev->vqs[idx]->irq_affinity);
732
733 return 0;
734 }
735
736 static const struct cpumask *
737 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx)
738 {
739 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
740
741 return &dev->vqs[idx]->irq_affinity;
742 }
743
744 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
745 unsigned int asid,
746 struct vhost_iotlb *iotlb)
747 {
748 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
749 int ret;
750
751 ret = vduse_domain_set_map(dev->domain, iotlb);
752 if (ret)
753 return ret;
754
755 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
756 if (ret) {
757 vduse_domain_clear_map(dev->domain, iotlb);
758 return ret;
759 }
760
761 return 0;
762 }
763
764 static void vduse_vdpa_free(struct vdpa_device *vdpa)
765 {
766 struct vduse_dev *dev = vdpa_to_vduse(vdpa);
767
768 dev->vdev = NULL;
769 }
770
771 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
772 .set_vq_address = vduse_vdpa_set_vq_address,
773 .kick_vq = vduse_vdpa_kick_vq,
774 .set_vq_cb = vduse_vdpa_set_vq_cb,
775 .set_vq_num = vduse_vdpa_set_vq_num,
776 .set_vq_ready = vduse_vdpa_set_vq_ready,
777 .get_vq_ready = vduse_vdpa_get_vq_ready,
778 .set_vq_state = vduse_vdpa_set_vq_state,
779 .get_vq_state = vduse_vdpa_get_vq_state,
780 .get_vq_align = vduse_vdpa_get_vq_align,
781 .get_device_features = vduse_vdpa_get_device_features,
782 .set_driver_features = vduse_vdpa_set_driver_features,
783 .get_driver_features = vduse_vdpa_get_driver_features,
784 .set_config_cb = vduse_vdpa_set_config_cb,
785 .get_vq_num_max = vduse_vdpa_get_vq_num_max,
786 .get_device_id = vduse_vdpa_get_device_id,
787 .get_vendor_id = vduse_vdpa_get_vendor_id,
788 .get_status = vduse_vdpa_get_status,
789 .set_status = vduse_vdpa_set_status,
790 .get_config_size = vduse_vdpa_get_config_size,
791 .get_config = vduse_vdpa_get_config,
792 .set_config = vduse_vdpa_set_config,
793 .get_generation = vduse_vdpa_get_generation,
794 .set_vq_affinity = vduse_vdpa_set_vq_affinity,
795 .get_vq_affinity = vduse_vdpa_get_vq_affinity,
796 .reset = vduse_vdpa_reset,
797 .set_map = vduse_vdpa_set_map,
798 .free = vduse_vdpa_free,
799 };
800
801 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
802 unsigned long offset, size_t size,
803 enum dma_data_direction dir,
804 unsigned long attrs)
805 {
806 struct vduse_dev *vdev = dev_to_vduse(dev);
807 struct vduse_iova_domain *domain = vdev->domain;
808
809 return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
810 }
811
812 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
813 size_t size, enum dma_data_direction dir,
814 unsigned long attrs)
815 {
816 struct vduse_dev *vdev = dev_to_vduse(dev);
817 struct vduse_iova_domain *domain = vdev->domain;
818
819 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
820 }
821
822 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
823 dma_addr_t *dma_addr, gfp_t flag,
824 unsigned long attrs)
825 {
826 struct vduse_dev *vdev = dev_to_vduse(dev);
827 struct vduse_iova_domain *domain = vdev->domain;
828 unsigned long iova;
829 void *addr;
830
831 *dma_addr = DMA_MAPPING_ERROR;
832 addr = vduse_domain_alloc_coherent(domain, size,
833 (dma_addr_t *)&iova, flag, attrs);
834 if (!addr)
835 return NULL;
836
837 *dma_addr = (dma_addr_t)iova;
838
839 return addr;
840 }
841
842 static void vduse_dev_free_coherent(struct device *dev, size_t size,
843 void *vaddr, dma_addr_t dma_addr,
844 unsigned long attrs)
845 {
846 struct vduse_dev *vdev = dev_to_vduse(dev);
847 struct vduse_iova_domain *domain = vdev->domain;
848
849 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
850 }
851
852 static size_t vduse_dev_max_mapping_size(struct device *dev)
853 {
854 struct vduse_dev *vdev = dev_to_vduse(dev);
855 struct vduse_iova_domain *domain = vdev->domain;
856
857 return domain->bounce_size;
858 }
859
860 static const struct dma_map_ops vduse_dev_dma_ops = {
861 .map_page = vduse_dev_map_page,
862 .unmap_page = vduse_dev_unmap_page,
863 .alloc = vduse_dev_alloc_coherent,
864 .free = vduse_dev_free_coherent,
865 .max_mapping_size = vduse_dev_max_mapping_size,
866 };
867
868 static unsigned int perm_to_file_flags(u8 perm)
869 {
870 unsigned int flags = 0;
871
872 switch (perm) {
873 case VDUSE_ACCESS_WO:
874 flags |= O_WRONLY;
875 break;
876 case VDUSE_ACCESS_RO:
877 flags |= O_RDONLY;
878 break;
879 case VDUSE_ACCESS_RW:
880 flags |= O_RDWR;
881 break;
882 default:
883 WARN(1, "invalidate vhost IOTLB permission\n");
884 break;
885 }
886
887 return flags;
888 }
889
890 static int vduse_kickfd_setup(struct vduse_dev *dev,
891 struct vduse_vq_eventfd *eventfd)
892 {
893 struct eventfd_ctx *ctx = NULL;
894 struct vduse_virtqueue *vq;
895 u32 index;
896
897 if (eventfd->index >= dev->vq_num)
898 return -EINVAL;
899
900 index = array_index_nospec(eventfd->index, dev->vq_num);
901 vq = dev->vqs[index];
902 if (eventfd->fd >= 0) {
903 ctx = eventfd_ctx_fdget(eventfd->fd);
904 if (IS_ERR(ctx))
905 return PTR_ERR(ctx);
906 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
907 return 0;
908
909 spin_lock(&vq->kick_lock);
910 if (vq->kickfd)
911 eventfd_ctx_put(vq->kickfd);
912 vq->kickfd = ctx;
913 if (vq->ready && vq->kicked && vq->kickfd) {
914 eventfd_signal(vq->kickfd);
915 vq->kicked = false;
916 }
917 spin_unlock(&vq->kick_lock);
918
919 return 0;
920 }
921
922 static bool vduse_dev_is_ready(struct vduse_dev *dev)
923 {
924 int i;
925
926 for (i = 0; i < dev->vq_num; i++)
927 if (!dev->vqs[i]->num_max)
928 return false;
929
930 return true;
931 }
932
933 static void vduse_dev_irq_inject(struct work_struct *work)
934 {
935 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
936
937 spin_lock_bh(&dev->irq_lock);
938 if (dev->config_cb.callback)
939 dev->config_cb.callback(dev->config_cb.private);
940 spin_unlock_bh(&dev->irq_lock);
941 }
942
943 static void vduse_vq_irq_inject(struct work_struct *work)
944 {
945 struct vduse_virtqueue *vq = container_of(work,
946 struct vduse_virtqueue, inject);
947
948 spin_lock_bh(&vq->irq_lock);
949 if (vq->ready && vq->cb.callback)
950 vq->cb.callback(vq->cb.private);
951 spin_unlock_bh(&vq->irq_lock);
952 }
953
954 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq)
955 {
956 bool signal = false;
957
958 if (!vq->cb.trigger)
959 return false;
960
961 spin_lock_irq(&vq->irq_lock);
962 if (vq->ready && vq->cb.trigger) {
963 eventfd_signal(vq->cb.trigger);
964 signal = true;
965 }
966 spin_unlock_irq(&vq->irq_lock);
967
968 return signal;
969 }
970
971 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
972 struct work_struct *irq_work,
973 int irq_effective_cpu)
974 {
975 int ret = -EINVAL;
976
977 down_read(&dev->rwsem);
978 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
979 goto unlock;
980
981 ret = 0;
982 if (irq_effective_cpu == IRQ_UNBOUND)
983 queue_work(vduse_irq_wq, irq_work);
984 else
985 queue_work_on(irq_effective_cpu,
986 vduse_irq_bound_wq, irq_work);
987 unlock:
988 up_read(&dev->rwsem);
989
990 return ret;
991 }
992
993 static int vduse_dev_dereg_umem(struct vduse_dev *dev,
994 u64 iova, u64 size)
995 {
996 int ret;
997
998 mutex_lock(&dev->mem_lock);
999 ret = -ENOENT;
1000 if (!dev->umem)
1001 goto unlock;
1002
1003 ret = -EINVAL;
1004 if (!dev->domain)
1005 goto unlock;
1006
1007 if (dev->umem->iova != iova || size != dev->domain->bounce_size)
1008 goto unlock;
1009
1010 vduse_domain_remove_user_bounce_pages(dev->domain);
1011 unpin_user_pages_dirty_lock(dev->umem->pages,
1012 dev->umem->npages, true);
1013 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm);
1014 mmdrop(dev->umem->mm);
1015 vfree(dev->umem->pages);
1016 kfree(dev->umem);
1017 dev->umem = NULL;
1018 ret = 0;
1019 unlock:
1020 mutex_unlock(&dev->mem_lock);
1021 return ret;
1022 }
1023
1024 static int vduse_dev_reg_umem(struct vduse_dev *dev,
1025 u64 iova, u64 uaddr, u64 size)
1026 {
1027 struct page **page_list = NULL;
1028 struct vduse_umem *umem = NULL;
1029 long pinned = 0;
1030 unsigned long npages, lock_limit;
1031 int ret;
1032
1033 if (!dev->domain || !dev->domain->bounce_map ||
1034 size != dev->domain->bounce_size ||
1035 iova != 0 || uaddr & ~PAGE_MASK)
1036 return -EINVAL;
1037
1038 mutex_lock(&dev->mem_lock);
1039 ret = -EEXIST;
1040 if (dev->umem)
1041 goto unlock;
1042
1043 ret = -ENOMEM;
1044 npages = size >> PAGE_SHIFT;
1045 page_list = __vmalloc(array_size(npages, sizeof(struct page *)),
1046 GFP_KERNEL_ACCOUNT);
1047 umem = kzalloc(sizeof(*umem), GFP_KERNEL);
1048 if (!page_list || !umem)
1049 goto unlock;
1050
1051 mmap_read_lock(current->mm);
1052
1053 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK));
1054 if (npages + atomic64_read(&current->mm->pinned_vm) > lock_limit)
1055 goto out;
1056
1057 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE,
1058 page_list);
1059 if (pinned != npages) {
1060 ret = pinned < 0 ? pinned : -ENOMEM;
1061 goto out;
1062 }
1063
1064 ret = vduse_domain_add_user_bounce_pages(dev->domain,
1065 page_list, pinned);
1066 if (ret)
1067 goto out;
1068
1069 atomic64_add(npages, &current->mm->pinned_vm);
1070
1071 umem->pages = page_list;
1072 umem->npages = pinned;
1073 umem->iova = iova;
1074 umem->mm = current->mm;
1075 mmgrab(current->mm);
1076
1077 dev->umem = umem;
1078 out:
1079 if (ret && pinned > 0)
1080 unpin_user_pages(page_list, pinned);
1081
1082 mmap_read_unlock(current->mm);
1083 unlock:
1084 if (ret) {
1085 vfree(page_list);
1086 kfree(umem);
1087 }
1088 mutex_unlock(&dev->mem_lock);
1089 return ret;
1090 }
1091
1092 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
1093 {
1094 int curr_cpu = vq->irq_effective_cpu;
1095
1096 while (true) {
1097 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity);
1098 if (cpu_online(curr_cpu))
1099 break;
1100
1101 if (curr_cpu >= nr_cpu_ids)
1102 curr_cpu = IRQ_UNBOUND;
1103 }
1104
1105 vq->irq_effective_cpu = curr_cpu;
1106 }
1107
1108 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
1109 unsigned long arg)
1110 {
1111 struct vduse_dev *dev = file->private_data;
1112 void __user *argp = (void __user *)arg;
1113 int ret;
1114
1115 if (unlikely(dev->broken))
1116 return -EPERM;
1117
1118 switch (cmd) {
1119 case VDUSE_IOTLB_GET_FD: {
1120 struct vduse_iotlb_entry entry;
1121 struct vhost_iotlb_map *map;
1122 struct vdpa_map_file *map_file;
1123 struct file *f = NULL;
1124
1125 ret = -EFAULT;
1126 if (copy_from_user(&entry, argp, sizeof(entry)))
1127 break;
1128
1129 ret = -EINVAL;
1130 if (entry.start > entry.last)
1131 break;
1132
1133 mutex_lock(&dev->domain_lock);
1134 if (!dev->domain) {
1135 mutex_unlock(&dev->domain_lock);
1136 break;
1137 }
1138 spin_lock(&dev->domain->iotlb_lock);
1139 map = vhost_iotlb_itree_first(dev->domain->iotlb,
1140 entry.start, entry.last);
1141 if (map) {
1142 map_file = (struct vdpa_map_file *)map->opaque;
1143 f = get_file(map_file->file);
1144 entry.offset = map_file->offset;
1145 entry.start = map->start;
1146 entry.last = map->last;
1147 entry.perm = map->perm;
1148 }
1149 spin_unlock(&dev->domain->iotlb_lock);
1150 mutex_unlock(&dev->domain_lock);
1151 ret = -EINVAL;
1152 if (!f)
1153 break;
1154
1155 ret = -EFAULT;
1156 if (copy_to_user(argp, &entry, sizeof(entry))) {
1157 fput(f);
1158 break;
1159 }
1160 ret = receive_fd(f, perm_to_file_flags(entry.perm));
1161 fput(f);
1162 break;
1163 }
1164 case VDUSE_DEV_GET_FEATURES:
1165 /*
1166 * Just mirror what driver wrote here.
1167 * The driver is expected to check FEATURE_OK later.
1168 */
1169 ret = put_user(dev->driver_features, (u64 __user *)argp);
1170 break;
1171 case VDUSE_DEV_SET_CONFIG: {
1172 struct vduse_config_data config;
1173 unsigned long size = offsetof(struct vduse_config_data,
1174 buffer);
1175
1176 ret = -EFAULT;
1177 if (copy_from_user(&config, argp, size))
1178 break;
1179
1180 ret = -EINVAL;
1181 if (config.offset > dev->config_size ||
1182 config.length == 0 ||
1183 config.length > dev->config_size - config.offset)
1184 break;
1185
1186 ret = -EFAULT;
1187 if (copy_from_user(dev->config + config.offset, argp + size,
1188 config.length))
1189 break;
1190
1191 ret = 0;
1192 break;
1193 }
1194 case VDUSE_DEV_INJECT_CONFIG_IRQ:
1195 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND);
1196 break;
1197 case VDUSE_VQ_SETUP: {
1198 struct vduse_vq_config config;
1199 u32 index;
1200
1201 ret = -EFAULT;
1202 if (copy_from_user(&config, argp, sizeof(config)))
1203 break;
1204
1205 ret = -EINVAL;
1206 if (config.index >= dev->vq_num)
1207 break;
1208
1209 if (!is_mem_zero((const char *)config.reserved,
1210 sizeof(config.reserved)))
1211 break;
1212
1213 index = array_index_nospec(config.index, dev->vq_num);
1214 dev->vqs[index]->num_max = config.max_size;
1215 ret = 0;
1216 break;
1217 }
1218 case VDUSE_VQ_GET_INFO: {
1219 struct vduse_vq_info vq_info;
1220 struct vduse_virtqueue *vq;
1221 u32 index;
1222
1223 ret = -EFAULT;
1224 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1225 break;
1226
1227 ret = -EINVAL;
1228 if (vq_info.index >= dev->vq_num)
1229 break;
1230
1231 index = array_index_nospec(vq_info.index, dev->vq_num);
1232 vq = dev->vqs[index];
1233 vq_info.desc_addr = vq->desc_addr;
1234 vq_info.driver_addr = vq->driver_addr;
1235 vq_info.device_addr = vq->device_addr;
1236 vq_info.num = vq->num;
1237
1238 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1239 vq_info.packed.last_avail_counter =
1240 vq->state.packed.last_avail_counter;
1241 vq_info.packed.last_avail_idx =
1242 vq->state.packed.last_avail_idx;
1243 vq_info.packed.last_used_counter =
1244 vq->state.packed.last_used_counter;
1245 vq_info.packed.last_used_idx =
1246 vq->state.packed.last_used_idx;
1247 } else
1248 vq_info.split.avail_index =
1249 vq->state.split.avail_index;
1250
1251 vq_info.ready = vq->ready;
1252
1253 ret = -EFAULT;
1254 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1255 break;
1256
1257 ret = 0;
1258 break;
1259 }
1260 case VDUSE_VQ_SETUP_KICKFD: {
1261 struct vduse_vq_eventfd eventfd;
1262
1263 ret = -EFAULT;
1264 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1265 break;
1266
1267 ret = vduse_kickfd_setup(dev, &eventfd);
1268 break;
1269 }
1270 case VDUSE_VQ_INJECT_IRQ: {
1271 u32 index;
1272
1273 ret = -EFAULT;
1274 if (get_user(index, (u32 __user *)argp))
1275 break;
1276
1277 ret = -EINVAL;
1278 if (index >= dev->vq_num)
1279 break;
1280
1281 ret = 0;
1282 index = array_index_nospec(index, dev->vq_num);
1283 if (!vduse_vq_signal_irqfd(dev->vqs[index])) {
1284 vduse_vq_update_effective_cpu(dev->vqs[index]);
1285 ret = vduse_dev_queue_irq_work(dev,
1286 &dev->vqs[index]->inject,
1287 dev->vqs[index]->irq_effective_cpu);
1288 }
1289 break;
1290 }
1291 case VDUSE_IOTLB_REG_UMEM: {
1292 struct vduse_iova_umem umem;
1293
1294 ret = -EFAULT;
1295 if (copy_from_user(&umem, argp, sizeof(umem)))
1296 break;
1297
1298 ret = -EINVAL;
1299 if (!is_mem_zero((const char *)umem.reserved,
1300 sizeof(umem.reserved)))
1301 break;
1302
1303 mutex_lock(&dev->domain_lock);
1304 ret = vduse_dev_reg_umem(dev, umem.iova,
1305 umem.uaddr, umem.size);
1306 mutex_unlock(&dev->domain_lock);
1307 break;
1308 }
1309 case VDUSE_IOTLB_DEREG_UMEM: {
1310 struct vduse_iova_umem umem;
1311
1312 ret = -EFAULT;
1313 if (copy_from_user(&umem, argp, sizeof(umem)))
1314 break;
1315
1316 ret = -EINVAL;
1317 if (!is_mem_zero((const char *)umem.reserved,
1318 sizeof(umem.reserved)))
1319 break;
1320 mutex_lock(&dev->domain_lock);
1321 ret = vduse_dev_dereg_umem(dev, umem.iova,
1322 umem.size);
1323 mutex_unlock(&dev->domain_lock);
1324 break;
1325 }
1326 case VDUSE_IOTLB_GET_INFO: {
1327 struct vduse_iova_info info;
1328 struct vhost_iotlb_map *map;
1329
1330 ret = -EFAULT;
1331 if (copy_from_user(&info, argp, sizeof(info)))
1332 break;
1333
1334 ret = -EINVAL;
1335 if (info.start > info.last)
1336 break;
1337
1338 if (!is_mem_zero((const char *)info.reserved,
1339 sizeof(info.reserved)))
1340 break;
1341
1342 mutex_lock(&dev->domain_lock);
1343 if (!dev->domain) {
1344 mutex_unlock(&dev->domain_lock);
1345 break;
1346 }
1347 spin_lock(&dev->domain->iotlb_lock);
1348 map = vhost_iotlb_itree_first(dev->domain->iotlb,
1349 info.start, info.last);
1350 if (map) {
1351 info.start = map->start;
1352 info.last = map->last;
1353 info.capability = 0;
1354 if (dev->domain->bounce_map && map->start == 0 &&
1355 map->last == dev->domain->bounce_size - 1)
1356 info.capability |= VDUSE_IOVA_CAP_UMEM;
1357 }
1358 spin_unlock(&dev->domain->iotlb_lock);
1359 mutex_unlock(&dev->domain_lock);
1360 if (!map)
1361 break;
1362
1363 ret = -EFAULT;
1364 if (copy_to_user(argp, &info, sizeof(info)))
1365 break;
1366
1367 ret = 0;
1368 break;
1369 }
1370 default:
1371 ret = -ENOIOCTLCMD;
1372 break;
1373 }
1374
1375 return ret;
1376 }
1377
1378 static int vduse_dev_release(struct inode *inode, struct file *file)
1379 {
1380 struct vduse_dev *dev = file->private_data;
1381
1382 mutex_lock(&dev->domain_lock);
1383 if (dev->domain)
1384 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size);
1385 mutex_unlock(&dev->domain_lock);
1386 spin_lock(&dev->msg_lock);
1387 /* Make sure the inflight messages can processed after reconncection */
1388 list_splice_init(&dev->recv_list, &dev->send_list);
1389 spin_unlock(&dev->msg_lock);
1390 dev->connected = false;
1391
1392 return 0;
1393 }
1394
1395 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1396 {
1397 struct vduse_dev *dev;
1398
1399 mutex_lock(&vduse_lock);
1400 dev = idr_find(&vduse_idr, minor);
1401 mutex_unlock(&vduse_lock);
1402
1403 return dev;
1404 }
1405
1406 static int vduse_dev_open(struct inode *inode, struct file *file)
1407 {
1408 int ret;
1409 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1410
1411 if (!dev)
1412 return -ENODEV;
1413
1414 ret = -EBUSY;
1415 mutex_lock(&dev->lock);
1416 if (dev->connected)
1417 goto unlock;
1418
1419 ret = 0;
1420 dev->connected = true;
1421 file->private_data = dev;
1422 unlock:
1423 mutex_unlock(&dev->lock);
1424
1425 return ret;
1426 }
1427
1428 static const struct file_operations vduse_dev_fops = {
1429 .owner = THIS_MODULE,
1430 .open = vduse_dev_open,
1431 .release = vduse_dev_release,
1432 .read_iter = vduse_dev_read_iter,
1433 .write_iter = vduse_dev_write_iter,
1434 .poll = vduse_dev_poll,
1435 .unlocked_ioctl = vduse_dev_ioctl,
1436 .compat_ioctl = compat_ptr_ioctl,
1437 .llseek = noop_llseek,
1438 };
1439
1440 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
1441 {
1442 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity));
1443 }
1444
1445 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq,
1446 const char *buf, size_t count)
1447 {
1448 cpumask_var_t new_value;
1449 int ret;
1450
1451 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL))
1452 return -ENOMEM;
1453
1454 ret = cpumask_parse(buf, new_value);
1455 if (ret)
1456 goto free_mask;
1457
1458 ret = -EINVAL;
1459 if (!cpumask_intersects(new_value, cpu_online_mask))
1460 goto free_mask;
1461
1462 cpumask_copy(&vq->irq_affinity, new_value);
1463 ret = count;
1464 free_mask:
1465 free_cpumask_var(new_value);
1466 return ret;
1467 }
1468
1469 struct vq_sysfs_entry {
1470 struct attribute attr;
1471 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf);
1472 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf,
1473 size_t count);
1474 };
1475
1476 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity);
1477
1478 static struct attribute *vq_attrs[] = {
1479 &irq_cb_affinity_attr.attr,
1480 NULL,
1481 };
1482 ATTRIBUTE_GROUPS(vq);
1483
1484 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr,
1485 char *buf)
1486 {
1487 struct vduse_virtqueue *vq = container_of(kobj,
1488 struct vduse_virtqueue, kobj);
1489 struct vq_sysfs_entry *entry = container_of(attr,
1490 struct vq_sysfs_entry, attr);
1491
1492 if (!entry->show)
1493 return -EIO;
1494
1495 return entry->show(vq, buf);
1496 }
1497
1498 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr,
1499 const char *buf, size_t count)
1500 {
1501 struct vduse_virtqueue *vq = container_of(kobj,
1502 struct vduse_virtqueue, kobj);
1503 struct vq_sysfs_entry *entry = container_of(attr,
1504 struct vq_sysfs_entry, attr);
1505
1506 if (!entry->store)
1507 return -EIO;
1508
1509 return entry->store(vq, buf, count);
1510 }
1511
1512 static const struct sysfs_ops vq_sysfs_ops = {
1513 .show = vq_attr_show,
1514 .store = vq_attr_store,
1515 };
1516
1517 static void vq_release(struct kobject *kobj)
1518 {
1519 struct vduse_virtqueue *vq = container_of(kobj,
1520 struct vduse_virtqueue, kobj);
1521 kfree(vq);
1522 }
1523
1524 static const struct kobj_type vq_type = {
1525 .release = vq_release,
1526 .sysfs_ops = &vq_sysfs_ops,
1527 .default_groups = vq_groups,
1528 };
1529
1530 static char *vduse_devnode(const struct device *dev, umode_t *mode)
1531 {
1532 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1533 }
1534
1535 static const struct class vduse_class = {
1536 .name = "vduse",
1537 .devnode = vduse_devnode,
1538 };
1539
1540 static void vduse_dev_deinit_vqs(struct vduse_dev *dev)
1541 {
1542 int i;
1543
1544 if (!dev->vqs)
1545 return;
1546
1547 for (i = 0; i < dev->vq_num; i++)
1548 kobject_put(&dev->vqs[i]->kobj);
1549 kfree(dev->vqs);
1550 }
1551
1552 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num)
1553 {
1554 int ret, i;
1555
1556 dev->vq_align = vq_align;
1557 dev->vq_num = vq_num;
1558 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1559 if (!dev->vqs)
1560 return -ENOMEM;
1561
1562 for (i = 0; i < vq_num; i++) {
1563 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL);
1564 if (!dev->vqs[i]) {
1565 ret = -ENOMEM;
1566 goto err;
1567 }
1568
1569 dev->vqs[i]->index = i;
1570 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND;
1571 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject);
1572 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work);
1573 spin_lock_init(&dev->vqs[i]->kick_lock);
1574 spin_lock_init(&dev->vqs[i]->irq_lock);
1575 cpumask_setall(&dev->vqs[i]->irq_affinity);
1576
1577 kobject_init(&dev->vqs[i]->kobj, &vq_type);
1578 ret = kobject_add(&dev->vqs[i]->kobj,
1579 &dev->dev->kobj, "vq%d", i);
1580 if (ret) {
1581 kfree(dev->vqs[i]);
1582 goto err;
1583 }
1584 }
1585
1586 return 0;
1587 err:
1588 while (i--)
1589 kobject_put(&dev->vqs[i]->kobj);
1590 kfree(dev->vqs);
1591 dev->vqs = NULL;
1592 return ret;
1593 }
1594
1595 static struct vduse_dev *vduse_dev_create(void)
1596 {
1597 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1598
1599 if (!dev)
1600 return NULL;
1601
1602 mutex_init(&dev->lock);
1603 mutex_init(&dev->mem_lock);
1604 mutex_init(&dev->domain_lock);
1605 spin_lock_init(&dev->msg_lock);
1606 INIT_LIST_HEAD(&dev->send_list);
1607 INIT_LIST_HEAD(&dev->recv_list);
1608 spin_lock_init(&dev->irq_lock);
1609 init_rwsem(&dev->rwsem);
1610
1611 INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1612 init_waitqueue_head(&dev->waitq);
1613
1614 return dev;
1615 }
1616
1617 static void vduse_dev_destroy(struct vduse_dev *dev)
1618 {
1619 kfree(dev);
1620 }
1621
1622 static struct vduse_dev *vduse_find_dev(const char *name)
1623 {
1624 struct vduse_dev *dev;
1625 int id;
1626
1627 idr_for_each_entry(&vduse_idr, dev, id)
1628 if (!strcmp(dev->name, name))
1629 return dev;
1630
1631 return NULL;
1632 }
1633
1634 static int vduse_destroy_dev(char *name)
1635 {
1636 struct vduse_dev *dev = vduse_find_dev(name);
1637
1638 if (!dev)
1639 return -EINVAL;
1640
1641 mutex_lock(&dev->lock);
1642 if (dev->vdev || dev->connected) {
1643 mutex_unlock(&dev->lock);
1644 return -EBUSY;
1645 }
1646 dev->connected = true;
1647 mutex_unlock(&dev->lock);
1648
1649 vduse_dev_reset(dev);
1650 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1651 idr_remove(&vduse_idr, dev->minor);
1652 kvfree(dev->config);
1653 vduse_dev_deinit_vqs(dev);
1654 if (dev->domain)
1655 vduse_domain_destroy(dev->domain);
1656 kfree(dev->name);
1657 vduse_dev_destroy(dev);
1658 module_put(THIS_MODULE);
1659
1660 return 0;
1661 }
1662
1663 static bool device_is_allowed(u32 device_id)
1664 {
1665 int i;
1666
1667 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1668 if (allowed_device_id[i] == device_id)
1669 return true;
1670
1671 return false;
1672 }
1673
1674 static bool features_is_valid(u64 features)
1675 {
1676 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1677 return false;
1678
1679 /* Now we only support read-only configuration space */
1680 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1681 return false;
1682
1683 return true;
1684 }
1685
1686 static bool vduse_validate_config(struct vduse_dev_config *config)
1687 {
1688 if (!is_mem_zero((const char *)config->reserved,
1689 sizeof(config->reserved)))
1690 return false;
1691
1692 if (config->vq_align > PAGE_SIZE)
1693 return false;
1694
1695 if (config->config_size > PAGE_SIZE)
1696 return false;
1697
1698 if (config->vq_num > 0xffff)
1699 return false;
1700
1701 if (!config->name[0])
1702 return false;
1703
1704 if (!device_is_allowed(config->device_id))
1705 return false;
1706
1707 if (!features_is_valid(config->features))
1708 return false;
1709
1710 return true;
1711 }
1712
1713 static ssize_t msg_timeout_show(struct device *device,
1714 struct device_attribute *attr, char *buf)
1715 {
1716 struct vduse_dev *dev = dev_get_drvdata(device);
1717
1718 return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1719 }
1720
1721 static ssize_t msg_timeout_store(struct device *device,
1722 struct device_attribute *attr,
1723 const char *buf, size_t count)
1724 {
1725 struct vduse_dev *dev = dev_get_drvdata(device);
1726 int ret;
1727
1728 ret = kstrtouint(buf, 10, &dev->msg_timeout);
1729 if (ret < 0)
1730 return ret;
1731
1732 return count;
1733 }
1734
1735 static DEVICE_ATTR_RW(msg_timeout);
1736
1737 static ssize_t bounce_size_show(struct device *device,
1738 struct device_attribute *attr, char *buf)
1739 {
1740 struct vduse_dev *dev = dev_get_drvdata(device);
1741
1742 return sysfs_emit(buf, "%u\n", dev->bounce_size);
1743 }
1744
1745 static ssize_t bounce_size_store(struct device *device,
1746 struct device_attribute *attr,
1747 const char *buf, size_t count)
1748 {
1749 struct vduse_dev *dev = dev_get_drvdata(device);
1750 unsigned int bounce_size;
1751 int ret;
1752
1753 ret = -EPERM;
1754 mutex_lock(&dev->domain_lock);
1755 if (dev->domain)
1756 goto unlock;
1757
1758 ret = kstrtouint(buf, 10, &bounce_size);
1759 if (ret < 0)
1760 goto unlock;
1761
1762 ret = -EINVAL;
1763 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE ||
1764 bounce_size < VDUSE_MIN_BOUNCE_SIZE)
1765 goto unlock;
1766
1767 dev->bounce_size = bounce_size & PAGE_MASK;
1768 ret = count;
1769 unlock:
1770 mutex_unlock(&dev->domain_lock);
1771 return ret;
1772 }
1773
1774 static DEVICE_ATTR_RW(bounce_size);
1775
1776 static struct attribute *vduse_dev_attrs[] = {
1777 &dev_attr_msg_timeout.attr,
1778 &dev_attr_bounce_size.attr,
1779 NULL
1780 };
1781
1782 ATTRIBUTE_GROUPS(vduse_dev);
1783
1784 static int vduse_create_dev(struct vduse_dev_config *config,
1785 void *config_buf, u64 api_version)
1786 {
1787 int ret;
1788 struct vduse_dev *dev;
1789
1790 ret = -EEXIST;
1791 if (vduse_find_dev(config->name))
1792 goto err;
1793
1794 ret = -ENOMEM;
1795 dev = vduse_dev_create();
1796 if (!dev)
1797 goto err;
1798
1799 dev->api_version = api_version;
1800 dev->device_features = config->features;
1801 dev->device_id = config->device_id;
1802 dev->vendor_id = config->vendor_id;
1803 dev->name = kstrdup(config->name, GFP_KERNEL);
1804 if (!dev->name)
1805 goto err_str;
1806
1807 dev->bounce_size = VDUSE_BOUNCE_SIZE;
1808 dev->config = config_buf;
1809 dev->config_size = config->config_size;
1810
1811 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1812 if (ret < 0)
1813 goto err_idr;
1814
1815 dev->minor = ret;
1816 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1817 dev->dev = device_create_with_groups(&vduse_class, NULL,
1818 MKDEV(MAJOR(vduse_major), dev->minor),
1819 dev, vduse_dev_groups, "%s", config->name);
1820 if (IS_ERR(dev->dev)) {
1821 ret = PTR_ERR(dev->dev);
1822 goto err_dev;
1823 }
1824
1825 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
1826 if (ret)
1827 goto err_vqs;
1828
1829 __module_get(THIS_MODULE);
1830
1831 return 0;
1832 err_vqs:
1833 device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1834 err_dev:
1835 idr_remove(&vduse_idr, dev->minor);
1836 err_idr:
1837 kfree(dev->name);
1838 err_str:
1839 vduse_dev_destroy(dev);
1840 err:
1841 return ret;
1842 }
1843
1844 static long vduse_ioctl(struct file *file, unsigned int cmd,
1845 unsigned long arg)
1846 {
1847 int ret;
1848 void __user *argp = (void __user *)arg;
1849 struct vduse_control *control = file->private_data;
1850
1851 mutex_lock(&vduse_lock);
1852 switch (cmd) {
1853 case VDUSE_GET_API_VERSION:
1854 ret = put_user(control->api_version, (u64 __user *)argp);
1855 break;
1856 case VDUSE_SET_API_VERSION: {
1857 u64 api_version;
1858
1859 ret = -EFAULT;
1860 if (get_user(api_version, (u64 __user *)argp))
1861 break;
1862
1863 ret = -EINVAL;
1864 if (api_version > VDUSE_API_VERSION)
1865 break;
1866
1867 ret = 0;
1868 control->api_version = api_version;
1869 break;
1870 }
1871 case VDUSE_CREATE_DEV: {
1872 struct vduse_dev_config config;
1873 unsigned long size = offsetof(struct vduse_dev_config, config);
1874 void *buf;
1875
1876 ret = -EFAULT;
1877 if (copy_from_user(&config, argp, size))
1878 break;
1879
1880 ret = -EINVAL;
1881 if (vduse_validate_config(&config) == false)
1882 break;
1883
1884 buf = vmemdup_user(argp + size, config.config_size);
1885 if (IS_ERR(buf)) {
1886 ret = PTR_ERR(buf);
1887 break;
1888 }
1889 config.name[VDUSE_NAME_MAX - 1] = '\0';
1890 ret = vduse_create_dev(&config, buf, control->api_version);
1891 if (ret)
1892 kvfree(buf);
1893 break;
1894 }
1895 case VDUSE_DESTROY_DEV: {
1896 char name[VDUSE_NAME_MAX];
1897
1898 ret = -EFAULT;
1899 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1900 break;
1901
1902 name[VDUSE_NAME_MAX - 1] = '\0';
1903 ret = vduse_destroy_dev(name);
1904 break;
1905 }
1906 default:
1907 ret = -EINVAL;
1908 break;
1909 }
1910 mutex_unlock(&vduse_lock);
1911
1912 return ret;
1913 }
1914
1915 static int vduse_release(struct inode *inode, struct file *file)
1916 {
1917 struct vduse_control *control = file->private_data;
1918
1919 kfree(control);
1920 return 0;
1921 }
1922
1923 static int vduse_open(struct inode *inode, struct file *file)
1924 {
1925 struct vduse_control *control;
1926
1927 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1928 if (!control)
1929 return -ENOMEM;
1930
1931 control->api_version = VDUSE_API_VERSION;
1932 file->private_data = control;
1933
1934 return 0;
1935 }
1936
1937 static const struct file_operations vduse_ctrl_fops = {
1938 .owner = THIS_MODULE,
1939 .open = vduse_open,
1940 .release = vduse_release,
1941 .unlocked_ioctl = vduse_ioctl,
1942 .compat_ioctl = compat_ptr_ioctl,
1943 .llseek = noop_llseek,
1944 };
1945
1946 struct vduse_mgmt_dev {
1947 struct vdpa_mgmt_dev mgmt_dev;
1948 struct device dev;
1949 };
1950
1951 static struct vduse_mgmt_dev *vduse_mgmt;
1952
1953 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1954 {
1955 struct vduse_vdpa *vdev;
1956 int ret;
1957
1958 if (dev->vdev)
1959 return -EEXIST;
1960
1961 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1962 &vduse_vdpa_config_ops, 1, 1, name, true);
1963 if (IS_ERR(vdev))
1964 return PTR_ERR(vdev);
1965
1966 dev->vdev = vdev;
1967 vdev->dev = dev;
1968 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1969 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1970 if (ret) {
1971 put_device(&vdev->vdpa.dev);
1972 return ret;
1973 }
1974 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1975 vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1976 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev;
1977
1978 return 0;
1979 }
1980
1981 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1982 const struct vdpa_dev_set_config *config)
1983 {
1984 struct vduse_dev *dev;
1985 int ret;
1986
1987 mutex_lock(&vduse_lock);
1988 dev = vduse_find_dev(name);
1989 if (!dev || !vduse_dev_is_ready(dev)) {
1990 mutex_unlock(&vduse_lock);
1991 return -EINVAL;
1992 }
1993 ret = vduse_dev_init_vdpa(dev, name);
1994 mutex_unlock(&vduse_lock);
1995 if (ret)
1996 return ret;
1997
1998 mutex_lock(&dev->domain_lock);
1999 if (!dev->domain)
2000 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
2001 dev->bounce_size);
2002 mutex_unlock(&dev->domain_lock);
2003 if (!dev->domain) {
2004 put_device(&dev->vdev->vdpa.dev);
2005 return -ENOMEM;
2006 }
2007
2008 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
2009 if (ret) {
2010 put_device(&dev->vdev->vdpa.dev);
2011 mutex_lock(&dev->domain_lock);
2012 vduse_domain_destroy(dev->domain);
2013 dev->domain = NULL;
2014 mutex_unlock(&dev->domain_lock);
2015 return ret;
2016 }
2017
2018 return 0;
2019 }
2020
2021 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
2022 {
2023 _vdpa_unregister_device(dev);
2024 }
2025
2026 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
2027 .dev_add = vdpa_dev_add,
2028 .dev_del = vdpa_dev_del,
2029 };
2030
2031 static struct virtio_device_id id_table[] = {
2032 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
2033 { 0 },
2034 };
2035
2036 static void vduse_mgmtdev_release(struct device *dev)
2037 {
2038 struct vduse_mgmt_dev *mgmt_dev;
2039
2040 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev);
2041 kfree(mgmt_dev);
2042 }
2043
2044 static int vduse_mgmtdev_init(void)
2045 {
2046 int ret;
2047
2048 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL);
2049 if (!vduse_mgmt)
2050 return -ENOMEM;
2051
2052 ret = dev_set_name(&vduse_mgmt->dev, "vduse");
2053 if (ret) {
2054 kfree(vduse_mgmt);
2055 return ret;
2056 }
2057
2058 vduse_mgmt->dev.release = vduse_mgmtdev_release;
2059
2060 ret = device_register(&vduse_mgmt->dev);
2061 if (ret)
2062 goto dev_reg_err;
2063
2064 vduse_mgmt->mgmt_dev.id_table = id_table;
2065 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops;
2066 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev;
2067 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev);
2068 if (ret)
2069 device_unregister(&vduse_mgmt->dev);
2070
2071 return ret;
2072
2073 dev_reg_err:
2074 put_device(&vduse_mgmt->dev);
2075 return ret;
2076 }
2077
2078 static void vduse_mgmtdev_exit(void)
2079 {
2080 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev);
2081 device_unregister(&vduse_mgmt->dev);
2082 }
2083
2084 static int vduse_init(void)
2085 {
2086 int ret;
2087 struct device *dev;
2088
2089 ret = class_register(&vduse_class);
2090 if (ret)
2091 return ret;
2092
2093 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
2094 if (ret)
2095 goto err_chardev_region;
2096
2097 /* /dev/vduse/control */
2098 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
2099 vduse_ctrl_cdev.owner = THIS_MODULE;
2100 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
2101 if (ret)
2102 goto err_ctrl_cdev;
2103
2104 dev = device_create(&vduse_class, NULL, vduse_major, NULL, "control");
2105 if (IS_ERR(dev)) {
2106 ret = PTR_ERR(dev);
2107 goto err_device;
2108 }
2109
2110 /* /dev/vduse/$DEVICE */
2111 cdev_init(&vduse_cdev, &vduse_dev_fops);
2112 vduse_cdev.owner = THIS_MODULE;
2113 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
2114 VDUSE_DEV_MAX - 1);
2115 if (ret)
2116 goto err_cdev;
2117
2118 ret = -ENOMEM;
2119 vduse_irq_wq = alloc_workqueue("vduse-irq",
2120 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
2121 if (!vduse_irq_wq)
2122 goto err_wq;
2123
2124 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0);
2125 if (!vduse_irq_bound_wq)
2126 goto err_bound_wq;
2127
2128 ret = vduse_domain_init();
2129 if (ret)
2130 goto err_domain;
2131
2132 ret = vduse_mgmtdev_init();
2133 if (ret)
2134 goto err_mgmtdev;
2135
2136 return 0;
2137 err_mgmtdev:
2138 vduse_domain_exit();
2139 err_domain:
2140 destroy_workqueue(vduse_irq_bound_wq);
2141 err_bound_wq:
2142 destroy_workqueue(vduse_irq_wq);
2143 err_wq:
2144 cdev_del(&vduse_cdev);
2145 err_cdev:
2146 device_destroy(&vduse_class, vduse_major);
2147 err_device:
2148 cdev_del(&vduse_ctrl_cdev);
2149 err_ctrl_cdev:
2150 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2151 err_chardev_region:
2152 class_unregister(&vduse_class);
2153 return ret;
2154 }
2155 module_init(vduse_init);
2156
2157 static void vduse_exit(void)
2158 {
2159 vduse_mgmtdev_exit();
2160 vduse_domain_exit();
2161 destroy_workqueue(vduse_irq_bound_wq);
2162 destroy_workqueue(vduse_irq_wq);
2163 cdev_del(&vduse_cdev);
2164 device_destroy(&vduse_class, vduse_major);
2165 cdev_del(&vduse_ctrl_cdev);
2166 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
2167 class_unregister(&vduse_class);
2168 }
2169 module_exit(vduse_exit);
2170
2171 MODULE_LICENSE(DRV_LICENSE);
2172 MODULE_AUTHOR(DRV_AUTHOR);
2173 MODULE_DESCRIPTION(DRV_DESC);
A mirror of Linus' kernel repository