2 * tools/testing/selftests/kvm/lib/kvm_util.c
4 * Copyright (C) 2018, Google LLC.
6 * This work is licensed under the terms of the GNU GPL, version 2.
11 #include "kvm_util_internal.h"
15 #include <sys/types.h>
17 #include <linux/kernel.h>
19 #define KVM_UTIL_PGS_PER_HUGEPG 512
20 #define KVM_UTIL_MIN_PFN 2
22 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
23 static void *align(void *x
, size_t size
)
25 size_t mask
= size
- 1;
26 TEST_ASSERT(size
!= 0 && !(size
& (size
- 1)),
27 "size not a power of 2: %lu", size
);
28 return (void *) (((size_t) x
+ mask
) & ~mask
);
40 * On success, the Value corresponding to the capability (KVM_CAP_*)
41 * specified by the value of cap. On failure a TEST_ASSERT failure
44 * Looks up and returns the value corresponding to the capability
45 * (KVM_CAP_*) given by cap.
47 int kvm_check_cap(long cap
)
52 kvm_fd
= open(KVM_DEV_PATH
, O_RDONLY
);
56 ret
= ioctl(kvm_fd
, KVM_CHECK_EXTENSION
, cap
);
57 TEST_ASSERT(ret
!= -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
58 " rc: %i errno: %i", ret
, errno
);
65 /* VM Enable Capability
68 * vm - Virtual Machine
73 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
75 * Enables a capability (KVM_CAP_*) on the VM.
77 int vm_enable_cap(struct kvm_vm
*vm
, struct kvm_enable_cap
*cap
)
81 ret
= ioctl(vm
->fd
, KVM_ENABLE_CAP
, cap
);
82 TEST_ASSERT(ret
== 0, "KVM_ENABLE_CAP IOCTL failed,\n"
83 " rc: %i errno: %i", ret
, errno
);
88 static void vm_open(struct kvm_vm
*vm
, int perm
)
90 vm
->kvm_fd
= open(KVM_DEV_PATH
, perm
);
94 vm
->fd
= ioctl(vm
->kvm_fd
, KVM_CREATE_VM
, NULL
);
95 TEST_ASSERT(vm
->fd
>= 0, "KVM_CREATE_VM ioctl failed, "
96 "rc: %i errno: %i", vm
->fd
, errno
);
99 const char * const vm_guest_mode_string
[] = {
100 "PA-bits:52, VA-bits:48, 4K pages",
101 "PA-bits:52, VA-bits:48, 64K pages",
102 "PA-bits:40, VA-bits:48, 4K pages",
103 "PA-bits:40, VA-bits:48, 64K pages",
110 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
111 * phy_pages - Physical memory pages
117 * Pointer to opaque structure that describes the created VM.
119 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
120 * When phy_pages is non-zero, a memory region of phy_pages physical pages
121 * is created and mapped starting at guest physical address 0. The file
122 * descriptor to control the created VM is created with the permissions
123 * given by perm (e.g. O_RDWR).
125 struct kvm_vm
*vm_create(enum vm_guest_mode mode
, uint64_t phy_pages
, int perm
)
130 vm
= calloc(1, sizeof(*vm
));
131 TEST_ASSERT(vm
!= NULL
, "Insufficient Memory");
136 /* Setup mode specific traits. */
138 case VM_MODE_P52V48_4K
:
139 vm
->pgtable_levels
= 4;
140 vm
->page_size
= 0x1000;
144 case VM_MODE_P52V48_64K
:
145 vm
->pgtable_levels
= 3;
147 vm
->page_size
= 0x10000;
151 case VM_MODE_P40V48_4K
:
152 vm
->pgtable_levels
= 4;
155 vm
->page_size
= 0x1000;
158 case VM_MODE_P40V48_64K
:
159 vm
->pgtable_levels
= 3;
162 vm
->page_size
= 0x10000;
166 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", mode
);
169 /* Limit to VA-bit canonical virtual addresses. */
170 vm
->vpages_valid
= sparsebit_alloc();
171 sparsebit_set_num(vm
->vpages_valid
,
172 0, (1ULL << (vm
->va_bits
- 1)) >> vm
->page_shift
);
173 sparsebit_set_num(vm
->vpages_valid
,
174 (~((1ULL << (vm
->va_bits
- 1)) - 1)) >> vm
->page_shift
,
175 (1ULL << (vm
->va_bits
- 1)) >> vm
->page_shift
);
177 /* Limit physical addresses to PA-bits. */
178 vm
->max_gfn
= ((1ULL << vm
->pa_bits
) >> vm
->page_shift
) - 1;
180 /* Allocate and setup memory for guest. */
181 vm
->vpages_mapped
= sparsebit_alloc();
183 vm_userspace_mem_region_add(vm
, VM_MEM_SRC_ANONYMOUS
,
193 * vm - VM that has been released before
198 * Reopens the file descriptors associated to the VM and reinstates the
199 * global state, such as the irqchip and the memory regions that are mapped
202 void kvm_vm_restart(struct kvm_vm
*vmp
, int perm
)
204 struct userspace_mem_region
*region
;
207 if (vmp
->has_irqchip
)
208 vm_create_irqchip(vmp
);
210 for (region
= vmp
->userspace_mem_region_head
; region
;
211 region
= region
->next
) {
212 int ret
= ioctl(vmp
->fd
, KVM_SET_USER_MEMORY_REGION
, ®ion
->region
);
213 TEST_ASSERT(ret
== 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
214 " rc: %i errno: %i\n"
215 " slot: %u flags: 0x%x\n"
216 " guest_phys_addr: 0x%lx size: 0x%lx",
217 ret
, errno
, region
->region
.slot
,
218 region
->region
.flags
,
219 region
->region
.guest_phys_addr
,
220 region
->region
.memory_size
);
224 void kvm_vm_get_dirty_log(struct kvm_vm
*vm
, int slot
, void *log
)
226 struct kvm_dirty_log args
= { .dirty_bitmap
= log
, .slot
= slot
};
229 ret
= ioctl(vm
->fd
, KVM_GET_DIRTY_LOG
, &args
);
230 TEST_ASSERT(ret
== 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
235 * Userspace Memory Region Find
238 * vm - Virtual Machine
239 * start - Starting VM physical address
240 * end - Ending VM physical address, inclusive.
245 * Pointer to overlapping region, NULL if no such region.
247 * Searches for a region with any physical memory that overlaps with
248 * any portion of the guest physical addresses from start to end
249 * inclusive. If multiple overlapping regions exist, a pointer to any
250 * of the regions is returned. Null is returned only when no overlapping
253 static struct userspace_mem_region
*
254 userspace_mem_region_find(struct kvm_vm
*vm
, uint64_t start
, uint64_t end
)
256 struct userspace_mem_region
*region
;
258 for (region
= vm
->userspace_mem_region_head
; region
;
259 region
= region
->next
) {
260 uint64_t existing_start
= region
->region
.guest_phys_addr
;
261 uint64_t existing_end
= region
->region
.guest_phys_addr
262 + region
->region
.memory_size
- 1;
263 if (start
<= existing_end
&& end
>= existing_start
)
271 * KVM Userspace Memory Region Find
274 * vm - Virtual Machine
275 * start - Starting VM physical address
276 * end - Ending VM physical address, inclusive.
281 * Pointer to overlapping region, NULL if no such region.
283 * Public interface to userspace_mem_region_find. Allows tests to look up
284 * the memslot datastructure for a given range of guest physical memory.
286 struct kvm_userspace_memory_region
*
287 kvm_userspace_memory_region_find(struct kvm_vm
*vm
, uint64_t start
,
290 struct userspace_mem_region
*region
;
292 region
= userspace_mem_region_find(vm
, start
, end
);
296 return ®ion
->region
;
303 * vm - Virtual Machine
309 * Pointer to VCPU structure
311 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
312 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
313 * for the specified vcpuid.
315 struct vcpu
*vcpu_find(struct kvm_vm
*vm
, uint32_t vcpuid
)
319 for (vcpup
= vm
->vcpu_head
; vcpup
; vcpup
= vcpup
->next
) {
320 if (vcpup
->id
== vcpuid
)
331 * vm - Virtual Machine
336 * Return: None, TEST_ASSERT failures for all error conditions
338 * Within the VM specified by vm, removes the VCPU given by vcpuid.
340 static void vm_vcpu_rm(struct kvm_vm
*vm
, uint32_t vcpuid
)
342 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
345 ret
= munmap(vcpu
->state
, sizeof(*vcpu
->state
));
346 TEST_ASSERT(ret
== 0, "munmap of VCPU fd failed, rc: %i "
347 "errno: %i", ret
, errno
);
349 TEST_ASSERT(ret
== 0, "Close of VCPU fd failed, rc: %i "
350 "errno: %i", ret
, errno
);
353 vcpu
->next
->prev
= vcpu
->prev
;
355 vcpu
->prev
->next
= vcpu
->next
;
357 vm
->vcpu_head
= vcpu
->next
;
361 void kvm_vm_release(struct kvm_vm
*vmp
)
365 while (vmp
->vcpu_head
)
366 vm_vcpu_rm(vmp
, vmp
->vcpu_head
->id
);
368 ret
= close(vmp
->fd
);
369 TEST_ASSERT(ret
== 0, "Close of vm fd failed,\n"
370 " vmp->fd: %i rc: %i errno: %i", vmp
->fd
, ret
, errno
);
373 TEST_ASSERT(ret
== 0, "Close of /dev/kvm fd failed,\n"
374 " vmp->kvm_fd: %i rc: %i errno: %i", vmp
->kvm_fd
, ret
, errno
);
378 * Destroys and frees the VM pointed to by vmp.
380 void kvm_vm_free(struct kvm_vm
*vmp
)
387 /* Free userspace_mem_regions. */
388 while (vmp
->userspace_mem_region_head
) {
389 struct userspace_mem_region
*region
390 = vmp
->userspace_mem_region_head
;
392 region
->region
.memory_size
= 0;
393 ret
= ioctl(vmp
->fd
, KVM_SET_USER_MEMORY_REGION
,
395 TEST_ASSERT(ret
== 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
396 "rc: %i errno: %i", ret
, errno
);
398 vmp
->userspace_mem_region_head
= region
->next
;
399 sparsebit_free(®ion
->unused_phy_pages
);
400 ret
= munmap(region
->mmap_start
, region
->mmap_size
);
401 TEST_ASSERT(ret
== 0, "munmap failed, rc: %i errno: %i",
407 /* Free sparsebit arrays. */
408 sparsebit_free(&vmp
->vpages_valid
);
409 sparsebit_free(&vmp
->vpages_mapped
);
413 /* Free the structure describing the VM. */
418 * Memory Compare, host virtual to guest virtual
421 * hva - Starting host virtual address
422 * vm - Virtual Machine
423 * gva - Starting guest virtual address
424 * len - number of bytes to compare
428 * Input/Output Args: None
431 * Returns 0 if the bytes starting at hva for a length of len
432 * are equal the guest virtual bytes starting at gva. Returns
433 * a value < 0, if bytes at hva are less than those at gva.
434 * Otherwise a value > 0 is returned.
436 * Compares the bytes starting at the host virtual address hva, for
437 * a length of len, to the guest bytes starting at the guest virtual
438 * address given by gva.
440 int kvm_memcmp_hva_gva(void *hva
, struct kvm_vm
*vm
, vm_vaddr_t gva
, size_t len
)
445 * Compare a batch of bytes until either a match is found
446 * or all the bytes have been compared.
448 for (uintptr_t offset
= 0; offset
< len
; offset
+= amt
) {
449 uintptr_t ptr1
= (uintptr_t)hva
+ offset
;
452 * Determine host address for guest virtual address
455 uintptr_t ptr2
= (uintptr_t)addr_gva2hva(vm
, gva
+ offset
);
458 * Determine amount to compare on this pass.
459 * Don't allow the comparsion to cross a page boundary.
462 if ((ptr1
>> vm
->page_shift
) != ((ptr1
+ amt
) >> vm
->page_shift
))
463 amt
= vm
->page_size
- (ptr1
% vm
->page_size
);
464 if ((ptr2
>> vm
->page_shift
) != ((ptr2
+ amt
) >> vm
->page_shift
))
465 amt
= vm
->page_size
- (ptr2
% vm
->page_size
);
467 assert((ptr1
>> vm
->page_shift
) == ((ptr1
+ amt
- 1) >> vm
->page_shift
));
468 assert((ptr2
>> vm
->page_shift
) == ((ptr2
+ amt
- 1) >> vm
->page_shift
));
471 * Perform the comparison. If there is a difference
472 * return that result to the caller, otherwise need
473 * to continue on looking for a mismatch.
475 int ret
= memcmp((void *)ptr1
, (void *)ptr2
, amt
);
481 * No mismatch found. Let the caller know the two memory
488 * VM Userspace Memory Region Add
491 * vm - Virtual Machine
492 * backing_src - Storage source for this region.
493 * NULL to use anonymous memory.
494 * guest_paddr - Starting guest physical address
495 * slot - KVM region slot
496 * npages - Number of physical pages
497 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
503 * Allocates a memory area of the number of pages specified by npages
504 * and maps it to the VM specified by vm, at a starting physical address
505 * given by guest_paddr. The region is created with a KVM region slot
506 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
507 * region is created with the flags given by flags.
509 void vm_userspace_mem_region_add(struct kvm_vm
*vm
,
510 enum vm_mem_backing_src_type src_type
,
511 uint64_t guest_paddr
, uint32_t slot
, uint64_t npages
,
515 unsigned long pmem_size
= 0;
516 struct userspace_mem_region
*region
;
517 size_t huge_page_size
= KVM_UTIL_PGS_PER_HUGEPG
* vm
->page_size
;
519 TEST_ASSERT((guest_paddr
% vm
->page_size
) == 0, "Guest physical "
520 "address not on a page boundary.\n"
521 " guest_paddr: 0x%lx vm->page_size: 0x%x",
522 guest_paddr
, vm
->page_size
);
523 TEST_ASSERT((((guest_paddr
>> vm
->page_shift
) + npages
) - 1)
524 <= vm
->max_gfn
, "Physical range beyond maximum "
525 "supported physical address,\n"
526 " guest_paddr: 0x%lx npages: 0x%lx\n"
527 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
528 guest_paddr
, npages
, vm
->max_gfn
, vm
->page_size
);
531 * Confirm a mem region with an overlapping address doesn't
534 region
= (struct userspace_mem_region
*) userspace_mem_region_find(
535 vm
, guest_paddr
, guest_paddr
+ npages
* vm
->page_size
);
537 TEST_ASSERT(false, "overlapping userspace_mem_region already "
539 " requested guest_paddr: 0x%lx npages: 0x%lx "
541 " existing guest_paddr: 0x%lx size: 0x%lx",
542 guest_paddr
, npages
, vm
->page_size
,
543 (uint64_t) region
->region
.guest_phys_addr
,
544 (uint64_t) region
->region
.memory_size
);
546 /* Confirm no region with the requested slot already exists. */
547 for (region
= vm
->userspace_mem_region_head
; region
;
548 region
= region
->next
) {
549 if (region
->region
.slot
== slot
)
551 if ((guest_paddr
<= (region
->region
.guest_phys_addr
552 + region
->region
.memory_size
))
553 && ((guest_paddr
+ npages
* vm
->page_size
)
554 >= region
->region
.guest_phys_addr
))
558 TEST_ASSERT(false, "A mem region with the requested slot "
559 "or overlapping physical memory range already exists.\n"
560 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
561 " existing slot: %u paddr: 0x%lx size: 0x%lx",
562 slot
, guest_paddr
, npages
,
564 (uint64_t) region
->region
.guest_phys_addr
,
565 (uint64_t) region
->region
.memory_size
);
567 /* Allocate and initialize new mem region structure. */
568 region
= calloc(1, sizeof(*region
));
569 TEST_ASSERT(region
!= NULL
, "Insufficient Memory");
570 region
->mmap_size
= npages
* vm
->page_size
;
572 /* Enough memory to align up to a huge page. */
573 if (src_type
== VM_MEM_SRC_ANONYMOUS_THP
)
574 region
->mmap_size
+= huge_page_size
;
575 region
->mmap_start
= mmap(NULL
, region
->mmap_size
,
576 PROT_READ
| PROT_WRITE
,
577 MAP_PRIVATE
| MAP_ANONYMOUS
578 | (src_type
== VM_MEM_SRC_ANONYMOUS_HUGETLB
? MAP_HUGETLB
: 0),
580 TEST_ASSERT(region
->mmap_start
!= MAP_FAILED
,
581 "test_malloc failed, mmap_start: %p errno: %i",
582 region
->mmap_start
, errno
);
584 /* Align THP allocation up to start of a huge page. */
585 region
->host_mem
= align(region
->mmap_start
,
586 src_type
== VM_MEM_SRC_ANONYMOUS_THP
? huge_page_size
: 1);
588 /* As needed perform madvise */
589 if (src_type
== VM_MEM_SRC_ANONYMOUS
|| src_type
== VM_MEM_SRC_ANONYMOUS_THP
) {
590 ret
= madvise(region
->host_mem
, npages
* vm
->page_size
,
591 src_type
== VM_MEM_SRC_ANONYMOUS
? MADV_NOHUGEPAGE
: MADV_HUGEPAGE
);
592 TEST_ASSERT(ret
== 0, "madvise failed,\n"
596 region
->host_mem
, npages
* vm
->page_size
, src_type
);
599 region
->unused_phy_pages
= sparsebit_alloc();
600 sparsebit_set_num(region
->unused_phy_pages
,
601 guest_paddr
>> vm
->page_shift
, npages
);
602 region
->region
.slot
= slot
;
603 region
->region
.flags
= flags
;
604 region
->region
.guest_phys_addr
= guest_paddr
;
605 region
->region
.memory_size
= npages
* vm
->page_size
;
606 region
->region
.userspace_addr
= (uintptr_t) region
->host_mem
;
607 ret
= ioctl(vm
->fd
, KVM_SET_USER_MEMORY_REGION
, ®ion
->region
);
608 TEST_ASSERT(ret
== 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
609 " rc: %i errno: %i\n"
610 " slot: %u flags: 0x%x\n"
611 " guest_phys_addr: 0x%lx size: 0x%lx",
612 ret
, errno
, slot
, flags
,
613 guest_paddr
, (uint64_t) region
->region
.memory_size
);
615 /* Add to linked-list of memory regions. */
616 if (vm
->userspace_mem_region_head
)
617 vm
->userspace_mem_region_head
->prev
= region
;
618 region
->next
= vm
->userspace_mem_region_head
;
619 vm
->userspace_mem_region_head
= region
;
626 * vm - Virtual Machine
627 * memslot - KVM memory slot ID
632 * Pointer to memory region structure that describe memory region
633 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
634 * on error (e.g. currently no memory region using memslot as a KVM
637 static struct userspace_mem_region
*
638 memslot2region(struct kvm_vm
*vm
, uint32_t memslot
)
640 struct userspace_mem_region
*region
;
642 for (region
= vm
->userspace_mem_region_head
; region
;
643 region
= region
->next
) {
644 if (region
->region
.slot
== memslot
)
647 if (region
== NULL
) {
648 fprintf(stderr
, "No mem region with the requested slot found,\n"
649 " requested slot: %u\n", memslot
);
650 fputs("---- vm dump ----\n", stderr
);
651 vm_dump(stderr
, vm
, 2);
652 TEST_ASSERT(false, "Mem region not found");
659 * VM Memory Region Flags Set
662 * vm - Virtual Machine
663 * flags - Starting guest physical address
669 * Sets the flags of the memory region specified by the value of slot,
670 * to the values given by flags.
672 void vm_mem_region_set_flags(struct kvm_vm
*vm
, uint32_t slot
, uint32_t flags
)
675 struct userspace_mem_region
*region
;
677 region
= memslot2region(vm
, slot
);
679 region
->region
.flags
= flags
;
681 ret
= ioctl(vm
->fd
, KVM_SET_USER_MEMORY_REGION
, ®ion
->region
);
683 TEST_ASSERT(ret
== 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
684 " rc: %i errno: %i slot: %u flags: 0x%x",
685 ret
, errno
, slot
, flags
);
698 * Returns the size of the structure pointed to by the return value
701 static int vcpu_mmap_sz(void)
705 dev_fd
= open(KVM_DEV_PATH
, O_RDONLY
);
709 ret
= ioctl(dev_fd
, KVM_GET_VCPU_MMAP_SIZE
, NULL
);
710 TEST_ASSERT(ret
>= sizeof(struct kvm_run
),
711 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
712 __func__
, ret
, errno
);
723 * vm - Virtual Machine
730 * Creates and adds to the VM specified by vm and virtual CPU with
731 * the ID given by vcpuid.
733 void vm_vcpu_add(struct kvm_vm
*vm
, uint32_t vcpuid
, int pgd_memslot
,
738 /* Confirm a vcpu with the specified id doesn't already exist. */
739 vcpu
= vcpu_find(vm
, vcpuid
);
741 TEST_ASSERT(false, "vcpu with the specified id "
743 " requested vcpuid: %u\n"
744 " existing vcpuid: %u state: %p",
745 vcpuid
, vcpu
->id
, vcpu
->state
);
747 /* Allocate and initialize new vcpu structure. */
748 vcpu
= calloc(1, sizeof(*vcpu
));
749 TEST_ASSERT(vcpu
!= NULL
, "Insufficient Memory");
751 vcpu
->fd
= ioctl(vm
->fd
, KVM_CREATE_VCPU
, vcpuid
);
752 TEST_ASSERT(vcpu
->fd
>= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
755 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu
->state
), "vcpu mmap size "
756 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
757 vcpu_mmap_sz(), sizeof(*vcpu
->state
));
758 vcpu
->state
= (struct kvm_run
*) mmap(NULL
, sizeof(*vcpu
->state
),
759 PROT_READ
| PROT_WRITE
, MAP_SHARED
, vcpu
->fd
, 0);
760 TEST_ASSERT(vcpu
->state
!= MAP_FAILED
, "mmap vcpu_state failed, "
761 "vcpu id: %u errno: %i", vcpuid
, errno
);
763 /* Add to linked-list of VCPUs. */
765 vm
->vcpu_head
->prev
= vcpu
;
766 vcpu
->next
= vm
->vcpu_head
;
767 vm
->vcpu_head
= vcpu
;
769 vcpu_setup(vm
, vcpuid
, pgd_memslot
, gdt_memslot
);
773 * VM Virtual Address Unused Gap
776 * vm - Virtual Machine
778 * vaddr_min - Minimum Virtual Address
783 * Lowest virtual address at or below vaddr_min, with at least
784 * sz unused bytes. TEST_ASSERT failure if no area of at least
785 * size sz is available.
787 * Within the VM specified by vm, locates the lowest starting virtual
788 * address >= vaddr_min, that has at least sz unallocated bytes. A
789 * TEST_ASSERT failure occurs for invalid input or no area of at least
790 * sz unallocated bytes >= vaddr_min is available.
792 static vm_vaddr_t
vm_vaddr_unused_gap(struct kvm_vm
*vm
, size_t sz
,
793 vm_vaddr_t vaddr_min
)
795 uint64_t pages
= (sz
+ vm
->page_size
- 1) >> vm
->page_shift
;
797 /* Determine lowest permitted virtual page index. */
798 uint64_t pgidx_start
= (vaddr_min
+ vm
->page_size
- 1) >> vm
->page_shift
;
799 if ((pgidx_start
* vm
->page_size
) < vaddr_min
)
802 /* Loop over section with enough valid virtual page indexes. */
803 if (!sparsebit_is_set_num(vm
->vpages_valid
,
805 pgidx_start
= sparsebit_next_set_num(vm
->vpages_valid
,
809 * Are there enough unused virtual pages available at
810 * the currently proposed starting virtual page index.
811 * If not, adjust proposed starting index to next
814 if (sparsebit_is_clear_num(vm
->vpages_mapped
,
817 pgidx_start
= sparsebit_next_clear_num(vm
->vpages_mapped
,
819 if (pgidx_start
== 0)
823 * If needed, adjust proposed starting virtual address,
824 * to next range of valid virtual addresses.
826 if (!sparsebit_is_set_num(vm
->vpages_valid
,
827 pgidx_start
, pages
)) {
828 pgidx_start
= sparsebit_next_set_num(
829 vm
->vpages_valid
, pgidx_start
, pages
);
830 if (pgidx_start
== 0)
833 } while (pgidx_start
!= 0);
836 TEST_ASSERT(false, "No vaddr of specified pages available, "
837 "pages: 0x%lx", pages
);
843 TEST_ASSERT(sparsebit_is_set_num(vm
->vpages_valid
,
845 "Unexpected, invalid virtual page index range,\n"
846 " pgidx_start: 0x%lx\n"
849 TEST_ASSERT(sparsebit_is_clear_num(vm
->vpages_mapped
,
851 "Unexpected, pages already mapped,\n"
852 " pgidx_start: 0x%lx\n"
856 return pgidx_start
* vm
->page_size
;
860 * VM Virtual Address Allocate
863 * vm - Virtual Machine
865 * vaddr_min - Minimum starting virtual address
866 * data_memslot - Memory region slot for data pages
867 * pgd_memslot - Memory region slot for new virtual translation tables
872 * Starting guest virtual address
874 * Allocates at least sz bytes within the virtual address space of the vm
875 * given by vm. The allocated bytes are mapped to a virtual address >=
876 * the address given by vaddr_min. Note that each allocation uses a
877 * a unique set of pages, with the minimum real allocation being at least
880 vm_vaddr_t
vm_vaddr_alloc(struct kvm_vm
*vm
, size_t sz
, vm_vaddr_t vaddr_min
,
881 uint32_t data_memslot
, uint32_t pgd_memslot
)
883 uint64_t pages
= (sz
>> vm
->page_shift
) + ((sz
% vm
->page_size
) != 0);
885 virt_pgd_alloc(vm
, pgd_memslot
);
888 * Find an unused range of virtual page addresses of at least
891 vm_vaddr_t vaddr_start
= vm_vaddr_unused_gap(vm
, sz
, vaddr_min
);
893 /* Map the virtual pages. */
894 for (vm_vaddr_t vaddr
= vaddr_start
; pages
> 0;
895 pages
--, vaddr
+= vm
->page_size
) {
898 paddr
= vm_phy_page_alloc(vm
,
899 KVM_UTIL_MIN_PFN
* vm
->page_size
, data_memslot
);
901 virt_pg_map(vm
, vaddr
, paddr
, pgd_memslot
);
903 sparsebit_set(vm
->vpages_mapped
,
904 vaddr
>> vm
->page_shift
);
911 * Map a range of VM virtual address to the VM's physical address
914 * vm - Virtual Machine
915 * vaddr - Virtuall address to map
916 * paddr - VM Physical Address
917 * size - The size of the range to map
918 * pgd_memslot - Memory region slot for new virtual translation tables
924 * Within the VM given by vm, creates a virtual translation for the
925 * page range starting at vaddr to the page range starting at paddr.
927 void virt_map(struct kvm_vm
*vm
, uint64_t vaddr
, uint64_t paddr
,
928 size_t size
, uint32_t pgd_memslot
)
930 size_t page_size
= vm
->page_size
;
931 size_t npages
= size
/ page_size
;
933 TEST_ASSERT(vaddr
+ size
> vaddr
, "Vaddr overflow");
934 TEST_ASSERT(paddr
+ size
> paddr
, "Paddr overflow");
937 virt_pg_map(vm
, vaddr
, paddr
, pgd_memslot
);
944 * Address VM Physical to Host Virtual
947 * vm - Virtual Machine
948 * gpa - VM physical address
953 * Equivalent host virtual address
955 * Locates the memory region containing the VM physical address given
956 * by gpa, within the VM given by vm. When found, the host virtual
957 * address providing the memory to the vm physical address is returned.
958 * A TEST_ASSERT failure occurs if no region containing gpa exists.
960 void *addr_gpa2hva(struct kvm_vm
*vm
, vm_paddr_t gpa
)
962 struct userspace_mem_region
*region
;
963 for (region
= vm
->userspace_mem_region_head
; region
;
964 region
= region
->next
) {
965 if ((gpa
>= region
->region
.guest_phys_addr
)
966 && (gpa
<= (region
->region
.guest_phys_addr
967 + region
->region
.memory_size
- 1)))
968 return (void *) ((uintptr_t) region
->host_mem
969 + (gpa
- region
->region
.guest_phys_addr
));
972 TEST_ASSERT(false, "No vm physical memory at 0x%lx", gpa
);
977 * Address Host Virtual to VM Physical
980 * vm - Virtual Machine
981 * hva - Host virtual address
986 * Equivalent VM physical address
988 * Locates the memory region containing the host virtual address given
989 * by hva, within the VM given by vm. When found, the equivalent
990 * VM physical address is returned. A TEST_ASSERT failure occurs if no
991 * region containing hva exists.
993 vm_paddr_t
addr_hva2gpa(struct kvm_vm
*vm
, void *hva
)
995 struct userspace_mem_region
*region
;
996 for (region
= vm
->userspace_mem_region_head
; region
;
997 region
= region
->next
) {
998 if ((hva
>= region
->host_mem
)
999 && (hva
<= (region
->host_mem
1000 + region
->region
.memory_size
- 1)))
1001 return (vm_paddr_t
) ((uintptr_t)
1002 region
->region
.guest_phys_addr
1003 + (hva
- (uintptr_t) region
->host_mem
));
1006 TEST_ASSERT(false, "No mapping to a guest physical address, "
1012 * VM Create IRQ Chip
1015 * vm - Virtual Machine
1021 * Creates an interrupt controller chip for the VM specified by vm.
1023 void vm_create_irqchip(struct kvm_vm
*vm
)
1027 ret
= ioctl(vm
->fd
, KVM_CREATE_IRQCHIP
, 0);
1028 TEST_ASSERT(ret
== 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1029 "rc: %i errno: %i", ret
, errno
);
1031 vm
->has_irqchip
= true;
1038 * vm - Virtual Machine
1044 * Pointer to structure that describes the state of the VCPU.
1046 * Locates and returns a pointer to a structure that describes the
1047 * state of the VCPU with the given vcpuid.
1049 struct kvm_run
*vcpu_state(struct kvm_vm
*vm
, uint32_t vcpuid
)
1051 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1052 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1061 * vm - Virtual Machine
1068 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1071 void vcpu_run(struct kvm_vm
*vm
, uint32_t vcpuid
)
1073 int ret
= _vcpu_run(vm
, vcpuid
);
1074 TEST_ASSERT(ret
== 0, "KVM_RUN IOCTL failed, "
1075 "rc: %i errno: %i", ret
, errno
);
1078 int _vcpu_run(struct kvm_vm
*vm
, uint32_t vcpuid
)
1080 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1083 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1085 rc
= ioctl(vcpu
->fd
, KVM_RUN
, NULL
);
1086 } while (rc
== -1 && errno
== EINTR
);
1091 * VM VCPU Set MP State
1094 * vm - Virtual Machine
1096 * mp_state - mp_state to be set
1102 * Sets the MP state of the VCPU given by vcpuid, to the state given
1105 void vcpu_set_mp_state(struct kvm_vm
*vm
, uint32_t vcpuid
,
1106 struct kvm_mp_state
*mp_state
)
1108 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1111 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1113 ret
= ioctl(vcpu
->fd
, KVM_SET_MP_STATE
, mp_state
);
1114 TEST_ASSERT(ret
== 0, "KVM_SET_MP_STATE IOCTL failed, "
1115 "rc: %i errno: %i", ret
, errno
);
1122 * vm - Virtual Machine
1126 * regs - current state of VCPU regs
1130 * Obtains the current register state for the VCPU specified by vcpuid
1131 * and stores it at the location given by regs.
1133 void vcpu_regs_get(struct kvm_vm
*vm
, uint32_t vcpuid
, struct kvm_regs
*regs
)
1135 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1138 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1140 ret
= ioctl(vcpu
->fd
, KVM_GET_REGS
, regs
);
1141 TEST_ASSERT(ret
== 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1149 * vm - Virtual Machine
1151 * regs - Values to set VCPU regs to
1157 * Sets the regs of the VCPU specified by vcpuid to the values
1160 void vcpu_regs_set(struct kvm_vm
*vm
, uint32_t vcpuid
, struct kvm_regs
*regs
)
1162 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1165 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1167 ret
= ioctl(vcpu
->fd
, KVM_SET_REGS
, regs
);
1168 TEST_ASSERT(ret
== 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1172 void vcpu_events_get(struct kvm_vm
*vm
, uint32_t vcpuid
,
1173 struct kvm_vcpu_events
*events
)
1175 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1178 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1180 ret
= ioctl(vcpu
->fd
, KVM_GET_VCPU_EVENTS
, events
);
1181 TEST_ASSERT(ret
== 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1185 void vcpu_events_set(struct kvm_vm
*vm
, uint32_t vcpuid
,
1186 struct kvm_vcpu_events
*events
)
1188 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1191 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1193 ret
= ioctl(vcpu
->fd
, KVM_SET_VCPU_EVENTS
, events
);
1194 TEST_ASSERT(ret
== 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1199 * VM VCPU System Regs Get
1202 * vm - Virtual Machine
1206 * sregs - current state of VCPU system regs
1210 * Obtains the current system register state for the VCPU specified by
1211 * vcpuid and stores it at the location given by sregs.
1213 void vcpu_sregs_get(struct kvm_vm
*vm
, uint32_t vcpuid
, struct kvm_sregs
*sregs
)
1215 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1218 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1220 ret
= ioctl(vcpu
->fd
, KVM_GET_SREGS
, sregs
);
1221 TEST_ASSERT(ret
== 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1226 * VM VCPU System Regs Set
1229 * vm - Virtual Machine
1231 * sregs - Values to set VCPU system regs to
1237 * Sets the system regs of the VCPU specified by vcpuid to the values
1240 void vcpu_sregs_set(struct kvm_vm
*vm
, uint32_t vcpuid
, struct kvm_sregs
*sregs
)
1242 int ret
= _vcpu_sregs_set(vm
, vcpuid
, sregs
);
1243 TEST_ASSERT(ret
== 0, "KVM_RUN IOCTL failed, "
1244 "rc: %i errno: %i", ret
, errno
);
1247 int _vcpu_sregs_set(struct kvm_vm
*vm
, uint32_t vcpuid
, struct kvm_sregs
*sregs
)
1249 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1252 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1254 return ioctl(vcpu
->fd
, KVM_SET_SREGS
, sregs
);
1261 * vm - Virtual Machine
1263 * cmd - Ioctl number
1264 * arg - Argument to pass to the ioctl
1268 * Issues an arbitrary ioctl on a VCPU fd.
1270 void vcpu_ioctl(struct kvm_vm
*vm
, uint32_t vcpuid
,
1271 unsigned long cmd
, void *arg
)
1273 struct vcpu
*vcpu
= vcpu_find(vm
, vcpuid
);
1276 TEST_ASSERT(vcpu
!= NULL
, "vcpu not found, vcpuid: %u", vcpuid
);
1278 ret
= ioctl(vcpu
->fd
, cmd
, arg
);
1279 TEST_ASSERT(ret
== 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1280 cmd
, ret
, errno
, strerror(errno
));
1287 * vm - Virtual Machine
1288 * cmd - Ioctl number
1289 * arg - Argument to pass to the ioctl
1293 * Issues an arbitrary ioctl on a VM fd.
1295 void vm_ioctl(struct kvm_vm
*vm
, unsigned long cmd
, void *arg
)
1299 ret
= ioctl(vm
->fd
, cmd
, arg
);
1300 TEST_ASSERT(ret
== 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1301 cmd
, ret
, errno
, strerror(errno
));
1308 * vm - Virtual Machine
1309 * indent - Left margin indent amount
1312 * stream - Output FILE stream
1316 * Dumps the current state of the VM given by vm, to the FILE stream
1319 void vm_dump(FILE *stream
, struct kvm_vm
*vm
, uint8_t indent
)
1321 struct userspace_mem_region
*region
;
1324 fprintf(stream
, "%*smode: 0x%x\n", indent
, "", vm
->mode
);
1325 fprintf(stream
, "%*sfd: %i\n", indent
, "", vm
->fd
);
1326 fprintf(stream
, "%*spage_size: 0x%x\n", indent
, "", vm
->page_size
);
1327 fprintf(stream
, "%*sMem Regions:\n", indent
, "");
1328 for (region
= vm
->userspace_mem_region_head
; region
;
1329 region
= region
->next
) {
1330 fprintf(stream
, "%*sguest_phys: 0x%lx size: 0x%lx "
1331 "host_virt: %p\n", indent
+ 2, "",
1332 (uint64_t) region
->region
.guest_phys_addr
,
1333 (uint64_t) region
->region
.memory_size
,
1335 fprintf(stream
, "%*sunused_phy_pages: ", indent
+ 2, "");
1336 sparsebit_dump(stream
, region
->unused_phy_pages
, 0);
1338 fprintf(stream
, "%*sMapped Virtual Pages:\n", indent
, "");
1339 sparsebit_dump(stream
, vm
->vpages_mapped
, indent
+ 2);
1340 fprintf(stream
, "%*spgd_created: %u\n", indent
, "",
1342 if (vm
->pgd_created
) {
1343 fprintf(stream
, "%*sVirtual Translation Tables:\n",
1345 virt_dump(stream
, vm
, indent
+ 4);
1347 fprintf(stream
, "%*sVCPUs:\n", indent
, "");
1348 for (vcpu
= vm
->vcpu_head
; vcpu
; vcpu
= vcpu
->next
)
1349 vcpu_dump(stream
, vm
, vcpu
->id
, indent
+ 2);
1352 /* Known KVM exit reasons */
1353 static struct exit_reason
{
1354 unsigned int reason
;
1356 } exit_reasons_known
[] = {
1357 {KVM_EXIT_UNKNOWN
, "UNKNOWN"},
1358 {KVM_EXIT_EXCEPTION
, "EXCEPTION"},
1359 {KVM_EXIT_IO
, "IO"},
1360 {KVM_EXIT_HYPERCALL
, "HYPERCALL"},
1361 {KVM_EXIT_DEBUG
, "DEBUG"},
1362 {KVM_EXIT_HLT
, "HLT"},
1363 {KVM_EXIT_MMIO
, "MMIO"},
1364 {KVM_EXIT_IRQ_WINDOW_OPEN
, "IRQ_WINDOW_OPEN"},
1365 {KVM_EXIT_SHUTDOWN
, "SHUTDOWN"},
1366 {KVM_EXIT_FAIL_ENTRY
, "FAIL_ENTRY"},
1367 {KVM_EXIT_INTR
, "INTR"},
1368 {KVM_EXIT_SET_TPR
, "SET_TPR"},
1369 {KVM_EXIT_TPR_ACCESS
, "TPR_ACCESS"},
1370 {KVM_EXIT_S390_SIEIC
, "S390_SIEIC"},
1371 {KVM_EXIT_S390_RESET
, "S390_RESET"},
1372 {KVM_EXIT_DCR
, "DCR"},
1373 {KVM_EXIT_NMI
, "NMI"},
1374 {KVM_EXIT_INTERNAL_ERROR
, "INTERNAL_ERROR"},
1375 {KVM_EXIT_OSI
, "OSI"},
1376 {KVM_EXIT_PAPR_HCALL
, "PAPR_HCALL"},
1377 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1378 {KVM_EXIT_MEMORY_NOT_PRESENT
, "MEMORY_NOT_PRESENT"},
1383 * Exit Reason String
1386 * exit_reason - Exit reason
1391 * Constant string pointer describing the exit reason.
1393 * Locates and returns a constant string that describes the KVM exit
1394 * reason given by exit_reason. If no such string is found, a constant
1395 * string of "Unknown" is returned.
1397 const char *exit_reason_str(unsigned int exit_reason
)
1401 for (n1
= 0; n1
< ARRAY_SIZE(exit_reasons_known
); n1
++) {
1402 if (exit_reason
== exit_reasons_known
[n1
].reason
)
1403 return exit_reasons_known
[n1
].name
;
1410 * Physical Contiguous Page Allocator
1413 * vm - Virtual Machine
1414 * num - number of pages
1415 * paddr_min - Physical address minimum
1416 * memslot - Memory region to allocate page from
1421 * Starting physical address
1423 * Within the VM specified by vm, locates a range of available physical
1424 * pages at or above paddr_min. If found, the pages are marked as in use
1425 * and thier base address is returned. A TEST_ASSERT failure occurs if
1426 * not enough pages are available at or above paddr_min.
1428 vm_paddr_t
vm_phy_pages_alloc(struct kvm_vm
*vm
, size_t num
,
1429 vm_paddr_t paddr_min
, uint32_t memslot
)
1431 struct userspace_mem_region
*region
;
1432 sparsebit_idx_t pg
, base
;
1434 TEST_ASSERT(num
> 0, "Must allocate at least one page");
1436 TEST_ASSERT((paddr_min
% vm
->page_size
) == 0, "Min physical address "
1437 "not divisible by page size.\n"
1438 " paddr_min: 0x%lx page_size: 0x%x",
1439 paddr_min
, vm
->page_size
);
1441 region
= memslot2region(vm
, memslot
);
1442 base
= pg
= paddr_min
>> vm
->page_shift
;
1445 for (; pg
< base
+ num
; ++pg
) {
1446 if (!sparsebit_is_set(region
->unused_phy_pages
, pg
)) {
1447 base
= pg
= sparsebit_next_set(region
->unused_phy_pages
, pg
);
1451 } while (pg
&& pg
!= base
+ num
);
1454 fprintf(stderr
, "No guest physical page available, "
1455 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
1456 paddr_min
, vm
->page_size
, memslot
);
1457 fputs("---- vm dump ----\n", stderr
);
1458 vm_dump(stderr
, vm
, 2);
1462 for (pg
= base
; pg
< base
+ num
; ++pg
)
1463 sparsebit_clear(region
->unused_phy_pages
, pg
);
1465 return base
* vm
->page_size
;
1468 vm_paddr_t
vm_phy_page_alloc(struct kvm_vm
*vm
, vm_paddr_t paddr_min
,
1471 return vm_phy_pages_alloc(vm
, 1, paddr_min
, memslot
);
1475 * Address Guest Virtual to Host Virtual
1478 * vm - Virtual Machine
1479 * gva - VM virtual address
1484 * Equivalent host virtual address
1486 void *addr_gva2hva(struct kvm_vm
*vm
, vm_vaddr_t gva
)
1488 return addr_gpa2hva(vm
, addr_gva2gpa(vm
, gva
));