tools/testing/selftests/kvm/include/kvm_util_base.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * tools/testing/selftests/kvm/include/kvm_util_base.h
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7 #ifndef SELFTEST_KVM_UTIL_BASE_H
   8 #define SELFTEST_KVM_UTIL_BASE_H
   9
  10 #include "test_util.h"
  11
  12 #include <linux/compiler.h>
  13 #include "linux/hashtable.h"
  14 #include "linux/list.h"
  15 #include <linux/kernel.h>
  16 #include <linux/kvm.h>
  17 #include "linux/rbtree.h"
  18 #include <linux/types.h>
  19
  20 #include <asm/atomic.h>
  21
  22 #include <sys/ioctl.h>
  23
  24 #include "sparsebit.h"
  25
  26 /*
  27  * Provide a version of static_assert() that is guaranteed to have an optional
  28  * message param.  If _ISOC11_SOURCE is defined, glibc (/usr/include/assert.h)
  29  * #undefs and #defines static_assert() as a direct alias to _Static_assert(),
  30  * i.e. effectively makes the message mandatory.  Many KVM selftests #define
  31  * _GNU_SOURCE for various reasons, and _GNU_SOURCE implies _ISOC11_SOURCE.  As
  32  * a result, static_assert() behavior is non-deterministic and may or may not
  33  * require a message depending on #include order.
  34  */
  35 #define __kvm_static_assert(expr, msg, ...) _Static_assert(expr, msg)
  36 #define kvm_static_assert(expr, ...) __kvm_static_assert(expr, ##__VA_ARGS__, #expr)
  37
  38 #define KVM_DEV_PATH "/dev/kvm"
  39 #define KVM_MAX_VCPUS 512
  40
  41 #define NSEC_PER_SEC 1000000000L
  42
  43 typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */
  44 typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */
  45
  46 struct userspace_mem_region {
  47         struct kvm_userspace_memory_region2 region;
  48         struct sparsebit *unused_phy_pages;
  49         int fd;
  50         off_t offset;
  51         enum vm_mem_backing_src_type backing_src_type;
  52         void *host_mem;
  53         void *host_alias;
  54         void *mmap_start;
  55         void *mmap_alias;
  56         size_t mmap_size;
  57         struct rb_node gpa_node;
  58         struct rb_node hva_node;
  59         struct hlist_node slot_node;
  60 };
  61
  62 struct kvm_vcpu {
  63         struct list_head list;
  64         uint32_t id;
  65         int fd;
  66         struct kvm_vm *vm;
  67         struct kvm_run *run;
  68 #ifdef __x86_64__
  69         struct kvm_cpuid2 *cpuid;
  70 #endif
  71         struct kvm_dirty_gfn *dirty_gfns;
  72         uint32_t fetch_index;
  73         uint32_t dirty_gfns_count;
  74 };
  75
  76 struct userspace_mem_regions {
  77         struct rb_root gpa_tree;
  78         struct rb_root hva_tree;
  79         DECLARE_HASHTABLE(slot_hash, 9);
  80 };
  81
  82 enum kvm_mem_region_type {
  83         MEM_REGION_CODE,
  84         MEM_REGION_DATA,
  85         MEM_REGION_PT,
  86         MEM_REGION_TEST_DATA,
  87         NR_MEM_REGIONS,
  88 };
  89
  90 struct kvm_vm {
  91         int mode;
  92         unsigned long type;
  93         int kvm_fd;
  94         int fd;
  95         unsigned int pgtable_levels;
  96         unsigned int page_size;
  97         unsigned int page_shift;
  98         unsigned int pa_bits;
  99         unsigned int va_bits;
 100         uint64_t max_gfn;
 101         struct list_head vcpus;
 102         struct userspace_mem_regions regions;
 103         struct sparsebit *vpages_valid;
 104         struct sparsebit *vpages_mapped;
 105         bool has_irqchip;
 106         bool pgd_created;
 107         vm_paddr_t ucall_mmio_addr;
 108         vm_paddr_t pgd;
 109         vm_vaddr_t gdt;
 110         vm_vaddr_t tss;
 111         vm_vaddr_t idt;
 112         vm_vaddr_t handlers;
 113         uint32_t dirty_ring_size;
 114
 115         /* Cache of information for binary stats interface */
 116         int stats_fd;
 117         struct kvm_stats_header stats_header;
 118         struct kvm_stats_desc *stats_desc;
 119
 120         /*
 121          * KVM region slots. These are the default memslots used by page
 122          * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
 123          * memslot.
 124          */
 125         uint32_t memslots[NR_MEM_REGIONS];
 126 };
 127
 128 struct vcpu_reg_sublist {
 129         const char *name;
 130         long capability;
 131         int feature;
 132         int feature_type;
 133         bool finalize;
 134         __u64 *regs;
 135         __u64 regs_n;
 136         __u64 *rejects_set;
 137         __u64 rejects_set_n;
 138         __u64 *skips_set;
 139         __u64 skips_set_n;
 140 };
 141
 142 struct vcpu_reg_list {
 143         char *name;
 144         struct vcpu_reg_sublist sublists[];
 145 };
 146
 147 #define for_each_sublist(c, s)          \
 148         for ((s) = &(c)->sublists[0]; (s)->regs; ++(s))
 149
 150 #define kvm_for_each_vcpu(vm, i, vcpu)                  \
 151         for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \
 152                 if (!((vcpu) = vm->vcpus[i]))           \
 153                         continue;                       \
 154                 else
 155
 156 struct userspace_mem_region *
 157 memslot2region(struct kvm_vm *vm, uint32_t memslot);
 158
 159 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
 160                                                              enum kvm_mem_region_type type)
 161 {
 162         assert(type < NR_MEM_REGIONS);
 163         return memslot2region(vm, vm->memslots[type]);
 164 }
 165
 166 /* Minimum allocated guest virtual and physical addresses */
 167 #define KVM_UTIL_MIN_VADDR              0x2000
 168 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR  0x180000
 169
 170 #define DEFAULT_GUEST_STACK_VADDR_MIN   0xab6000
 171 #define DEFAULT_STACK_PGS               5
 172
 173 enum vm_guest_mode {
 174         VM_MODE_P52V48_4K,
 175         VM_MODE_P52V48_64K,
 176         VM_MODE_P48V48_4K,
 177         VM_MODE_P48V48_16K,
 178         VM_MODE_P48V48_64K,
 179         VM_MODE_P40V48_4K,
 180         VM_MODE_P40V48_16K,
 181         VM_MODE_P40V48_64K,
 182         VM_MODE_PXXV48_4K,      /* For 48bits VA but ANY bits PA */
 183         VM_MODE_P47V64_4K,
 184         VM_MODE_P44V64_4K,
 185         VM_MODE_P36V48_4K,
 186         VM_MODE_P36V48_16K,
 187         VM_MODE_P36V48_64K,
 188         VM_MODE_P36V47_16K,
 189         NUM_VM_MODES,
 190 };
 191
 192 struct vm_shape {
 193         enum vm_guest_mode mode;
 194         unsigned int type;
 195 };
 196
 197 #define VM_TYPE_DEFAULT                 0
 198
 199 #define VM_SHAPE(__mode)                        \
 200 ({                                              \
 201         struct vm_shape shape = {               \
 202                 .mode = (__mode),               \
 203                 .type = VM_TYPE_DEFAULT         \
 204         };                                      \
 205                                                 \
 206         shape;                                  \
 207 })
 208
 209 #if defined(__aarch64__)
 210
 211 extern enum vm_guest_mode vm_mode_default;
 212
 213 #define VM_MODE_DEFAULT                 vm_mode_default
 214 #define MIN_PAGE_SHIFT                  12U
 215 #define ptes_per_page(page_size)        ((page_size) / 8)
 216
 217 #elif defined(__x86_64__)
 218
 219 #define VM_MODE_DEFAULT                 VM_MODE_PXXV48_4K
 220 #define MIN_PAGE_SHIFT                  12U
 221 #define ptes_per_page(page_size)        ((page_size) / 8)
 222
 223 #elif defined(__s390x__)
 224
 225 #define VM_MODE_DEFAULT                 VM_MODE_P44V64_4K
 226 #define MIN_PAGE_SHIFT                  12U
 227 #define ptes_per_page(page_size)        ((page_size) / 16)
 228
 229 #elif defined(__riscv)
 230
 231 #if __riscv_xlen == 32
 232 #error "RISC-V 32-bit kvm selftests not supported"
 233 #endif
 234
 235 #define VM_MODE_DEFAULT                 VM_MODE_P40V48_4K
 236 #define MIN_PAGE_SHIFT                  12U
 237 #define ptes_per_page(page_size)        ((page_size) / 8)
 238
 239 #endif
 240
 241 #define VM_SHAPE_DEFAULT        VM_SHAPE(VM_MODE_DEFAULT)
 242
 243 #define MIN_PAGE_SIZE           (1U << MIN_PAGE_SHIFT)
 244 #define PTES_PER_MIN_PAGE       ptes_per_page(MIN_PAGE_SIZE)
 245
 246 struct vm_guest_mode_params {
 247         unsigned int pa_bits;
 248         unsigned int va_bits;
 249         unsigned int page_size;
 250         unsigned int page_shift;
 251 };
 252 extern const struct vm_guest_mode_params vm_guest_mode_params[];
 253
 254 int open_path_or_exit(const char *path, int flags);
 255 int open_kvm_dev_path_or_exit(void);
 256
 257 bool get_kvm_param_bool(const char *param);
 258 bool get_kvm_intel_param_bool(const char *param);
 259 bool get_kvm_amd_param_bool(const char *param);
 260
 261 unsigned int kvm_check_cap(long cap);
 262
 263 static inline bool kvm_has_cap(long cap)
 264 {
 265         return kvm_check_cap(cap);
 266 }
 267
 268 #define __KVM_SYSCALL_ERROR(_name, _ret) \
 269         "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
 270
 271 /*
 272  * Use the "inner", double-underscore macro when reporting errors from within
 273  * other macros so that the name of ioctl() and not its literal numeric value
 274  * is printed on error.  The "outer" macro is strongly preferred when reporting
 275  * errors "directly", i.e. without an additional layer of macros, as it reduces
 276  * the probability of passing in the wrong string.
 277  */
 278 #define __KVM_IOCTL_ERROR(_name, _ret)  __KVM_SYSCALL_ERROR(_name, _ret)
 279 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)
 280
 281 #define kvm_do_ioctl(fd, cmd, arg)                                              \
 282 ({                                                                              \
 283         kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd));   \
 284         ioctl(fd, cmd, arg);                                                    \
 285 })
 286
 287 #define __kvm_ioctl(kvm_fd, cmd, arg)                           \
 288         kvm_do_ioctl(kvm_fd, cmd, arg)
 289
 290 #define kvm_ioctl(kvm_fd, cmd, arg)                             \
 291 ({                                                              \
 292         int ret = __kvm_ioctl(kvm_fd, cmd, arg);                \
 293                                                                 \
 294         TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret));        \
 295 })
 296
 297 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
 298
 299 #define __vm_ioctl(vm, cmd, arg)                                \
 300 ({                                                              \
 301         static_assert_is_vm(vm);                                \
 302         kvm_do_ioctl((vm)->fd, cmd, arg);                       \
 303 })
 304
 305 /*
 306  * Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if
 307  * the ioctl() failed because KVM killed/bugged the VM.  To detect a dead VM,
 308  * probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before
 309  * selftests existed and (b) should never outright fail, i.e. is supposed to
 310  * return 0 or 1.  If KVM kills a VM, KVM returns -EIO for all ioctl()s for the
 311  * VM and its vCPUs, including KVM_CHECK_EXTENSION.
 312  */
 313 #define __TEST_ASSERT_VM_VCPU_IOCTL(cond, name, ret, vm)                                \
 314 do {                                                                                    \
 315         int __errno = errno;                                                            \
 316                                                                                         \
 317         static_assert_is_vm(vm);                                                        \
 318                                                                                         \
 319         if (cond)                                                                       \
 320                 break;                                                                  \
 321                                                                                         \
 322         if (errno == EIO &&                                                             \
 323             __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)KVM_CAP_USER_MEMORY) < 0) {     \
 324                 TEST_ASSERT(errno == EIO, "KVM killed the VM, should return -EIO");     \
 325                 TEST_FAIL("KVM killed/bugged the VM, check the kernel log for clues");  \
 326         }                                                                               \
 327         errno = __errno;                                                                \
 328         TEST_ASSERT(cond, __KVM_IOCTL_ERROR(name, ret));                                \
 329 } while (0)
 330
 331 #define TEST_ASSERT_VM_VCPU_IOCTL(cond, cmd, ret, vm)           \
 332         __TEST_ASSERT_VM_VCPU_IOCTL(cond, #cmd, ret, vm)
 333
 334 #define vm_ioctl(vm, cmd, arg)                                  \
 335 ({                                                              \
 336         int ret = __vm_ioctl(vm, cmd, arg);                     \
 337                                                                 \
 338         __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, vm);               \
 339 })
 340
 341 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
 342
 343 #define __vcpu_ioctl(vcpu, cmd, arg)                            \
 344 ({                                                              \
 345         static_assert_is_vcpu(vcpu);                            \
 346         kvm_do_ioctl((vcpu)->fd, cmd, arg);                     \
 347 })
 348
 349 #define vcpu_ioctl(vcpu, cmd, arg)                              \
 350 ({                                                              \
 351         int ret = __vcpu_ioctl(vcpu, cmd, arg);                 \
 352                                                                 \
 353         __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm);       \
 354 })
 355
 356 /*
 357  * Looks up and returns the value corresponding to the capability
 358  * (KVM_CAP_*) given by cap.
 359  */
 360 static inline int vm_check_cap(struct kvm_vm *vm, long cap)
 361 {
 362         int ret =  __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap);
 363
 364         TEST_ASSERT_VM_VCPU_IOCTL(ret >= 0, KVM_CHECK_EXTENSION, ret, vm);
 365         return ret;
 366 }
 367
 368 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 369 {
 370         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 371
 372         return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 373 }
 374 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 375 {
 376         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 377
 378         vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 379 }
 380
 381 static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
 382                                             uint64_t size, uint64_t attributes)
 383 {
 384         struct kvm_memory_attributes attr = {
 385                 .attributes = attributes,
 386                 .address = gpa,
 387                 .size = size,
 388                 .flags = 0,
 389         };
 390
 391         /*
 392          * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes.  These flows
 393          * need significant enhancements to support multiple attributes.
 394          */
 395         TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,
 396                     "Update me to support multiple attributes!");
 397
 398         vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);
 399 }
 400
 401
 402 static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,
 403                                       uint64_t size)
 404 {
 405         vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
 406 }
 407
 408 static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,
 409                                      uint64_t size)
 410 {
 411         vm_set_memory_attributes(vm, gpa, size, 0);
 412 }
 413
 414 void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,
 415                             bool punch_hole);
 416
 417 static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,
 418                                            uint64_t size)
 419 {
 420         vm_guest_mem_fallocate(vm, gpa, size, true);
 421 }
 422
 423 static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,
 424                                          uint64_t size)
 425 {
 426         vm_guest_mem_fallocate(vm, gpa, size, false);
 427 }
 428
 429 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
 430 const char *vm_guest_mode_string(uint32_t i);
 431
 432 void kvm_vm_free(struct kvm_vm *vmp);
 433 void kvm_vm_restart(struct kvm_vm *vmp);
 434 void kvm_vm_release(struct kvm_vm *vmp);
 435 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva,
 436                        size_t len);
 437 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
 438 int kvm_memfd_alloc(size_t size, bool hugepages);
 439
 440 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
 441
 442 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
 443 {
 444         struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
 445
 446         vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args);
 447 }
 448
 449 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
 450                                           uint64_t first_page, uint32_t num_pages)
 451 {
 452         struct kvm_clear_dirty_log args = {
 453                 .dirty_bitmap = log,
 454                 .slot = slot,
 455                 .first_page = first_page,
 456                 .num_pages = num_pages
 457         };
 458
 459         vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);
 460 }
 461
 462 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
 463 {
 464         return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);
 465 }
 466
 467 static inline int vm_get_stats_fd(struct kvm_vm *vm)
 468 {
 469         int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL);
 470
 471         TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_GET_STATS_FD, fd, vm);
 472         return fd;
 473 }
 474
 475 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header)
 476 {
 477         ssize_t ret;
 478
 479         ret = pread(stats_fd, header, sizeof(*header), 0);
 480         TEST_ASSERT(ret == sizeof(*header),
 481                     "Failed to read '%lu' header bytes, ret = '%ld'",
 482                     sizeof(*header), ret);
 483 }
 484
 485 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
 486                                               struct kvm_stats_header *header);
 487
 488 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header)
 489 {
 490          /*
 491           * The base size of the descriptor is defined by KVM's ABI, but the
 492           * size of the name field is variable, as far as KVM's ABI is
 493           * concerned. For a given instance of KVM, the name field is the same
 494           * size for all stats and is provided in the overall stats header.
 495           */
 496         return sizeof(struct kvm_stats_desc) + header->name_size;
 497 }
 498
 499 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats,
 500                                                           int index,
 501                                                           struct kvm_stats_header *header)
 502 {
 503         /*
 504          * Note, size_desc includes the size of the name field, which is
 505          * variable. i.e. this is NOT equivalent to &stats_desc[i].
 506          */
 507         return (void *)stats + index * get_stats_descriptor_size(header);
 508 }
 509
 510 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
 511                     struct kvm_stats_desc *desc, uint64_t *data,
 512                     size_t max_elements);
 513
 514 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data,
 515                    size_t max_elements);
 516
 517 static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name)
 518 {
 519         uint64_t data;
 520
 521         __vm_get_stat(vm, stat_name, &data, 1);
 522         return data;
 523 }
 524
 525 void vm_create_irqchip(struct kvm_vm *vm);
 526
 527 static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
 528                                         uint64_t flags)
 529 {
 530         struct kvm_create_guest_memfd guest_memfd = {
 531                 .size = size,
 532                 .flags = flags,
 533         };
 534
 535         return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
 536 }
 537
 538 static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
 539                                         uint64_t flags)
 540 {
 541         int fd = __vm_create_guest_memfd(vm, size, flags);
 542
 543         TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd));
 544         return fd;
 545 }
 546
 547 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 548                                uint64_t gpa, uint64_t size, void *hva);
 549 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 550                                 uint64_t gpa, uint64_t size, void *hva);
 551 void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 552                                 uint64_t gpa, uint64_t size, void *hva,
 553                                 uint32_t guest_memfd, uint64_t guest_memfd_offset);
 554 int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 555                                  uint64_t gpa, uint64_t size, void *hva,
 556                                  uint32_t guest_memfd, uint64_t guest_memfd_offset);
 557
 558 void vm_userspace_mem_region_add(struct kvm_vm *vm,
 559         enum vm_mem_backing_src_type src_type,
 560         uint64_t guest_paddr, uint32_t slot, uint64_t npages,
 561         uint32_t flags);
 562 void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
 563                 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
 564                 uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset);
 565
 566 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
 567 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
 568 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
 569 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
 570 void vm_populate_vaddr_bitmap(struct kvm_vm *vm);
 571 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 572 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 573 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
 574                             enum kvm_mem_region_type type);
 575 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
 576 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
 577                                  enum kvm_mem_region_type type);
 578 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
 579
 580 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 581               unsigned int npages);
 582 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);
 583 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);
 584 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
 585 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);
 586
 587 void vcpu_run(struct kvm_vcpu *vcpu);
 588 int _vcpu_run(struct kvm_vcpu *vcpu);
 589
 590 static inline int __vcpu_run(struct kvm_vcpu *vcpu)
 591 {
 592         return __vcpu_ioctl(vcpu, KVM_RUN, NULL);
 593 }
 594
 595 void vcpu_run_complete_io(struct kvm_vcpu *vcpu);
 596 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);
 597
 598 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,
 599                                    uint64_t arg0)
 600 {
 601         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 602
 603         vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap);
 604 }
 605
 606 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu,
 607                                         struct kvm_guest_debug *debug)
 608 {
 609         vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug);
 610 }
 611
 612 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu,
 613                                      struct kvm_mp_state *mp_state)
 614 {
 615         vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state);
 616 }
 617 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu,
 618                                      struct kvm_mp_state *mp_state)
 619 {
 620         vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state);
 621 }
 622
 623 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 624 {
 625         vcpu_ioctl(vcpu, KVM_GET_REGS, regs);
 626 }
 627
 628 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 629 {
 630         vcpu_ioctl(vcpu, KVM_SET_REGS, regs);
 631 }
 632 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 633 {
 634         vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs);
 635
 636 }
 637 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 638 {
 639         vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 640 }
 641 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 642 {
 643         return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 644 }
 645 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 646 {
 647         vcpu_ioctl(vcpu, KVM_GET_FPU, fpu);
 648 }
 649 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 650 {
 651         vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);
 652 }
 653
 654 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 655 {
 656         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
 657
 658         return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 659 }
 660 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 661 {
 662         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 663
 664         return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 665 }
 666 static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 667 {
 668         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
 669
 670         vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 671 }
 672 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 673 {
 674         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 675
 676         vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 677 }
 678
 679 #ifdef __KVM_HAVE_VCPU_EVENTS
 680 static inline void vcpu_events_get(struct kvm_vcpu *vcpu,
 681                                    struct kvm_vcpu_events *events)
 682 {
 683         vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events);
 684 }
 685 static inline void vcpu_events_set(struct kvm_vcpu *vcpu,
 686                                    struct kvm_vcpu_events *events)
 687 {
 688         vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events);
 689 }
 690 #endif
 691 #ifdef __x86_64__
 692 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu,
 693                                          struct kvm_nested_state *state)
 694 {
 695         vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state);
 696 }
 697 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 698                                           struct kvm_nested_state *state)
 699 {
 700         return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 701 }
 702
 703 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 704                                          struct kvm_nested_state *state)
 705 {
 706         vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 707 }
 708 #endif
 709 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)
 710 {
 711         int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL);
 712
 713         TEST_ASSERT_VM_VCPU_IOCTL(fd >= 0, KVM_CHECK_EXTENSION, fd, vcpu->vm);
 714         return fd;
 715 }
 716
 717 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);
 718
 719 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
 720 {
 721         int ret = __kvm_has_device_attr(dev_fd, group, attr);
 722
 723         TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
 724 }
 725
 726 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);
 727
 728 static inline void kvm_device_attr_get(int dev_fd, uint32_t group,
 729                                        uint64_t attr, void *val)
 730 {
 731         int ret = __kvm_device_attr_get(dev_fd, group, attr, val);
 732
 733         TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));
 734 }
 735
 736 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);
 737
 738 static inline void kvm_device_attr_set(int dev_fd, uint32_t group,
 739                                        uint64_t attr, void *val)
 740 {
 741         int ret = __kvm_device_attr_set(dev_fd, group, attr, val);
 742
 743         TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));
 744 }
 745
 746 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 747                                          uint64_t attr)
 748 {
 749         return __kvm_has_device_attr(vcpu->fd, group, attr);
 750 }
 751
 752 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 753                                         uint64_t attr)
 754 {
 755         kvm_has_device_attr(vcpu->fd, group, attr);
 756 }
 757
 758 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 759                                          uint64_t attr, void *val)
 760 {
 761         return __kvm_device_attr_get(vcpu->fd, group, attr, val);
 762 }
 763
 764 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 765                                         uint64_t attr, void *val)
 766 {
 767         kvm_device_attr_get(vcpu->fd, group, attr, val);
 768 }
 769
 770 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 771                                          uint64_t attr, void *val)
 772 {
 773         return __kvm_device_attr_set(vcpu->fd, group, attr, val);
 774 }
 775
 776 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 777                                         uint64_t attr, void *val)
 778 {
 779         kvm_device_attr_set(vcpu->fd, group, attr, val);
 780 }
 781
 782 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);
 783 int __kvm_create_device(struct kvm_vm *vm, uint64_t type);
 784
 785 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)
 786 {
 787         int fd = __kvm_create_device(vm, type);
 788
 789         TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd));
 790         return fd;
 791 }
 792
 793 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
 794
 795 /*
 796  * VM VCPU Args Set
 797  *
 798  * Input Args:
 799  *   vm - Virtual Machine
 800  *   num - number of arguments
 801  *   ... - arguments, each of type uint64_t
 802  *
 803  * Output Args: None
 804  *
 805  * Return: None
 806  *
 807  * Sets the first @num input parameters for the function at @vcpu's entry point,
 808  * per the C calling convention of the architecture, to the values given as
 809  * variable args. Each of the variable args is expected to be of type uint64_t.
 810  * The maximum @num can be is specific to the architecture.
 811  */
 812 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
 813
 814 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 815 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 816
 817 #define KVM_MAX_IRQ_ROUTES              4096
 818
 819 struct kvm_irq_routing *kvm_gsi_routing_create(void);
 820 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
 821                 uint32_t gsi, uint32_t pin);
 822 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 823 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 824
 825 const char *exit_reason_str(unsigned int exit_reason);
 826
 827 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
 828                              uint32_t memslot);
 829 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
 830                               vm_paddr_t paddr_min, uint32_t memslot);
 831 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
 832
 833 /*
 834  * ____vm_create() does KVM_CREATE_VM and little else.  __vm_create() also
 835  * loads the test binary into guest memory and creates an IRQ chip (x86 only).
 836  * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
 837  * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
 838  */
 839 struct kvm_vm *____vm_create(struct vm_shape shape);
 840 struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
 841                            uint64_t nr_extra_pages);
 842
 843 static inline struct kvm_vm *vm_create_barebones(void)
 844 {
 845         return ____vm_create(VM_SHAPE_DEFAULT);
 846 }
 847
 848 #ifdef __x86_64__
 849 static inline struct kvm_vm *vm_create_barebones_protected_vm(void)
 850 {
 851         const struct vm_shape shape = {
 852                 .mode = VM_MODE_DEFAULT,
 853                 .type = KVM_X86_SW_PROTECTED_VM,
 854         };
 855
 856         return ____vm_create(shape);
 857 }
 858 #endif
 859
 860 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
 861 {
 862         return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);
 863 }
 864
 865 struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
 866                                       uint64_t extra_mem_pages,
 867                                       void *guest_code, struct kvm_vcpu *vcpus[]);
 868
 869 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
 870                                                   void *guest_code,
 871                                                   struct kvm_vcpu *vcpus[])
 872 {
 873         return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0,
 874                                       guest_code, vcpus);
 875 }
 876
 877
 878 struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
 879                                                struct kvm_vcpu **vcpu,
 880                                                uint64_t extra_mem_pages,
 881                                                void *guest_code);
 882
 883 /*
 884  * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages
 885  * additional pages of guest memory.  Returns the VM and vCPU (via out param).
 886  */
 887 static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 888                                                        uint64_t extra_mem_pages,
 889                                                        void *guest_code)
 890 {
 891         return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,
 892                                                extra_mem_pages, guest_code);
 893 }
 894
 895 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 896                                                      void *guest_code)
 897 {
 898         return __vm_create_with_one_vcpu(vcpu, 0, guest_code);
 899 }
 900
 901 static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape,
 902                                                            struct kvm_vcpu **vcpu,
 903                                                            void *guest_code)
 904 {
 905         return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code);
 906 }
 907
 908 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
 909
 910 void kvm_pin_this_task_to_pcpu(uint32_t pcpu);
 911 void kvm_print_vcpu_pinning_help(void);
 912 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
 913                             int nr_vcpus);
 914
 915 unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
 916 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size);
 917 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages);
 918 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages);
 919 static inline unsigned int
 920 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
 921 {
 922         unsigned int n;
 923         n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages));
 924 #ifdef __s390x__
 925         /* s390 requires 1M aligned guest sizes */
 926         n = (n + 255) & ~255;
 927 #endif
 928         return n;
 929 }
 930
 931 #define sync_global_to_guest(vm, g) ({                          \
 932         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 933         memcpy(_p, &(g), sizeof(g));                            \
 934 })
 935
 936 #define sync_global_from_guest(vm, g) ({                        \
 937         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 938         memcpy(&(g), _p, sizeof(g));                            \
 939 })
 940
 941 /*
 942  * Write a global value, but only in the VM's (guest's) domain.  Primarily used
 943  * for "globals" that hold per-VM values (VMs always duplicate code and global
 944  * data into their own region of physical memory), but can be used anytime it's
 945  * undesirable to change the host's copy of the global.
 946  */
 947 #define write_guest_global(vm, g, val) ({                       \
 948         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 949         typeof(g) _val = val;                                   \
 950                                                                 \
 951         memcpy(_p, &(_val), sizeof(g));                         \
 952 })
 953
 954 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);
 955
 956 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,
 957                     uint8_t indent);
 958
 959 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
 960                              uint8_t indent)
 961 {
 962         vcpu_arch_dump(stream, vcpu, indent);
 963 }
 964
 965 /*
 966  * Adds a vCPU with reasonable defaults (e.g. a stack)
 967  *
 968  * Input Args:
 969  *   vm - Virtual Machine
 970  *   vcpu_id - The id of the VCPU to add to the VM.
 971  *   guest_code - The vCPU's entry point
 972  */
 973 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 974                                   void *guest_code);
 975
 976 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 977                                            void *guest_code)
 978 {
 979         return vm_arch_vcpu_add(vm, vcpu_id, guest_code);
 980 }
 981
 982 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */
 983 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);
 984
 985 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,
 986                                                 uint32_t vcpu_id)
 987 {
 988         return vm_arch_vcpu_recreate(vm, vcpu_id);
 989 }
 990
 991 void vcpu_arch_free(struct kvm_vcpu *vcpu);
 992
 993 void virt_arch_pgd_alloc(struct kvm_vm *vm);
 994
 995 static inline void virt_pgd_alloc(struct kvm_vm *vm)
 996 {
 997         virt_arch_pgd_alloc(vm);
 998 }
 999
1000 /*
1001  * VM Virtual Page Map
1002  *
1003  * Input Args:
1004  *   vm - Virtual Machine
1005  *   vaddr - VM Virtual Address
1006  *   paddr - VM Physical Address
1007  *   memslot - Memory region slot for new virtual translation tables
1008  *
1009  * Output Args: None
1010  *
1011  * Return: None
1012  *
1013  * Within @vm, creates a virtual translation for the page starting
1014  * at @vaddr to the page starting at @paddr.
1015  */
1016 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);
1017
1018 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
1019 {
1020         virt_arch_pg_map(vm, vaddr, paddr);
1021 }
1022
1023
1024 /*
1025  * Address Guest Virtual to Guest Physical
1026  *
1027  * Input Args:
1028  *   vm - Virtual Machine
1029  *   gva - VM virtual address
1030  *
1031  * Output Args: None
1032  *
1033  * Return:
1034  *   Equivalent VM physical address
1035  *
1036  * Returns the VM physical address of the translated VM virtual
1037  * address given by @gva.
1038  */
1039 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);
1040
1041 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
1042 {
1043         return addr_arch_gva2gpa(vm, gva);
1044 }
1045
1046 /*
1047  * Virtual Translation Tables Dump
1048  *
1049  * Input Args:
1050  *   stream - Output FILE stream
1051  *   vm     - Virtual Machine
1052  *   indent - Left margin indent amount
1053  *
1054  * Output Args: None
1055  *
1056  * Return: None
1057  *
1058  * Dumps to the FILE stream given by @stream, the contents of all the
1059  * virtual translation tables for the VM given by @vm.
1060  */
1061 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
1062
1063 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1064 {
1065         virt_arch_dump(stream, vm, indent);
1066 }
1067
1068
1069 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)
1070 {
1071         return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);
1072 }
1073
1074 /*
1075  * Arch hook that is invoked via a constructor, i.e. before exeucting main(),
1076  * to allow for arch-specific setup that is common to all tests, e.g. computing
1077  * the default guest "mode".
1078  */
1079 void kvm_selftest_arch_init(void);
1080
1081 void kvm_arch_vm_post_create(struct kvm_vm *vm);
1082
1083 #endif /* SELFTEST_KVM_UTIL_BASE_H */