tools/testing/selftests/kvm/include/kvm_util_base.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * tools/testing/selftests/kvm/include/kvm_util_base.h
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7 #ifndef SELFTEST_KVM_UTIL_BASE_H
   8 #define SELFTEST_KVM_UTIL_BASE_H
   9
  10 #include "test_util.h"
  11
  12 #include <linux/compiler.h>
  13 #include "linux/hashtable.h"
  14 #include "linux/list.h"
  15 #include <linux/kernel.h>
  16 #include <linux/kvm.h>
  17 #include "linux/rbtree.h"
  18 #include <linux/types.h>
  19
  20 #include <asm/atomic.h>
  21
  22 #include <sys/ioctl.h>
  23
  24 #include "sparsebit.h"
  25
  26 /*
  27  * Provide a version of static_assert() that is guaranteed to have an optional
  28  * message param.  If _ISOC11_SOURCE is defined, glibc (/usr/include/assert.h)
  29  * #undefs and #defines static_assert() as a direct alias to _Static_assert(),
  30  * i.e. effectively makes the message mandatory.  Many KVM selftests #define
  31  * _GNU_SOURCE for various reasons, and _GNU_SOURCE implies _ISOC11_SOURCE.  As
  32  * a result, static_assert() behavior is non-deterministic and may or may not
  33  * require a message depending on #include order.
  34  */
  35 #define __kvm_static_assert(expr, msg, ...) _Static_assert(expr, msg)
  36 #define kvm_static_assert(expr, ...) __kvm_static_assert(expr, ##__VA_ARGS__, #expr)
  37
  38 #define KVM_DEV_PATH "/dev/kvm"
  39 #define KVM_MAX_VCPUS 512
  40
  41 #define NSEC_PER_SEC 1000000000L
  42
  43 typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */
  44 typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */
  45
  46 struct userspace_mem_region {
  47         struct kvm_userspace_memory_region region;
  48         struct sparsebit *unused_phy_pages;
  49         int fd;
  50         off_t offset;
  51         enum vm_mem_backing_src_type backing_src_type;
  52         void *host_mem;
  53         void *host_alias;
  54         void *mmap_start;
  55         void *mmap_alias;
  56         size_t mmap_size;
  57         struct rb_node gpa_node;
  58         struct rb_node hva_node;
  59         struct hlist_node slot_node;
  60 };
  61
  62 struct kvm_vcpu {
  63         struct list_head list;
  64         uint32_t id;
  65         int fd;
  66         struct kvm_vm *vm;
  67         struct kvm_run *run;
  68 #ifdef __x86_64__
  69         struct kvm_cpuid2 *cpuid;
  70 #endif
  71         struct kvm_dirty_gfn *dirty_gfns;
  72         uint32_t fetch_index;
  73         uint32_t dirty_gfns_count;
  74 };
  75
  76 struct userspace_mem_regions {
  77         struct rb_root gpa_tree;
  78         struct rb_root hva_tree;
  79         DECLARE_HASHTABLE(slot_hash, 9);
  80 };
  81
  82 enum kvm_mem_region_type {
  83         MEM_REGION_CODE,
  84         MEM_REGION_DATA,
  85         MEM_REGION_PT,
  86         MEM_REGION_TEST_DATA,
  87         NR_MEM_REGIONS,
  88 };
  89
  90 struct kvm_vm {
  91         int mode;
  92         unsigned long type;
  93         int kvm_fd;
  94         int fd;
  95         unsigned int pgtable_levels;
  96         unsigned int page_size;
  97         unsigned int page_shift;
  98         unsigned int pa_bits;
  99         unsigned int va_bits;
 100         uint64_t max_gfn;
 101         struct list_head vcpus;
 102         struct userspace_mem_regions regions;
 103         struct sparsebit *vpages_valid;
 104         struct sparsebit *vpages_mapped;
 105         bool has_irqchip;
 106         bool pgd_created;
 107         vm_paddr_t ucall_mmio_addr;
 108         vm_paddr_t pgd;
 109         vm_vaddr_t gdt;
 110         vm_vaddr_t tss;
 111         vm_vaddr_t idt;
 112         vm_vaddr_t handlers;
 113         uint32_t dirty_ring_size;
 114
 115         /* Cache of information for binary stats interface */
 116         int stats_fd;
 117         struct kvm_stats_header stats_header;
 118         struct kvm_stats_desc *stats_desc;
 119
 120         /*
 121          * KVM region slots. These are the default memslots used by page
 122          * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE]
 123          * memslot.
 124          */
 125         uint32_t memslots[NR_MEM_REGIONS];
 126 };
 127
 128 struct vcpu_reg_sublist {
 129         const char *name;
 130         long capability;
 131         int feature;
 132         bool finalize;
 133         __u64 *regs;
 134         __u64 regs_n;
 135         __u64 *rejects_set;
 136         __u64 rejects_set_n;
 137         __u64 *skips_set;
 138         __u64 skips_set_n;
 139 };
 140
 141 struct vcpu_reg_list {
 142         char *name;
 143         struct vcpu_reg_sublist sublists[];
 144 };
 145
 146 #define for_each_sublist(c, s)          \
 147         for ((s) = &(c)->sublists[0]; (s)->regs; ++(s))
 148
 149 #define kvm_for_each_vcpu(vm, i, vcpu)                  \
 150         for ((i) = 0; (i) <= (vm)->last_vcpu_id; (i)++) \
 151                 if (!((vcpu) = vm->vcpus[i]))           \
 152                         continue;                       \
 153                 else
 154
 155 struct userspace_mem_region *
 156 memslot2region(struct kvm_vm *vm, uint32_t memslot);
 157
 158 static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm,
 159                                                              enum kvm_mem_region_type type)
 160 {
 161         assert(type < NR_MEM_REGIONS);
 162         return memslot2region(vm, vm->memslots[type]);
 163 }
 164
 165 /* Minimum allocated guest virtual and physical addresses */
 166 #define KVM_UTIL_MIN_VADDR              0x2000
 167 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR  0x180000
 168
 169 #define DEFAULT_GUEST_STACK_VADDR_MIN   0xab6000
 170 #define DEFAULT_STACK_PGS               5
 171
 172 enum vm_guest_mode {
 173         VM_MODE_P52V48_4K,
 174         VM_MODE_P52V48_64K,
 175         VM_MODE_P48V48_4K,
 176         VM_MODE_P48V48_16K,
 177         VM_MODE_P48V48_64K,
 178         VM_MODE_P40V48_4K,
 179         VM_MODE_P40V48_16K,
 180         VM_MODE_P40V48_64K,
 181         VM_MODE_PXXV48_4K,      /* For 48bits VA but ANY bits PA */
 182         VM_MODE_P47V64_4K,
 183         VM_MODE_P44V64_4K,
 184         VM_MODE_P36V48_4K,
 185         VM_MODE_P36V48_16K,
 186         VM_MODE_P36V48_64K,
 187         VM_MODE_P36V47_16K,
 188         NUM_VM_MODES,
 189 };
 190
 191 #if defined(__aarch64__)
 192
 193 extern enum vm_guest_mode vm_mode_default;
 194
 195 #define VM_MODE_DEFAULT                 vm_mode_default
 196 #define MIN_PAGE_SHIFT                  12U
 197 #define ptes_per_page(page_size)        ((page_size) / 8)
 198
 199 #elif defined(__x86_64__)
 200
 201 #define VM_MODE_DEFAULT                 VM_MODE_PXXV48_4K
 202 #define MIN_PAGE_SHIFT                  12U
 203 #define ptes_per_page(page_size)        ((page_size) / 8)
 204
 205 #elif defined(__s390x__)
 206
 207 #define VM_MODE_DEFAULT                 VM_MODE_P44V64_4K
 208 #define MIN_PAGE_SHIFT                  12U
 209 #define ptes_per_page(page_size)        ((page_size) / 16)
 210
 211 #elif defined(__riscv)
 212
 213 #if __riscv_xlen == 32
 214 #error "RISC-V 32-bit kvm selftests not supported"
 215 #endif
 216
 217 #define VM_MODE_DEFAULT                 VM_MODE_P40V48_4K
 218 #define MIN_PAGE_SHIFT                  12U
 219 #define ptes_per_page(page_size)        ((page_size) / 8)
 220
 221 #endif
 222
 223 #define MIN_PAGE_SIZE           (1U << MIN_PAGE_SHIFT)
 224 #define PTES_PER_MIN_PAGE       ptes_per_page(MIN_PAGE_SIZE)
 225
 226 struct vm_guest_mode_params {
 227         unsigned int pa_bits;
 228         unsigned int va_bits;
 229         unsigned int page_size;
 230         unsigned int page_shift;
 231 };
 232 extern const struct vm_guest_mode_params vm_guest_mode_params[];
 233
 234 int open_path_or_exit(const char *path, int flags);
 235 int open_kvm_dev_path_or_exit(void);
 236
 237 bool get_kvm_param_bool(const char *param);
 238 bool get_kvm_intel_param_bool(const char *param);
 239 bool get_kvm_amd_param_bool(const char *param);
 240
 241 unsigned int kvm_check_cap(long cap);
 242
 243 static inline bool kvm_has_cap(long cap)
 244 {
 245         return kvm_check_cap(cap);
 246 }
 247
 248 #define __KVM_SYSCALL_ERROR(_name, _ret) \
 249         "%s failed, rc: %i errno: %i (%s)", (_name), (_ret), errno, strerror(errno)
 250
 251 #define __KVM_IOCTL_ERROR(_name, _ret)  __KVM_SYSCALL_ERROR(_name, _ret)
 252 #define KVM_IOCTL_ERROR(_ioctl, _ret) __KVM_IOCTL_ERROR(#_ioctl, _ret)
 253
 254 #define kvm_do_ioctl(fd, cmd, arg)                                              \
 255 ({                                                                              \
 256         kvm_static_assert(!_IOC_SIZE(cmd) || sizeof(*arg) == _IOC_SIZE(cmd));   \
 257         ioctl(fd, cmd, arg);                                                    \
 258 })
 259
 260 #define __kvm_ioctl(kvm_fd, cmd, arg)                           \
 261         kvm_do_ioctl(kvm_fd, cmd, arg)
 262
 263
 264 #define _kvm_ioctl(kvm_fd, cmd, name, arg)                      \
 265 ({                                                              \
 266         int ret = __kvm_ioctl(kvm_fd, cmd, arg);                \
 267                                                                 \
 268         TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret));        \
 269 })
 270
 271 #define kvm_ioctl(kvm_fd, cmd, arg) \
 272         _kvm_ioctl(kvm_fd, cmd, #cmd, arg)
 273
 274 static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { }
 275
 276 #define __vm_ioctl(vm, cmd, arg)                                \
 277 ({                                                              \
 278         static_assert_is_vm(vm);                                \
 279         kvm_do_ioctl((vm)->fd, cmd, arg);                       \
 280 })
 281
 282 #define _vm_ioctl(vm, cmd, name, arg)                           \
 283 ({                                                              \
 284         int ret = __vm_ioctl(vm, cmd, arg);                     \
 285                                                                 \
 286         TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret));        \
 287 })
 288
 289 #define vm_ioctl(vm, cmd, arg)                                  \
 290         _vm_ioctl(vm, cmd, #cmd, arg)
 291
 292
 293 static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
 294
 295 #define __vcpu_ioctl(vcpu, cmd, arg)                            \
 296 ({                                                              \
 297         static_assert_is_vcpu(vcpu);                            \
 298         kvm_do_ioctl((vcpu)->fd, cmd, arg);                     \
 299 })
 300
 301 #define _vcpu_ioctl(vcpu, cmd, name, arg)                       \
 302 ({                                                              \
 303         int ret = __vcpu_ioctl(vcpu, cmd, arg);                 \
 304                                                                 \
 305         TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(name, ret));        \
 306 })
 307
 308 #define vcpu_ioctl(vcpu, cmd, arg)                              \
 309         _vcpu_ioctl(vcpu, cmd, #cmd, arg)
 310
 311 /*
 312  * Looks up and returns the value corresponding to the capability
 313  * (KVM_CAP_*) given by cap.
 314  */
 315 static inline int vm_check_cap(struct kvm_vm *vm, long cap)
 316 {
 317         int ret =  __vm_ioctl(vm, KVM_CHECK_EXTENSION, (void *)cap);
 318
 319         TEST_ASSERT(ret >= 0, KVM_IOCTL_ERROR(KVM_CHECK_EXTENSION, ret));
 320         return ret;
 321 }
 322
 323 static inline int __vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 324 {
 325         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 326
 327         return __vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 328 }
 329 static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 330 {
 331         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 332
 333         vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 334 }
 335
 336 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
 337 const char *vm_guest_mode_string(uint32_t i);
 338
 339 void kvm_vm_free(struct kvm_vm *vmp);
 340 void kvm_vm_restart(struct kvm_vm *vmp);
 341 void kvm_vm_release(struct kvm_vm *vmp);
 342 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva,
 343                        size_t len);
 344 void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename);
 345 int kvm_memfd_alloc(size_t size, bool hugepages);
 346
 347 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
 348
 349 static inline void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
 350 {
 351         struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
 352
 353         vm_ioctl(vm, KVM_GET_DIRTY_LOG, &args);
 354 }
 355
 356 static inline void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
 357                                           uint64_t first_page, uint32_t num_pages)
 358 {
 359         struct kvm_clear_dirty_log args = {
 360                 .dirty_bitmap = log,
 361                 .slot = slot,
 362                 .first_page = first_page,
 363                 .num_pages = num_pages
 364         };
 365
 366         vm_ioctl(vm, KVM_CLEAR_DIRTY_LOG, &args);
 367 }
 368
 369 static inline uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
 370 {
 371         return __vm_ioctl(vm, KVM_RESET_DIRTY_RINGS, NULL);
 372 }
 373
 374 static inline int vm_get_stats_fd(struct kvm_vm *vm)
 375 {
 376         int fd = __vm_ioctl(vm, KVM_GET_STATS_FD, NULL);
 377
 378         TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_GET_STATS_FD, fd));
 379         return fd;
 380 }
 381
 382 static inline void read_stats_header(int stats_fd, struct kvm_stats_header *header)
 383 {
 384         ssize_t ret;
 385
 386         ret = pread(stats_fd, header, sizeof(*header), 0);
 387         TEST_ASSERT(ret == sizeof(*header),
 388                     "Failed to read '%lu' header bytes, ret = '%ld'",
 389                     sizeof(*header), ret);
 390 }
 391
 392 struct kvm_stats_desc *read_stats_descriptors(int stats_fd,
 393                                               struct kvm_stats_header *header);
 394
 395 static inline ssize_t get_stats_descriptor_size(struct kvm_stats_header *header)
 396 {
 397          /*
 398           * The base size of the descriptor is defined by KVM's ABI, but the
 399           * size of the name field is variable, as far as KVM's ABI is
 400           * concerned. For a given instance of KVM, the name field is the same
 401           * size for all stats and is provided in the overall stats header.
 402           */
 403         return sizeof(struct kvm_stats_desc) + header->name_size;
 404 }
 405
 406 static inline struct kvm_stats_desc *get_stats_descriptor(struct kvm_stats_desc *stats,
 407                                                           int index,
 408                                                           struct kvm_stats_header *header)
 409 {
 410         /*
 411          * Note, size_desc includes the size of the name field, which is
 412          * variable. i.e. this is NOT equivalent to &stats_desc[i].
 413          */
 414         return (void *)stats + index * get_stats_descriptor_size(header);
 415 }
 416
 417 void read_stat_data(int stats_fd, struct kvm_stats_header *header,
 418                     struct kvm_stats_desc *desc, uint64_t *data,
 419                     size_t max_elements);
 420
 421 void __vm_get_stat(struct kvm_vm *vm, const char *stat_name, uint64_t *data,
 422                    size_t max_elements);
 423
 424 static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name)
 425 {
 426         uint64_t data;
 427
 428         __vm_get_stat(vm, stat_name, &data, 1);
 429         return data;
 430 }
 431
 432 void vm_create_irqchip(struct kvm_vm *vm);
 433
 434 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 435                                uint64_t gpa, uint64_t size, void *hva);
 436 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 437                                 uint64_t gpa, uint64_t size, void *hva);
 438 void vm_userspace_mem_region_add(struct kvm_vm *vm,
 439         enum vm_mem_backing_src_type src_type,
 440         uint64_t guest_paddr, uint32_t slot, uint64_t npages,
 441         uint32_t flags);
 442
 443 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
 444 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
 445 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot);
 446 struct kvm_vcpu *__vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id);
 447 void vm_populate_vaddr_bitmap(struct kvm_vm *vm);
 448 vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 449 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min);
 450 vm_vaddr_t __vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
 451                             enum kvm_mem_region_type type);
 452 vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages);
 453 vm_vaddr_t __vm_vaddr_alloc_page(struct kvm_vm *vm,
 454                                  enum kvm_mem_region_type type);
 455 vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm);
 456
 457 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 458               unsigned int npages);
 459 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa);
 460 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva);
 461 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
 462 void *addr_gpa2alias(struct kvm_vm *vm, vm_paddr_t gpa);
 463
 464 void vcpu_run(struct kvm_vcpu *vcpu);
 465 int _vcpu_run(struct kvm_vcpu *vcpu);
 466
 467 static inline int __vcpu_run(struct kvm_vcpu *vcpu)
 468 {
 469         return __vcpu_ioctl(vcpu, KVM_RUN, NULL);
 470 }
 471
 472 void vcpu_run_complete_io(struct kvm_vcpu *vcpu);
 473 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu);
 474
 475 static inline void vcpu_enable_cap(struct kvm_vcpu *vcpu, uint32_t cap,
 476                                    uint64_t arg0)
 477 {
 478         struct kvm_enable_cap enable_cap = { .cap = cap, .args = { arg0 } };
 479
 480         vcpu_ioctl(vcpu, KVM_ENABLE_CAP, &enable_cap);
 481 }
 482
 483 static inline void vcpu_guest_debug_set(struct kvm_vcpu *vcpu,
 484                                         struct kvm_guest_debug *debug)
 485 {
 486         vcpu_ioctl(vcpu, KVM_SET_GUEST_DEBUG, debug);
 487 }
 488
 489 static inline void vcpu_mp_state_get(struct kvm_vcpu *vcpu,
 490                                      struct kvm_mp_state *mp_state)
 491 {
 492         vcpu_ioctl(vcpu, KVM_GET_MP_STATE, mp_state);
 493 }
 494 static inline void vcpu_mp_state_set(struct kvm_vcpu *vcpu,
 495                                      struct kvm_mp_state *mp_state)
 496 {
 497         vcpu_ioctl(vcpu, KVM_SET_MP_STATE, mp_state);
 498 }
 499
 500 static inline void vcpu_regs_get(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 501 {
 502         vcpu_ioctl(vcpu, KVM_GET_REGS, regs);
 503 }
 504
 505 static inline void vcpu_regs_set(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 506 {
 507         vcpu_ioctl(vcpu, KVM_SET_REGS, regs);
 508 }
 509 static inline void vcpu_sregs_get(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 510 {
 511         vcpu_ioctl(vcpu, KVM_GET_SREGS, sregs);
 512
 513 }
 514 static inline void vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 515 {
 516         vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 517 }
 518 static inline int _vcpu_sregs_set(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 519 {
 520         return __vcpu_ioctl(vcpu, KVM_SET_SREGS, sregs);
 521 }
 522 static inline void vcpu_fpu_get(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 523 {
 524         vcpu_ioctl(vcpu, KVM_GET_FPU, fpu);
 525 }
 526 static inline void vcpu_fpu_set(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 527 {
 528         vcpu_ioctl(vcpu, KVM_SET_FPU, fpu);
 529 }
 530
 531 static inline int __vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 532 {
 533         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
 534
 535         return __vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 536 }
 537 static inline int __vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 538 {
 539         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 540
 541         return __vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 542 }
 543 static inline void vcpu_get_reg(struct kvm_vcpu *vcpu, uint64_t id, void *addr)
 544 {
 545         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)addr };
 546
 547         vcpu_ioctl(vcpu, KVM_GET_ONE_REG, &reg);
 548 }
 549 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, uint64_t id, uint64_t val)
 550 {
 551         struct kvm_one_reg reg = { .id = id, .addr = (uint64_t)&val };
 552
 553         vcpu_ioctl(vcpu, KVM_SET_ONE_REG, &reg);
 554 }
 555
 556 #ifdef __KVM_HAVE_VCPU_EVENTS
 557 static inline void vcpu_events_get(struct kvm_vcpu *vcpu,
 558                                    struct kvm_vcpu_events *events)
 559 {
 560         vcpu_ioctl(vcpu, KVM_GET_VCPU_EVENTS, events);
 561 }
 562 static inline void vcpu_events_set(struct kvm_vcpu *vcpu,
 563                                    struct kvm_vcpu_events *events)
 564 {
 565         vcpu_ioctl(vcpu, KVM_SET_VCPU_EVENTS, events);
 566 }
 567 #endif
 568 #ifdef __x86_64__
 569 static inline void vcpu_nested_state_get(struct kvm_vcpu *vcpu,
 570                                          struct kvm_nested_state *state)
 571 {
 572         vcpu_ioctl(vcpu, KVM_GET_NESTED_STATE, state);
 573 }
 574 static inline int __vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 575                                           struct kvm_nested_state *state)
 576 {
 577         return __vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 578 }
 579
 580 static inline void vcpu_nested_state_set(struct kvm_vcpu *vcpu,
 581                                          struct kvm_nested_state *state)
 582 {
 583         vcpu_ioctl(vcpu, KVM_SET_NESTED_STATE, state);
 584 }
 585 #endif
 586 static inline int vcpu_get_stats_fd(struct kvm_vcpu *vcpu)
 587 {
 588         int fd = __vcpu_ioctl(vcpu, KVM_GET_STATS_FD, NULL);
 589
 590         TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_GET_STATS_FD, fd));
 591         return fd;
 592 }
 593
 594 int __kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr);
 595
 596 static inline void kvm_has_device_attr(int dev_fd, uint32_t group, uint64_t attr)
 597 {
 598         int ret = __kvm_has_device_attr(dev_fd, group, attr);
 599
 600         TEST_ASSERT(!ret, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
 601 }
 602
 603 int __kvm_device_attr_get(int dev_fd, uint32_t group, uint64_t attr, void *val);
 604
 605 static inline void kvm_device_attr_get(int dev_fd, uint32_t group,
 606                                        uint64_t attr, void *val)
 607 {
 608         int ret = __kvm_device_attr_get(dev_fd, group, attr, val);
 609
 610         TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_GET_DEVICE_ATTR, ret));
 611 }
 612
 613 int __kvm_device_attr_set(int dev_fd, uint32_t group, uint64_t attr, void *val);
 614
 615 static inline void kvm_device_attr_set(int dev_fd, uint32_t group,
 616                                        uint64_t attr, void *val)
 617 {
 618         int ret = __kvm_device_attr_set(dev_fd, group, attr, val);
 619
 620         TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_SET_DEVICE_ATTR, ret));
 621 }
 622
 623 static inline int __vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 624                                          uint64_t attr)
 625 {
 626         return __kvm_has_device_attr(vcpu->fd, group, attr);
 627 }
 628
 629 static inline void vcpu_has_device_attr(struct kvm_vcpu *vcpu, uint32_t group,
 630                                         uint64_t attr)
 631 {
 632         kvm_has_device_attr(vcpu->fd, group, attr);
 633 }
 634
 635 static inline int __vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 636                                          uint64_t attr, void *val)
 637 {
 638         return __kvm_device_attr_get(vcpu->fd, group, attr, val);
 639 }
 640
 641 static inline void vcpu_device_attr_get(struct kvm_vcpu *vcpu, uint32_t group,
 642                                         uint64_t attr, void *val)
 643 {
 644         kvm_device_attr_get(vcpu->fd, group, attr, val);
 645 }
 646
 647 static inline int __vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 648                                          uint64_t attr, void *val)
 649 {
 650         return __kvm_device_attr_set(vcpu->fd, group, attr, val);
 651 }
 652
 653 static inline void vcpu_device_attr_set(struct kvm_vcpu *vcpu, uint32_t group,
 654                                         uint64_t attr, void *val)
 655 {
 656         kvm_device_attr_set(vcpu->fd, group, attr, val);
 657 }
 658
 659 int __kvm_test_create_device(struct kvm_vm *vm, uint64_t type);
 660 int __kvm_create_device(struct kvm_vm *vm, uint64_t type);
 661
 662 static inline int kvm_create_device(struct kvm_vm *vm, uint64_t type)
 663 {
 664         int fd = __kvm_create_device(vm, type);
 665
 666         TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_DEVICE, fd));
 667         return fd;
 668 }
 669
 670 void *vcpu_map_dirty_ring(struct kvm_vcpu *vcpu);
 671
 672 /*
 673  * VM VCPU Args Set
 674  *
 675  * Input Args:
 676  *   vm - Virtual Machine
 677  *   num - number of arguments
 678  *   ... - arguments, each of type uint64_t
 679  *
 680  * Output Args: None
 681  *
 682  * Return: None
 683  *
 684  * Sets the first @num input parameters for the function at @vcpu's entry point,
 685  * per the C calling convention of the architecture, to the values given as
 686  * variable args. Each of the variable args is expected to be of type uint64_t.
 687  * The maximum @num can be is specific to the architecture.
 688  */
 689 void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...);
 690
 691 void kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 692 int _kvm_irq_line(struct kvm_vm *vm, uint32_t irq, int level);
 693
 694 #define KVM_MAX_IRQ_ROUTES              4096
 695
 696 struct kvm_irq_routing *kvm_gsi_routing_create(void);
 697 void kvm_gsi_routing_irqchip_add(struct kvm_irq_routing *routing,
 698                 uint32_t gsi, uint32_t pin);
 699 int _kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 700 void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing);
 701
 702 const char *exit_reason_str(unsigned int exit_reason);
 703
 704 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
 705                              uint32_t memslot);
 706 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
 707                               vm_paddr_t paddr_min, uint32_t memslot);
 708 vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
 709
 710 /*
 711  * ____vm_create() does KVM_CREATE_VM and little else.  __vm_create() also
 712  * loads the test binary into guest memory and creates an IRQ chip (x86 only).
 713  * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
 714  * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
 715  */
 716 struct kvm_vm *____vm_create(enum vm_guest_mode mode);
 717 struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
 718                            uint64_t nr_extra_pages);
 719
 720 static inline struct kvm_vm *vm_create_barebones(void)
 721 {
 722         return ____vm_create(VM_MODE_DEFAULT);
 723 }
 724
 725 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
 726 {
 727         return __vm_create(VM_MODE_DEFAULT, nr_runnable_vcpus, 0);
 728 }
 729
 730 struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
 731                                       uint64_t extra_mem_pages,
 732                                       void *guest_code, struct kvm_vcpu *vcpus[]);
 733
 734 static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
 735                                                   void *guest_code,
 736                                                   struct kvm_vcpu *vcpus[])
 737 {
 738         return __vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, 0,
 739                                       guest_code, vcpus);
 740 }
 741
 742 /*
 743  * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages
 744  * additional pages of guest memory.  Returns the VM and vCPU (via out param).
 745  */
 746 struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 747                                          uint64_t extra_mem_pages,
 748                                          void *guest_code);
 749
 750 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 751                                                      void *guest_code)
 752 {
 753         return __vm_create_with_one_vcpu(vcpu, 0, guest_code);
 754 }
 755
 756 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
 757
 758 void kvm_pin_this_task_to_pcpu(uint32_t pcpu);
 759 void kvm_print_vcpu_pinning_help(void);
 760 void kvm_parse_vcpu_pinning(const char *pcpus_string, uint32_t vcpu_to_pcpu[],
 761                             int nr_vcpus);
 762
 763 unsigned long vm_compute_max_gfn(struct kvm_vm *vm);
 764 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size);
 765 unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages);
 766 unsigned int vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages);
 767 static inline unsigned int
 768 vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
 769 {
 770         unsigned int n;
 771         n = vm_num_guest_pages(mode, vm_num_host_pages(mode, num_guest_pages));
 772 #ifdef __s390x__
 773         /* s390 requires 1M aligned guest sizes */
 774         n = (n + 255) & ~255;
 775 #endif
 776         return n;
 777 }
 778
 779 struct kvm_userspace_memory_region *
 780 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
 781                                  uint64_t end);
 782
 783 #define sync_global_to_guest(vm, g) ({                          \
 784         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 785         memcpy(_p, &(g), sizeof(g));                            \
 786 })
 787
 788 #define sync_global_from_guest(vm, g) ({                        \
 789         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 790         memcpy(&(g), _p, sizeof(g));                            \
 791 })
 792
 793 /*
 794  * Write a global value, but only in the VM's (guest's) domain.  Primarily used
 795  * for "globals" that hold per-VM values (VMs always duplicate code and global
 796  * data into their own region of physical memory), but can be used anytime it's
 797  * undesirable to change the host's copy of the global.
 798  */
 799 #define write_guest_global(vm, g, val) ({                       \
 800         typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));     \
 801         typeof(g) _val = val;                                   \
 802                                                                 \
 803         memcpy(_p, &(_val), sizeof(g));                         \
 804 })
 805
 806 void assert_on_unhandled_exception(struct kvm_vcpu *vcpu);
 807
 808 void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu,
 809                     uint8_t indent);
 810
 811 static inline void vcpu_dump(FILE *stream, struct kvm_vcpu *vcpu,
 812                              uint8_t indent)
 813 {
 814         vcpu_arch_dump(stream, vcpu, indent);
 815 }
 816
 817 /*
 818  * Adds a vCPU with reasonable defaults (e.g. a stack)
 819  *
 820  * Input Args:
 821  *   vm - Virtual Machine
 822  *   vcpu_id - The id of the VCPU to add to the VM.
 823  *   guest_code - The vCPU's entry point
 824  */
 825 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 826                                   void *guest_code);
 827
 828 static inline struct kvm_vcpu *vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
 829                                            void *guest_code)
 830 {
 831         return vm_arch_vcpu_add(vm, vcpu_id, guest_code);
 832 }
 833
 834 /* Re-create a vCPU after restarting a VM, e.g. for state save/restore tests. */
 835 struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, uint32_t vcpu_id);
 836
 837 static inline struct kvm_vcpu *vm_vcpu_recreate(struct kvm_vm *vm,
 838                                                 uint32_t vcpu_id)
 839 {
 840         return vm_arch_vcpu_recreate(vm, vcpu_id);
 841 }
 842
 843 void vcpu_arch_free(struct kvm_vcpu *vcpu);
 844
 845 void virt_arch_pgd_alloc(struct kvm_vm *vm);
 846
 847 static inline void virt_pgd_alloc(struct kvm_vm *vm)
 848 {
 849         virt_arch_pgd_alloc(vm);
 850 }
 851
 852 /*
 853  * VM Virtual Page Map
 854  *
 855  * Input Args:
 856  *   vm - Virtual Machine
 857  *   vaddr - VM Virtual Address
 858  *   paddr - VM Physical Address
 859  *   memslot - Memory region slot for new virtual translation tables
 860  *
 861  * Output Args: None
 862  *
 863  * Return: None
 864  *
 865  * Within @vm, creates a virtual translation for the page starting
 866  * at @vaddr to the page starting at @paddr.
 867  */
 868 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr);
 869
 870 static inline void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
 871 {
 872         virt_arch_pg_map(vm, vaddr, paddr);
 873 }
 874
 875
 876 /*
 877  * Address Guest Virtual to Guest Physical
 878  *
 879  * Input Args:
 880  *   vm - Virtual Machine
 881  *   gva - VM virtual address
 882  *
 883  * Output Args: None
 884  *
 885  * Return:
 886  *   Equivalent VM physical address
 887  *
 888  * Returns the VM physical address of the translated VM virtual
 889  * address given by @gva.
 890  */
 891 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva);
 892
 893 static inline vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 894 {
 895         return addr_arch_gva2gpa(vm, gva);
 896 }
 897
 898 /*
 899  * Virtual Translation Tables Dump
 900  *
 901  * Input Args:
 902  *   stream - Output FILE stream
 903  *   vm     - Virtual Machine
 904  *   indent - Left margin indent amount
 905  *
 906  * Output Args: None
 907  *
 908  * Return: None
 909  *
 910  * Dumps to the FILE stream given by @stream, the contents of all the
 911  * virtual translation tables for the VM given by @vm.
 912  */
 913 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent);
 914
 915 static inline void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 916 {
 917         virt_arch_dump(stream, vm, indent);
 918 }
 919
 920
 921 static inline int __vm_disable_nx_huge_pages(struct kvm_vm *vm)
 922 {
 923         return __vm_enable_cap(vm, KVM_CAP_VM_DISABLE_NX_HUGE_PAGES, 0);
 924 }
 925
 926 /*
 927  * Arch hook that is invoked via a constructor, i.e. before exeucting main(),
 928  * to allow for arch-specific setup that is common to all tests, e.g. computing
 929  * the default guest "mode".
 930  */
 931 void kvm_selftest_arch_init(void);
 932
 933 void kvm_arch_vm_post_create(struct kvm_vm *vm);
 934
 935 #endif /* SELFTEST_KVM_UTIL_BASE_H */