]> git.ipfire.org Git - thirdparty/linux.git/blob - arch/x86/kvm/svm/svm.c
projects / thirdparty / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block
[thirdparty/linux.git] / arch / x86 / kvm / svm / svm.c
1 #define pr_fmt(fmt) "SVM: " fmt
2
3 #include <linux/kvm_host.h>
4
5 #include "irq.h"
6 #include "mmu.h"
7 #include "kvm_cache_regs.h"
8 #include "x86.h"
9 #include "cpuid.h"
10 #include "pmu.h"
11
12 #include <linux/module.h>
13 #include <linux/mod_devicetable.h>
14 #include <linux/kernel.h>
15 #include <linux/vmalloc.h>
16 #include <linux/highmem.h>
17 #include <linux/amd-iommu.h>
18 #include <linux/sched.h>
19 #include <linux/trace_events.h>
20 #include <linux/slab.h>
21 #include <linux/hashtable.h>
22 #include <linux/frame.h>
23 #include <linux/psp-sev.h>
24 #include <linux/file.h>
25 #include <linux/pagemap.h>
26 #include <linux/swap.h>
27 #include <linux/rwsem.h>
28
29 #include <asm/apic.h>
30 #include <asm/perf_event.h>
31 #include <asm/tlbflush.h>
32 #include <asm/desc.h>
33 #include <asm/debugreg.h>
34 #include <asm/kvm_para.h>
35 #include <asm/irq_remapping.h>
36 #include <asm/spec-ctrl.h>
37 #include <asm/cpu_device_id.h>
38
39 #include <asm/virtext.h>
40 #include "trace.h"
41
42 #include "svm.h"
43
44 #define __ex(x) __kvm_handle_fault_on_reboot(x)
45
46 MODULE_AUTHOR("Qumranet");
47 MODULE_LICENSE("GPL");
48
49 #ifdef MODULE
50 static const struct x86_cpu_id svm_cpu_id[] = {
51 X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
52 {}
53 };
54 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
55 #endif
56
57 #define IOPM_ALLOC_ORDER 2
58 #define MSRPM_ALLOC_ORDER 1
59
60 #define SEG_TYPE_LDT 2
61 #define SEG_TYPE_BUSY_TSS16 3
62
63 #define SVM_FEATURE_LBRV (1 << 1)
64 #define SVM_FEATURE_SVML (1 << 2)
65 #define SVM_FEATURE_TSC_RATE (1 << 4)
66 #define SVM_FEATURE_VMCB_CLEAN (1 << 5)
67 #define SVM_FEATURE_FLUSH_ASID (1 << 6)
68 #define SVM_FEATURE_DECODE_ASSIST (1 << 7)
69 #define SVM_FEATURE_PAUSE_FILTER (1 << 10)
70
71 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
72
73 #define TSC_RATIO_RSVD 0xffffff0000000000ULL
74 #define TSC_RATIO_MIN 0x0000000000000001ULL
75 #define TSC_RATIO_MAX 0x000000ffffffffffULL
76
77 static bool erratum_383_found __read_mostly;
78
79 u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
80
81 /*
82 * Set osvw_len to higher value when updated Revision Guides
83 * are published and we know what the new status bits are
84 */
85 static uint64_t osvw_len = 4, osvw_status;
86
87 static DEFINE_PER_CPU(u64, current_tsc_ratio);
88 #define TSC_RATIO_DEFAULT 0x0100000000ULL
89
90 static const struct svm_direct_access_msrs {
91 u32 index; /* Index of the MSR */
92 bool always; /* True if intercept is always on */
93 } direct_access_msrs[] = {
94 { .index = MSR_STAR, .always = true },
95 { .index = MSR_IA32_SYSENTER_CS, .always = true },
96 #ifdef CONFIG_X86_64
97 { .index = MSR_GS_BASE, .always = true },
98 { .index = MSR_FS_BASE, .always = true },
99 { .index = MSR_KERNEL_GS_BASE, .always = true },
100 { .index = MSR_LSTAR, .always = true },
101 { .index = MSR_CSTAR, .always = true },
102 { .index = MSR_SYSCALL_MASK, .always = true },
103 #endif
104 { .index = MSR_IA32_SPEC_CTRL, .always = false },
105 { .index = MSR_IA32_PRED_CMD, .always = false },
106 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
107 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
108 { .index = MSR_IA32_LASTINTFROMIP, .always = false },
109 { .index = MSR_IA32_LASTINTTOIP, .always = false },
110 { .index = MSR_INVALID, .always = false },
111 };
112
113 /* enable NPT for AMD64 and X86 with PAE */
114 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
115 bool npt_enabled = true;
116 #else
117 bool npt_enabled;
118 #endif
119
120 /*
121 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
122 * pause_filter_count: On processors that support Pause filtering(indicated
123 * by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
124 * count value. On VMRUN this value is loaded into an internal counter.
125 * Each time a pause instruction is executed, this counter is decremented
126 * until it reaches zero at which time a #VMEXIT is generated if pause
127 * intercept is enabled. Refer to AMD APM Vol 2 Section 15.14.4 Pause
128 * Intercept Filtering for more details.
129 * This also indicate if ple logic enabled.
130 *
131 * pause_filter_thresh: In addition, some processor families support advanced
132 * pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
133 * the amount of time a guest is allowed to execute in a pause loop.
134 * In this mode, a 16-bit pause filter threshold field is added in the
135 * VMCB. The threshold value is a cycle count that is used to reset the
136 * pause counter. As with simple pause filtering, VMRUN loads the pause
137 * count value from VMCB into an internal counter. Then, on each pause
138 * instruction the hardware checks the elapsed number of cycles since
139 * the most recent pause instruction against the pause filter threshold.
140 * If the elapsed cycle count is greater than the pause filter threshold,
141 * then the internal pause count is reloaded from the VMCB and execution
142 * continues. If the elapsed cycle count is less than the pause filter
143 * threshold, then the internal pause count is decremented. If the count
144 * value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
145 * triggered. If advanced pause filtering is supported and pause filter
146 * threshold field is set to zero, the filter will operate in the simpler,
147 * count only mode.
148 */
149
150 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
151 module_param(pause_filter_thresh, ushort, 0444);
152
153 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
154 module_param(pause_filter_count, ushort, 0444);
155
156 /* Default doubles per-vcpu window every exit. */
157 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
158 module_param(pause_filter_count_grow, ushort, 0444);
159
160 /* Default resets per-vcpu window every exit to pause_filter_count. */
161 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
162 module_param(pause_filter_count_shrink, ushort, 0444);
163
164 /* Default is to compute the maximum so we can never overflow. */
165 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
166 module_param(pause_filter_count_max, ushort, 0444);
167
168 /* allow nested paging (virtualized MMU) for all guests */
169 static int npt = true;
170 module_param(npt, int, S_IRUGO);
171
172 /* allow nested virtualization in KVM/SVM */
173 static int nested = true;
174 module_param(nested, int, S_IRUGO);
175
176 /* enable/disable Next RIP Save */
177 static int nrips = true;
178 module_param(nrips, int, 0444);
179
180 /* enable/disable Virtual VMLOAD VMSAVE */
181 static int vls = true;
182 module_param(vls, int, 0444);
183
184 /* enable/disable Virtual GIF */
185 static int vgif = true;
186 module_param(vgif, int, 0444);
187
188 /* enable/disable SEV support */
189 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
190 module_param(sev, int, 0444);
191
192 static bool __read_mostly dump_invalid_vmcb = 0;
193 module_param(dump_invalid_vmcb, bool, 0644);
194
195 static u8 rsm_ins_bytes[] = "\x0f\xaa";
196
197 static void svm_complete_interrupts(struct vcpu_svm *svm);
198
199 static unsigned long iopm_base;
200
201 struct kvm_ldttss_desc {
202 u16 limit0;
203 u16 base0;
204 unsigned base1:8, type:5, dpl:2, p:1;
205 unsigned limit1:4, zero0:3, g:1, base2:8;
206 u32 base3;
207 u32 zero1;
208 } __attribute__((packed));
209
210 DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
211
212 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
213
214 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
215 #define MSRS_RANGE_SIZE 2048
216 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
217
218 u32 svm_msrpm_offset(u32 msr)
219 {
220 u32 offset;
221 int i;
222
223 for (i = 0; i < NUM_MSR_MAPS; i++) {
224 if (msr < msrpm_ranges[i] ||
225 msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
226 continue;
227
228 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
229 offset += (i * MSRS_RANGE_SIZE); /* add range offset */
230
231 /* Now we have the u8 offset - but need the u32 offset */
232 return offset / 4;
233 }
234
235 /* MSR not in any range */
236 return MSR_INVALID;
237 }
238
239 #define MAX_INST_SIZE 15
240
241 static inline void clgi(void)
242 {
243 asm volatile (__ex("clgi"));
244 }
245
246 static inline void stgi(void)
247 {
248 asm volatile (__ex("stgi"));
249 }
250
251 static inline void invlpga(unsigned long addr, u32 asid)
252 {
253 asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
254 }
255
256 static int get_npt_level(struct kvm_vcpu *vcpu)
257 {
258 #ifdef CONFIG_X86_64
259 return PT64_ROOT_4LEVEL;
260 #else
261 return PT32E_ROOT_LEVEL;
262 #endif
263 }
264
265 void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
266 {
267 vcpu->arch.efer = efer;
268
269 if (!npt_enabled) {
270 /* Shadow paging assumes NX to be available. */
271 efer |= EFER_NX;
272
273 if (!(efer & EFER_LMA))
274 efer &= ~EFER_LME;
275 }
276
277 to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
278 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
279 }
280
281 static int is_external_interrupt(u32 info)
282 {
283 info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
284 return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
285 }
286
287 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
288 {
289 struct vcpu_svm *svm = to_svm(vcpu);
290 u32 ret = 0;
291
292 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
293 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
294 return ret;
295 }
296
297 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
298 {
299 struct vcpu_svm *svm = to_svm(vcpu);
300
301 if (mask == 0)
302 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
303 else
304 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
305
306 }
307
308 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
309 {
310 struct vcpu_svm *svm = to_svm(vcpu);
311
312 if (nrips && svm->vmcb->control.next_rip != 0) {
313 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
314 svm->next_rip = svm->vmcb->control.next_rip;
315 }
316
317 if (!svm->next_rip) {
318 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
319 return 0;
320 } else {
321 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
322 pr_err("%s: ip 0x%lx next 0x%llx\n",
323 __func__, kvm_rip_read(vcpu), svm->next_rip);
324 kvm_rip_write(vcpu, svm->next_rip);
325 }
326 svm_set_interrupt_shadow(vcpu, 0);
327
328 return 1;
329 }
330
331 static void svm_queue_exception(struct kvm_vcpu *vcpu)
332 {
333 struct vcpu_svm *svm = to_svm(vcpu);
334 unsigned nr = vcpu->arch.exception.nr;
335 bool has_error_code = vcpu->arch.exception.has_error_code;
336 bool reinject = vcpu->arch.exception.injected;
337 u32 error_code = vcpu->arch.exception.error_code;
338
339 /*
340 * If we are within a nested VM we'd better #VMEXIT and let the guest
341 * handle the exception
342 */
343 if (!reinject &&
344 nested_svm_check_exception(svm, nr, has_error_code, error_code))
345 return;
346
347 kvm_deliver_exception_payload(&svm->vcpu);
348
349 if (nr == BP_VECTOR && !nrips) {
350 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
351
352 /*
353 * For guest debugging where we have to reinject #BP if some
354 * INT3 is guest-owned:
355 * Emulate nRIP by moving RIP forward. Will fail if injection
356 * raises a fault that is not intercepted. Still better than
357 * failing in all cases.
358 */
359 (void)skip_emulated_instruction(&svm->vcpu);
360 rip = kvm_rip_read(&svm->vcpu);
361 svm->int3_rip = rip + svm->vmcb->save.cs.base;
362 svm->int3_injected = rip - old_rip;
363 }
364
365 svm->vmcb->control.event_inj = nr
366 | SVM_EVTINJ_VALID
367 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
368 | SVM_EVTINJ_TYPE_EXEPT;
369 svm->vmcb->control.event_inj_err = error_code;
370 }
371
372 static void svm_init_erratum_383(void)
373 {
374 u32 low, high;
375 int err;
376 u64 val;
377
378 if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
379 return;
380
381 /* Use _safe variants to not break nested virtualization */
382 val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
383 if (err)
384 return;
385
386 val |= (1ULL << 47);
387
388 low = lower_32_bits(val);
389 high = upper_32_bits(val);
390
391 native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
392
393 erratum_383_found = true;
394 }
395
396 static void svm_init_osvw(struct kvm_vcpu *vcpu)
397 {
398 /*
399 * Guests should see errata 400 and 415 as fixed (assuming that
400 * HLT and IO instructions are intercepted).
401 */
402 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
403 vcpu->arch.osvw.status = osvw_status & ~(6ULL);
404
405 /*
406 * By increasing VCPU's osvw.length to 3 we are telling the guest that
407 * all osvw.status bits inside that length, including bit 0 (which is
408 * reserved for erratum 298), are valid. However, if host processor's
409 * osvw_len is 0 then osvw_status[0] carries no information. We need to
410 * be conservative here and therefore we tell the guest that erratum 298
411 * is present (because we really don't know).
412 */
413 if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
414 vcpu->arch.osvw.status |= 1;
415 }
416
417 static int has_svm(void)
418 {
419 const char *msg;
420
421 if (!cpu_has_svm(&msg)) {
422 printk(KERN_INFO "has_svm: %s\n", msg);
423 return 0;
424 }
425
426 return 1;
427 }
428
429 static void svm_hardware_disable(void)
430 {
431 /* Make sure we clean up behind us */
432 if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
433 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
434
435 cpu_svm_disable();
436
437 amd_pmu_disable_virt();
438 }
439
440 static int svm_hardware_enable(void)
441 {
442
443 struct svm_cpu_data *sd;
444 uint64_t efer;
445 struct desc_struct *gdt;
446 int me = raw_smp_processor_id();
447
448 rdmsrl(MSR_EFER, efer);
449 if (efer & EFER_SVME)
450 return -EBUSY;
451
452 if (!has_svm()) {
453 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
454 return -EINVAL;
455 }
456 sd = per_cpu(svm_data, me);
457 if (!sd) {
458 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
459 return -EINVAL;
460 }
461
462 sd->asid_generation = 1;
463 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
464 sd->next_asid = sd->max_asid + 1;
465 sd->min_asid = max_sev_asid + 1;
466
467 gdt = get_current_gdt_rw();
468 sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
469
470 wrmsrl(MSR_EFER, efer | EFER_SVME);
471
472 wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
473
474 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
475 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
476 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
477 }
478
479
480 /*
481 * Get OSVW bits.
482 *
483 * Note that it is possible to have a system with mixed processor
484 * revisions and therefore different OSVW bits. If bits are not the same
485 * on different processors then choose the worst case (i.e. if erratum
486 * is present on one processor and not on another then assume that the
487 * erratum is present everywhere).
488 */
489 if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
490 uint64_t len, status = 0;
491 int err;
492
493 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
494 if (!err)
495 status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
496 &err);
497
498 if (err)
499 osvw_status = osvw_len = 0;
500 else {
501 if (len < osvw_len)
502 osvw_len = len;
503 osvw_status |= status;
504 osvw_status &= (1ULL << osvw_len) - 1;
505 }
506 } else
507 osvw_status = osvw_len = 0;
508
509 svm_init_erratum_383();
510
511 amd_pmu_enable_virt();
512
513 return 0;
514 }
515
516 static void svm_cpu_uninit(int cpu)
517 {
518 struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
519
520 if (!sd)
521 return;
522
523 per_cpu(svm_data, raw_smp_processor_id()) = NULL;
524 kfree(sd->sev_vmcbs);
525 __free_page(sd->save_area);
526 kfree(sd);
527 }
528
529 static int svm_cpu_init(int cpu)
530 {
531 struct svm_cpu_data *sd;
532
533 sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
534 if (!sd)
535 return -ENOMEM;
536 sd->cpu = cpu;
537 sd->save_area = alloc_page(GFP_KERNEL);
538 if (!sd->save_area)
539 goto free_cpu_data;
540
541 if (svm_sev_enabled()) {
542 sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
543 sizeof(void *),
544 GFP_KERNEL);
545 if (!sd->sev_vmcbs)
546 goto free_save_area;
547 }
548
549 per_cpu(svm_data, cpu) = sd;
550
551 return 0;
552
553 free_save_area:
554 __free_page(sd->save_area);
555 free_cpu_data:
556 kfree(sd);
557 return -ENOMEM;
558
559 }
560
561 static bool valid_msr_intercept(u32 index)
562 {
563 int i;
564
565 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
566 if (direct_access_msrs[i].index == index)
567 return true;
568
569 return false;
570 }
571
572 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
573 {
574 u8 bit_write;
575 unsigned long tmp;
576 u32 offset;
577 u32 *msrpm;
578
579 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
580 to_svm(vcpu)->msrpm;
581
582 offset = svm_msrpm_offset(msr);
583 bit_write = 2 * (msr & 0x0f) + 1;
584 tmp = msrpm[offset];
585
586 BUG_ON(offset == MSR_INVALID);
587
588 return !!test_bit(bit_write, &tmp);
589 }
590
591 static void set_msr_interception(u32 *msrpm, unsigned msr,
592 int read, int write)
593 {
594 u8 bit_read, bit_write;
595 unsigned long tmp;
596 u32 offset;
597
598 /*
599 * If this warning triggers extend the direct_access_msrs list at the
600 * beginning of the file
601 */
602 WARN_ON(!valid_msr_intercept(msr));
603
604 offset = svm_msrpm_offset(msr);
605 bit_read = 2 * (msr & 0x0f);
606 bit_write = 2 * (msr & 0x0f) + 1;
607 tmp = msrpm[offset];
608
609 BUG_ON(offset == MSR_INVALID);
610
611 read ? clear_bit(bit_read, &tmp) : set_bit(bit_read, &tmp);
612 write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
613
614 msrpm[offset] = tmp;
615 }
616
617 static void svm_vcpu_init_msrpm(u32 *msrpm)
618 {
619 int i;
620
621 memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
622
623 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
624 if (!direct_access_msrs[i].always)
625 continue;
626
627 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
628 }
629 }
630
631 static void add_msr_offset(u32 offset)
632 {
633 int i;
634
635 for (i = 0; i < MSRPM_OFFSETS; ++i) {
636
637 /* Offset already in list? */
638 if (msrpm_offsets[i] == offset)
639 return;
640
641 /* Slot used by another offset? */
642 if (msrpm_offsets[i] != MSR_INVALID)
643 continue;
644
645 /* Add offset to list */
646 msrpm_offsets[i] = offset;
647
648 return;
649 }
650
651 /*
652 * If this BUG triggers the msrpm_offsets table has an overflow. Just
653 * increase MSRPM_OFFSETS in this case.
654 */
655 BUG();
656 }
657
658 static void init_msrpm_offsets(void)
659 {
660 int i;
661
662 memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
663
664 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
665 u32 offset;
666
667 offset = svm_msrpm_offset(direct_access_msrs[i].index);
668 BUG_ON(offset == MSR_INVALID);
669
670 add_msr_offset(offset);
671 }
672 }
673
674 static void svm_enable_lbrv(struct vcpu_svm *svm)
675 {
676 u32 *msrpm = svm->msrpm;
677
678 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
679 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
680 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
681 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
682 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
683 }
684
685 static void svm_disable_lbrv(struct vcpu_svm *svm)
686 {
687 u32 *msrpm = svm->msrpm;
688
689 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
690 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
691 set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
692 set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
693 set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
694 }
695
696 void disable_nmi_singlestep(struct vcpu_svm *svm)
697 {
698 svm->nmi_singlestep = false;
699
700 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
701 /* Clear our flags if they were not set by the guest */
702 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
703 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
704 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
705 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
706 }
707 }
708
709 static void grow_ple_window(struct kvm_vcpu *vcpu)
710 {
711 struct vcpu_svm *svm = to_svm(vcpu);
712 struct vmcb_control_area *control = &svm->vmcb->control;
713 int old = control->pause_filter_count;
714
715 control->pause_filter_count = __grow_ple_window(old,
716 pause_filter_count,
717 pause_filter_count_grow,
718 pause_filter_count_max);
719
720 if (control->pause_filter_count != old) {
721 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
722 trace_kvm_ple_window_update(vcpu->vcpu_id,
723 control->pause_filter_count, old);
724 }
725 }
726
727 static void shrink_ple_window(struct kvm_vcpu *vcpu)
728 {
729 struct vcpu_svm *svm = to_svm(vcpu);
730 struct vmcb_control_area *control = &svm->vmcb->control;
731 int old = control->pause_filter_count;
732
733 control->pause_filter_count =
734 __shrink_ple_window(old,
735 pause_filter_count,
736 pause_filter_count_shrink,
737 pause_filter_count);
738 if (control->pause_filter_count != old) {
739 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
740 trace_kvm_ple_window_update(vcpu->vcpu_id,
741 control->pause_filter_count, old);
742 }
743 }
744
745 /*
746 * The default MMIO mask is a single bit (excluding the present bit),
747 * which could conflict with the memory encryption bit. Check for
748 * memory encryption support and override the default MMIO mask if
749 * memory encryption is enabled.
750 */
751 static __init void svm_adjust_mmio_mask(void)
752 {
753 unsigned int enc_bit, mask_bit;
754 u64 msr, mask;
755
756 /* If there is no memory encryption support, use existing mask */
757 if (cpuid_eax(0x80000000) < 0x8000001f)
758 return;
759
760 /* If memory encryption is not enabled, use existing mask */
761 rdmsrl(MSR_K8_SYSCFG, msr);
762 if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT))
763 return;
764
765 enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
766 mask_bit = boot_cpu_data.x86_phys_bits;
767
768 /* Increment the mask bit if it is the same as the encryption bit */
769 if (enc_bit == mask_bit)
770 mask_bit++;
771
772 /*
773 * If the mask bit location is below 52, then some bits above the
774 * physical addressing limit will always be reserved, so use the
775 * rsvd_bits() function to generate the mask. This mask, along with
776 * the present bit, will be used to generate a page fault with
777 * PFER.RSV = 1.
778 *
779 * If the mask bit location is 52 (or above), then clear the mask.
780 */
781 mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
782
783 kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
784 }
785
786 static void svm_hardware_teardown(void)
787 {
788 int cpu;
789
790 if (svm_sev_enabled())
791 sev_hardware_teardown();
792
793 for_each_possible_cpu(cpu)
794 svm_cpu_uninit(cpu);
795
796 __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
797 iopm_base = 0;
798 }
799
800 static __init void svm_set_cpu_caps(void)
801 {
802 kvm_set_cpu_caps();
803
804 supported_xss = 0;
805
806 /* CPUID 0x80000001 and 0x8000000A (SVM features) */
807 if (nested) {
808 kvm_cpu_cap_set(X86_FEATURE_SVM);
809
810 if (nrips)
811 kvm_cpu_cap_set(X86_FEATURE_NRIPS);
812
813 if (npt_enabled)
814 kvm_cpu_cap_set(X86_FEATURE_NPT);
815 }
816
817 /* CPUID 0x80000008 */
818 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
819 boot_cpu_has(X86_FEATURE_AMD_SSBD))
820 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
821 }
822
823 static __init int svm_hardware_setup(void)
824 {
825 int cpu;
826 struct page *iopm_pages;
827 void *iopm_va;
828 int r;
829
830 iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
831
832 if (!iopm_pages)
833 return -ENOMEM;
834
835 iopm_va = page_address(iopm_pages);
836 memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
837 iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
838
839 init_msrpm_offsets();
840
841 supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR);
842
843 if (boot_cpu_has(X86_FEATURE_NX))
844 kvm_enable_efer_bits(EFER_NX);
845
846 if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
847 kvm_enable_efer_bits(EFER_FFXSR);
848
849 if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
850 kvm_has_tsc_control = true;
851 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
852 kvm_tsc_scaling_ratio_frac_bits = 32;
853 }
854
855 /* Check for pause filtering support */
856 if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
857 pause_filter_count = 0;
858 pause_filter_thresh = 0;
859 } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
860 pause_filter_thresh = 0;
861 }
862
863 if (nested) {
864 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
865 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
866 }
867
868 if (sev) {
869 if (boot_cpu_has(X86_FEATURE_SEV) &&
870 IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
871 r = sev_hardware_setup();
872 if (r)
873 sev = false;
874 } else {
875 sev = false;
876 }
877 }
878
879 svm_adjust_mmio_mask();
880
881 for_each_possible_cpu(cpu) {
882 r = svm_cpu_init(cpu);
883 if (r)
884 goto err;
885 }
886
887 if (!boot_cpu_has(X86_FEATURE_NPT))
888 npt_enabled = false;
889
890 if (npt_enabled && !npt)
891 npt_enabled = false;
892
893 kvm_configure_mmu(npt_enabled, PT_PDPE_LEVEL);
894 pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis");
895
896 if (nrips) {
897 if (!boot_cpu_has(X86_FEATURE_NRIPS))
898 nrips = false;
899 }
900
901 if (avic) {
902 if (!npt_enabled ||
903 !boot_cpu_has(X86_FEATURE_AVIC) ||
904 !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
905 avic = false;
906 } else {
907 pr_info("AVIC enabled\n");
908
909 amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
910 }
911 }
912
913 if (vls) {
914 if (!npt_enabled ||
915 !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
916 !IS_ENABLED(CONFIG_X86_64)) {
917 vls = false;
918 } else {
919 pr_info("Virtual VMLOAD VMSAVE supported\n");
920 }
921 }
922
923 if (vgif) {
924 if (!boot_cpu_has(X86_FEATURE_VGIF))
925 vgif = false;
926 else
927 pr_info("Virtual GIF supported\n");
928 }
929
930 svm_set_cpu_caps();
931
932 return 0;
933
934 err:
935 svm_hardware_teardown();
936 return r;
937 }
938
939 static void init_seg(struct vmcb_seg *seg)
940 {
941 seg->selector = 0;
942 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
943 SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
944 seg->limit = 0xffff;
945 seg->base = 0;
946 }
947
948 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
949 {
950 seg->selector = 0;
951 seg->attrib = SVM_SELECTOR_P_MASK | type;
952 seg->limit = 0xffff;
953 seg->base = 0;
954 }
955
956 static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
957 {
958 struct vcpu_svm *svm = to_svm(vcpu);
959
960 if (is_guest_mode(vcpu))
961 return svm->nested.hsave->control.tsc_offset;
962
963 return vcpu->arch.tsc_offset;
964 }
965
966 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
967 {
968 struct vcpu_svm *svm = to_svm(vcpu);
969 u64 g_tsc_offset = 0;
970
971 if (is_guest_mode(vcpu)) {
972 /* Write L1's TSC offset. */
973 g_tsc_offset = svm->vmcb->control.tsc_offset -
974 svm->nested.hsave->control.tsc_offset;
975 svm->nested.hsave->control.tsc_offset = offset;
976 }
977
978 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
979 svm->vmcb->control.tsc_offset - g_tsc_offset,
980 offset);
981
982 svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
983
984 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
985 return svm->vmcb->control.tsc_offset;
986 }
987
988 static void init_vmcb(struct vcpu_svm *svm)
989 {
990 struct vmcb_control_area *control = &svm->vmcb->control;
991 struct vmcb_save_area *save = &svm->vmcb->save;
992
993 svm->vcpu.arch.hflags = 0;
994
995 set_cr_intercept(svm, INTERCEPT_CR0_READ);
996 set_cr_intercept(svm, INTERCEPT_CR3_READ);
997 set_cr_intercept(svm, INTERCEPT_CR4_READ);
998 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
999 set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1000 set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1001 if (!kvm_vcpu_apicv_active(&svm->vcpu))
1002 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1003
1004 set_dr_intercepts(svm);
1005
1006 set_exception_intercept(svm, PF_VECTOR);
1007 set_exception_intercept(svm, UD_VECTOR);
1008 set_exception_intercept(svm, MC_VECTOR);
1009 set_exception_intercept(svm, AC_VECTOR);
1010 set_exception_intercept(svm, DB_VECTOR);
1011 /*
1012 * Guest access to VMware backdoor ports could legitimately
1013 * trigger #GP because of TSS I/O permission bitmap.
1014 * We intercept those #GP and allow access to them anyway
1015 * as VMware does.
1016 */
1017 if (enable_vmware_backdoor)
1018 set_exception_intercept(svm, GP_VECTOR);
1019
1020 set_intercept(svm, INTERCEPT_INTR);
1021 set_intercept(svm, INTERCEPT_NMI);
1022 set_intercept(svm, INTERCEPT_SMI);
1023 set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1024 set_intercept(svm, INTERCEPT_RDPMC);
1025 set_intercept(svm, INTERCEPT_CPUID);
1026 set_intercept(svm, INTERCEPT_INVD);
1027 set_intercept(svm, INTERCEPT_INVLPG);
1028 set_intercept(svm, INTERCEPT_INVLPGA);
1029 set_intercept(svm, INTERCEPT_IOIO_PROT);
1030 set_intercept(svm, INTERCEPT_MSR_PROT);
1031 set_intercept(svm, INTERCEPT_TASK_SWITCH);
1032 set_intercept(svm, INTERCEPT_SHUTDOWN);
1033 set_intercept(svm, INTERCEPT_VMRUN);
1034 set_intercept(svm, INTERCEPT_VMMCALL);
1035 set_intercept(svm, INTERCEPT_VMLOAD);
1036 set_intercept(svm, INTERCEPT_VMSAVE);
1037 set_intercept(svm, INTERCEPT_STGI);
1038 set_intercept(svm, INTERCEPT_CLGI);
1039 set_intercept(svm, INTERCEPT_SKINIT);
1040 set_intercept(svm, INTERCEPT_WBINVD);
1041 set_intercept(svm, INTERCEPT_XSETBV);
1042 set_intercept(svm, INTERCEPT_RDPRU);
1043 set_intercept(svm, INTERCEPT_RSM);
1044
1045 if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1046 set_intercept(svm, INTERCEPT_MONITOR);
1047 set_intercept(svm, INTERCEPT_MWAIT);
1048 }
1049
1050 if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1051 set_intercept(svm, INTERCEPT_HLT);
1052
1053 control->iopm_base_pa = __sme_set(iopm_base);
1054 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1055 control->int_ctl = V_INTR_MASKING_MASK;
1056
1057 init_seg(&save->es);
1058 init_seg(&save->ss);
1059 init_seg(&save->ds);
1060 init_seg(&save->fs);
1061 init_seg(&save->gs);
1062
1063 save->cs.selector = 0xf000;
1064 save->cs.base = 0xffff0000;
1065 /* Executable/Readable Code Segment */
1066 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1067 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1068 save->cs.limit = 0xffff;
1069
1070 save->gdtr.limit = 0xffff;
1071 save->idtr.limit = 0xffff;
1072
1073 init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1074 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1075
1076 svm_set_efer(&svm->vcpu, 0);
1077 save->dr6 = 0xffff0ff0;
1078 kvm_set_rflags(&svm->vcpu, 2);
1079 save->rip = 0x0000fff0;
1080 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1081
1082 /*
1083 * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1084 * It also updates the guest-visible cr0 value.
1085 */
1086 svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1087 kvm_mmu_reset_context(&svm->vcpu);
1088
1089 save->cr4 = X86_CR4_PAE;
1090 /* rdx = ?? */
1091
1092 if (npt_enabled) {
1093 /* Setup VMCB for Nested Paging */
1094 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1095 clr_intercept(svm, INTERCEPT_INVLPG);
1096 clr_exception_intercept(svm, PF_VECTOR);
1097 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1098 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1099 save->g_pat = svm->vcpu.arch.pat;
1100 save->cr3 = 0;
1101 save->cr4 = 0;
1102 }
1103 svm->asid_generation = 0;
1104
1105 svm->nested.vmcb = 0;
1106 svm->vcpu.arch.hflags = 0;
1107
1108 if (pause_filter_count) {
1109 control->pause_filter_count = pause_filter_count;
1110 if (pause_filter_thresh)
1111 control->pause_filter_thresh = pause_filter_thresh;
1112 set_intercept(svm, INTERCEPT_PAUSE);
1113 } else {
1114 clr_intercept(svm, INTERCEPT_PAUSE);
1115 }
1116
1117 if (kvm_vcpu_apicv_active(&svm->vcpu))
1118 avic_init_vmcb(svm);
1119
1120 /*
1121 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1122 * in VMCB and clear intercepts to avoid #VMEXIT.
1123 */
1124 if (vls) {
1125 clr_intercept(svm, INTERCEPT_VMLOAD);
1126 clr_intercept(svm, INTERCEPT_VMSAVE);
1127 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1128 }
1129
1130 if (vgif) {
1131 clr_intercept(svm, INTERCEPT_STGI);
1132 clr_intercept(svm, INTERCEPT_CLGI);
1133 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1134 }
1135
1136 if (sev_guest(svm->vcpu.kvm)) {
1137 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1138 clr_exception_intercept(svm, UD_VECTOR);
1139 }
1140
1141 mark_all_dirty(svm->vmcb);
1142
1143 enable_gif(svm);
1144
1145 }
1146
1147 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1148 {
1149 struct vcpu_svm *svm = to_svm(vcpu);
1150 u32 dummy;
1151 u32 eax = 1;
1152
1153 svm->spec_ctrl = 0;
1154 svm->virt_spec_ctrl = 0;
1155
1156 if (!init_event) {
1157 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1158 MSR_IA32_APICBASE_ENABLE;
1159 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1160 svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1161 }
1162 init_vmcb(svm);
1163
1164 kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, false);
1165 kvm_rdx_write(vcpu, eax);
1166
1167 if (kvm_vcpu_apicv_active(vcpu) && !init_event)
1168 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1169 }
1170
1171 static int svm_create_vcpu(struct kvm_vcpu *vcpu)
1172 {
1173 struct vcpu_svm *svm;
1174 struct page *page;
1175 struct page *msrpm_pages;
1176 struct page *hsave_page;
1177 struct page *nested_msrpm_pages;
1178 int err;
1179
1180 BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1181 svm = to_svm(vcpu);
1182
1183 err = -ENOMEM;
1184 page = alloc_page(GFP_KERNEL_ACCOUNT);
1185 if (!page)
1186 goto out;
1187
1188 msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
1189 if (!msrpm_pages)
1190 goto free_page1;
1191
1192 nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
1193 if (!nested_msrpm_pages)
1194 goto free_page2;
1195
1196 hsave_page = alloc_page(GFP_KERNEL_ACCOUNT);
1197 if (!hsave_page)
1198 goto free_page3;
1199
1200 err = avic_init_vcpu(svm);
1201 if (err)
1202 goto free_page4;
1203
1204 /* We initialize this flag to true to make sure that the is_running
1205 * bit would be set the first time the vcpu is loaded.
1206 */
1207 if (irqchip_in_kernel(vcpu->kvm) && kvm_apicv_activated(vcpu->kvm))
1208 svm->avic_is_running = true;
1209
1210 svm->nested.hsave = page_address(hsave_page);
1211
1212 svm->msrpm = page_address(msrpm_pages);
1213 svm_vcpu_init_msrpm(svm->msrpm);
1214
1215 svm->nested.msrpm = page_address(nested_msrpm_pages);
1216 svm_vcpu_init_msrpm(svm->nested.msrpm);
1217
1218 svm->vmcb = page_address(page);
1219 clear_page(svm->vmcb);
1220 svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
1221 svm->asid_generation = 0;
1222 init_vmcb(svm);
1223
1224 svm_init_osvw(vcpu);
1225 vcpu->arch.microcode_version = 0x01000065;
1226
1227 return 0;
1228
1229 free_page4:
1230 __free_page(hsave_page);
1231 free_page3:
1232 __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1233 free_page2:
1234 __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1235 free_page1:
1236 __free_page(page);
1237 out:
1238 return err;
1239 }
1240
1241 static void svm_clear_current_vmcb(struct vmcb *vmcb)
1242 {
1243 int i;
1244
1245 for_each_online_cpu(i)
1246 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
1247 }
1248
1249 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1250 {
1251 struct vcpu_svm *svm = to_svm(vcpu);
1252
1253 /*
1254 * The vmcb page can be recycled, causing a false negative in
1255 * svm_vcpu_load(). So, ensure that no logical CPU has this
1256 * vmcb page recorded as its current vmcb.
1257 */
1258 svm_clear_current_vmcb(svm->vmcb);
1259
1260 __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
1261 __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1262 __free_page(virt_to_page(svm->nested.hsave));
1263 __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1264 }
1265
1266 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1267 {
1268 struct vcpu_svm *svm = to_svm(vcpu);
1269 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1270 int i;
1271
1272 if (unlikely(cpu != vcpu->cpu)) {
1273 svm->asid_generation = 0;
1274 mark_all_dirty(svm->vmcb);
1275 }
1276
1277 #ifdef CONFIG_X86_64
1278 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1279 #endif
1280 savesegment(fs, svm->host.fs);
1281 savesegment(gs, svm->host.gs);
1282 svm->host.ldt = kvm_read_ldt();
1283
1284 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1285 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1286
1287 if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1288 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1289 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1290 __this_cpu_write(current_tsc_ratio, tsc_ratio);
1291 wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1292 }
1293 }
1294 /* This assumes that the kernel never uses MSR_TSC_AUX */
1295 if (static_cpu_has(X86_FEATURE_RDTSCP))
1296 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1297
1298 if (sd->current_vmcb != svm->vmcb) {
1299 sd->current_vmcb = svm->vmcb;
1300 indirect_branch_prediction_barrier();
1301 }
1302 avic_vcpu_load(vcpu, cpu);
1303 }
1304
1305 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1306 {
1307 struct vcpu_svm *svm = to_svm(vcpu);
1308 int i;
1309
1310 avic_vcpu_put(vcpu);
1311
1312 ++vcpu->stat.host_state_reload;
1313 kvm_load_ldt(svm->host.ldt);
1314 #ifdef CONFIG_X86_64
1315 loadsegment(fs, svm->host.fs);
1316 wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
1317 load_gs_index(svm->host.gs);
1318 #else
1319 #ifdef CONFIG_X86_32_LAZY_GS
1320 loadsegment(gs, svm->host.gs);
1321 #endif
1322 #endif
1323 for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1324 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1325 }
1326
1327 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1328 {
1329 struct vcpu_svm *svm = to_svm(vcpu);
1330 unsigned long rflags = svm->vmcb->save.rflags;
1331
1332 if (svm->nmi_singlestep) {
1333 /* Hide our flags if they were not set by the guest */
1334 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1335 rflags &= ~X86_EFLAGS_TF;
1336 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1337 rflags &= ~X86_EFLAGS_RF;
1338 }
1339 return rflags;
1340 }
1341
1342 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1343 {
1344 if (to_svm(vcpu)->nmi_singlestep)
1345 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1346
1347 /*
1348 * Any change of EFLAGS.VM is accompanied by a reload of SS
1349 * (caused by either a task switch or an inter-privilege IRET),
1350 * so we do not need to update the CPL here.
1351 */
1352 to_svm(vcpu)->vmcb->save.rflags = rflags;
1353 }
1354
1355 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1356 {
1357 switch (reg) {
1358 case VCPU_EXREG_PDPTR:
1359 BUG_ON(!npt_enabled);
1360 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1361 break;
1362 default:
1363 WARN_ON_ONCE(1);
1364 }
1365 }
1366
1367 static inline void svm_enable_vintr(struct vcpu_svm *svm)
1368 {
1369 struct vmcb_control_area *control;
1370
1371 /* The following fields are ignored when AVIC is enabled */
1372 WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu));
1373
1374 /*
1375 * This is just a dummy VINTR to actually cause a vmexit to happen.
1376 * Actual injection of virtual interrupts happens through EVENTINJ.
1377 */
1378 control = &svm->vmcb->control;
1379 control->int_vector = 0x0;
1380 control->int_ctl &= ~V_INTR_PRIO_MASK;
1381 control->int_ctl |= V_IRQ_MASK |
1382 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1383 mark_dirty(svm->vmcb, VMCB_INTR);
1384 }
1385
1386 static void svm_set_vintr(struct vcpu_svm *svm)
1387 {
1388 set_intercept(svm, INTERCEPT_VINTR);
1389 if (is_intercept(svm, INTERCEPT_VINTR))
1390 svm_enable_vintr(svm);
1391 }
1392
1393 static void svm_clear_vintr(struct vcpu_svm *svm)
1394 {
1395 clr_intercept(svm, INTERCEPT_VINTR);
1396
1397 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
1398 mark_dirty(svm->vmcb, VMCB_INTR);
1399 }
1400
1401 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1402 {
1403 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1404
1405 switch (seg) {
1406 case VCPU_SREG_CS: return &save->cs;
1407 case VCPU_SREG_DS: return &save->ds;
1408 case VCPU_SREG_ES: return &save->es;
1409 case VCPU_SREG_FS: return &save->fs;
1410 case VCPU_SREG_GS: return &save->gs;
1411 case VCPU_SREG_SS: return &save->ss;
1412 case VCPU_SREG_TR: return &save->tr;
1413 case VCPU_SREG_LDTR: return &save->ldtr;
1414 }
1415 BUG();
1416 return NULL;
1417 }
1418
1419 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1420 {
1421 struct vmcb_seg *s = svm_seg(vcpu, seg);
1422
1423 return s->base;
1424 }
1425
1426 static void svm_get_segment(struct kvm_vcpu *vcpu,
1427 struct kvm_segment *var, int seg)
1428 {
1429 struct vmcb_seg *s = svm_seg(vcpu, seg);
1430
1431 var->base = s->base;
1432 var->limit = s->limit;
1433 var->selector = s->selector;
1434 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1435 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1436 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1437 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1438 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1439 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1440 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1441
1442 /*
1443 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1444 * However, the SVM spec states that the G bit is not observed by the
1445 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1446 * So let's synthesize a legal G bit for all segments, this helps
1447 * running KVM nested. It also helps cross-vendor migration, because
1448 * Intel's vmentry has a check on the 'G' bit.
1449 */
1450 var->g = s->limit > 0xfffff;
1451
1452 /*
1453 * AMD's VMCB does not have an explicit unusable field, so emulate it
1454 * for cross vendor migration purposes by "not present"
1455 */
1456 var->unusable = !var->present;
1457
1458 switch (seg) {
1459 case VCPU_SREG_TR:
1460 /*
1461 * Work around a bug where the busy flag in the tr selector
1462 * isn't exposed
1463 */
1464 var->type |= 0x2;
1465 break;
1466 case VCPU_SREG_DS:
1467 case VCPU_SREG_ES:
1468 case VCPU_SREG_FS:
1469 case VCPU_SREG_GS:
1470 /*
1471 * The accessed bit must always be set in the segment
1472 * descriptor cache, although it can be cleared in the
1473 * descriptor, the cached bit always remains at 1. Since
1474 * Intel has a check on this, set it here to support
1475 * cross-vendor migration.
1476 */
1477 if (!var->unusable)
1478 var->type |= 0x1;
1479 break;
1480 case VCPU_SREG_SS:
1481 /*
1482 * On AMD CPUs sometimes the DB bit in the segment
1483 * descriptor is left as 1, although the whole segment has
1484 * been made unusable. Clear it here to pass an Intel VMX
1485 * entry check when cross vendor migrating.
1486 */
1487 if (var->unusable)
1488 var->db = 0;
1489 /* This is symmetric with svm_set_segment() */
1490 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1491 break;
1492 }
1493 }
1494
1495 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1496 {
1497 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1498
1499 return save->cpl;
1500 }
1501
1502 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1503 {
1504 struct vcpu_svm *svm = to_svm(vcpu);
1505
1506 dt->size = svm->vmcb->save.idtr.limit;
1507 dt->address = svm->vmcb->save.idtr.base;
1508 }
1509
1510 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1511 {
1512 struct vcpu_svm *svm = to_svm(vcpu);
1513
1514 svm->vmcb->save.idtr.limit = dt->size;
1515 svm->vmcb->save.idtr.base = dt->address ;
1516 mark_dirty(svm->vmcb, VMCB_DT);
1517 }
1518
1519 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1520 {
1521 struct vcpu_svm *svm = to_svm(vcpu);
1522
1523 dt->size = svm->vmcb->save.gdtr.limit;
1524 dt->address = svm->vmcb->save.gdtr.base;
1525 }
1526
1527 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1528 {
1529 struct vcpu_svm *svm = to_svm(vcpu);
1530
1531 svm->vmcb->save.gdtr.limit = dt->size;
1532 svm->vmcb->save.gdtr.base = dt->address ;
1533 mark_dirty(svm->vmcb, VMCB_DT);
1534 }
1535
1536 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1537 {
1538 }
1539
1540 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1541 {
1542 }
1543
1544 static void update_cr0_intercept(struct vcpu_svm *svm)
1545 {
1546 ulong gcr0 = svm->vcpu.arch.cr0;
1547 u64 *hcr0 = &svm->vmcb->save.cr0;
1548
1549 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1550 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1551
1552 mark_dirty(svm->vmcb, VMCB_CR);
1553
1554 if (gcr0 == *hcr0) {
1555 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1556 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1557 } else {
1558 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1559 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1560 }
1561 }
1562
1563 void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1564 {
1565 struct vcpu_svm *svm = to_svm(vcpu);
1566
1567 #ifdef CONFIG_X86_64
1568 if (vcpu->arch.efer & EFER_LME) {
1569 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1570 vcpu->arch.efer |= EFER_LMA;
1571 svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1572 }
1573
1574 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1575 vcpu->arch.efer &= ~EFER_LMA;
1576 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1577 }
1578 }
1579 #endif
1580 vcpu->arch.cr0 = cr0;
1581
1582 if (!npt_enabled)
1583 cr0 |= X86_CR0_PG | X86_CR0_WP;
1584
1585 /*
1586 * re-enable caching here because the QEMU bios
1587 * does not do it - this results in some delay at
1588 * reboot
1589 */
1590 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1591 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1592 svm->vmcb->save.cr0 = cr0;
1593 mark_dirty(svm->vmcb, VMCB_CR);
1594 update_cr0_intercept(svm);
1595 }
1596
1597 int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1598 {
1599 unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1600 unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1601
1602 if (cr4 & X86_CR4_VMXE)
1603 return 1;
1604
1605 if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1606 svm_flush_tlb(vcpu, true);
1607
1608 vcpu->arch.cr4 = cr4;
1609 if (!npt_enabled)
1610 cr4 |= X86_CR4_PAE;
1611 cr4 |= host_cr4_mce;
1612 to_svm(vcpu)->vmcb->save.cr4 = cr4;
1613 mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1614 return 0;
1615 }
1616
1617 static void svm_set_segment(struct kvm_vcpu *vcpu,
1618 struct kvm_segment *var, int seg)
1619 {
1620 struct vcpu_svm *svm = to_svm(vcpu);
1621 struct vmcb_seg *s = svm_seg(vcpu, seg);
1622
1623 s->base = var->base;
1624 s->limit = var->limit;
1625 s->selector = var->selector;
1626 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1627 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1628 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1629 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1630 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1631 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1632 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1633 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1634
1635 /*
1636 * This is always accurate, except if SYSRET returned to a segment
1637 * with SS.DPL != 3. Intel does not have this quirk, and always
1638 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1639 * would entail passing the CPL to userspace and back.
1640 */
1641 if (seg == VCPU_SREG_SS)
1642 /* This is symmetric with svm_get_segment() */
1643 svm->vmcb->save.cpl = (var->dpl & 3);
1644
1645 mark_dirty(svm->vmcb, VMCB_SEG);
1646 }
1647
1648 static void update_bp_intercept(struct kvm_vcpu *vcpu)
1649 {
1650 struct vcpu_svm *svm = to_svm(vcpu);
1651
1652 clr_exception_intercept(svm, BP_VECTOR);
1653
1654 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1655 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1656 set_exception_intercept(svm, BP_VECTOR);
1657 } else
1658 vcpu->guest_debug = 0;
1659 }
1660
1661 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1662 {
1663 if (sd->next_asid > sd->max_asid) {
1664 ++sd->asid_generation;
1665 sd->next_asid = sd->min_asid;
1666 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1667 }
1668
1669 svm->asid_generation = sd->asid_generation;
1670 svm->vmcb->control.asid = sd->next_asid++;
1671
1672 mark_dirty(svm->vmcb, VMCB_ASID);
1673 }
1674
1675 static void svm_set_dr6(struct vcpu_svm *svm, unsigned long value)
1676 {
1677 struct vmcb *vmcb = svm->vmcb;
1678
1679 if (unlikely(value != vmcb->save.dr6)) {
1680 vmcb->save.dr6 = value;
1681 mark_dirty(vmcb, VMCB_DR);
1682 }
1683 }
1684
1685 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1686 {
1687 struct vcpu_svm *svm = to_svm(vcpu);
1688
1689 get_debugreg(vcpu->arch.db[0], 0);
1690 get_debugreg(vcpu->arch.db[1], 1);
1691 get_debugreg(vcpu->arch.db[2], 2);
1692 get_debugreg(vcpu->arch.db[3], 3);
1693 /*
1694 * We cannot reset svm->vmcb->save.dr6 to DR6_FIXED_1|DR6_RTM here,
1695 * because db_interception might need it. We can do it before vmentry.
1696 */
1697 vcpu->arch.dr6 = svm->vmcb->save.dr6;
1698 vcpu->arch.dr7 = svm->vmcb->save.dr7;
1699 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1700 set_dr_intercepts(svm);
1701 }
1702
1703 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1704 {
1705 struct vcpu_svm *svm = to_svm(vcpu);
1706
1707 svm->vmcb->save.dr7 = value;
1708 mark_dirty(svm->vmcb, VMCB_DR);
1709 }
1710
1711 static int pf_interception(struct vcpu_svm *svm)
1712 {
1713 u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
1714 u64 error_code = svm->vmcb->control.exit_info_1;
1715
1716 return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
1717 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1718 svm->vmcb->control.insn_bytes : NULL,
1719 svm->vmcb->control.insn_len);
1720 }
1721
1722 static int npf_interception(struct vcpu_svm *svm)
1723 {
1724 u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
1725 u64 error_code = svm->vmcb->control.exit_info_1;
1726
1727 trace_kvm_page_fault(fault_address, error_code);
1728 return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1729 static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1730 svm->vmcb->control.insn_bytes : NULL,
1731 svm->vmcb->control.insn_len);
1732 }
1733
1734 static int db_interception(struct vcpu_svm *svm)
1735 {
1736 struct kvm_run *kvm_run = svm->vcpu.run;
1737 struct kvm_vcpu *vcpu = &svm->vcpu;
1738
1739 if (!(svm->vcpu.guest_debug &
1740 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1741 !svm->nmi_singlestep) {
1742 u32 payload = (svm->vmcb->save.dr6 ^ DR6_RTM) & ~DR6_FIXED_1;
1743 kvm_queue_exception_p(&svm->vcpu, DB_VECTOR, payload);
1744 return 1;
1745 }
1746
1747 if (svm->nmi_singlestep) {
1748 disable_nmi_singlestep(svm);
1749 /* Make sure we check for pending NMIs upon entry */
1750 kvm_make_request(KVM_REQ_EVENT, vcpu);
1751 }
1752
1753 if (svm->vcpu.guest_debug &
1754 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1755 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1756 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
1757 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
1758 kvm_run->debug.arch.pc =
1759 svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1760 kvm_run->debug.arch.exception = DB_VECTOR;
1761 return 0;
1762 }
1763
1764 return 1;
1765 }
1766
1767 static int bp_interception(struct vcpu_svm *svm)
1768 {
1769 struct kvm_run *kvm_run = svm->vcpu.run;
1770
1771 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1772 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1773 kvm_run->debug.arch.exception = BP_VECTOR;
1774 return 0;
1775 }
1776
1777 static int ud_interception(struct vcpu_svm *svm)
1778 {
1779 return handle_ud(&svm->vcpu);
1780 }
1781
1782 static int ac_interception(struct vcpu_svm *svm)
1783 {
1784 kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1785 return 1;
1786 }
1787
1788 static int gp_interception(struct vcpu_svm *svm)
1789 {
1790 struct kvm_vcpu *vcpu = &svm->vcpu;
1791 u32 error_code = svm->vmcb->control.exit_info_1;
1792
1793 WARN_ON_ONCE(!enable_vmware_backdoor);
1794
1795 /*
1796 * VMware backdoor emulation on #GP interception only handles IN{S},
1797 * OUT{S}, and RDPMC, none of which generate a non-zero error code.
1798 */
1799 if (error_code) {
1800 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
1801 return 1;
1802 }
1803 return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
1804 }
1805
1806 static bool is_erratum_383(void)
1807 {
1808 int err, i;
1809 u64 value;
1810
1811 if (!erratum_383_found)
1812 return false;
1813
1814 value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1815 if (err)
1816 return false;
1817
1818 /* Bit 62 may or may not be set for this mce */
1819 value &= ~(1ULL << 62);
1820
1821 if (value != 0xb600000000010015ULL)
1822 return false;
1823
1824 /* Clear MCi_STATUS registers */
1825 for (i = 0; i < 6; ++i)
1826 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1827
1828 value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1829 if (!err) {
1830 u32 low, high;
1831
1832 value &= ~(1ULL << 2);
1833 low = lower_32_bits(value);
1834 high = upper_32_bits(value);
1835
1836 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1837 }
1838
1839 /* Flush tlb to evict multi-match entries */
1840 __flush_tlb_all();
1841
1842 return true;
1843 }
1844
1845 static void svm_handle_mce(struct vcpu_svm *svm)
1846 {
1847 if (is_erratum_383()) {
1848 /*
1849 * Erratum 383 triggered. Guest state is corrupt so kill the
1850 * guest.
1851 */
1852 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1853
1854 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1855
1856 return;
1857 }
1858
1859 /*
1860 * On an #MC intercept the MCE handler is not called automatically in
1861 * the host. So do it by hand here.
1862 */
1863 asm volatile (
1864 "int $0x12\n");
1865 /* not sure if we ever come back to this point */
1866
1867 return;
1868 }
1869
1870 static int mc_interception(struct vcpu_svm *svm)
1871 {
1872 return 1;
1873 }
1874
1875 static int shutdown_interception(struct vcpu_svm *svm)
1876 {
1877 struct kvm_run *kvm_run = svm->vcpu.run;
1878
1879 /*
1880 * VMCB is undefined after a SHUTDOWN intercept
1881 * so reinitialize it.
1882 */
1883 clear_page(svm->vmcb);
1884 init_vmcb(svm);
1885
1886 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1887 return 0;
1888 }
1889
1890 static int io_interception(struct vcpu_svm *svm)
1891 {
1892 struct kvm_vcpu *vcpu = &svm->vcpu;
1893 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1894 int size, in, string;
1895 unsigned port;
1896
1897 ++svm->vcpu.stat.io_exits;
1898 string = (io_info & SVM_IOIO_STR_MASK) != 0;
1899 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1900 if (string)
1901 return kvm_emulate_instruction(vcpu, 0);
1902
1903 port = io_info >> 16;
1904 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1905 svm->next_rip = svm->vmcb->control.exit_info_2;
1906
1907 return kvm_fast_pio(&svm->vcpu, size, port, in);
1908 }
1909
1910 static int nmi_interception(struct vcpu_svm *svm)
1911 {
1912 return 1;
1913 }
1914
1915 static int intr_interception(struct vcpu_svm *svm)
1916 {
1917 ++svm->vcpu.stat.irq_exits;
1918 return 1;
1919 }
1920
1921 static int nop_on_interception(struct vcpu_svm *svm)
1922 {
1923 return 1;
1924 }
1925
1926 static int halt_interception(struct vcpu_svm *svm)
1927 {
1928 return kvm_emulate_halt(&svm->vcpu);
1929 }
1930
1931 static int vmmcall_interception(struct vcpu_svm *svm)
1932 {
1933 return kvm_emulate_hypercall(&svm->vcpu);
1934 }
1935
1936 static int vmload_interception(struct vcpu_svm *svm)
1937 {
1938 struct vmcb *nested_vmcb;
1939 struct kvm_host_map map;
1940 int ret;
1941
1942 if (nested_svm_check_permissions(svm))
1943 return 1;
1944
1945 ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
1946 if (ret) {
1947 if (ret == -EINVAL)
1948 kvm_inject_gp(&svm->vcpu, 0);
1949 return 1;
1950 }
1951
1952 nested_vmcb = map.hva;
1953
1954 ret = kvm_skip_emulated_instruction(&svm->vcpu);
1955
1956 nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
1957 kvm_vcpu_unmap(&svm->vcpu, &map, true);
1958
1959 return ret;
1960 }
1961
1962 static int vmsave_interception(struct vcpu_svm *svm)
1963 {
1964 struct vmcb *nested_vmcb;
1965 struct kvm_host_map map;
1966 int ret;
1967
1968 if (nested_svm_check_permissions(svm))
1969 return 1;
1970
1971 ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
1972 if (ret) {
1973 if (ret == -EINVAL)
1974 kvm_inject_gp(&svm->vcpu, 0);
1975 return 1;
1976 }
1977
1978 nested_vmcb = map.hva;
1979
1980 ret = kvm_skip_emulated_instruction(&svm->vcpu);
1981
1982 nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
1983 kvm_vcpu_unmap(&svm->vcpu, &map, true);
1984
1985 return ret;
1986 }
1987
1988 static int vmrun_interception(struct vcpu_svm *svm)
1989 {
1990 if (nested_svm_check_permissions(svm))
1991 return 1;
1992
1993 return nested_svm_vmrun(svm);
1994 }
1995
1996 static int stgi_interception(struct vcpu_svm *svm)
1997 {
1998 int ret;
1999
2000 if (nested_svm_check_permissions(svm))
2001 return 1;
2002
2003 /*
2004 * If VGIF is enabled, the STGI intercept is only added to
2005 * detect the opening of the SMI/NMI window; remove it now.
2006 */
2007 if (vgif_enabled(svm))
2008 clr_intercept(svm, INTERCEPT_STGI);
2009
2010 ret = kvm_skip_emulated_instruction(&svm->vcpu);
2011 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2012
2013 enable_gif(svm);
2014
2015 return ret;
2016 }
2017
2018 static int clgi_interception(struct vcpu_svm *svm)
2019 {
2020 int ret;
2021
2022 if (nested_svm_check_permissions(svm))
2023 return 1;
2024
2025 ret = kvm_skip_emulated_instruction(&svm->vcpu);
2026
2027 disable_gif(svm);
2028
2029 /* After a CLGI no interrupts should come */
2030 if (!kvm_vcpu_apicv_active(&svm->vcpu))
2031 svm_clear_vintr(svm);
2032
2033 return ret;
2034 }
2035
2036 static int invlpga_interception(struct vcpu_svm *svm)
2037 {
2038 struct kvm_vcpu *vcpu = &svm->vcpu;
2039
2040 trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
2041 kvm_rax_read(&svm->vcpu));
2042
2043 /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2044 kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
2045
2046 return kvm_skip_emulated_instruction(&svm->vcpu);
2047 }
2048
2049 static int skinit_interception(struct vcpu_svm *svm)
2050 {
2051 trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
2052
2053 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2054 return 1;
2055 }
2056
2057 static int wbinvd_interception(struct vcpu_svm *svm)
2058 {
2059 return kvm_emulate_wbinvd(&svm->vcpu);
2060 }
2061
2062 static int xsetbv_interception(struct vcpu_svm *svm)
2063 {
2064 u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2065 u32 index = kvm_rcx_read(&svm->vcpu);
2066
2067 if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2068 return kvm_skip_emulated_instruction(&svm->vcpu);
2069 }
2070
2071 return 1;
2072 }
2073
2074 static int rdpru_interception(struct vcpu_svm *svm)
2075 {
2076 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2077 return 1;
2078 }
2079
2080 static int task_switch_interception(struct vcpu_svm *svm)
2081 {
2082 u16 tss_selector;
2083 int reason;
2084 int int_type = svm->vmcb->control.exit_int_info &
2085 SVM_EXITINTINFO_TYPE_MASK;
2086 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2087 uint32_t type =
2088 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2089 uint32_t idt_v =
2090 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2091 bool has_error_code = false;
2092 u32 error_code = 0;
2093
2094 tss_selector = (u16)svm->vmcb->control.exit_info_1;
2095
2096 if (svm->vmcb->control.exit_info_2 &
2097 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2098 reason = TASK_SWITCH_IRET;
2099 else if (svm->vmcb->control.exit_info_2 &
2100 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2101 reason = TASK_SWITCH_JMP;
2102 else if (idt_v)
2103 reason = TASK_SWITCH_GATE;
2104 else
2105 reason = TASK_SWITCH_CALL;
2106
2107 if (reason == TASK_SWITCH_GATE) {
2108 switch (type) {
2109 case SVM_EXITINTINFO_TYPE_NMI:
2110 svm->vcpu.arch.nmi_injected = false;
2111 break;
2112 case SVM_EXITINTINFO_TYPE_EXEPT:
2113 if (svm->vmcb->control.exit_info_2 &
2114 (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2115 has_error_code = true;
2116 error_code =
2117 (u32)svm->vmcb->control.exit_info_2;
2118 }
2119 kvm_clear_exception_queue(&svm->vcpu);
2120 break;
2121 case SVM_EXITINTINFO_TYPE_INTR:
2122 kvm_clear_interrupt_queue(&svm->vcpu);
2123 break;
2124 default:
2125 break;
2126 }
2127 }
2128
2129 if (reason != TASK_SWITCH_GATE ||
2130 int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2131 (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2132 (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2133 if (!skip_emulated_instruction(&svm->vcpu))
2134 return 0;
2135 }
2136
2137 if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2138 int_vec = -1;
2139
2140 return kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2141 has_error_code, error_code);
2142 }
2143
2144 static int cpuid_interception(struct vcpu_svm *svm)
2145 {
2146 return kvm_emulate_cpuid(&svm->vcpu);
2147 }
2148
2149 static int iret_interception(struct vcpu_svm *svm)
2150 {
2151 ++svm->vcpu.stat.nmi_window_exits;
2152 clr_intercept(svm, INTERCEPT_IRET);
2153 svm->vcpu.arch.hflags |= HF_IRET_MASK;
2154 svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2155 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2156 return 1;
2157 }
2158
2159 static int invlpg_interception(struct vcpu_svm *svm)
2160 {
2161 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2162 return kvm_emulate_instruction(&svm->vcpu, 0);
2163
2164 kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2165 return kvm_skip_emulated_instruction(&svm->vcpu);
2166 }
2167
2168 static int emulate_on_interception(struct vcpu_svm *svm)
2169 {
2170 return kvm_emulate_instruction(&svm->vcpu, 0);
2171 }
2172
2173 static int rsm_interception(struct vcpu_svm *svm)
2174 {
2175 return kvm_emulate_instruction_from_buffer(&svm->vcpu, rsm_ins_bytes, 2);
2176 }
2177
2178 static int rdpmc_interception(struct vcpu_svm *svm)
2179 {
2180 int err;
2181
2182 if (!nrips)
2183 return emulate_on_interception(svm);
2184
2185 err = kvm_rdpmc(&svm->vcpu);
2186 return kvm_complete_insn_gp(&svm->vcpu, err);
2187 }
2188
2189 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2190 unsigned long val)
2191 {
2192 unsigned long cr0 = svm->vcpu.arch.cr0;
2193 bool ret = false;
2194 u64 intercept;
2195
2196 intercept = svm->nested.intercept;
2197
2198 if (!is_guest_mode(&svm->vcpu) ||
2199 (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
2200 return false;
2201
2202 cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2203 val &= ~SVM_CR0_SELECTIVE_MASK;
2204
2205 if (cr0 ^ val) {
2206 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2207 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2208 }
2209
2210 return ret;
2211 }
2212
2213 #define CR_VALID (1ULL << 63)
2214
2215 static int cr_interception(struct vcpu_svm *svm)
2216 {
2217 int reg, cr;
2218 unsigned long val;
2219 int err;
2220
2221 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2222 return emulate_on_interception(svm);
2223
2224 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2225 return emulate_on_interception(svm);
2226
2227 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2228 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2229 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2230 else
2231 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2232
2233 err = 0;
2234 if (cr >= 16) { /* mov to cr */
2235 cr -= 16;
2236 val = kvm_register_read(&svm->vcpu, reg);
2237 switch (cr) {
2238 case 0:
2239 if (!check_selective_cr0_intercepted(svm, val))
2240 err = kvm_set_cr0(&svm->vcpu, val);
2241 else
2242 return 1;
2243
2244 break;
2245 case 3:
2246 err = kvm_set_cr3(&svm->vcpu, val);
2247 break;
2248 case 4:
2249 err = kvm_set_cr4(&svm->vcpu, val);
2250 break;
2251 case 8:
2252 err = kvm_set_cr8(&svm->vcpu, val);
2253 break;
2254 default:
2255 WARN(1, "unhandled write to CR%d", cr);
2256 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2257 return 1;
2258 }
2259 } else { /* mov from cr */
2260 switch (cr) {
2261 case 0:
2262 val = kvm_read_cr0(&svm->vcpu);
2263 break;
2264 case 2:
2265 val = svm->vcpu.arch.cr2;
2266 break;
2267 case 3:
2268 val = kvm_read_cr3(&svm->vcpu);
2269 break;
2270 case 4:
2271 val = kvm_read_cr4(&svm->vcpu);
2272 break;
2273 case 8:
2274 val = kvm_get_cr8(&svm->vcpu);
2275 break;
2276 default:
2277 WARN(1, "unhandled read from CR%d", cr);
2278 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2279 return 1;
2280 }
2281 kvm_register_write(&svm->vcpu, reg, val);
2282 }
2283 return kvm_complete_insn_gp(&svm->vcpu, err);
2284 }
2285
2286 static int dr_interception(struct vcpu_svm *svm)
2287 {
2288 int reg, dr;
2289 unsigned long val;
2290
2291 if (svm->vcpu.guest_debug == 0) {
2292 /*
2293 * No more DR vmexits; force a reload of the debug registers
2294 * and reenter on this instruction. The next vmexit will
2295 * retrieve the full state of the debug registers.
2296 */
2297 clr_dr_intercepts(svm);
2298 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2299 return 1;
2300 }
2301
2302 if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2303 return emulate_on_interception(svm);
2304
2305 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2306 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2307
2308 if (dr >= 16) { /* mov to DRn */
2309 if (!kvm_require_dr(&svm->vcpu, dr - 16))
2310 return 1;
2311 val = kvm_register_read(&svm->vcpu, reg);
2312 kvm_set_dr(&svm->vcpu, dr - 16, val);
2313 } else {
2314 if (!kvm_require_dr(&svm->vcpu, dr))
2315 return 1;
2316 kvm_get_dr(&svm->vcpu, dr, &val);
2317 kvm_register_write(&svm->vcpu, reg, val);
2318 }
2319
2320 return kvm_skip_emulated_instruction(&svm->vcpu);
2321 }
2322
2323 static int cr8_write_interception(struct vcpu_svm *svm)
2324 {
2325 struct kvm_run *kvm_run = svm->vcpu.run;
2326 int r;
2327
2328 u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2329 /* instruction emulation calls kvm_set_cr8() */
2330 r = cr_interception(svm);
2331 if (lapic_in_kernel(&svm->vcpu))
2332 return r;
2333 if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2334 return r;
2335 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2336 return 0;
2337 }
2338
2339 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
2340 {
2341 msr->data = 0;
2342
2343 switch (msr->index) {
2344 case MSR_F10H_DECFG:
2345 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
2346 msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
2347 break;
2348 default:
2349 return 1;
2350 }
2351
2352 return 0;
2353 }
2354
2355 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2356 {
2357 struct vcpu_svm *svm = to_svm(vcpu);
2358
2359 switch (msr_info->index) {
2360 case MSR_STAR:
2361 msr_info->data = svm->vmcb->save.star;
2362 break;
2363 #ifdef CONFIG_X86_64
2364 case MSR_LSTAR:
2365 msr_info->data = svm->vmcb->save.lstar;
2366 break;
2367 case MSR_CSTAR:
2368 msr_info->data = svm->vmcb->save.cstar;
2369 break;
2370 case MSR_KERNEL_GS_BASE:
2371 msr_info->data = svm->vmcb->save.kernel_gs_base;
2372 break;
2373 case MSR_SYSCALL_MASK:
2374 msr_info->data = svm->vmcb->save.sfmask;
2375 break;
2376 #endif
2377 case MSR_IA32_SYSENTER_CS:
2378 msr_info->data = svm->vmcb->save.sysenter_cs;
2379 break;
2380 case MSR_IA32_SYSENTER_EIP:
2381 msr_info->data = svm->sysenter_eip;
2382 break;
2383 case MSR_IA32_SYSENTER_ESP:
2384 msr_info->data = svm->sysenter_esp;
2385 break;
2386 case MSR_TSC_AUX:
2387 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
2388 return 1;
2389 msr_info->data = svm->tsc_aux;
2390 break;
2391 /*
2392 * Nobody will change the following 5 values in the VMCB so we can
2393 * safely return them on rdmsr. They will always be 0 until LBRV is
2394 * implemented.
2395 */
2396 case MSR_IA32_DEBUGCTLMSR:
2397 msr_info->data = svm->vmcb->save.dbgctl;
2398 break;
2399 case MSR_IA32_LASTBRANCHFROMIP:
2400 msr_info->data = svm->vmcb->save.br_from;
2401 break;
2402 case MSR_IA32_LASTBRANCHTOIP:
2403 msr_info->data = svm->vmcb->save.br_to;
2404 break;
2405 case MSR_IA32_LASTINTFROMIP:
2406 msr_info->data = svm->vmcb->save.last_excp_from;
2407 break;
2408 case MSR_IA32_LASTINTTOIP:
2409 msr_info->data = svm->vmcb->save.last_excp_to;
2410 break;
2411 case MSR_VM_HSAVE_PA:
2412 msr_info->data = svm->nested.hsave_msr;
2413 break;
2414 case MSR_VM_CR:
2415 msr_info->data = svm->nested.vm_cr_msr;
2416 break;
2417 case MSR_IA32_SPEC_CTRL:
2418 if (!msr_info->host_initiated &&
2419 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
2420 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
2421 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
2422 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
2423 return 1;
2424
2425 msr_info->data = svm->spec_ctrl;
2426 break;
2427 case MSR_AMD64_VIRT_SPEC_CTRL:
2428 if (!msr_info->host_initiated &&
2429 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2430 return 1;
2431
2432 msr_info->data = svm->virt_spec_ctrl;
2433 break;
2434 case MSR_F15H_IC_CFG: {
2435
2436 int family, model;
2437
2438 family = guest_cpuid_family(vcpu);
2439 model = guest_cpuid_model(vcpu);
2440
2441 if (family < 0 || model < 0)
2442 return kvm_get_msr_common(vcpu, msr_info);
2443
2444 msr_info->data = 0;
2445
2446 if (family == 0x15 &&
2447 (model >= 0x2 && model < 0x20))
2448 msr_info->data = 0x1E;
2449 }
2450 break;
2451 case MSR_F10H_DECFG:
2452 msr_info->data = svm->msr_decfg;
2453 break;
2454 default:
2455 return kvm_get_msr_common(vcpu, msr_info);
2456 }
2457 return 0;
2458 }
2459
2460 static int rdmsr_interception(struct vcpu_svm *svm)
2461 {
2462 return kvm_emulate_rdmsr(&svm->vcpu);
2463 }
2464
2465 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2466 {
2467 struct vcpu_svm *svm = to_svm(vcpu);
2468 int svm_dis, chg_mask;
2469
2470 if (data & ~SVM_VM_CR_VALID_MASK)
2471 return 1;
2472
2473 chg_mask = SVM_VM_CR_VALID_MASK;
2474
2475 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2476 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2477
2478 svm->nested.vm_cr_msr &= ~chg_mask;
2479 svm->nested.vm_cr_msr |= (data & chg_mask);
2480
2481 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2482
2483 /* check for svm_disable while efer.svme is set */
2484 if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2485 return 1;
2486
2487 return 0;
2488 }
2489
2490 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2491 {
2492 struct vcpu_svm *svm = to_svm(vcpu);
2493
2494 u32 ecx = msr->index;
2495 u64 data = msr->data;
2496 switch (ecx) {
2497 case MSR_IA32_CR_PAT:
2498 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
2499 return 1;
2500 vcpu->arch.pat = data;
2501 svm->vmcb->save.g_pat = data;
2502 mark_dirty(svm->vmcb, VMCB_NPT);
2503 break;
2504 case MSR_IA32_SPEC_CTRL:
2505 if (!msr->host_initiated &&
2506 !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
2507 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_STIBP) &&
2508 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
2509 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
2510 return 1;
2511
2512 if (data & ~kvm_spec_ctrl_valid_bits(vcpu))
2513 return 1;
2514
2515 svm->spec_ctrl = data;
2516 if (!data)
2517 break;
2518
2519 /*
2520 * For non-nested:
2521 * When it's written (to non-zero) for the first time, pass
2522 * it through.
2523 *
2524 * For nested:
2525 * The handling of the MSR bitmap for L2 guests is done in
2526 * nested_svm_vmrun_msrpm.
2527 * We update the L1 MSR bit as well since it will end up
2528 * touching the MSR anyway now.
2529 */
2530 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
2531 break;
2532 case MSR_IA32_PRED_CMD:
2533 if (!msr->host_initiated &&
2534 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
2535 return 1;
2536
2537 if (data & ~PRED_CMD_IBPB)
2538 return 1;
2539 if (!boot_cpu_has(X86_FEATURE_AMD_IBPB))
2540 return 1;
2541 if (!data)
2542 break;
2543
2544 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2545 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
2546 break;
2547 case MSR_AMD64_VIRT_SPEC_CTRL:
2548 if (!msr->host_initiated &&
2549 !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
2550 return 1;
2551
2552 if (data & ~SPEC_CTRL_SSBD)
2553 return 1;
2554
2555 svm->virt_spec_ctrl = data;
2556 break;
2557 case MSR_STAR:
2558 svm->vmcb->save.star = data;
2559 break;
2560 #ifdef CONFIG_X86_64
2561 case MSR_LSTAR:
2562 svm->vmcb->save.lstar = data;
2563 break;
2564 case MSR_CSTAR:
2565 svm->vmcb->save.cstar = data;
2566 break;
2567 case MSR_KERNEL_GS_BASE:
2568 svm->vmcb->save.kernel_gs_base = data;
2569 break;
2570 case MSR_SYSCALL_MASK:
2571 svm->vmcb->save.sfmask = data;
2572 break;
2573 #endif
2574 case MSR_IA32_SYSENTER_CS:
2575 svm->vmcb->save.sysenter_cs = data;
2576 break;
2577 case MSR_IA32_SYSENTER_EIP:
2578 svm->sysenter_eip = data;
2579 svm->vmcb->save.sysenter_eip = data;
2580 break;
2581 case MSR_IA32_SYSENTER_ESP:
2582 svm->sysenter_esp = data;
2583 svm->vmcb->save.sysenter_esp = data;
2584 break;
2585 case MSR_TSC_AUX:
2586 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
2587 return 1;
2588
2589 /*
2590 * This is rare, so we update the MSR here instead of using
2591 * direct_access_msrs. Doing that would require a rdmsr in
2592 * svm_vcpu_put.
2593 */
2594 svm->tsc_aux = data;
2595 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2596 break;
2597 case MSR_IA32_DEBUGCTLMSR:
2598 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
2599 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
2600 __func__, data);
2601 break;
2602 }
2603 if (data & DEBUGCTL_RESERVED_BITS)
2604 return 1;
2605
2606 svm->vmcb->save.dbgctl = data;
2607 mark_dirty(svm->vmcb, VMCB_LBR);
2608 if (data & (1ULL<<0))
2609 svm_enable_lbrv(svm);
2610 else
2611 svm_disable_lbrv(svm);
2612 break;
2613 case MSR_VM_HSAVE_PA:
2614 svm->nested.hsave_msr = data;
2615 break;
2616 case MSR_VM_CR:
2617 return svm_set_vm_cr(vcpu, data);
2618 case MSR_VM_IGNNE:
2619 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
2620 break;
2621 case MSR_F10H_DECFG: {
2622 struct kvm_msr_entry msr_entry;
2623
2624 msr_entry.index = msr->index;
2625 if (svm_get_msr_feature(&msr_entry))
2626 return 1;
2627
2628 /* Check the supported bits */
2629 if (data & ~msr_entry.data)
2630 return 1;
2631
2632 /* Don't allow the guest to change a bit, #GP */
2633 if (!msr->host_initiated && (data ^ msr_entry.data))
2634 return 1;
2635
2636 svm->msr_decfg = data;
2637 break;
2638 }
2639 case MSR_IA32_APICBASE:
2640 if (kvm_vcpu_apicv_active(vcpu))
2641 avic_update_vapic_bar(to_svm(vcpu), data);
2642 /* Fall through */
2643 default:
2644 return kvm_set_msr_common(vcpu, msr);
2645 }
2646 return 0;
2647 }
2648
2649 static int wrmsr_interception(struct vcpu_svm *svm)
2650 {
2651 return kvm_emulate_wrmsr(&svm->vcpu);
2652 }
2653
2654 static int msr_interception(struct vcpu_svm *svm)
2655 {
2656 if (svm->vmcb->control.exit_info_1)
2657 return wrmsr_interception(svm);
2658 else
2659 return rdmsr_interception(svm);
2660 }
2661
2662 static int interrupt_window_interception(struct vcpu_svm *svm)
2663 {
2664 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2665 svm_clear_vintr(svm);
2666
2667 /*
2668 * For AVIC, the only reason to end up here is ExtINTs.
2669 * In this case AVIC was temporarily disabled for
2670 * requesting the IRQ window and we have to re-enable it.
2671 */
2672 svm_toggle_avic_for_irq_window(&svm->vcpu, true);
2673
2674 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2675 mark_dirty(svm->vmcb, VMCB_INTR);
2676 ++svm->vcpu.stat.irq_window_exits;
2677 return 1;
2678 }
2679
2680 static int pause_interception(struct vcpu_svm *svm)
2681 {
2682 struct kvm_vcpu *vcpu = &svm->vcpu;
2683 bool in_kernel = (svm_get_cpl(vcpu) == 0);
2684
2685 if (pause_filter_thresh)
2686 grow_ple_window(vcpu);
2687
2688 kvm_vcpu_on_spin(vcpu, in_kernel);
2689 return 1;
2690 }
2691
2692 static int nop_interception(struct vcpu_svm *svm)
2693 {
2694 return kvm_skip_emulated_instruction(&(svm->vcpu));
2695 }
2696
2697 static int monitor_interception(struct vcpu_svm *svm)
2698 {
2699 printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
2700 return nop_interception(svm);
2701 }
2702
2703 static int mwait_interception(struct vcpu_svm *svm)
2704 {
2705 printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
2706 return nop_interception(svm);
2707 }
2708
2709 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
2710 [SVM_EXIT_READ_CR0] = cr_interception,
2711 [SVM_EXIT_READ_CR3] = cr_interception,
2712 [SVM_EXIT_READ_CR4] = cr_interception,
2713 [SVM_EXIT_READ_CR8] = cr_interception,
2714 [SVM_EXIT_CR0_SEL_WRITE] = cr_interception,
2715 [SVM_EXIT_WRITE_CR0] = cr_interception,
2716 [SVM_EXIT_WRITE_CR3] = cr_interception,
2717 [SVM_EXIT_WRITE_CR4] = cr_interception,
2718 [SVM_EXIT_WRITE_CR8] = cr8_write_interception,
2719 [SVM_EXIT_READ_DR0] = dr_interception,
2720 [SVM_EXIT_READ_DR1] = dr_interception,
2721 [SVM_EXIT_READ_DR2] = dr_interception,
2722 [SVM_EXIT_READ_DR3] = dr_interception,
2723 [SVM_EXIT_READ_DR4] = dr_interception,
2724 [SVM_EXIT_READ_DR5] = dr_interception,
2725 [SVM_EXIT_READ_DR6] = dr_interception,
2726 [SVM_EXIT_READ_DR7] = dr_interception,
2727 [SVM_EXIT_WRITE_DR0] = dr_interception,
2728 [SVM_EXIT_WRITE_DR1] = dr_interception,
2729 [SVM_EXIT_WRITE_DR2] = dr_interception,
2730 [SVM_EXIT_WRITE_DR3] = dr_interception,
2731 [SVM_EXIT_WRITE_DR4] = dr_interception,
2732 [SVM_EXIT_WRITE_DR5] = dr_interception,
2733 [SVM_EXIT_WRITE_DR6] = dr_interception,
2734 [SVM_EXIT_WRITE_DR7] = dr_interception,
2735 [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception,
2736 [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception,
2737 [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception,
2738 [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception,
2739 [SVM_EXIT_EXCP_BASE + MC_VECTOR] = mc_interception,
2740 [SVM_EXIT_EXCP_BASE + AC_VECTOR] = ac_interception,
2741 [SVM_EXIT_EXCP_BASE + GP_VECTOR] = gp_interception,
2742 [SVM_EXIT_INTR] = intr_interception,
2743 [SVM_EXIT_NMI] = nmi_interception,
2744 [SVM_EXIT_SMI] = nop_on_interception,
2745 [SVM_EXIT_INIT] = nop_on_interception,
2746 [SVM_EXIT_VINTR] = interrupt_window_interception,
2747 [SVM_EXIT_RDPMC] = rdpmc_interception,
2748 [SVM_EXIT_CPUID] = cpuid_interception,
2749 [SVM_EXIT_IRET] = iret_interception,
2750 [SVM_EXIT_INVD] = emulate_on_interception,
2751 [SVM_EXIT_PAUSE] = pause_interception,
2752 [SVM_EXIT_HLT] = halt_interception,
2753 [SVM_EXIT_INVLPG] = invlpg_interception,
2754 [SVM_EXIT_INVLPGA] = invlpga_interception,
2755 [SVM_EXIT_IOIO] = io_interception,
2756 [SVM_EXIT_MSR] = msr_interception,
2757 [SVM_EXIT_TASK_SWITCH] = task_switch_interception,
2758 [SVM_EXIT_SHUTDOWN] = shutdown_interception,
2759 [SVM_EXIT_VMRUN] = vmrun_interception,
2760 [SVM_EXIT_VMMCALL] = vmmcall_interception,
2761 [SVM_EXIT_VMLOAD] = vmload_interception,
2762 [SVM_EXIT_VMSAVE] = vmsave_interception,
2763 [SVM_EXIT_STGI] = stgi_interception,
2764 [SVM_EXIT_CLGI] = clgi_interception,
2765 [SVM_EXIT_SKINIT] = skinit_interception,
2766 [SVM_EXIT_WBINVD] = wbinvd_interception,
2767 [SVM_EXIT_MONITOR] = monitor_interception,
2768 [SVM_EXIT_MWAIT] = mwait_interception,
2769 [SVM_EXIT_XSETBV] = xsetbv_interception,
2770 [SVM_EXIT_RDPRU] = rdpru_interception,
2771 [SVM_EXIT_NPF] = npf_interception,
2772 [SVM_EXIT_RSM] = rsm_interception,
2773 [SVM_EXIT_AVIC_INCOMPLETE_IPI] = avic_incomplete_ipi_interception,
2774 [SVM_EXIT_AVIC_UNACCELERATED_ACCESS] = avic_unaccelerated_access_interception,
2775 };
2776
2777 static void dump_vmcb(struct kvm_vcpu *vcpu)
2778 {
2779 struct vcpu_svm *svm = to_svm(vcpu);
2780 struct vmcb_control_area *control = &svm->vmcb->control;
2781 struct vmcb_save_area *save = &svm->vmcb->save;
2782
2783 if (!dump_invalid_vmcb) {
2784 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
2785 return;
2786 }
2787
2788 pr_err("VMCB Control Area:\n");
2789 pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
2790 pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
2791 pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
2792 pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
2793 pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
2794 pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
2795 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
2796 pr_err("%-20s%d\n", "pause filter threshold:",
2797 control->pause_filter_thresh);
2798 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
2799 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
2800 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
2801 pr_err("%-20s%d\n", "asid:", control->asid);
2802 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
2803 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
2804 pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
2805 pr_err("%-20s%08x\n", "int_state:", control->int_state);
2806 pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
2807 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
2808 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
2809 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
2810 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
2811 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
2812 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
2813 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
2814 pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
2815 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
2816 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
2817 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
2818 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
2819 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
2820 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
2821 pr_err("VMCB State Save Area:\n");
2822 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2823 "es:",
2824 save->es.selector, save->es.attrib,
2825 save->es.limit, save->es.base);
2826 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2827 "cs:",
2828 save->cs.selector, save->cs.attrib,
2829 save->cs.limit, save->cs.base);
2830 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2831 "ss:",
2832 save->ss.selector, save->ss.attrib,
2833 save->ss.limit, save->ss.base);
2834 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2835 "ds:",
2836 save->ds.selector, save->ds.attrib,
2837 save->ds.limit, save->ds.base);
2838 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2839 "fs:",
2840 save->fs.selector, save->fs.attrib,
2841 save->fs.limit, save->fs.base);
2842 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2843 "gs:",
2844 save->gs.selector, save->gs.attrib,
2845 save->gs.limit, save->gs.base);
2846 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2847 "gdtr:",
2848 save->gdtr.selector, save->gdtr.attrib,
2849 save->gdtr.limit, save->gdtr.base);
2850 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2851 "ldtr:",
2852 save->ldtr.selector, save->ldtr.attrib,
2853 save->ldtr.limit, save->ldtr.base);
2854 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2855 "idtr:",
2856 save->idtr.selector, save->idtr.attrib,
2857 save->idtr.limit, save->idtr.base);
2858 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
2859 "tr:",
2860 save->tr.selector, save->tr.attrib,
2861 save->tr.limit, save->tr.base);
2862 pr_err("cpl: %d efer: %016llx\n",
2863 save->cpl, save->efer);
2864 pr_err("%-15s %016llx %-13s %016llx\n",
2865 "cr0:", save->cr0, "cr2:", save->cr2);
2866 pr_err("%-15s %016llx %-13s %016llx\n",
2867 "cr3:", save->cr3, "cr4:", save->cr4);
2868 pr_err("%-15s %016llx %-13s %016llx\n",
2869 "dr6:", save->dr6, "dr7:", save->dr7);
2870 pr_err("%-15s %016llx %-13s %016llx\n",
2871 "rip:", save->rip, "rflags:", save->rflags);
2872 pr_err("%-15s %016llx %-13s %016llx\n",
2873 "rsp:", save->rsp, "rax:", save->rax);
2874 pr_err("%-15s %016llx %-13s %016llx\n",
2875 "star:", save->star, "lstar:", save->lstar);
2876 pr_err("%-15s %016llx %-13s %016llx\n",
2877 "cstar:", save->cstar, "sfmask:", save->sfmask);
2878 pr_err("%-15s %016llx %-13s %016llx\n",
2879 "kernel_gs_base:", save->kernel_gs_base,
2880 "sysenter_cs:", save->sysenter_cs);
2881 pr_err("%-15s %016llx %-13s %016llx\n",
2882 "sysenter_esp:", save->sysenter_esp,
2883 "sysenter_eip:", save->sysenter_eip);
2884 pr_err("%-15s %016llx %-13s %016llx\n",
2885 "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
2886 pr_err("%-15s %016llx %-13s %016llx\n",
2887 "br_from:", save->br_from, "br_to:", save->br_to);
2888 pr_err("%-15s %016llx %-13s %016llx\n",
2889 "excp_from:", save->last_excp_from,
2890 "excp_to:", save->last_excp_to);
2891 }
2892
2893 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
2894 {
2895 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
2896
2897 *info1 = control->exit_info_1;
2898 *info2 = control->exit_info_2;
2899 }
2900
2901 static int handle_exit(struct kvm_vcpu *vcpu,
2902 enum exit_fastpath_completion exit_fastpath)
2903 {
2904 struct vcpu_svm *svm = to_svm(vcpu);
2905 struct kvm_run *kvm_run = vcpu->run;
2906 u32 exit_code = svm->vmcb->control.exit_code;
2907
2908 trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
2909
2910 if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
2911 vcpu->arch.cr0 = svm->vmcb->save.cr0;
2912 if (npt_enabled)
2913 vcpu->arch.cr3 = svm->vmcb->save.cr3;
2914
2915 if (unlikely(svm->nested.exit_required)) {
2916 nested_svm_vmexit(svm);
2917 svm->nested.exit_required = false;
2918
2919 return 1;
2920 }
2921
2922 if (is_guest_mode(vcpu)) {
2923 int vmexit;
2924
2925 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
2926 svm->vmcb->control.exit_info_1,
2927 svm->vmcb->control.exit_info_2,
2928 svm->vmcb->control.exit_int_info,
2929 svm->vmcb->control.exit_int_info_err,
2930 KVM_ISA_SVM);
2931
2932 vmexit = nested_svm_exit_special(svm);
2933
2934 if (vmexit == NESTED_EXIT_CONTINUE)
2935 vmexit = nested_svm_exit_handled(svm);
2936
2937 if (vmexit == NESTED_EXIT_DONE)
2938 return 1;
2939 }
2940
2941 svm_complete_interrupts(svm);
2942
2943 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
2944 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
2945 kvm_run->fail_entry.hardware_entry_failure_reason
2946 = svm->vmcb->control.exit_code;
2947 dump_vmcb(vcpu);
2948 return 0;
2949 }
2950
2951 if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
2952 exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
2953 exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
2954 exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
2955 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
2956 "exit_code 0x%x\n",
2957 __func__, svm->vmcb->control.exit_int_info,
2958 exit_code);
2959
2960 if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) {
2961 kvm_skip_emulated_instruction(vcpu);
2962 return 1;
2963 } else if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
2964 || !svm_exit_handlers[exit_code]) {
2965 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
2966 dump_vmcb(vcpu);
2967 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2968 vcpu->run->internal.suberror =
2969 KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
2970 vcpu->run->internal.ndata = 1;
2971 vcpu->run->internal.data[0] = exit_code;
2972 return 0;
2973 }
2974
2975 #ifdef CONFIG_RETPOLINE
2976 if (exit_code == SVM_EXIT_MSR)
2977 return msr_interception(svm);
2978 else if (exit_code == SVM_EXIT_VINTR)
2979 return interrupt_window_interception(svm);
2980 else if (exit_code == SVM_EXIT_INTR)
2981 return intr_interception(svm);
2982 else if (exit_code == SVM_EXIT_HLT)
2983 return halt_interception(svm);
2984 else if (exit_code == SVM_EXIT_NPF)
2985 return npf_interception(svm);
2986 #endif
2987 return svm_exit_handlers[exit_code](svm);
2988 }
2989
2990 static void reload_tss(struct kvm_vcpu *vcpu)
2991 {
2992 int cpu = raw_smp_processor_id();
2993
2994 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2995 sd->tss_desc->type = 9; /* available 32/64-bit TSS */
2996 load_TR_desc();
2997 }
2998
2999 static void pre_svm_run(struct vcpu_svm *svm)
3000 {
3001 int cpu = raw_smp_processor_id();
3002
3003 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3004
3005 if (sev_guest(svm->vcpu.kvm))
3006 return pre_sev_run(svm, cpu);
3007
3008 /* FIXME: handle wraparound of asid_generation */
3009 if (svm->asid_generation != sd->asid_generation)
3010 new_asid(svm, sd);
3011 }
3012
3013 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3014 {
3015 struct vcpu_svm *svm = to_svm(vcpu);
3016
3017 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3018 vcpu->arch.hflags |= HF_NMI_MASK;
3019 set_intercept(svm, INTERCEPT_IRET);
3020 ++vcpu->stat.nmi_injections;
3021 }
3022
3023 static void svm_set_irq(struct kvm_vcpu *vcpu)
3024 {
3025 struct vcpu_svm *svm = to_svm(vcpu);
3026
3027 BUG_ON(!(gif_set(svm)));
3028
3029 trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3030 ++vcpu->stat.irq_injections;
3031
3032 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3033 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3034 }
3035
3036 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3037 {
3038 struct vcpu_svm *svm = to_svm(vcpu);
3039
3040 if (svm_nested_virtualize_tpr(vcpu))
3041 return;
3042
3043 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3044
3045 if (irr == -1)
3046 return;
3047
3048 if (tpr >= irr)
3049 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3050 }
3051
3052 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3053 {
3054 struct vcpu_svm *svm = to_svm(vcpu);
3055 struct vmcb *vmcb = svm->vmcb;
3056 int ret;
3057 ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3058 !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3059 ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3060
3061 return ret;
3062 }
3063
3064 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3065 {
3066 struct vcpu_svm *svm = to_svm(vcpu);
3067
3068 return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3069 }
3070
3071 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3072 {
3073 struct vcpu_svm *svm = to_svm(vcpu);
3074
3075 if (masked) {
3076 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3077 set_intercept(svm, INTERCEPT_IRET);
3078 } else {
3079 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3080 clr_intercept(svm, INTERCEPT_IRET);
3081 }
3082 }
3083
3084 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3085 {
3086 struct vcpu_svm *svm = to_svm(vcpu);
3087 struct vmcb *vmcb = svm->vmcb;
3088
3089 if (!gif_set(svm) ||
3090 (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3091 return 0;
3092
3093 if (is_guest_mode(vcpu) && (svm->vcpu.arch.hflags & HF_VINTR_MASK))
3094 return !!(svm->vcpu.arch.hflags & HF_HIF_MASK);
3095 else
3096 return !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3097 }
3098
3099 static void enable_irq_window(struct kvm_vcpu *vcpu)
3100 {
3101 struct vcpu_svm *svm = to_svm(vcpu);
3102
3103 /*
3104 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3105 * 1, because that's a separate STGI/VMRUN intercept. The next time we
3106 * get that intercept, this function will be called again though and
3107 * we'll get the vintr intercept. However, if the vGIF feature is
3108 * enabled, the STGI interception will not occur. Enable the irq
3109 * window under the assumption that the hardware will set the GIF.
3110 */
3111 if (vgif_enabled(svm) || gif_set(svm)) {
3112 /*
3113 * IRQ window is not needed when AVIC is enabled,
3114 * unless we have pending ExtINT since it cannot be injected
3115 * via AVIC. In such case, we need to temporarily disable AVIC,
3116 * and fallback to injecting IRQ via V_IRQ.
3117 */
3118 svm_toggle_avic_for_irq_window(vcpu, false);
3119 svm_set_vintr(svm);
3120 }
3121 }
3122
3123 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3124 {
3125 struct vcpu_svm *svm = to_svm(vcpu);
3126
3127 if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3128 == HF_NMI_MASK)
3129 return; /* IRET will cause a vm exit */
3130
3131 if (!gif_set(svm)) {
3132 if (vgif_enabled(svm))
3133 set_intercept(svm, INTERCEPT_STGI);
3134 return; /* STGI will cause a vm exit */
3135 }
3136
3137 if (svm->nested.exit_required)
3138 return; /* we're not going to run the guest yet */
3139
3140 /*
3141 * Something prevents NMI from been injected. Single step over possible
3142 * problem (IRET or exception injection or interrupt shadow)
3143 */
3144 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3145 svm->nmi_singlestep = true;
3146 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3147 }
3148
3149 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3150 {
3151 return 0;
3152 }
3153
3154 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
3155 {
3156 return 0;
3157 }
3158
3159 void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
3160 {
3161 struct vcpu_svm *svm = to_svm(vcpu);
3162
3163 if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3164 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3165 else
3166 svm->asid_generation--;
3167 }
3168
3169 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3170 {
3171 struct vcpu_svm *svm = to_svm(vcpu);
3172
3173 invlpga(gva, svm->vmcb->control.asid);
3174 }
3175
3176 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3177 {
3178 }
3179
3180 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3181 {
3182 struct vcpu_svm *svm = to_svm(vcpu);
3183
3184 if (svm_nested_virtualize_tpr(vcpu))
3185 return;
3186
3187 if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3188 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3189 kvm_set_cr8(vcpu, cr8);
3190 }
3191 }
3192
3193 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3194 {
3195 struct vcpu_svm *svm = to_svm(vcpu);
3196 u64 cr8;
3197
3198 if (svm_nested_virtualize_tpr(vcpu) ||
3199 kvm_vcpu_apicv_active(vcpu))
3200 return;
3201
3202 cr8 = kvm_get_cr8(vcpu);
3203 svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3204 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3205 }
3206
3207 static void svm_complete_interrupts(struct vcpu_svm *svm)
3208 {
3209 u8 vector;
3210 int type;
3211 u32 exitintinfo = svm->vmcb->control.exit_int_info;
3212 unsigned int3_injected = svm->int3_injected;
3213
3214 svm->int3_injected = 0;
3215
3216 /*
3217 * If we've made progress since setting HF_IRET_MASK, we've
3218 * executed an IRET and can allow NMI injection.
3219 */
3220 if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3221 && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3222 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3223 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3224 }
3225
3226 svm->vcpu.arch.nmi_injected = false;
3227 kvm_clear_exception_queue(&svm->vcpu);
3228 kvm_clear_interrupt_queue(&svm->vcpu);
3229
3230 if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3231 return;
3232
3233 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3234
3235 vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3236 type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3237
3238 switch (type) {
3239 case SVM_EXITINTINFO_TYPE_NMI:
3240 svm->vcpu.arch.nmi_injected = true;
3241 break;
3242 case SVM_EXITINTINFO_TYPE_EXEPT:
3243 /*
3244 * In case of software exceptions, do not reinject the vector,
3245 * but re-execute the instruction instead. Rewind RIP first
3246 * if we emulated INT3 before.
3247 */
3248 if (kvm_exception_is_soft(vector)) {
3249 if (vector == BP_VECTOR && int3_injected &&
3250 kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3251 kvm_rip_write(&svm->vcpu,
3252 kvm_rip_read(&svm->vcpu) -
3253 int3_injected);
3254 break;
3255 }
3256 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3257 u32 err = svm->vmcb->control.exit_int_info_err;
3258 kvm_requeue_exception_e(&svm->vcpu, vector, err);
3259
3260 } else
3261 kvm_requeue_exception(&svm->vcpu, vector);
3262 break;
3263 case SVM_EXITINTINFO_TYPE_INTR:
3264 kvm_queue_interrupt(&svm->vcpu, vector, false);
3265 break;
3266 default:
3267 break;
3268 }
3269 }
3270
3271 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3272 {
3273 struct vcpu_svm *svm = to_svm(vcpu);
3274 struct vmcb_control_area *control = &svm->vmcb->control;
3275
3276 control->exit_int_info = control->event_inj;
3277 control->exit_int_info_err = control->event_inj_err;
3278 control->event_inj = 0;
3279 svm_complete_interrupts(svm);
3280 }
3281
3282 void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs);
3283
3284 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3285 {
3286 struct vcpu_svm *svm = to_svm(vcpu);
3287
3288 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3289 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3290 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3291
3292 /*
3293 * A vmexit emulation is required before the vcpu can be executed
3294 * again.
3295 */
3296 if (unlikely(svm->nested.exit_required))
3297 return;
3298
3299 /*
3300 * Disable singlestep if we're injecting an interrupt/exception.
3301 * We don't want our modified rflags to be pushed on the stack where
3302 * we might not be able to easily reset them if we disabled NMI
3303 * singlestep later.
3304 */
3305 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
3306 /*
3307 * Event injection happens before external interrupts cause a
3308 * vmexit and interrupts are disabled here, so smp_send_reschedule
3309 * is enough to force an immediate vmexit.
3310 */
3311 disable_nmi_singlestep(svm);
3312 smp_send_reschedule(vcpu->cpu);
3313 }
3314
3315 pre_svm_run(svm);
3316
3317 sync_lapic_to_cr8(vcpu);
3318
3319 svm->vmcb->save.cr2 = vcpu->arch.cr2;
3320
3321 /*
3322 * Run with all-zero DR6 unless needed, so that we can get the exact cause
3323 * of a #DB.
3324 */
3325 if (unlikely(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))
3326 svm_set_dr6(svm, vcpu->arch.dr6);
3327 else
3328 svm_set_dr6(svm, DR6_FIXED_1 | DR6_RTM);
3329
3330 clgi();
3331 kvm_load_guest_xsave_state(vcpu);
3332
3333 if (lapic_in_kernel(vcpu) &&
3334 vcpu->arch.apic->lapic_timer.timer_advance_ns)
3335 kvm_wait_lapic_expire(vcpu);
3336
3337 /*
3338 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
3339 * it's non-zero. Since vmentry is serialising on affected CPUs, there
3340 * is no need to worry about the conditional branch over the wrmsr
3341 * being speculatively taken.
3342 */
3343 x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
3344
3345 __svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
3346
3347 #ifdef CONFIG_X86_64
3348 wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3349 #else
3350 loadsegment(fs, svm->host.fs);
3351 #ifndef CONFIG_X86_32_LAZY_GS
3352 loadsegment(gs, svm->host.gs);
3353 #endif
3354 #endif
3355
3356 /*
3357 * We do not use IBRS in the kernel. If this vCPU has used the
3358 * SPEC_CTRL MSR it may have left it on; save the value and
3359 * turn it off. This is much more efficient than blindly adding
3360 * it to the atomic save/restore list. Especially as the former
3361 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
3362 *
3363 * For non-nested case:
3364 * If the L01 MSR bitmap does not intercept the MSR, then we need to
3365 * save it.
3366 *
3367 * For nested case:
3368 * If the L02 MSR bitmap does not intercept the MSR, then we need to
3369 * save it.
3370 */
3371 if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
3372 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
3373
3374 reload_tss(vcpu);
3375
3376 x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
3377
3378 vcpu->arch.cr2 = svm->vmcb->save.cr2;
3379 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3380 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3381 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3382
3383 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3384 kvm_before_interrupt(&svm->vcpu);
3385
3386 kvm_load_host_xsave_state(vcpu);
3387 stgi();
3388
3389 /* Any pending NMI will happen here */
3390
3391 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3392 kvm_after_interrupt(&svm->vcpu);
3393
3394 sync_cr8_to_lapic(vcpu);
3395
3396 svm->next_rip = 0;
3397
3398 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3399
3400 /* if exit due to PF check for async PF */
3401 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3402 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
3403
3404 if (npt_enabled) {
3405 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3406 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3407 }
3408
3409 /*
3410 * We need to handle MC intercepts here before the vcpu has a chance to
3411 * change the physical cpu
3412 */
3413 if (unlikely(svm->vmcb->control.exit_code ==
3414 SVM_EXIT_EXCP_BASE + MC_VECTOR))
3415 svm_handle_mce(svm);
3416
3417 mark_all_clean(svm->vmcb);
3418 }
3419
3420 static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root)
3421 {
3422 struct vcpu_svm *svm = to_svm(vcpu);
3423 bool update_guest_cr3 = true;
3424 unsigned long cr3;
3425
3426 cr3 = __sme_set(root);
3427 if (npt_enabled) {
3428 svm->vmcb->control.nested_cr3 = cr3;
3429 mark_dirty(svm->vmcb, VMCB_NPT);
3430
3431 /* Loading L2's CR3 is handled by enter_svm_guest_mode. */
3432 if (is_guest_mode(vcpu))
3433 update_guest_cr3 = false;
3434 else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3435 cr3 = vcpu->arch.cr3;
3436 else /* CR3 is already up-to-date. */
3437 update_guest_cr3 = false;
3438 }
3439
3440 if (update_guest_cr3) {
3441 svm->vmcb->save.cr3 = cr3;
3442 mark_dirty(svm->vmcb, VMCB_CR);
3443 }
3444 }
3445
3446 static int is_disabled(void)
3447 {
3448 u64 vm_cr;
3449
3450 rdmsrl(MSR_VM_CR, vm_cr);
3451 if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
3452 return 1;
3453
3454 return 0;
3455 }
3456
3457 static void
3458 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
3459 {
3460 /*
3461 * Patch in the VMMCALL instruction:
3462 */
3463 hypercall[0] = 0x0f;
3464 hypercall[1] = 0x01;
3465 hypercall[2] = 0xd9;
3466 }
3467
3468 static int __init svm_check_processor_compat(void)
3469 {
3470 return 0;
3471 }
3472
3473 static bool svm_cpu_has_accelerated_tpr(void)
3474 {
3475 return false;
3476 }
3477
3478 static bool svm_has_emulated_msr(int index)
3479 {
3480 switch (index) {
3481 case MSR_IA32_MCG_EXT_CTL:
3482 case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
3483 return false;
3484 default:
3485 break;
3486 }
3487
3488 return true;
3489 }
3490
3491 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
3492 {
3493 return 0;
3494 }
3495
3496 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
3497 {
3498 struct vcpu_svm *svm = to_svm(vcpu);
3499
3500 vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
3501 boot_cpu_has(X86_FEATURE_XSAVE) &&
3502 boot_cpu_has(X86_FEATURE_XSAVES);
3503
3504 /* Update nrips enabled cache */
3505 svm->nrips_enabled = kvm_cpu_cap_has(X86_FEATURE_NRIPS) &&
3506 guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
3507
3508 if (!kvm_vcpu_apicv_active(vcpu))
3509 return;
3510
3511 /*
3512 * AVIC does not work with an x2APIC mode guest. If the X2APIC feature
3513 * is exposed to the guest, disable AVIC.
3514 */
3515 if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC))
3516 kvm_request_apicv_update(vcpu->kvm, false,
3517 APICV_INHIBIT_REASON_X2APIC);
3518
3519 /*
3520 * Currently, AVIC does not work with nested virtualization.
3521 * So, we disable AVIC when cpuid for SVM is set in the L1 guest.
3522 */
3523 if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM))
3524 kvm_request_apicv_update(vcpu->kvm, false,
3525 APICV_INHIBIT_REASON_NESTED);
3526 }
3527
3528 static bool svm_has_wbinvd_exit(void)
3529 {
3530 return true;
3531 }
3532
3533 #define PRE_EX(exit) { .exit_code = (exit), \
3534 .stage = X86_ICPT_PRE_EXCEPT, }
3535 #define POST_EX(exit) { .exit_code = (exit), \
3536 .stage = X86_ICPT_POST_EXCEPT, }
3537 #define POST_MEM(exit) { .exit_code = (exit), \
3538 .stage = X86_ICPT_POST_MEMACCESS, }
3539
3540 static const struct __x86_intercept {
3541 u32 exit_code;
3542 enum x86_intercept_stage stage;
3543 } x86_intercept_map[] = {
3544 [x86_intercept_cr_read] = POST_EX(SVM_EXIT_READ_CR0),
3545 [x86_intercept_cr_write] = POST_EX(SVM_EXIT_WRITE_CR0),
3546 [x86_intercept_clts] = POST_EX(SVM_EXIT_WRITE_CR0),
3547 [x86_intercept_lmsw] = POST_EX(SVM_EXIT_WRITE_CR0),
3548 [x86_intercept_smsw] = POST_EX(SVM_EXIT_READ_CR0),
3549 [x86_intercept_dr_read] = POST_EX(SVM_EXIT_READ_DR0),
3550 [x86_intercept_dr_write] = POST_EX(SVM_EXIT_WRITE_DR0),
3551 [x86_intercept_sldt] = POST_EX(SVM_EXIT_LDTR_READ),
3552 [x86_intercept_str] = POST_EX(SVM_EXIT_TR_READ),
3553 [x86_intercept_lldt] = POST_EX(SVM_EXIT_LDTR_WRITE),
3554 [x86_intercept_ltr] = POST_EX(SVM_EXIT_TR_WRITE),
3555 [x86_intercept_sgdt] = POST_EX(SVM_EXIT_GDTR_READ),
3556 [x86_intercept_sidt] = POST_EX(SVM_EXIT_IDTR_READ),
3557 [x86_intercept_lgdt] = POST_EX(SVM_EXIT_GDTR_WRITE),
3558 [x86_intercept_lidt] = POST_EX(SVM_EXIT_IDTR_WRITE),
3559 [x86_intercept_vmrun] = POST_EX(SVM_EXIT_VMRUN),
3560 [x86_intercept_vmmcall] = POST_EX(SVM_EXIT_VMMCALL),
3561 [x86_intercept_vmload] = POST_EX(SVM_EXIT_VMLOAD),
3562 [x86_intercept_vmsave] = POST_EX(SVM_EXIT_VMSAVE),
3563 [x86_intercept_stgi] = POST_EX(SVM_EXIT_STGI),
3564 [x86_intercept_clgi] = POST_EX(SVM_EXIT_CLGI),
3565 [x86_intercept_skinit] = POST_EX(SVM_EXIT_SKINIT),
3566 [x86_intercept_invlpga] = POST_EX(SVM_EXIT_INVLPGA),
3567 [x86_intercept_rdtscp] = POST_EX(SVM_EXIT_RDTSCP),
3568 [x86_intercept_monitor] = POST_MEM(SVM_EXIT_MONITOR),
3569 [x86_intercept_mwait] = POST_EX(SVM_EXIT_MWAIT),
3570 [x86_intercept_invlpg] = POST_EX(SVM_EXIT_INVLPG),
3571 [x86_intercept_invd] = POST_EX(SVM_EXIT_INVD),
3572 [x86_intercept_wbinvd] = POST_EX(SVM_EXIT_WBINVD),
3573 [x86_intercept_wrmsr] = POST_EX(SVM_EXIT_MSR),
3574 [x86_intercept_rdtsc] = POST_EX(SVM_EXIT_RDTSC),
3575 [x86_intercept_rdmsr] = POST_EX(SVM_EXIT_MSR),
3576 [x86_intercept_rdpmc] = POST_EX(SVM_EXIT_RDPMC),
3577 [x86_intercept_cpuid] = PRE_EX(SVM_EXIT_CPUID),
3578 [x86_intercept_rsm] = PRE_EX(SVM_EXIT_RSM),
3579 [x86_intercept_pause] = PRE_EX(SVM_EXIT_PAUSE),
3580 [x86_intercept_pushf] = PRE_EX(SVM_EXIT_PUSHF),
3581 [x86_intercept_popf] = PRE_EX(SVM_EXIT_POPF),
3582 [x86_intercept_intn] = PRE_EX(SVM_EXIT_SWINT),
3583 [x86_intercept_iret] = PRE_EX(SVM_EXIT_IRET),
3584 [x86_intercept_icebp] = PRE_EX(SVM_EXIT_ICEBP),
3585 [x86_intercept_hlt] = POST_EX(SVM_EXIT_HLT),
3586 [x86_intercept_in] = POST_EX(SVM_EXIT_IOIO),
3587 [x86_intercept_ins] = POST_EX(SVM_EXIT_IOIO),
3588 [x86_intercept_out] = POST_EX(SVM_EXIT_IOIO),
3589 [x86_intercept_outs] = POST_EX(SVM_EXIT_IOIO),
3590 [x86_intercept_xsetbv] = PRE_EX(SVM_EXIT_XSETBV),
3591 };
3592
3593 #undef PRE_EX
3594 #undef POST_EX
3595 #undef POST_MEM
3596
3597 static int svm_check_intercept(struct kvm_vcpu *vcpu,
3598 struct x86_instruction_info *info,
3599 enum x86_intercept_stage stage,
3600 struct x86_exception *exception)
3601 {
3602 struct vcpu_svm *svm = to_svm(vcpu);
3603 int vmexit, ret = X86EMUL_CONTINUE;
3604 struct __x86_intercept icpt_info;
3605 struct vmcb *vmcb = svm->vmcb;
3606
3607 if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
3608 goto out;
3609
3610 icpt_info = x86_intercept_map[info->intercept];
3611
3612 if (stage != icpt_info.stage)
3613 goto out;
3614
3615 switch (icpt_info.exit_code) {
3616 case SVM_EXIT_READ_CR0:
3617 if (info->intercept == x86_intercept_cr_read)
3618 icpt_info.exit_code += info->modrm_reg;
3619 break;
3620 case SVM_EXIT_WRITE_CR0: {
3621 unsigned long cr0, val;
3622 u64 intercept;
3623
3624 if (info->intercept == x86_intercept_cr_write)
3625 icpt_info.exit_code += info->modrm_reg;
3626
3627 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
3628 info->intercept == x86_intercept_clts)
3629 break;
3630
3631 intercept = svm->nested.intercept;
3632
3633 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
3634 break;
3635
3636 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
3637 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK;
3638
3639 if (info->intercept == x86_intercept_lmsw) {
3640 cr0 &= 0xfUL;
3641 val &= 0xfUL;
3642 /* lmsw can't clear PE - catch this here */
3643 if (cr0 & X86_CR0_PE)
3644 val |= X86_CR0_PE;
3645 }
3646
3647 if (cr0 ^ val)
3648 icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3649
3650 break;
3651 }
3652 case SVM_EXIT_READ_DR0:
3653 case SVM_EXIT_WRITE_DR0:
3654 icpt_info.exit_code += info->modrm_reg;
3655 break;
3656 case SVM_EXIT_MSR:
3657 if (info->intercept == x86_intercept_wrmsr)
3658 vmcb->control.exit_info_1 = 1;
3659 else
3660 vmcb->control.exit_info_1 = 0;
3661 break;
3662 case SVM_EXIT_PAUSE:
3663 /*
3664 * We get this for NOP only, but pause
3665 * is rep not, check this here
3666 */
3667 if (info->rep_prefix != REPE_PREFIX)
3668 goto out;
3669 break;
3670 case SVM_EXIT_IOIO: {
3671 u64 exit_info;
3672 u32 bytes;
3673
3674 if (info->intercept == x86_intercept_in ||
3675 info->intercept == x86_intercept_ins) {
3676 exit_info = ((info->src_val & 0xffff) << 16) |
3677 SVM_IOIO_TYPE_MASK;
3678 bytes = info->dst_bytes;
3679 } else {
3680 exit_info = (info->dst_val & 0xffff) << 16;
3681 bytes = info->src_bytes;
3682 }
3683
3684 if (info->intercept == x86_intercept_outs ||
3685 info->intercept == x86_intercept_ins)
3686 exit_info |= SVM_IOIO_STR_MASK;
3687
3688 if (info->rep_prefix)
3689 exit_info |= SVM_IOIO_REP_MASK;
3690
3691 bytes = min(bytes, 4u);
3692
3693 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
3694
3695 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
3696
3697 vmcb->control.exit_info_1 = exit_info;
3698 vmcb->control.exit_info_2 = info->next_rip;
3699
3700 break;
3701 }
3702 default:
3703 break;
3704 }
3705
3706 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
3707 if (static_cpu_has(X86_FEATURE_NRIPS))
3708 vmcb->control.next_rip = info->next_rip;
3709 vmcb->control.exit_code = icpt_info.exit_code;
3710 vmexit = nested_svm_exit_handled(svm);
3711
3712 ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
3713 : X86EMUL_CONTINUE;
3714
3715 out:
3716 return ret;
3717 }
3718
3719 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu,
3720 enum exit_fastpath_completion *exit_fastpath)
3721 {
3722 if (!is_guest_mode(vcpu) &&
3723 to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR &&
3724 to_svm(vcpu)->vmcb->control.exit_info_1)
3725 *exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
3726 }
3727
3728 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
3729 {
3730 if (pause_filter_thresh)
3731 shrink_ple_window(vcpu);
3732 }
3733
3734 static void svm_setup_mce(struct kvm_vcpu *vcpu)
3735 {
3736 /* [63:9] are reserved. */
3737 vcpu->arch.mcg_cap &= 0x1ff;
3738 }
3739
3740 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
3741 {
3742 struct vcpu_svm *svm = to_svm(vcpu);
3743
3744 /* Per APM Vol.2 15.22.2 "Response to SMI" */
3745 if (!gif_set(svm))
3746 return 0;
3747
3748 if (is_guest_mode(&svm->vcpu) &&
3749 svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
3750 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
3751 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
3752 svm->nested.exit_required = true;
3753 return 0;
3754 }
3755
3756 return 1;
3757 }
3758
3759 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
3760 {
3761 struct vcpu_svm *svm = to_svm(vcpu);
3762 int ret;
3763
3764 if (is_guest_mode(vcpu)) {
3765 /* FED8h - SVM Guest */
3766 put_smstate(u64, smstate, 0x7ed8, 1);
3767 /* FEE0h - SVM Guest VMCB Physical Address */
3768 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
3769
3770 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3771 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3772 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3773
3774 ret = nested_svm_vmexit(svm);
3775 if (ret)
3776 return ret;
3777 }
3778 return 0;
3779 }
3780
3781 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
3782 {
3783 struct vcpu_svm *svm = to_svm(vcpu);
3784 struct vmcb *nested_vmcb;
3785 struct kvm_host_map map;
3786 u64 guest;
3787 u64 vmcb;
3788
3789 guest = GET_SMSTATE(u64, smstate, 0x7ed8);
3790 vmcb = GET_SMSTATE(u64, smstate, 0x7ee0);
3791
3792 if (guest) {
3793 if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL)
3794 return 1;
3795 nested_vmcb = map.hva;
3796 enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map);
3797 }
3798 return 0;
3799 }
3800
3801 static int enable_smi_window(struct kvm_vcpu *vcpu)
3802 {
3803 struct vcpu_svm *svm = to_svm(vcpu);
3804
3805 if (!gif_set(svm)) {
3806 if (vgif_enabled(svm))
3807 set_intercept(svm, INTERCEPT_STGI);
3808 /* STGI will cause a vm exit */
3809 return 1;
3810 }
3811 return 0;
3812 }
3813
3814 static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
3815 {
3816 unsigned long cr4 = kvm_read_cr4(vcpu);
3817 bool smep = cr4 & X86_CR4_SMEP;
3818 bool smap = cr4 & X86_CR4_SMAP;
3819 bool is_user = svm_get_cpl(vcpu) == 3;
3820
3821 /*
3822 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
3823 *
3824 * Errata:
3825 * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
3826 * possible that CPU microcode implementing DecodeAssist will fail
3827 * to read bytes of instruction which caused #NPF. In this case,
3828 * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
3829 * return 0 instead of the correct guest instruction bytes.
3830 *
3831 * This happens because CPU microcode reading instruction bytes
3832 * uses a special opcode which attempts to read data using CPL=0
3833 * priviledges. The microcode reads CS:RIP and if it hits a SMAP
3834 * fault, it gives up and returns no instruction bytes.
3835 *
3836 * Detection:
3837 * We reach here in case CPU supports DecodeAssist, raised #NPF and
3838 * returned 0 in GuestIntrBytes field of the VMCB.
3839 * First, errata can only be triggered in case vCPU CR4.SMAP=1.
3840 * Second, if vCPU CR4.SMEP=1, errata could only be triggered
3841 * in case vCPU CPL==3 (Because otherwise guest would have triggered
3842 * a SMEP fault instead of #NPF).
3843 * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
3844 * As most guests enable SMAP if they have also enabled SMEP, use above
3845 * logic in order to attempt minimize false-positive of detecting errata
3846 * while still preserving all cases semantic correctness.
3847 *
3848 * Workaround:
3849 * To determine what instruction the guest was executing, the hypervisor
3850 * will have to decode the instruction at the instruction pointer.
3851 *
3852 * In non SEV guest, hypervisor will be able to read the guest
3853 * memory to decode the instruction pointer when insn_len is zero
3854 * so we return true to indicate that decoding is possible.
3855 *
3856 * But in the SEV guest, the guest memory is encrypted with the
3857 * guest specific key and hypervisor will not be able to decode the
3858 * instruction pointer so we will not able to workaround it. Lets
3859 * print the error and request to kill the guest.
3860 */
3861 if (smap && (!smep || is_user)) {
3862 if (!sev_guest(vcpu->kvm))
3863 return true;
3864
3865 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
3866 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3867 }
3868
3869 return false;
3870 }
3871
3872 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
3873 {
3874 struct vcpu_svm *svm = to_svm(vcpu);
3875
3876 /*
3877 * TODO: Last condition latch INIT signals on vCPU when
3878 * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
3879 * To properly emulate the INIT intercept, SVM should implement
3880 * kvm_x86_ops.check_nested_events() and call nested_svm_vmexit()
3881 * there if an INIT signal is pending.
3882 */
3883 return !gif_set(svm) ||
3884 (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
3885 }
3886
3887 static void svm_vm_destroy(struct kvm *kvm)
3888 {
3889 avic_vm_destroy(kvm);
3890 sev_vm_destroy(kvm);
3891 }
3892
3893 static int svm_vm_init(struct kvm *kvm)
3894 {
3895 if (avic) {
3896 int ret = avic_vm_init(kvm);
3897 if (ret)
3898 return ret;
3899 }
3900
3901 kvm_apicv_init(kvm, avic);
3902 return 0;
3903 }
3904
3905 static struct kvm_x86_ops svm_x86_ops __initdata = {
3906 .hardware_unsetup = svm_hardware_teardown,
3907 .hardware_enable = svm_hardware_enable,
3908 .hardware_disable = svm_hardware_disable,
3909 .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
3910 .has_emulated_msr = svm_has_emulated_msr,
3911
3912 .vcpu_create = svm_create_vcpu,
3913 .vcpu_free = svm_free_vcpu,
3914 .vcpu_reset = svm_vcpu_reset,
3915
3916 .vm_size = sizeof(struct kvm_svm),
3917 .vm_init = svm_vm_init,
3918 .vm_destroy = svm_vm_destroy,
3919
3920 .prepare_guest_switch = svm_prepare_guest_switch,
3921 .vcpu_load = svm_vcpu_load,
3922 .vcpu_put = svm_vcpu_put,
3923 .vcpu_blocking = svm_vcpu_blocking,
3924 .vcpu_unblocking = svm_vcpu_unblocking,
3925
3926 .update_bp_intercept = update_bp_intercept,
3927 .get_msr_feature = svm_get_msr_feature,
3928 .get_msr = svm_get_msr,
3929 .set_msr = svm_set_msr,
3930 .get_segment_base = svm_get_segment_base,
3931 .get_segment = svm_get_segment,
3932 .set_segment = svm_set_segment,
3933 .get_cpl = svm_get_cpl,
3934 .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
3935 .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
3936 .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
3937 .set_cr0 = svm_set_cr0,
3938 .set_cr4 = svm_set_cr4,
3939 .set_efer = svm_set_efer,
3940 .get_idt = svm_get_idt,
3941 .set_idt = svm_set_idt,
3942 .get_gdt = svm_get_gdt,
3943 .set_gdt = svm_set_gdt,
3944 .set_dr7 = svm_set_dr7,
3945 .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
3946 .cache_reg = svm_cache_reg,
3947 .get_rflags = svm_get_rflags,
3948 .set_rflags = svm_set_rflags,
3949
3950 .tlb_flush = svm_flush_tlb,
3951 .tlb_flush_gva = svm_flush_tlb_gva,
3952
3953 .run = svm_vcpu_run,
3954 .handle_exit = handle_exit,
3955 .skip_emulated_instruction = skip_emulated_instruction,
3956 .update_emulated_instruction = NULL,
3957 .set_interrupt_shadow = svm_set_interrupt_shadow,
3958 .get_interrupt_shadow = svm_get_interrupt_shadow,
3959 .patch_hypercall = svm_patch_hypercall,
3960 .set_irq = svm_set_irq,
3961 .set_nmi = svm_inject_nmi,
3962 .queue_exception = svm_queue_exception,
3963 .cancel_injection = svm_cancel_injection,
3964 .interrupt_allowed = svm_interrupt_allowed,
3965 .nmi_allowed = svm_nmi_allowed,
3966 .get_nmi_mask = svm_get_nmi_mask,
3967 .set_nmi_mask = svm_set_nmi_mask,
3968 .enable_nmi_window = enable_nmi_window,
3969 .enable_irq_window = enable_irq_window,
3970 .update_cr8_intercept = update_cr8_intercept,
3971 .set_virtual_apic_mode = svm_set_virtual_apic_mode,
3972 .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
3973 .check_apicv_inhibit_reasons = svm_check_apicv_inhibit_reasons,
3974 .pre_update_apicv_exec_ctrl = svm_pre_update_apicv_exec_ctrl,
3975 .load_eoi_exitmap = svm_load_eoi_exitmap,
3976 .hwapic_irr_update = svm_hwapic_irr_update,
3977 .hwapic_isr_update = svm_hwapic_isr_update,
3978 .sync_pir_to_irr = kvm_lapic_find_highest_irr,
3979 .apicv_post_state_restore = avic_post_state_restore,
3980
3981 .set_tss_addr = svm_set_tss_addr,
3982 .set_identity_map_addr = svm_set_identity_map_addr,
3983 .get_tdp_level = get_npt_level,
3984 .get_mt_mask = svm_get_mt_mask,
3985
3986 .get_exit_info = svm_get_exit_info,
3987
3988 .cpuid_update = svm_cpuid_update,
3989
3990 .has_wbinvd_exit = svm_has_wbinvd_exit,
3991
3992 .read_l1_tsc_offset = svm_read_l1_tsc_offset,
3993 .write_l1_tsc_offset = svm_write_l1_tsc_offset,
3994
3995 .load_mmu_pgd = svm_load_mmu_pgd,
3996
3997 .check_intercept = svm_check_intercept,
3998 .handle_exit_irqoff = svm_handle_exit_irqoff,
3999
4000 .request_immediate_exit = __kvm_request_immediate_exit,
4001
4002 .sched_in = svm_sched_in,
4003
4004 .pmu_ops = &amd_pmu_ops,
4005 .deliver_posted_interrupt = svm_deliver_avic_intr,
4006 .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
4007 .update_pi_irte = svm_update_pi_irte,
4008 .setup_mce = svm_setup_mce,
4009
4010 .smi_allowed = svm_smi_allowed,
4011 .pre_enter_smm = svm_pre_enter_smm,
4012 .pre_leave_smm = svm_pre_leave_smm,
4013 .enable_smi_window = enable_smi_window,
4014
4015 .mem_enc_op = svm_mem_enc_op,
4016 .mem_enc_reg_region = svm_register_enc_region,
4017 .mem_enc_unreg_region = svm_unregister_enc_region,
4018
4019 .nested_enable_evmcs = NULL,
4020 .nested_get_evmcs_version = NULL,
4021
4022 .need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
4023
4024 .apic_init_signal_blocked = svm_apic_init_signal_blocked,
4025
4026 .check_nested_events = svm_check_nested_events,
4027 };
4028
4029 static struct kvm_x86_init_ops svm_init_ops __initdata = {
4030 .cpu_has_kvm_support = has_svm,
4031 .disabled_by_bios = is_disabled,
4032 .hardware_setup = svm_hardware_setup,
4033 .check_processor_compatibility = svm_check_processor_compat,
4034
4035 .runtime_ops = &svm_x86_ops,
4036 };
4037
4038 static int __init svm_init(void)
4039 {
4040 return kvm_init(&svm_init_ops, sizeof(struct vcpu_svm),
4041 __alignof__(struct vcpu_svm), THIS_MODULE);
4042 }
4043
4044 static void __exit svm_exit(void)
4045 {
4046 kvm_exit();
4047 }
4048
4049 module_init(svm_init)
4050 module_exit(svm_exit)
A mirror of Linus' kernel repository