1 // SPDX-License-Identifier: GPL-2.0-only
3 * X86 specific Hyper-V initialization code.
5 * Copyright (C) 2016, Microsoft, Inc.
7 * Author : K. Y. Srinivasan <kys@microsoft.com>
10 #include <linux/acpi.h>
11 #include <linux/efi.h>
12 #include <linux/types.h>
15 #include <asm/hypervisor.h>
16 #include <asm/hyperv-tlfs.h>
17 #include <asm/mshyperv.h>
18 #include <linux/version.h>
19 #include <linux/vmalloc.h>
21 #include <linux/hyperv.h>
22 #include <linux/slab.h>
23 #include <linux/kernel.h>
24 #include <linux/cpuhotplug.h>
25 #include <linux/syscore_ops.h>
26 #include <clocksource/hyperv_timer.h>
28 void *hv_hypercall_pg
;
29 EXPORT_SYMBOL_GPL(hv_hypercall_pg
);
31 /* Storage to save the hypercall page temporarily for hibernation */
32 static void *hv_hypercall_pg_saved
;
35 EXPORT_SYMBOL_GPL(hv_vp_index
);
37 struct hv_vp_assist_page
**hv_vp_assist_page
;
38 EXPORT_SYMBOL_GPL(hv_vp_assist_page
);
40 void __percpu
**hyperv_pcpu_input_arg
;
41 EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg
);
44 EXPORT_SYMBOL_GPL(hv_max_vp_index
);
46 void *hv_alloc_hyperv_page(void)
48 BUILD_BUG_ON(PAGE_SIZE
!= HV_HYP_PAGE_SIZE
);
50 return (void *)__get_free_page(GFP_KERNEL
);
52 EXPORT_SYMBOL_GPL(hv_alloc_hyperv_page
);
54 void *hv_alloc_hyperv_zeroed_page(void)
56 BUILD_BUG_ON(PAGE_SIZE
!= HV_HYP_PAGE_SIZE
);
58 return (void *)__get_free_page(GFP_KERNEL
| __GFP_ZERO
);
60 EXPORT_SYMBOL_GPL(hv_alloc_hyperv_zeroed_page
);
62 void hv_free_hyperv_page(unsigned long addr
)
66 EXPORT_SYMBOL_GPL(hv_free_hyperv_page
);
68 static int hv_cpu_init(unsigned int cpu
)
71 struct hv_vp_assist_page
**hvp
= &hv_vp_assist_page
[smp_processor_id()];
75 input_arg
= (void **)this_cpu_ptr(hyperv_pcpu_input_arg
);
76 pg
= alloc_page(GFP_KERNEL
);
79 *input_arg
= page_address(pg
);
81 hv_get_vp_index(msr_vp_index
);
83 hv_vp_index
[smp_processor_id()] = msr_vp_index
;
85 if (msr_vp_index
> hv_max_vp_index
)
86 hv_max_vp_index
= msr_vp_index
;
88 if (!hv_vp_assist_page
)
92 * The VP ASSIST PAGE is an "overlay" page (see Hyper-V TLFS's Section
93 * 5.2.1 "GPA Overlay Pages"). Here it must be zeroed out to make sure
94 * we always write the EOI MSR in hv_apic_eoi_write() *after* the
95 * EOI optimization is disabled in hv_cpu_die(), otherwise a CPU may
96 * not be stopped in the case of CPU offlining and the VM will hang.
99 *hvp
= __vmalloc(PAGE_SIZE
, GFP_KERNEL
| __GFP_ZERO
,
106 val
= vmalloc_to_pfn(*hvp
);
107 val
= (val
<< HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT
) |
108 HV_X64_MSR_VP_ASSIST_PAGE_ENABLE
;
110 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE
, val
);
116 static void (*hv_reenlightenment_cb
)(void);
118 static void hv_reenlightenment_notify(struct work_struct
*dummy
)
120 struct hv_tsc_emulation_status emu_status
;
122 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS
, *(u64
*)&emu_status
);
124 /* Don't issue the callback if TSC accesses are not emulated */
125 if (hv_reenlightenment_cb
&& emu_status
.inprogress
)
126 hv_reenlightenment_cb();
128 static DECLARE_DELAYED_WORK(hv_reenlightenment_work
, hv_reenlightenment_notify
);
130 void hyperv_stop_tsc_emulation(void)
133 struct hv_tsc_emulation_status emu_status
;
135 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS
, *(u64
*)&emu_status
);
136 emu_status
.inprogress
= 0;
137 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS
, *(u64
*)&emu_status
);
139 rdmsrl(HV_X64_MSR_TSC_FREQUENCY
, freq
);
140 tsc_khz
= div64_u64(freq
, 1000);
142 EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation
);
144 static inline bool hv_reenlightenment_available(void)
147 * Check for required features and priviliges to make TSC frequency
148 * change notifications work.
150 return ms_hyperv
.features
& HV_X64_ACCESS_FREQUENCY_MSRS
&&
151 ms_hyperv
.misc_features
& HV_FEATURE_FREQUENCY_MSRS_AVAILABLE
&&
152 ms_hyperv
.features
& HV_X64_ACCESS_REENLIGHTENMENT
;
155 __visible
void __irq_entry
hyperv_reenlightenment_intr(struct pt_regs
*regs
)
159 inc_irq_stat(irq_hv_reenlightenment_count
);
161 schedule_delayed_work(&hv_reenlightenment_work
, HZ
/10);
166 void set_hv_tscchange_cb(void (*cb
)(void))
168 struct hv_reenlightenment_control re_ctrl
= {
169 .vector
= HYPERV_REENLIGHTENMENT_VECTOR
,
171 .target_vp
= hv_vp_index
[smp_processor_id()]
173 struct hv_tsc_emulation_control emu_ctrl
= {.enabled
= 1};
175 if (!hv_reenlightenment_available()) {
176 pr_warn("Hyper-V: reenlightenment support is unavailable\n");
180 hv_reenlightenment_cb
= cb
;
182 /* Make sure callback is registered before we write to MSRs */
185 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL
, *((u64
*)&re_ctrl
));
186 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL
, *((u64
*)&emu_ctrl
));
188 EXPORT_SYMBOL_GPL(set_hv_tscchange_cb
);
190 void clear_hv_tscchange_cb(void)
192 struct hv_reenlightenment_control re_ctrl
;
194 if (!hv_reenlightenment_available())
197 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL
, *(u64
*)&re_ctrl
);
199 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL
, *(u64
*)&re_ctrl
);
201 hv_reenlightenment_cb
= NULL
;
203 EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb
);
205 static int hv_cpu_die(unsigned int cpu
)
207 struct hv_reenlightenment_control re_ctrl
;
208 unsigned int new_cpu
;
211 void *input_pg
= NULL
;
213 local_irq_save(flags
);
214 input_arg
= (void **)this_cpu_ptr(hyperv_pcpu_input_arg
);
215 input_pg
= *input_arg
;
217 local_irq_restore(flags
);
218 free_page((unsigned long)input_pg
);
220 if (hv_vp_assist_page
&& hv_vp_assist_page
[cpu
])
221 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE
, 0);
223 if (hv_reenlightenment_cb
== NULL
)
226 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL
, *((u64
*)&re_ctrl
));
227 if (re_ctrl
.target_vp
== hv_vp_index
[cpu
]) {
228 /* Reassign to some other online CPU */
229 new_cpu
= cpumask_any_but(cpu_online_mask
, cpu
);
231 re_ctrl
.target_vp
= hv_vp_index
[new_cpu
];
232 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL
, *((u64
*)&re_ctrl
));
238 static int __init
hv_pci_init(void)
240 int gen2vm
= efi_enabled(EFI_BOOT
);
243 * For Generation-2 VM, we exit from pci_arch_init() by returning 0.
244 * The purpose is to suppress the harmless warning:
245 * "PCI: Fatal: No config space access function found"
250 /* For Generation-1 VM, we'll proceed in pci_arch_init(). */
254 static int hv_suspend(void)
256 union hv_x64_msr_hypercall_contents hypercall_msr
;
259 * Reset the hypercall page as it is going to be invalidated
260 * accross hibernation. Setting hv_hypercall_pg to NULL ensures
261 * that any subsequent hypercall operation fails safely instead of
262 * crashing due to an access of an invalid page. The hypercall page
263 * pointer is restored on resume.
265 hv_hypercall_pg_saved
= hv_hypercall_pg
;
266 hv_hypercall_pg
= NULL
;
268 /* Disable the hypercall page in the hypervisor */
269 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
270 hypercall_msr
.enable
= 0;
271 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
276 static void hv_resume(void)
278 union hv_x64_msr_hypercall_contents hypercall_msr
;
280 /* Re-enable the hypercall page */
281 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
282 hypercall_msr
.enable
= 1;
283 hypercall_msr
.guest_physical_address
=
284 vmalloc_to_pfn(hv_hypercall_pg_saved
);
285 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
287 hv_hypercall_pg
= hv_hypercall_pg_saved
;
288 hv_hypercall_pg_saved
= NULL
;
291 static struct syscore_ops hv_syscore_ops
= {
292 .suspend
= hv_suspend
,
297 * This function is to be invoked early in the boot sequence after the
298 * hypervisor has been detected.
300 * 1. Setup the hypercall page.
301 * 2. Register Hyper-V specific clocksource.
302 * 3. Setup Hyper-V specific APIC entry points.
304 void __init
hyperv_init(void)
306 u64 guest_id
, required_msrs
;
307 union hv_x64_msr_hypercall_contents hypercall_msr
;
310 if (x86_hyper_type
!= X86_HYPER_MS_HYPERV
)
313 /* Absolutely required MSRs */
314 required_msrs
= HV_X64_MSR_HYPERCALL_AVAILABLE
|
315 HV_X64_MSR_VP_INDEX_AVAILABLE
;
317 if ((ms_hyperv
.features
& required_msrs
) != required_msrs
)
321 * Allocate the per-CPU state for the hypercall input arg.
322 * If this allocation fails, we will not be able to setup
323 * (per-CPU) hypercall input page and thus this failure is
326 hyperv_pcpu_input_arg
= alloc_percpu(void *);
328 BUG_ON(hyperv_pcpu_input_arg
== NULL
);
330 /* Allocate percpu VP index */
331 hv_vp_index
= kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index
),
336 for (i
= 0; i
< num_possible_cpus(); i
++)
337 hv_vp_index
[i
] = VP_INVAL
;
339 hv_vp_assist_page
= kcalloc(num_possible_cpus(),
340 sizeof(*hv_vp_assist_page
), GFP_KERNEL
);
341 if (!hv_vp_assist_page
) {
342 ms_hyperv
.hints
&= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED
;
346 cpuhp
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "x86/hyperv_init:online",
347 hv_cpu_init
, hv_cpu_die
);
349 goto free_vp_assist_page
;
352 * Setup the hypercall page and enable hypercalls.
353 * 1. Register the guest ID
354 * 2. Enable the hypercall and register the hypercall page
356 guest_id
= generate_guest_id(0, LINUX_VERSION_CODE
, 0);
357 wrmsrl(HV_X64_MSR_GUEST_OS_ID
, guest_id
);
359 hv_hypercall_pg
= __vmalloc(PAGE_SIZE
, GFP_KERNEL
, PAGE_KERNEL_RX
);
360 if (hv_hypercall_pg
== NULL
) {
361 wrmsrl(HV_X64_MSR_GUEST_OS_ID
, 0);
362 goto remove_cpuhp_state
;
365 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
366 hypercall_msr
.enable
= 1;
367 hypercall_msr
.guest_physical_address
= vmalloc_to_pfn(hv_hypercall_pg
);
368 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
371 * Ignore any errors in setting up stimer clockevents
372 * as we can run with the LAPIC timer as a fallback.
374 (void)hv_stimer_alloc();
378 x86_init
.pci
.arch_init
= hv_pci_init
;
380 register_syscore_ops(&hv_syscore_ops
);
385 cpuhp_remove_state(cpuhp
);
387 kfree(hv_vp_assist_page
);
388 hv_vp_assist_page
= NULL
;
395 * This routine is called before kexec/kdump, it does the required cleanup.
397 void hyperv_cleanup(void)
399 union hv_x64_msr_hypercall_contents hypercall_msr
;
401 unregister_syscore_ops(&hv_syscore_ops
);
403 /* Reset our OS id */
404 wrmsrl(HV_X64_MSR_GUEST_OS_ID
, 0);
407 * Reset hypercall page reference before reset the page,
408 * let hypercall operations fail safely rather than
409 * panic the kernel for using invalid hypercall page
411 hv_hypercall_pg
= NULL
;
413 /* Reset the hypercall page */
414 hypercall_msr
.as_uint64
= 0;
415 wrmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
417 /* Reset the TSC page */
418 hypercall_msr
.as_uint64
= 0;
419 wrmsrl(HV_X64_MSR_REFERENCE_TSC
, hypercall_msr
.as_uint64
);
421 EXPORT_SYMBOL_GPL(hyperv_cleanup
);
423 void hyperv_report_panic(struct pt_regs
*regs
, long err
, bool in_die
)
425 static bool panic_reported
;
428 if (in_die
&& !panic_on_oops
)
432 * We prefer to report panic on 'die' chain as we have proper
433 * registers to report, but if we miss it (e.g. on BUG()) we need
434 * to report it on 'panic'.
438 panic_reported
= true;
440 rdmsrl(HV_X64_MSR_GUEST_OS_ID
, guest_id
);
442 wrmsrl(HV_X64_MSR_CRASH_P0
, err
);
443 wrmsrl(HV_X64_MSR_CRASH_P1
, guest_id
);
444 wrmsrl(HV_X64_MSR_CRASH_P2
, regs
->ip
);
445 wrmsrl(HV_X64_MSR_CRASH_P3
, regs
->ax
);
446 wrmsrl(HV_X64_MSR_CRASH_P4
, regs
->sp
);
449 * Let Hyper-V know there is crash data available
451 wrmsrl(HV_X64_MSR_CRASH_CTL
, HV_CRASH_CTL_CRASH_NOTIFY
);
453 EXPORT_SYMBOL_GPL(hyperv_report_panic
);
456 * hyperv_report_panic_msg - report panic message to Hyper-V
457 * @pa: physical address of the panic page containing the message
458 * @size: size of the message in the page
460 void hyperv_report_panic_msg(phys_addr_t pa
, size_t size
)
463 * P3 to contain the physical address of the panic page & P4 to
464 * contain the size of the panic data in that page. Rest of the
465 * registers are no-op when the NOTIFY_MSG flag is set.
467 wrmsrl(HV_X64_MSR_CRASH_P0
, 0);
468 wrmsrl(HV_X64_MSR_CRASH_P1
, 0);
469 wrmsrl(HV_X64_MSR_CRASH_P2
, 0);
470 wrmsrl(HV_X64_MSR_CRASH_P3
, pa
);
471 wrmsrl(HV_X64_MSR_CRASH_P4
, size
);
474 * Let Hyper-V know there is crash data available along with
477 wrmsrl(HV_X64_MSR_CRASH_CTL
,
478 (HV_CRASH_CTL_CRASH_NOTIFY
| HV_CRASH_CTL_CRASH_NOTIFY_MSG
));
480 EXPORT_SYMBOL_GPL(hyperv_report_panic_msg
);
482 bool hv_is_hyperv_initialized(void)
484 union hv_x64_msr_hypercall_contents hypercall_msr
;
487 * Ensure that we're really on Hyper-V, and not a KVM or Xen
488 * emulation of Hyper-V
490 if (x86_hyper_type
!= X86_HYPER_MS_HYPERV
)
494 * Verify that earlier initialization succeeded by checking
495 * that the hypercall page is setup
497 hypercall_msr
.as_uint64
= 0;
498 rdmsrl(HV_X64_MSR_HYPERCALL
, hypercall_msr
.as_uint64
);
500 return hypercall_msr
.enable
;
502 EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized
);
504 bool hv_is_hibernation_supported(void)
506 return acpi_sleep_state_supported(ACPI_STATE_S4
);
508 EXPORT_SYMBOL_GPL(hv_is_hibernation_supported
);