[thirdparty/linux.git] / arch / x86 / hyperv / hv_init.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * X86 specific Hyper-V initialization code.
 *
 * Copyright (C) 2016, Microsoft, Inc.
 *
 * Author : K. Y. Srinivasan <kys@microsoft.com>
 */

#include <linux/acpi.h>
#include <linux/efi.h>
#include <linux/types.h>
#include <asm/apic.h>
#include <asm/desc.h>
#include <asm/hypervisor.h>
#include <asm/hyperv-tlfs.h>
#include <asm/mshyperv.h>
#include <linux/version.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/hyperv.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/cpuhotplug.h>
#include <linux/syscore_ops.h>
#include <clocksource/hyperv_timer.h>

void *hv_hypercall_pg;
EXPORT_SYMBOL_GPL(hv_hypercall_pg);

/* Storage to save the hypercall page temporarily for hibernation */
static void *hv_hypercall_pg_saved;

u32 *hv_vp_index;
EXPORT_SYMBOL_GPL(hv_vp_index);

struct hv_vp_assist_page **hv_vp_assist_page;
EXPORT_SYMBOL_GPL(hv_vp_assist_page);

void  __percpu **hyperv_pcpu_input_arg;
EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg);

u32 hv_max_vp_index;
EXPORT_SYMBOL_GPL(hv_max_vp_index);

void *hv_alloc_hyperv_page(void)
{
	BUILD_BUG_ON(PAGE_SIZE != HV_HYP_PAGE_SIZE);

	return (void *)__get_free_page(GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(hv_alloc_hyperv_page);

void *hv_alloc_hyperv_zeroed_page(void)
{
        BUILD_BUG_ON(PAGE_SIZE != HV_HYP_PAGE_SIZE);

        return (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
}
EXPORT_SYMBOL_GPL(hv_alloc_hyperv_zeroed_page);

void hv_free_hyperv_page(unsigned long addr)
{
	free_page(addr);
}
EXPORT_SYMBOL_GPL(hv_free_hyperv_page);

static int hv_cpu_init(unsigned int cpu)
{
	u64 msr_vp_index;
	struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()];
	void **input_arg;
	struct page *pg;

	input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
	/* hv_cpu_init() can be called with IRQs disabled from hv_resume() */
	pg = alloc_page(irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL);
	if (unlikely(!pg))
		return -ENOMEM;
	*input_arg = page_address(pg);

	hv_get_vp_index(msr_vp_index);

	hv_vp_index[smp_processor_id()] = msr_vp_index;

	if (msr_vp_index > hv_max_vp_index)
		hv_max_vp_index = msr_vp_index;

	if (!hv_vp_assist_page)
		return 0;

	/*
	 * The VP ASSIST PAGE is an "overlay" page (see Hyper-V TLFS's Section
	 * 5.2.1 "GPA Overlay Pages"). Here it must be zeroed out to make sure
	 * we always write the EOI MSR in hv_apic_eoi_write() *after* the
	 * EOI optimization is disabled in hv_cpu_die(), otherwise a CPU may
	 * not be stopped in the case of CPU offlining and the VM will hang.
	 */
	if (!*hvp) {
		*hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO,
				 PAGE_KERNEL);
	}

	if (*hvp) {
		u64 val;

		val = vmalloc_to_pfn(*hvp);
		val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) |
			HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;

		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val);
	}

	return 0;
}

static void (*hv_reenlightenment_cb)(void);

static void hv_reenlightenment_notify(struct work_struct *dummy)
{
	struct hv_tsc_emulation_status emu_status;

	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);

	/* Don't issue the callback if TSC accesses are not emulated */
	if (hv_reenlightenment_cb && emu_status.inprogress)
		hv_reenlightenment_cb();
}
static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify);

void hyperv_stop_tsc_emulation(void)
{
	u64 freq;
	struct hv_tsc_emulation_status emu_status;

	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
	emu_status.inprogress = 0;
	wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);

	rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
	tsc_khz = div64_u64(freq, 1000);
}
EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);

static inline bool hv_reenlightenment_available(void)
{
	/*
	 * Check for required features and priviliges to make TSC frequency
	 * change notifications work.
	 */
	return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
		ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
		ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT;
}

__visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs)
{
	entering_ack_irq();

	inc_irq_stat(irq_hv_reenlightenment_count);

	schedule_delayed_work(&hv_reenlightenment_work, HZ/10);

	exiting_irq();
}

void set_hv_tscchange_cb(void (*cb)(void))
{
	struct hv_reenlightenment_control re_ctrl = {
		.vector = HYPERV_REENLIGHTENMENT_VECTOR,
		.enabled = 1,
		.target_vp = hv_vp_index[smp_processor_id()]
	};
	struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};

	if (!hv_reenlightenment_available()) {
		pr_warn("Hyper-V: reenlightenment support is unavailable\n");
		return;
	}

	hv_reenlightenment_cb = cb;

	/* Make sure callback is registered before we write to MSRs */
	wmb();

	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
	wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
}
EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);

void clear_hv_tscchange_cb(void)
{
	struct hv_reenlightenment_control re_ctrl;

	if (!hv_reenlightenment_available())
		return;

	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
	re_ctrl.enabled = 0;
	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);

	hv_reenlightenment_cb = NULL;
}
EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb);

static int hv_cpu_die(unsigned int cpu)
{
	struct hv_reenlightenment_control re_ctrl;
	unsigned int new_cpu;
	unsigned long flags;
	void **input_arg;
	void *input_pg = NULL;

	local_irq_save(flags);
	input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
	input_pg = *input_arg;
	*input_arg = NULL;
	local_irq_restore(flags);
	free_page((unsigned long)input_pg);

	if (hv_vp_assist_page && hv_vp_assist_page[cpu])
		wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0);

	if (hv_reenlightenment_cb == NULL)
		return 0;

	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
	if (re_ctrl.target_vp == hv_vp_index[cpu]) {
		/*
		 * Reassign reenlightenment notifications to some other online
		 * CPU or just disable the feature if there are no online CPUs
		 * left (happens on hibernation).
		 */
		new_cpu = cpumask_any_but(cpu_online_mask, cpu);

		if (new_cpu < nr_cpu_ids)
			re_ctrl.target_vp = hv_vp_index[new_cpu];
		else
			re_ctrl.enabled = 0;

		wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
	}

	return 0;
}

static int __init hv_pci_init(void)
{
	int gen2vm = efi_enabled(EFI_BOOT);

	/*
	 * For Generation-2 VM, we exit from pci_arch_init() by returning 0.
	 * The purpose is to suppress the harmless warning:
	 * "PCI: Fatal: No config space access function found"
	 */
	if (gen2vm)
		return 0;

	/* For Generation-1 VM, we'll proceed in pci_arch_init().  */
	return 1;
}

static int hv_suspend(void)
{
	union hv_x64_msr_hypercall_contents hypercall_msr;
	int ret;

	/*
	 * Reset the hypercall page as it is going to be invalidated
	 * accross hibernation. Setting hv_hypercall_pg to NULL ensures
	 * that any subsequent hypercall operation fails safely instead of
	 * crashing due to an access of an invalid page. The hypercall page
	 * pointer is restored on resume.
	 */
	hv_hypercall_pg_saved = hv_hypercall_pg;
	hv_hypercall_pg = NULL;

	/* Disable the hypercall page in the hypervisor */
	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	hypercall_msr.enable = 0;
	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	ret = hv_cpu_die(0);
	return ret;
}

static void hv_resume(void)
{
	union hv_x64_msr_hypercall_contents hypercall_msr;
	int ret;

	ret = hv_cpu_init(0);
	WARN_ON(ret);

	/* Re-enable the hypercall page */
	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	hypercall_msr.enable = 1;
	hypercall_msr.guest_physical_address =
		vmalloc_to_pfn(hv_hypercall_pg_saved);
	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	hv_hypercall_pg = hv_hypercall_pg_saved;
	hv_hypercall_pg_saved = NULL;

	/*
	 * Reenlightenment notifications are disabled by hv_cpu_die(0),
	 * reenable them here if hv_reenlightenment_cb was previously set.
	 */
	if (hv_reenlightenment_cb)
		set_hv_tscchange_cb(hv_reenlightenment_cb);
}

/* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */
static struct syscore_ops hv_syscore_ops = {
	.suspend	= hv_suspend,
	.resume		= hv_resume,
};

/*
 * This function is to be invoked early in the boot sequence after the
 * hypervisor has been detected.
 *
 * 1. Setup the hypercall page.
 * 2. Register Hyper-V specific clocksource.
 * 3. Setup Hyper-V specific APIC entry points.
 */
void __init hyperv_init(void)
{
	u64 guest_id, required_msrs;
	union hv_x64_msr_hypercall_contents hypercall_msr;
	int cpuhp, i;

	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
		return;

	/* Absolutely required MSRs */
	required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE |
		HV_X64_MSR_VP_INDEX_AVAILABLE;

	if ((ms_hyperv.features & required_msrs) != required_msrs)
		return;

	/*
	 * Allocate the per-CPU state for the hypercall input arg.
	 * If this allocation fails, we will not be able to setup
	 * (per-CPU) hypercall input page and thus this failure is
	 * fatal on Hyper-V.
	 */
	hyperv_pcpu_input_arg = alloc_percpu(void  *);

	BUG_ON(hyperv_pcpu_input_arg == NULL);

	/* Allocate percpu VP index */
	hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index),
				    GFP_KERNEL);
	if (!hv_vp_index)
		return;

	for (i = 0; i < num_possible_cpus(); i++)
		hv_vp_index[i] = VP_INVAL;

	hv_vp_assist_page = kcalloc(num_possible_cpus(),
				    sizeof(*hv_vp_assist_page), GFP_KERNEL);
	if (!hv_vp_assist_page) {
		ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
		goto free_vp_index;
	}

	cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online",
				  hv_cpu_init, hv_cpu_die);
	if (cpuhp < 0)
		goto free_vp_assist_page;

	/*
	 * Setup the hypercall page and enable hypercalls.
	 * 1. Register the guest ID
	 * 2. Enable the hypercall and register the hypercall page
	 */
	guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0);
	wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);

	hv_hypercall_pg  = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);
	if (hv_hypercall_pg == NULL) {
		wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
		goto remove_cpuhp_state;
	}

	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	hypercall_msr.enable = 1;
	hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	/*
	 * Ignore any errors in setting up stimer clockevents
	 * as we can run with the LAPIC timer as a fallback.
	 */
	(void)hv_stimer_alloc();

	hv_apic_init();

	x86_init.pci.arch_init = hv_pci_init;

	register_syscore_ops(&hv_syscore_ops);

	return;

remove_cpuhp_state:
	cpuhp_remove_state(cpuhp);
free_vp_assist_page:
	kfree(hv_vp_assist_page);
	hv_vp_assist_page = NULL;
free_vp_index:
	kfree(hv_vp_index);
	hv_vp_index = NULL;
}

/*
 * This routine is called before kexec/kdump, it does the required cleanup.
 */
void hyperv_cleanup(void)
{
	union hv_x64_msr_hypercall_contents hypercall_msr;

	unregister_syscore_ops(&hv_syscore_ops);

	/* Reset our OS id */
	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);

	/*
	 * Reset hypercall page reference before reset the page,
	 * let hypercall operations fail safely rather than
	 * panic the kernel for using invalid hypercall page
	 */
	hv_hypercall_pg = NULL;

	/* Reset the hypercall page */
	hypercall_msr.as_uint64 = 0;
	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	/* Reset the TSC page */
	hypercall_msr.as_uint64 = 0;
	wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64);
}
EXPORT_SYMBOL_GPL(hyperv_cleanup);

void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die)
{
	static bool panic_reported;
	u64 guest_id;

	if (in_die && !panic_on_oops)
		return;

	/*
	 * We prefer to report panic on 'die' chain as we have proper
	 * registers to report, but if we miss it (e.g. on BUG()) we need
	 * to report it on 'panic'.
	 */
	if (panic_reported)
		return;
	panic_reported = true;

	rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);

	wrmsrl(HV_X64_MSR_CRASH_P0, err);
	wrmsrl(HV_X64_MSR_CRASH_P1, guest_id);
	wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip);
	wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax);
	wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp);

	/*
	 * Let Hyper-V know there is crash data available
	 */
	wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
}
EXPORT_SYMBOL_GPL(hyperv_report_panic);

/**
 * hyperv_report_panic_msg - report panic message to Hyper-V
 * @pa: physical address of the panic page containing the message
 * @size: size of the message in the page
 */
void hyperv_report_panic_msg(phys_addr_t pa, size_t size)
{
	/*
	 * P3 to contain the physical address of the panic page & P4 to
	 * contain the size of the panic data in that page. Rest of the
	 * registers are no-op when the NOTIFY_MSG flag is set.
	 */
	wrmsrl(HV_X64_MSR_CRASH_P0, 0);
	wrmsrl(HV_X64_MSR_CRASH_P1, 0);
	wrmsrl(HV_X64_MSR_CRASH_P2, 0);
	wrmsrl(HV_X64_MSR_CRASH_P3, pa);
	wrmsrl(HV_X64_MSR_CRASH_P4, size);

	/*
	 * Let Hyper-V know there is crash data available along with
	 * the panic message.
	 */
	wrmsrl(HV_X64_MSR_CRASH_CTL,
	       (HV_CRASH_CTL_CRASH_NOTIFY | HV_CRASH_CTL_CRASH_NOTIFY_MSG));
}
EXPORT_SYMBOL_GPL(hyperv_report_panic_msg);

bool hv_is_hyperv_initialized(void)
{
	union hv_x64_msr_hypercall_contents hypercall_msr;

	/*
	 * Ensure that we're really on Hyper-V, and not a KVM or Xen
	 * emulation of Hyper-V
	 */
	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
		return false;

	/*
	 * Verify that earlier initialization succeeded by checking
	 * that the hypercall page is setup
	 */
	hypercall_msr.as_uint64 = 0;
	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

	return hypercall_msr.enable;
}
EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);

bool hv_is_hibernation_supported(void)
{
	return acpi_sleep_state_supported(ACPI_STATE_S4);
}
EXPORT_SYMBOL_GPL(hv_is_hibernation_supported);
Commit	Line	Data
43aa3132	1	// SPDX-License-Identifier: GPL-2.0-only
8730046c S	2	/*
	3	* X86 specific Hyper-V initialization code.
	4	*
	5	* Copyright (C) 2016, Microsoft, Inc.
	6	*
	7	* Author : K. Y. Srinivasan <kys@microsoft.com>
8730046c S	8	*/
8730046c S	9
b96f8653	10	#include <linux/acpi.h>
2f285f46	11	#include <linux/efi.h>
8730046c	12	#include <linux/types.h>
93286261 VK	13	#include <asm/apic.h>
93286261 VK	14	#include <asm/desc.h>
8730046c	15	#include <asm/hypervisor.h>
5a485803	16	#include <asm/hyperv-tlfs.h>
8730046c S	17	#include <asm/mshyperv.h>
	18	#include <linux/version.h>
	19	#include <linux/vmalloc.h>
	20	#include <linux/mm.h>
67071816	21	#include <linux/hyperv.h>
7415aea6	22	#include <linux/slab.h>
f3a99e76	23	#include <linux/kernel.h>
7415aea6	24	#include <linux/cpuhotplug.h>
05bd330a	25	#include <linux/syscore_ops.h>
dd2cb348	26	#include <clocksource/hyperv_timer.h>
8730046c	27
fc53662f VK	28	void *hv_hypercall_pg;
fc53662f VK	29	EXPORT_SYMBOL_GPL(hv_hypercall_pg);
dee863b5	30
05bd330a DC	31	/* Storage to save the hypercall page temporarily for hibernation */
	32	static void *hv_hypercall_pg_saved;
	33
7415aea6 VK	34	u32 *hv_vp_index;
	35	EXPORT_SYMBOL_GPL(hv_vp_index);
	36
a46d15cc VK	37	struct hv_vp_assist_page **hv_vp_assist_page;
	38	EXPORT_SYMBOL_GPL(hv_vp_assist_page);
	39
68bb7bfb S	40	void __percpu **hyperv_pcpu_input_arg;
	41	EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg);
	42
a3b74243	43	u32 hv_max_vp_index;
c8ccf759	44	EXPORT_SYMBOL_GPL(hv_max_vp_index);
a3b74243	45
8c3e44bd MN	46	void *hv_alloc_hyperv_page(void)
	47	{
	48	BUILD_BUG_ON(PAGE_SIZE != HV_HYP_PAGE_SIZE);
	49
	50	return (void *)__get_free_page(GFP_KERNEL);
	51	}
	52	EXPORT_SYMBOL_GPL(hv_alloc_hyperv_page);
	53
fa36dcdf HP	54	void *hv_alloc_hyperv_zeroed_page(void)
	55	{
	56	BUILD_BUG_ON(PAGE_SIZE != HV_HYP_PAGE_SIZE);
	57
	58	return (void *)__get_free_page(GFP_KERNEL \| __GFP_ZERO);
	59	}
	60	EXPORT_SYMBOL_GPL(hv_alloc_hyperv_zeroed_page);
	61
8c3e44bd MN	62	void hv_free_hyperv_page(unsigned long addr)
	63	{
	64	free_page(addr);
	65	}
	66	EXPORT_SYMBOL_GPL(hv_free_hyperv_page);
	67
7415aea6 VK	68	static int hv_cpu_init(unsigned int cpu)
	69	{
	70	u64 msr_vp_index;
a46d15cc	71	struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()];
68bb7bfb	72	void **input_arg;
534c89c2	73	struct page *pg;
68bb7bfb S	74
68bb7bfb S	75	input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
421f090c DC	76	/* hv_cpu_init() can be called with IRQs disabled from hv_resume() */
421f090c DC	77	pg = alloc_page(irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL);
534c89c2 KL	78	if (unlikely(!pg))
	79	return -ENOMEM;
	80	*input_arg = page_address(pg);
7415aea6 VK	81
	82	hv_get_vp_index(msr_vp_index);
	83
	84	hv_vp_index[smp_processor_id()] = msr_vp_index;
	85
a3b74243 VK	86	if (msr_vp_index > hv_max_vp_index)
	87	hv_max_vp_index = msr_vp_index;
	88
a46d15cc VK	89	if (!hv_vp_assist_page)
	90	return 0;
	91
e320ab3c DC	92	/*
	93	* The VP ASSIST PAGE is an "overlay" page (see Hyper-V TLFS's Section
	94	* 5.2.1 "GPA Overlay Pages"). Here it must be zeroed out to make sure
	95	* we always write the EOI MSR in hv_apic_eoi_write() after the
	96	* EOI optimization is disabled in hv_cpu_die(), otherwise a CPU may
	97	* not be stopped in the case of CPU offlining and the VM will hang.
	98	*/
	99	if (!*hvp) {
	100	*hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL \| __GFP_ZERO,
	101	PAGE_KERNEL);
	102	}
a46d15cc VK	103
	104	if (*hvp) {
	105	u64 val;
	106
	107	val = vmalloc_to_pfn(*hvp);
	108	val = (val << HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT) \|
	109	HV_X64_MSR_VP_ASSIST_PAGE_ENABLE;
	110
	111	wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, val);
	112	}
	113
7415aea6 VK	114	return 0;
	115	}
	116
93286261 VK	117	static void (*hv_reenlightenment_cb)(void);
	118
	119	static void hv_reenlightenment_notify(struct work_struct *dummy)
	120	{
	121	struct hv_tsc_emulation_status emu_status;
	122
	123	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, (u64 )&emu_status);
	124
	125	/* Don't issue the callback if TSC accesses are not emulated */
	126	if (hv_reenlightenment_cb && emu_status.inprogress)
	127	hv_reenlightenment_cb();
	128	}
	129	static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify);
	130
	131	void hyperv_stop_tsc_emulation(void)
	132	{
	133	u64 freq;
	134	struct hv_tsc_emulation_status emu_status;
	135
	136	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, (u64 )&emu_status);
	137	emu_status.inprogress = 0;
	138	wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, (u64 )&emu_status);
	139
	140	rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
	141	tsc_khz = div64_u64(freq, 1000);
	142	}
	143	EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);
	144
	145	static inline bool hv_reenlightenment_available(void)
	146	{
	147	/*
	148	* Check for required features and priviliges to make TSC frequency
	149	* change notifications work.
	150	*/
	151	return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
	152	ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
	153	ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT;
	154	}
	155
	156	__visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs)
	157	{
	158	entering_ack_irq();
	159
51d4e5da VK	160	inc_irq_stat(irq_hv_reenlightenment_count);
51d4e5da VK	161
93286261 VK	162	schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
	163
	164	exiting_irq();
	165	}
	166
	167	void set_hv_tscchange_cb(void (*cb)(void))
	168	{
	169	struct hv_reenlightenment_control re_ctrl = {
	170	.vector = HYPERV_REENLIGHTENMENT_VECTOR,
	171	.enabled = 1,
	172	.target_vp = hv_vp_index[smp_processor_id()]
	173	};
	174	struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};
	175
	176	if (!hv_reenlightenment_available()) {
	177	pr_warn("Hyper-V: reenlightenment support is unavailable\n");
	178	return;
	179	}
	180
	181	hv_reenlightenment_cb = cb;
	182
	183	/* Make sure callback is registered before we write to MSRs */
	184	wmb();
	185
	186	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, ((u64 )&re_ctrl));
	187	wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, ((u64 )&emu_ctrl));
	188	}
	189	EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);
	190
	191	void clear_hv_tscchange_cb(void)
	192	{
	193	struct hv_reenlightenment_control re_ctrl;
	194
	195	if (!hv_reenlightenment_available())
	196	return;
	197
	198	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, (u64 )&re_ctrl);
	199	re_ctrl.enabled = 0;
	200	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, (u64 )&re_ctrl);
	201
	202	hv_reenlightenment_cb = NULL;
	203	}
	204	EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb);
	205
e7c4e36c VK	206	static int hv_cpu_die(unsigned int cpu)
	207	{
	208	struct hv_reenlightenment_control re_ctrl;
	209	unsigned int new_cpu;
68bb7bfb S	210	unsigned long flags;
	211	void **input_arg;
	212	void *input_pg = NULL;
	213
	214	local_irq_save(flags);
	215	input_arg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg);
	216	input_pg = *input_arg;
	217	*input_arg = NULL;
	218	local_irq_restore(flags);
	219	free_page((unsigned long)input_pg);
e7c4e36c	220
a46d15cc VK	221	if (hv_vp_assist_page && hv_vp_assist_page[cpu])
	222	wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, 0);
	223
e7c4e36c VK	224	if (hv_reenlightenment_cb == NULL)
	225	return 0;
	226
	227	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, ((u64 )&re_ctrl));
	228	if (re_ctrl.target_vp == hv_vp_index[cpu]) {
38dce419 VK	229	/*
	230	* Reassign reenlightenment notifications to some other online
	231	* CPU or just disable the feature if there are no online CPUs
	232	* left (happens on hibernation).
	233	*/
e7c4e36c VK	234	new_cpu = cpumask_any_but(cpu_online_mask, cpu);
e7c4e36c VK	235
38dce419 VK	236	if (new_cpu < nr_cpu_ids)
	237	re_ctrl.target_vp = hv_vp_index[new_cpu];
	238	else
	239	re_ctrl.enabled = 0;
	240
e7c4e36c VK	241	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, ((u64 )&re_ctrl));
	242	}
	243
	244	return 0;
	245	}
	246
2f285f46 DC	247	static int __init hv_pci_init(void)
	248	{
	249	int gen2vm = efi_enabled(EFI_BOOT);
	250
	251	/*
	252	* For Generation-2 VM, we exit from pci_arch_init() by returning 0.
	253	* The purpose is to suppress the harmless warning:
	254	* "PCI: Fatal: No config space access function found"
	255	*/
	256	if (gen2vm)
	257	return 0;
	258
	259	/* For Generation-1 VM, we'll proceed in pci_arch_init(). */
	260	return 1;
	261	}
	262
05bd330a DC	263	static int hv_suspend(void)
	264	{
	265	union hv_x64_msr_hypercall_contents hypercall_msr;
421f090c	266	int ret;
05bd330a DC	267
	268	/*
	269	* Reset the hypercall page as it is going to be invalidated
	270	* accross hibernation. Setting hv_hypercall_pg to NULL ensures
	271	* that any subsequent hypercall operation fails safely instead of
	272	* crashing due to an access of an invalid page. The hypercall page
	273	* pointer is restored on resume.
	274	*/
	275	hv_hypercall_pg_saved = hv_hypercall_pg;
	276	hv_hypercall_pg = NULL;
	277
	278	/* Disable the hypercall page in the hypervisor */
	279	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	280	hypercall_msr.enable = 0;
	281	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	282
421f090c DC	283	ret = hv_cpu_die(0);
421f090c DC	284	return ret;
05bd330a DC	285	}
	286
	287	static void hv_resume(void)
	288	{
	289	union hv_x64_msr_hypercall_contents hypercall_msr;
421f090c DC	290	int ret;
	291
	292	ret = hv_cpu_init(0);
	293	WARN_ON(ret);
05bd330a DC	294
	295	/* Re-enable the hypercall page */
	296	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	297	hypercall_msr.enable = 1;
	298	hypercall_msr.guest_physical_address =
	299	vmalloc_to_pfn(hv_hypercall_pg_saved);
	300	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	301
	302	hv_hypercall_pg = hv_hypercall_pg_saved;
	303	hv_hypercall_pg_saved = NULL;
38dce419 VK	304
	305	/*
	306	* Reenlightenment notifications are disabled by hv_cpu_die(0),
	307	* reenable them here if hv_reenlightenment_cb was previously set.
	308	*/
	309	if (hv_reenlightenment_cb)
	310	set_hv_tscchange_cb(hv_reenlightenment_cb);
05bd330a DC	311	}
05bd330a DC	312
421f090c	313	/* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */
05bd330a DC	314	static struct syscore_ops hv_syscore_ops = {
	315	.suspend = hv_suspend,
	316	.resume = hv_resume,
	317	};
	318
8730046c S	319	/*
	320	* This function is to be invoked early in the boot sequence after the
	321	* hypervisor has been detected.
	322	*
	323	* 1. Setup the hypercall page.
63ed4e0c	324	* 2. Register Hyper-V specific clocksource.
6b48cb5f	325	* 3. Setup Hyper-V specific APIC entry points.
8730046c	326	*/
6b48cb5f	327	void __init hyperv_init(void)
8730046c	328	{
89a8f6d4	329	u64 guest_id, required_msrs;
8730046c	330	union hv_x64_msr_hypercall_contents hypercall_msr;
1268ed0c	331	int cpuhp, i;
8730046c	332
03b2a320	333	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
8730046c S	334	return;
8730046c S	335
89a8f6d4 VK	336	/* Absolutely required MSRs */
	337	required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE \|
	338	HV_X64_MSR_VP_INDEX_AVAILABLE;
	339
	340	if ((ms_hyperv.features & required_msrs) != required_msrs)
	341	return;
	342
68bb7bfb S	343	/*
	344	* Allocate the per-CPU state for the hypercall input arg.
	345	* If this allocation fails, we will not be able to setup
	346	* (per-CPU) hypercall input page and thus this failure is
	347	* fatal on Hyper-V.
	348	*/
	349	hyperv_pcpu_input_arg = alloc_percpu(void *);
	350
	351	BUG_ON(hyperv_pcpu_input_arg == NULL);
	352
7415aea6 VK	353	/* Allocate percpu VP index */
	354	hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index),
	355	GFP_KERNEL);
	356	if (!hv_vp_index)
	357	return;
	358
1268ed0c S	359	for (i = 0; i < num_possible_cpus(); i++)
	360	hv_vp_index[i] = VP_INVAL;
	361
a46d15cc VK	362	hv_vp_assist_page = kcalloc(num_possible_cpus(),
	363	sizeof(*hv_vp_assist_page), GFP_KERNEL);
	364	if (!hv_vp_assist_page) {
	365	ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
7415aea6	366	goto free_vp_index;
a46d15cc VK	367	}
	368
	369	cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online",
	370	hv_cpu_init, hv_cpu_die);
	371	if (cpuhp < 0)
	372	goto free_vp_assist_page;
7415aea6	373
8730046c S	374	/*
	375	* Setup the hypercall page and enable hypercalls.
	376	* 1. Register the guest ID
	377	* 2. Enable the hypercall and register the hypercall page
	378	*/
	379	guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0);
	380	wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
	381
fc53662f VK	382	hv_hypercall_pg = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);
fc53662f VK	383	if (hv_hypercall_pg == NULL) {
8730046c	384	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
a46d15cc	385	goto remove_cpuhp_state;
8730046c S	386	}
	387
	388	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	389	hypercall_msr.enable = 1;
fc53662f	390	hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
8730046c	391	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
63ed4e0c	392
4df4cb9e MK	393	/*
	394	* Ignore any errors in setting up stimer clockevents
	395	* as we can run with the LAPIC timer as a fallback.
	396	*/
	397	(void)hv_stimer_alloc();
	398
6b48cb5f S	399	hv_apic_init();
6b48cb5f S	400
2f285f46 DC	401	x86_init.pci.arch_init = hv_pci_init;
2f285f46 DC	402
05bd330a DC	403	register_syscore_ops(&hv_syscore_ops);
05bd330a DC	404
7415aea6 VK	405	return;
7415aea6 VK	406
a46d15cc VK	407	remove_cpuhp_state:
	408	cpuhp_remove_state(cpuhp);
	409	free_vp_assist_page:
	410	kfree(hv_vp_assist_page);
	411	hv_vp_assist_page = NULL;
7415aea6 VK	412	free_vp_index:
	413	kfree(hv_vp_index);
	414	hv_vp_index = NULL;
8730046c	415	}
6ab42a66	416
d6f3609d VK	417	/*
	418	* This routine is called before kexec/kdump, it does the required cleanup.
	419	*/
	420	void hyperv_cleanup(void)
	421	{
	422	union hv_x64_msr_hypercall_contents hypercall_msr;
	423
05bd330a DC	424	unregister_syscore_ops(&hv_syscore_ops);
05bd330a DC	425
d6f3609d VK	426	/* Reset our OS id */
	427	wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
	428
179fb36a KS	429	/*
	430	* Reset hypercall page reference before reset the page,
	431	* let hypercall operations fail safely rather than
	432	* panic the kernel for using invalid hypercall page
	433	*/
	434	hv_hypercall_pg = NULL;
	435
d6f3609d VK	436	/* Reset the hypercall page */
	437	hypercall_msr.as_uint64 = 0;
	438	wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
5647dbf8 VK	439
	440	/* Reset the TSC page */
	441	hypercall_msr.as_uint64 = 0;
	442	wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64);
d6f3609d VK	443	}
	444	EXPORT_SYMBOL_GPL(hyperv_cleanup);
	445
f3a99e76	446	void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die)
d058fa7e S	447	{
d058fa7e S	448	static bool panic_reported;
7ed4325a	449	u64 guest_id;
d058fa7e	450
f3a99e76 TL	451	if (in_die && !panic_on_oops)
	452	return;
	453
d058fa7e S	454	/*
	455	* We prefer to report panic on 'die' chain as we have proper
	456	* registers to report, but if we miss it (e.g. on BUG()) we need
	457	* to report it on 'panic'.
	458	*/
	459	if (panic_reported)
	460	return;
	461	panic_reported = true;
	462
7ed4325a S	463	rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
	464
	465	wrmsrl(HV_X64_MSR_CRASH_P0, err);
	466	wrmsrl(HV_X64_MSR_CRASH_P1, guest_id);
	467	wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip);
	468	wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax);
	469	wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp);
d058fa7e S	470
	471	/*
	472	* Let Hyper-V know there is crash data available
	473	*/
	474	wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
	475	}
	476	EXPORT_SYMBOL_GPL(hyperv_report_panic);
73638cdd	477
81b18bce SM	478	/**
	479	* hyperv_report_panic_msg - report panic message to Hyper-V
	480	* @pa: physical address of the panic page containing the message
	481	* @size: size of the message in the page
	482	*/
	483	void hyperv_report_panic_msg(phys_addr_t pa, size_t size)
	484	{
	485	/*
	486	* P3 to contain the physical address of the panic page & P4 to
	487	* contain the size of the panic data in that page. Rest of the
	488	* registers are no-op when the NOTIFY_MSG flag is set.
	489	*/
	490	wrmsrl(HV_X64_MSR_CRASH_P0, 0);
	491	wrmsrl(HV_X64_MSR_CRASH_P1, 0);
	492	wrmsrl(HV_X64_MSR_CRASH_P2, 0);
	493	wrmsrl(HV_X64_MSR_CRASH_P3, pa);
	494	wrmsrl(HV_X64_MSR_CRASH_P4, size);
	495
	496	/*
	497	* Let Hyper-V know there is crash data available along with
	498	* the panic message.
	499	*/
	500	wrmsrl(HV_X64_MSR_CRASH_CTL,
	501	(HV_CRASH_CTL_CRASH_NOTIFY \| HV_CRASH_CTL_CRASH_NOTIFY_MSG));
	502	}
	503	EXPORT_SYMBOL_GPL(hyperv_report_panic_msg);
	504
4a5f3cde	505	bool hv_is_hyperv_initialized(void)
73638cdd S	506	{
	507	union hv_x64_msr_hypercall_contents hypercall_msr;
	508
4a5f3cde MK	509	/*
	510	* Ensure that we're really on Hyper-V, and not a KVM or Xen
	511	* emulation of Hyper-V
	512	*/
	513	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
	514	return false;
	515
	516	/*
	517	* Verify that earlier initialization succeeded by checking
	518	* that the hypercall page is setup
	519	*/
73638cdd S	520	hypercall_msr.as_uint64 = 0;
	521	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
	522
4a5f3cde	523	return hypercall_msr.enable;
73638cdd	524	}
4a5f3cde	525	EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);
b96f8653 DC	526
	527	bool hv_is_hibernation_supported(void)
	528	{
	529	return acpi_sleep_state_supported(ACPI_STATE_S4);
	530	}
	531	EXPORT_SYMBOL_GPL(hv_is_hibernation_supported);