]>
Commit | Line | Data |
---|---|---|
3b20eb23 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
3e7ee490 | 2 | /* |
3e7ee490 HJ |
3 | * Copyright (c) 2009, Microsoft Corporation. |
4 | * | |
3e7ee490 HJ |
5 | * Authors: |
6 | * Haiyang Zhang <haiyangz@microsoft.com> | |
7 | * Hank Janssen <hjanssen@microsoft.com> | |
3e7ee490 | 8 | */ |
0a46618d HJ |
9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
10 | ||
a0086dc5 GKH |
11 | #include <linux/kernel.h> |
12 | #include <linux/mm.h> | |
5a0e3ad6 | 13 | #include <linux/slab.h> |
b7c947f0 | 14 | #include <linux/vmalloc.h> |
46a97191 | 15 | #include <linux/hyperv.h> |
83ba0c4f | 16 | #include <linux/version.h> |
248e742a | 17 | #include <linux/random.h> |
4061ed9e | 18 | #include <linux/clockchips.h> |
4061ed9e | 19 | #include <asm/mshyperv.h> |
0f2a6619 | 20 | #include "hyperv_vmbus.h" |
3e7ee490 | 21 | |
454f18a9 | 22 | /* The one and only */ |
a3cadf38 | 23 | struct hv_context hv_context; |
3e7ee490 | 24 | |
248e742a MK |
25 | /* |
26 | * If false, we're using the old mechanism for stimer0 interrupts | |
27 | * where it sends a VMbus message when it expires. The old | |
28 | * mechanism is used when running on older versions of Hyper-V | |
29 | * that don't support Direct Mode. While Hyper-V provides | |
30 | * four stimer's per CPU, Linux uses only stimer0. | |
31 | */ | |
32 | static bool direct_mode_enabled; | |
33 | static int stimer0_irq; | |
34 | static int stimer0_vector; | |
35 | ||
4061ed9e S |
36 | #define HV_TIMER_FREQUENCY (10 * 1000 * 1000) /* 100ns period */ |
37 | #define HV_MAX_MAX_DELTA_TICKS 0xffffffff | |
38 | #define HV_MIN_DELTA_TICKS 1 | |
39 | ||
3e189519 | 40 | /* |
d44890c8 | 41 | * hv_init - Main initialization routine. |
0831ad04 GKH |
42 | * |
43 | * This routine must be called before any other routines in here are called | |
44 | */ | |
d44890c8 | 45 | int hv_init(void) |
3e7ee490 | 46 | { |
37cdd991 SH |
47 | hv_context.cpu_context = alloc_percpu(struct hv_per_cpu_context); |
48 | if (!hv_context.cpu_context) | |
49 | return -ENOMEM; | |
50 | ||
248e742a | 51 | direct_mode_enabled = ms_hyperv.misc_features & |
7dc9b6b8 | 52 | HV_STIMER_DIRECT_MODE_AVAILABLE; |
5433e003 | 53 | return 0; |
3e7ee490 HJ |
54 | } |
55 | ||
3e189519 | 56 | /* |
d44890c8 | 57 | * hv_post_message - Post a message using the hypervisor message IPC. |
0831ad04 GKH |
58 | * |
59 | * This involves a hypercall. | |
60 | */ | |
415f0a02 | 61 | int hv_post_message(union hv_connection_id connection_id, |
b8dfb264 HZ |
62 | enum hv_message_type message_type, |
63 | void *payload, size_t payload_size) | |
3e7ee490 | 64 | { |
b8dfb264 | 65 | struct hv_input_post_message *aligned_msg; |
37cdd991 | 66 | struct hv_per_cpu_context *hv_cpu; |
a108393d | 67 | u64 status; |
3e7ee490 | 68 | |
b8dfb264 | 69 | if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) |
39594abc | 70 | return -EMSGSIZE; |
3e7ee490 | 71 | |
37cdd991 SH |
72 | hv_cpu = get_cpu_ptr(hv_context.cpu_context); |
73 | aligned_msg = hv_cpu->post_msg_page; | |
b8dfb264 | 74 | aligned_msg->connectionid = connection_id; |
b29ef354 | 75 | aligned_msg->reserved = 0; |
b8dfb264 HZ |
76 | aligned_msg->message_type = message_type; |
77 | aligned_msg->payload_size = payload_size; | |
78 | memcpy((void *)aligned_msg->payload, payload, payload_size); | |
3e7ee490 | 79 | |
a108393d | 80 | status = hv_do_hypercall(HVCALL_POST_MESSAGE, aligned_msg, NULL); |
3e7ee490 | 81 | |
13b9abfc MK |
82 | /* Preemption must remain disabled until after the hypercall |
83 | * so some other thread can't get scheduled onto this cpu and | |
84 | * corrupt the per-cpu post_msg_page | |
85 | */ | |
86 | put_cpu_ptr(hv_cpu); | |
87 | ||
a108393d | 88 | return status & 0xFFFF; |
3e7ee490 HJ |
89 | } |
90 | ||
248e742a MK |
91 | /* |
92 | * ISR for when stimer0 is operating in Direct Mode. Direct Mode | |
93 | * does not use VMbus or any VMbus messages, so process here and not | |
94 | * in the VMbus driver code. | |
95 | */ | |
96 | ||
97 | static void hv_stimer0_isr(void) | |
98 | { | |
99 | struct hv_per_cpu_context *hv_cpu; | |
100 | ||
101 | hv_cpu = this_cpu_ptr(hv_context.cpu_context); | |
102 | hv_cpu->clk_evt->event_handler(hv_cpu->clk_evt); | |
103 | add_interrupt_randomness(stimer0_vector, 0); | |
104 | } | |
105 | ||
4061ed9e S |
106 | static int hv_ce_set_next_event(unsigned long delta, |
107 | struct clock_event_device *evt) | |
108 | { | |
a5a1d1c2 | 109 | u64 current_tick; |
4061ed9e | 110 | |
bc609cb4 | 111 | WARN_ON(!clockevent_state_oneshot(evt)); |
4061ed9e | 112 | |
e546d778 | 113 | current_tick = hyperv_cs->read(NULL); |
4061ed9e | 114 | current_tick += delta; |
619a4c8b | 115 | hv_init_timer(0, current_tick); |
4061ed9e S |
116 | return 0; |
117 | } | |
118 | ||
bc609cb4 VK |
119 | static int hv_ce_shutdown(struct clock_event_device *evt) |
120 | { | |
619a4c8b MK |
121 | hv_init_timer(0, 0); |
122 | hv_init_timer_config(0, 0); | |
248e742a MK |
123 | if (direct_mode_enabled) |
124 | hv_disable_stimer0_percpu_irq(stimer0_irq); | |
bc609cb4 VK |
125 | |
126 | return 0; | |
127 | } | |
128 | ||
129 | static int hv_ce_set_oneshot(struct clock_event_device *evt) | |
4061ed9e | 130 | { |
0aa67255 | 131 | union hv_stimer_config timer_cfg; |
4061ed9e | 132 | |
248e742a | 133 | timer_cfg.as_uint64 = 0; |
bc609cb4 VK |
134 | timer_cfg.enable = 1; |
135 | timer_cfg.auto_enable = 1; | |
248e742a MK |
136 | if (direct_mode_enabled) { |
137 | /* | |
138 | * When it expires, the timer will directly interrupt | |
139 | * on the specified hardware vector/IRQ. | |
140 | */ | |
141 | timer_cfg.direct_mode = 1; | |
142 | timer_cfg.apic_vector = stimer0_vector; | |
143 | hv_enable_stimer0_percpu_irq(stimer0_irq); | |
144 | } else { | |
145 | /* | |
146 | * When it expires, the timer will generate a VMbus message, | |
147 | * to be handled by the normal VMbus interrupt handler. | |
148 | */ | |
149 | timer_cfg.direct_mode = 0; | |
150 | timer_cfg.sintx = VMBUS_MESSAGE_SINT; | |
151 | } | |
619a4c8b | 152 | hv_init_timer_config(0, timer_cfg.as_uint64); |
bc609cb4 | 153 | return 0; |
4061ed9e S |
154 | } |
155 | ||
156 | static void hv_init_clockevent_device(struct clock_event_device *dev, int cpu) | |
157 | { | |
158 | dev->name = "Hyper-V clockevent"; | |
159 | dev->features = CLOCK_EVT_FEAT_ONESHOT; | |
160 | dev->cpumask = cpumask_of(cpu); | |
161 | dev->rating = 1000; | |
e086748c VK |
162 | /* |
163 | * Avoid settint dev->owner = THIS_MODULE deliberately as doing so will | |
164 | * result in clockevents_config_and_register() taking additional | |
165 | * references to the hv_vmbus module making it impossible to unload. | |
166 | */ | |
4061ed9e | 167 | |
bc609cb4 VK |
168 | dev->set_state_shutdown = hv_ce_shutdown; |
169 | dev->set_state_oneshot = hv_ce_set_oneshot; | |
4061ed9e S |
170 | dev->set_next_event = hv_ce_set_next_event; |
171 | } | |
172 | ||
2608fb65 JW |
173 | |
174 | int hv_synic_alloc(void) | |
175 | { | |
2608fb65 | 176 | int cpu; |
f25a7ece MK |
177 | struct hv_per_cpu_context *hv_cpu; |
178 | ||
179 | /* | |
180 | * First, zero all per-cpu memory areas so hv_synic_free() can | |
181 | * detect what memory has been allocated and cleanup properly | |
182 | * after any failures. | |
183 | */ | |
184 | for_each_present_cpu(cpu) { | |
185 | hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); | |
186 | memset(hv_cpu, 0, sizeof(*hv_cpu)); | |
187 | } | |
2608fb65 | 188 | |
6396bb22 | 189 | hv_context.hv_numa_map = kcalloc(nr_node_ids, sizeof(struct cpumask), |
597ff72f | 190 | GFP_KERNEL); |
9f01ec53 S |
191 | if (hv_context.hv_numa_map == NULL) { |
192 | pr_err("Unable to allocate NUMA map\n"); | |
193 | goto err; | |
194 | } | |
195 | ||
421b8f20 | 196 | for_each_present_cpu(cpu) { |
f25a7ece | 197 | hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); |
37cdd991 | 198 | |
37cdd991 SH |
199 | tasklet_init(&hv_cpu->msg_dpc, |
200 | vmbus_on_msg_dpc, (unsigned long) hv_cpu); | |
201 | ||
202 | hv_cpu->clk_evt = kzalloc(sizeof(struct clock_event_device), | |
203 | GFP_KERNEL); | |
204 | if (hv_cpu->clk_evt == NULL) { | |
4061ed9e S |
205 | pr_err("Unable to allocate clock event device\n"); |
206 | goto err; | |
207 | } | |
37cdd991 | 208 | hv_init_clockevent_device(hv_cpu->clk_evt, cpu); |
9f01ec53 | 209 | |
37cdd991 | 210 | hv_cpu->synic_message_page = |
2608fb65 | 211 | (void *)get_zeroed_page(GFP_ATOMIC); |
37cdd991 | 212 | if (hv_cpu->synic_message_page == NULL) { |
2608fb65 JW |
213 | pr_err("Unable to allocate SYNIC message page\n"); |
214 | goto err; | |
215 | } | |
216 | ||
37cdd991 SH |
217 | hv_cpu->synic_event_page = (void *)get_zeroed_page(GFP_ATOMIC); |
218 | if (hv_cpu->synic_event_page == NULL) { | |
2608fb65 JW |
219 | pr_err("Unable to allocate SYNIC event page\n"); |
220 | goto err; | |
221 | } | |
b29ef354 | 222 | |
37cdd991 SH |
223 | hv_cpu->post_msg_page = (void *)get_zeroed_page(GFP_ATOMIC); |
224 | if (hv_cpu->post_msg_page == NULL) { | |
b29ef354 S |
225 | pr_err("Unable to allocate post msg page\n"); |
226 | goto err; | |
227 | } | |
3c7630d3 | 228 | |
37cdd991 | 229 | INIT_LIST_HEAD(&hv_cpu->chan_list); |
2608fb65 JW |
230 | } |
231 | ||
248e742a MK |
232 | if (direct_mode_enabled && |
233 | hv_setup_stimer0_irq(&stimer0_irq, &stimer0_vector, | |
234 | hv_stimer0_isr)) | |
235 | goto err; | |
236 | ||
2608fb65 JW |
237 | return 0; |
238 | err: | |
57208632 MK |
239 | /* |
240 | * Any memory allocations that succeeded will be freed when | |
241 | * the caller cleans up by calling hv_synic_free() | |
242 | */ | |
2608fb65 JW |
243 | return -ENOMEM; |
244 | } | |
245 | ||
2608fb65 JW |
246 | |
247 | void hv_synic_free(void) | |
248 | { | |
249 | int cpu; | |
250 | ||
37cdd991 SH |
251 | for_each_present_cpu(cpu) { |
252 | struct hv_per_cpu_context *hv_cpu | |
253 | = per_cpu_ptr(hv_context.cpu_context, cpu); | |
254 | ||
57208632 MK |
255 | kfree(hv_cpu->clk_evt); |
256 | free_page((unsigned long)hv_cpu->synic_event_page); | |
257 | free_page((unsigned long)hv_cpu->synic_message_page); | |
258 | free_page((unsigned long)hv_cpu->post_msg_page); | |
37cdd991 SH |
259 | } |
260 | ||
9f01ec53 | 261 | kfree(hv_context.hv_numa_map); |
2608fb65 JW |
262 | } |
263 | ||
3e189519 | 264 | /* |
68cb8117 | 265 | * hv_synic_init - Initialize the Synthetic Interrupt Controller. |
0831ad04 GKH |
266 | * |
267 | * If it is already initialized by another entity (ie x2v shim), we need to | |
268 | * retrieve the initialized message and event pages. Otherwise, we create and | |
269 | * initialize the message and event pages. | |
270 | */ | |
76d36ab7 | 271 | int hv_synic_init(unsigned int cpu) |
3e7ee490 | 272 | { |
37cdd991 SH |
273 | struct hv_per_cpu_context *hv_cpu |
274 | = per_cpu_ptr(hv_context.cpu_context, cpu); | |
eacb1b4d GKH |
275 | union hv_synic_simp simp; |
276 | union hv_synic_siefp siefp; | |
b8dfb264 | 277 | union hv_synic_sint shared_sint; |
eacb1b4d | 278 | union hv_synic_scontrol sctrl; |
a73e6b7c | 279 | |
a73e6b7c | 280 | /* Setup the Synic's message page */ |
155e4a2f | 281 | hv_get_simp(simp.as_uint64); |
f6feebe0 | 282 | simp.simp_enabled = 1; |
37cdd991 | 283 | simp.base_simp_gpa = virt_to_phys(hv_cpu->synic_message_page) |
a73e6b7c | 284 | >> PAGE_SHIFT; |
3e7ee490 | 285 | |
155e4a2f | 286 | hv_set_simp(simp.as_uint64); |
3e7ee490 | 287 | |
a73e6b7c | 288 | /* Setup the Synic's event page */ |
8e307bf8 | 289 | hv_get_siefp(siefp.as_uint64); |
f6feebe0 | 290 | siefp.siefp_enabled = 1; |
37cdd991 | 291 | siefp.base_siefp_gpa = virt_to_phys(hv_cpu->synic_event_page) |
a73e6b7c HJ |
292 | >> PAGE_SHIFT; |
293 | ||
8e307bf8 | 294 | hv_set_siefp(siefp.as_uint64); |
0831ad04 | 295 | |
0831ad04 | 296 | /* Setup the shared SINT. */ |
619a4c8b | 297 | hv_get_synint_state(VMBUS_MESSAGE_SINT, shared_sint.as_uint64); |
3e7ee490 | 298 | |
302a3c0f | 299 | shared_sint.vector = HYPERVISOR_CALLBACK_VECTOR; |
b8dfb264 | 300 | shared_sint.masked = false; |
7dc9b6b8 | 301 | if (ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED) |
6c248aad S |
302 | shared_sint.auto_eoi = false; |
303 | else | |
304 | shared_sint.auto_eoi = true; | |
3e7ee490 | 305 | |
619a4c8b | 306 | hv_set_synint_state(VMBUS_MESSAGE_SINT, shared_sint.as_uint64); |
3e7ee490 | 307 | |
454f18a9 | 308 | /* Enable the global synic bit */ |
06d1d98a | 309 | hv_get_synic_state(sctrl.as_uint64); |
f6feebe0 | 310 | sctrl.enable = 1; |
3e7ee490 | 311 | |
06d1d98a | 312 | hv_set_synic_state(sctrl.as_uint64); |
3e7ee490 | 313 | |
4061ed9e S |
314 | /* |
315 | * Register the per-cpu clockevent source. | |
316 | */ | |
7dc9b6b8 | 317 | if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) |
37cdd991 | 318 | clockevents_config_and_register(hv_cpu->clk_evt, |
4061ed9e S |
319 | HV_TIMER_FREQUENCY, |
320 | HV_MIN_DELTA_TICKS, | |
321 | HV_MAX_MAX_DELTA_TICKS); | |
76d36ab7 | 322 | return 0; |
3e7ee490 HJ |
323 | } |
324 | ||
e086748c VK |
325 | /* |
326 | * hv_synic_clockevents_cleanup - Cleanup clockevent devices | |
327 | */ | |
328 | void hv_synic_clockevents_cleanup(void) | |
329 | { | |
330 | int cpu; | |
331 | ||
7dc9b6b8 | 332 | if (!(ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE)) |
e086748c VK |
333 | return; |
334 | ||
248e742a MK |
335 | if (direct_mode_enabled) |
336 | hv_remove_stimer0_irq(stimer0_irq); | |
337 | ||
37cdd991 SH |
338 | for_each_present_cpu(cpu) { |
339 | struct hv_per_cpu_context *hv_cpu | |
340 | = per_cpu_ptr(hv_context.cpu_context, cpu); | |
341 | ||
342 | clockevents_unbind_device(hv_cpu->clk_evt, cpu); | |
343 | } | |
e086748c VK |
344 | } |
345 | ||
3e189519 | 346 | /* |
d44890c8 | 347 | * hv_synic_cleanup - Cleanup routine for hv_synic_init(). |
0831ad04 | 348 | */ |
76d36ab7 | 349 | int hv_synic_cleanup(unsigned int cpu) |
3e7ee490 | 350 | { |
b8dfb264 | 351 | union hv_synic_sint shared_sint; |
eacb1b4d GKH |
352 | union hv_synic_simp simp; |
353 | union hv_synic_siefp siefp; | |
e72e7ac5 | 354 | union hv_synic_scontrol sctrl; |
523b9408 VK |
355 | struct vmbus_channel *channel, *sc; |
356 | bool channel_found = false; | |
357 | unsigned long flags; | |
3e7ee490 | 358 | |
a3cadf38 S |
359 | hv_get_synic_state(sctrl.as_uint64); |
360 | if (sctrl.enable != 1) | |
76d36ab7 | 361 | return -EFAULT; |
3e7ee490 | 362 | |
523b9408 VK |
363 | /* |
364 | * Search for channels which are bound to the CPU we're about to | |
365 | * cleanup. In case we find one and vmbus is still connected we need to | |
366 | * fail, this will effectively prevent CPU offlining. There is no way | |
367 | * we can re-bind channels to different CPUs for now. | |
368 | */ | |
369 | mutex_lock(&vmbus_connection.channel_mutex); | |
370 | list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { | |
371 | if (channel->target_cpu == cpu) { | |
372 | channel_found = true; | |
373 | break; | |
374 | } | |
375 | spin_lock_irqsave(&channel->lock, flags); | |
376 | list_for_each_entry(sc, &channel->sc_list, sc_list) { | |
377 | if (sc->target_cpu == cpu) { | |
378 | channel_found = true; | |
379 | break; | |
380 | } | |
381 | } | |
382 | spin_unlock_irqrestore(&channel->lock, flags); | |
383 | if (channel_found) | |
384 | break; | |
385 | } | |
386 | mutex_unlock(&vmbus_connection.channel_mutex); | |
387 | ||
388 | if (channel_found && vmbus_connection.conn_state == CONNECTED) | |
389 | return -EBUSY; | |
390 | ||
e086748c | 391 | /* Turn off clockevent device */ |
7dc9b6b8 | 392 | if (ms_hyperv.features & HV_MSR_SYNTIMER_AVAILABLE) { |
37cdd991 SH |
393 | struct hv_per_cpu_context *hv_cpu |
394 | = this_cpu_ptr(hv_context.cpu_context); | |
395 | ||
396 | clockevents_unbind_device(hv_cpu->clk_evt, cpu); | |
397 | hv_ce_shutdown(hv_cpu->clk_evt); | |
6ffc4b85 | 398 | } |
e086748c | 399 | |
619a4c8b | 400 | hv_get_synint_state(VMBUS_MESSAGE_SINT, shared_sint.as_uint64); |
3e7ee490 | 401 | |
b8dfb264 | 402 | shared_sint.masked = 1; |
3e7ee490 | 403 | |
7692fd4d | 404 | /* Need to correctly cleanup in the case of SMP!!! */ |
454f18a9 | 405 | /* Disable the interrupt */ |
619a4c8b | 406 | hv_set_synint_state(VMBUS_MESSAGE_SINT, shared_sint.as_uint64); |
3e7ee490 | 407 | |
155e4a2f | 408 | hv_get_simp(simp.as_uint64); |
f6feebe0 HZ |
409 | simp.simp_enabled = 0; |
410 | simp.base_simp_gpa = 0; | |
3e7ee490 | 411 | |
155e4a2f | 412 | hv_set_simp(simp.as_uint64); |
3e7ee490 | 413 | |
8e307bf8 | 414 | hv_get_siefp(siefp.as_uint64); |
f6feebe0 HZ |
415 | siefp.siefp_enabled = 0; |
416 | siefp.base_siefp_gpa = 0; | |
3e7ee490 | 417 | |
8e307bf8 | 418 | hv_set_siefp(siefp.as_uint64); |
3e7ee490 | 419 | |
e72e7ac5 | 420 | /* Disable the global synic bit */ |
e72e7ac5 | 421 | sctrl.enable = 0; |
06d1d98a | 422 | hv_set_synic_state(sctrl.as_uint64); |
76d36ab7 VK |
423 | |
424 | return 0; | |
3e7ee490 | 425 | } |