]> git.ipfire.org Git - thirdparty/linux.git/blame - arch/x86/kvm/lapic.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'hyperv-fixes-signed' of git://git.kernel.org/pub/scm/linux/kernel/git...
[thirdparty/linux.git] / arch / x86 / kvm / lapic.c
CommitLineData
20c8ccb1 1// SPDX-License-Identifier: GPL-2.0-only
97222cc8
ED		 2
3/*
4 * Local APIC virtualization
5 *
6 * Copyright (C) 2006 Qumranet, Inc.
7 * Copyright (C) 2007 Novell
8 * Copyright (C) 2007 Intel
9611c187 9 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
97222cc8
ED		 10 *
11 * Authors:
12 * Dor Laor <dor.laor@qumranet.com>
13 * Gregory Haskins <ghaskins@novell.com>
14 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
15 *
16 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
97222cc8
ED		 17 */
18
edf88417 19#include <linux/kvm_host.h>
97222cc8
ED		 20#include <linux/kvm.h>
21#include <linux/mm.h>
22#include <linux/highmem.h>
23#include <linux/smp.h>
24#include <linux/hrtimer.h>
25#include <linux/io.h>
1767e931 26#include <linux/export.h>
6f6d6a1a 27#include <linux/math64.h>
5a0e3ad6 28#include <linux/slab.h>
97222cc8
ED		 29#include <asm/processor.h>
30#include <asm/msr.h>
31#include <asm/page.h>
32#include <asm/current.h>
33#include <asm/apicdef.h>
d0659d94 34#include <asm/delay.h>
60063497 35#include <linux/atomic.h>
c5cc421b 36#include <linux/jump_label.h>
5fdbf976 37#include "kvm_cache_regs.h"
97222cc8 38#include "irq.h"
229456fc 39#include "trace.h"
fc61b800 40#include "x86.h"
00b27a3e 41#include "cpuid.h"
5c919412 42#include "hyperv.h"
97222cc8 43
b682b814
MT		 44#ifndef CONFIG_X86_64
45#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
46#else
47#define mod_64(x, y) ((x) % (y))
48#endif
49
97222cc8
ED		 50#define PRId64 "d"
51#define PRIx64 "llx"
52#define PRIu64 "u"
53#define PRIo64 "o"
54
97222cc8 55/* 14 is the version for Xeon and Pentium 8.4.8*/
1e6e2755 56#define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
97222cc8
ED		 57#define LAPIC_MMIO_LENGTH (1 << 12)
58/* followed define is not in apicdef.h */
59#define APIC_SHORT_MASK 0xc0000
60#define APIC_DEST_NOSHORT 0x0
61#define APIC_DEST_MASK 0x800
62#define MAX_APIC_VECTOR 256
ecba9a52 63#define APIC_VECTORS_PER_REG 32
97222cc8 64
394457a9
NA		 65#define APIC_BROADCAST 0xFF
66#define X2APIC_BROADCAST 0xFFFFFFFFul
67
3b8a5df6 68#define LAPIC_TIMER_ADVANCE_ADJUST_DONE 100
548f7fb2 69#define LAPIC_TIMER_ADVANCE_ADJUST_INIT 1000
3b8a5df6
WL		 70/* step-by-step approximation to mitigate fluctuation */
71#define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
72
a0c9a822
MT		 73static inline int apic_test_vector(int vec, void *bitmap)
74{
75 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
76}
77
10606919
YZ		 78bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
79{
80 struct kvm_lapic *apic = vcpu->arch.apic;
81
82 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
83 apic_test_vector(vector, apic->regs + APIC_IRR);
84}
85
8680b94b
MT		 86static inline int __apic_test_and_set_vector(int vec, void *bitmap)
87{
88 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
89}
90
91static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
92{
93 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
94}
95
c5cc421b 96struct static_key_deferred apic_hw_disabled __read_mostly;
f8c1ea10
GN		 97struct static_key_deferred apic_sw_disabled __read_mostly;
98
97222cc8
ED		 99static inline int apic_enabled(struct kvm_lapic *apic)
100{
c48f1496 101 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
54e9818f
GN		 102}
103
97222cc8
ED		 104#define LVT_MASK \
105 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
106
107#define LINT_MASK \
108 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
109 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
110
6e500439
RK		 111static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
112{
113 return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
114}
115
116static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
117{
118 return apic->vcpu->vcpu_id;
119}
120
0c5f81da
WL		 121bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
122{
123 return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
124}
125EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt);
126
127static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
128{
129 return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
130}
131
e45115b6
RK		 132static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
133 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
134 switch (map->mode) {
135 case KVM_APIC_MODE_X2APIC: {
136 u32 offset = (dest_id >> 16) * 16;
0ca52e7b 137 u32 max_apic_id = map->max_apic_id;
e45115b6
RK		 138
139 if (offset <= max_apic_id) {
140 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
141
1d487e9b 142 offset = array_index_nospec(offset, map->max_apic_id + 1);
e45115b6
RK		 143 *cluster = &map->phys_map[offset];
144 *mask = dest_id & (0xffff >> (16 - cluster_size));
145 } else {
146 *mask = 0;
147 }
3b5a5ffa 148
e45115b6
RK		 149 return true;
150 }
151 case KVM_APIC_MODE_XAPIC_FLAT:
152 *cluster = map->xapic_flat_map;
153 *mask = dest_id & 0xff;
154 return true;
155 case KVM_APIC_MODE_XAPIC_CLUSTER:
444fdad8 156 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
e45115b6
RK		 157 *mask = dest_id & 0xf;
158 return true;
159 default:
160 /* Not optimized. */
161 return false;
162 }
3548a259
RK		 163}
164
af1bae54 165static void kvm_apic_map_free(struct rcu_head *rcu)
3b5a5ffa 166{
af1bae54 167 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
3b5a5ffa 168
af1bae54 169 kvfree(map);
3b5a5ffa
RK		 170}
171
1e08ec4a
GN		 172static void recalculate_apic_map(struct kvm *kvm)
173{
174 struct kvm_apic_map *new, *old = NULL;
175 struct kvm_vcpu *vcpu;
176 int i;
6e500439 177 u32 max_id = 255; /* enough space for any xAPIC ID */
1e08ec4a
GN		 178
179 mutex_lock(&kvm->arch.apic_map_lock);
180
0ca52e7b
RK		 181 kvm_for_each_vcpu(i, vcpu, kvm)
182 if (kvm_apic_present(vcpu))
6e500439 183 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
0ca52e7b 184
a7c3e901 185 new = kvzalloc(sizeof(struct kvm_apic_map) +
254272ce
BG		 186 sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
187 GFP_KERNEL_ACCOUNT);
0ca52e7b 188
1e08ec4a
GN		 189 if (!new)
190 goto out;
191
0ca52e7b
RK		 192 new->max_apic_id = max_id;
193
173beedc
NA		 194 kvm_for_each_vcpu(i, vcpu, kvm) {
195 struct kvm_lapic *apic = vcpu->arch.apic;
e45115b6
RK		 196 struct kvm_lapic **cluster;
197 u16 mask;
5bd5db38
RK		 198 u32 ldr;
199 u8 xapic_id;
200 u32 x2apic_id;
1e08ec4a 201
df04d1d1
RK		 202 if (!kvm_apic_present(vcpu))
203 continue;
204
5bd5db38
RK		 205 xapic_id = kvm_xapic_id(apic);
206 x2apic_id = kvm_x2apic_id(apic);
207
208 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
209 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
210 x2apic_id <= new->max_apic_id)
211 new->phys_map[x2apic_id] = apic;
212 /*
213 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
214 * prevent them from masking VCPUs with APIC ID <= 0xff.
215 */
216 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
217 new->phys_map[xapic_id] = apic;
3548a259 218
6e500439
RK		 219 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
220
3b5a5ffa
RK		 221 if (apic_x2apic_mode(apic)) {
222 new->mode |= KVM_APIC_MODE_X2APIC;
223 } else if (ldr) {
224 ldr = GET_APIC_LOGICAL_ID(ldr);
dfb95954 225 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
3b5a5ffa
RK		 226 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
227 else
228 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
229 }
230
e45115b6 231 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
3548a259
RK		 232 continue;
233
e45115b6
RK		 234 if (mask)
235 cluster[ffs(mask) - 1] = apic;
1e08ec4a
GN		 236 }
237out:
238 old = rcu_dereference_protected(kvm->arch.apic_map,
239 lockdep_is_held(&kvm->arch.apic_map_lock));
240 rcu_assign_pointer(kvm->arch.apic_map, new);
241 mutex_unlock(&kvm->arch.apic_map_lock);
242
243 if (old)
af1bae54 244 call_rcu(&old->rcu, kvm_apic_map_free);
c7c9c56c 245
b053b2ae 246 kvm_make_scan_ioapic_request(kvm);
1e08ec4a
GN		 247}
248
1e1b6c26
NA		 249static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
250{
e462755c 251 bool enabled = val & APIC_SPIV_APIC_ENABLED;
1e1b6c26 252
1e6e2755 253 kvm_lapic_set_reg(apic, APIC_SPIV, val);
e462755c
RK		 254
255 if (enabled != apic->sw_enabled) {
256 apic->sw_enabled = enabled;
eb1ff0a9 257 if (enabled)
1e1b6c26 258 static_key_slow_dec_deferred(&apic_sw_disabled);
eb1ff0a9 259 else
1e1b6c26
NA		 260 static_key_slow_inc(&apic_sw_disabled.key);
261 }
262}
263
a92e2543 264static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
1e08ec4a 265{
1e6e2755 266 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
1e08ec4a
GN		 267 recalculate_apic_map(apic->vcpu->kvm);
268}
269
270static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
271{
1e6e2755 272 kvm_lapic_set_reg(apic, APIC_LDR, id);
1e08ec4a
GN		 273 recalculate_apic_map(apic->vcpu->kvm);
274}
275
e872fa94
DDAG		 276static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
277{
278 return ((id >> 4) << 16) | (1 << (id & 0xf));
279}
280
a92e2543 281static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
257b9a5f 282{
e872fa94 283 u32 ldr = kvm_apic_calc_x2apic_ldr(id);
257b9a5f 284
6e500439
RK		 285 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
286
a92e2543 287 kvm_lapic_set_reg(apic, APIC_ID, id);
1e6e2755 288 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
257b9a5f
RK		 289 recalculate_apic_map(apic->vcpu->kvm);
290}
291
97222cc8
ED		 292static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
293{
dfb95954 294 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
97222cc8
ED		 295}
296
297static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
298{
dfb95954 299 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
97222cc8
ED		 300}
301
a3e06bbe
LJ		 302static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
303{
f30ebc31 304 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
a3e06bbe
LJ		 305}
306
97222cc8
ED		 307static inline int apic_lvtt_period(struct kvm_lapic *apic)
308{
f30ebc31 309 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
a3e06bbe
LJ		 310}
311
312static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
313{
f30ebc31 314 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
97222cc8
ED		 315}
316
cc6e462c
JK		 317static inline int apic_lvt_nmi_mode(u32 lvt_val)
318{
319 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
320}
321
fc61b800
GN		 322void kvm_apic_set_version(struct kvm_vcpu *vcpu)
323{
324 struct kvm_lapic *apic = vcpu->arch.apic;
325 struct kvm_cpuid_entry2 *feat;
326 u32 v = APIC_VERSION;
327
bce87cce 328 if (!lapic_in_kernel(vcpu))
fc61b800
GN		 329 return;
330
0bcc3fb9
VK		 331 /*
332 * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
333 * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
334 * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
335 * version first and level-triggered interrupts never get EOIed in
336 * IOAPIC.
337 */
fc61b800 338 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
0bcc3fb9
VK		 339 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
340 !ioapic_in_kernel(vcpu->kvm))
fc61b800 341 v |= APIC_LVR_DIRECTED_EOI;
1e6e2755 342 kvm_lapic_set_reg(apic, APIC_LVR, v);
fc61b800
GN		 343}
344
1e6e2755 345static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
a3e06bbe 346 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
97222cc8
ED		 347 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
348 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
349 LINT_MASK, LINT_MASK, /* LVT0-1 */
350 LVT_MASK /* LVTERR */
351};
352
353static int find_highest_vector(void *bitmap)
354{
ecba9a52
TY		 355 int vec;
356 u32 *reg;
97222cc8 357
ecba9a52
TY		 358 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
359 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
360 reg = bitmap + REG_POS(vec);
361 if (*reg)
810e6def 362 return __fls(*reg) + vec;
ecba9a52 363 }
97222cc8 364
ecba9a52 365 return -1;
97222cc8
ED		 366}
367
8680b94b
MT		 368static u8 count_vectors(void *bitmap)
369{
ecba9a52
TY		 370 int vec;
371 u32 *reg;
8680b94b 372 u8 count = 0;
ecba9a52
TY		 373
374 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
375 reg = bitmap + REG_POS(vec);
376 count += hweight32(*reg);
377 }
378
8680b94b
MT		 379 return count;
380}
381
e7387b0e 382bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
a20ed54d 383{
810e6def 384 u32 i, vec;
e7387b0e
LA		 385 u32 pir_val, irr_val, prev_irr_val;
386 int max_updated_irr;
387
388 max_updated_irr = -1;
389 *max_irr = -1;
a20ed54d 390
810e6def 391 for (i = vec = 0; i <= 7; i++, vec += 32) {
ad361091 392 pir_val = READ_ONCE(pir[i]);
810e6def 393 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
ad361091 394 if (pir_val) {
e7387b0e 395 prev_irr_val = irr_val;
810e6def
PB		 396 irr_val |= xchg(&pir[i], 0);
397 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
e7387b0e
LA		 398 if (prev_irr_val != irr_val) {
399 max_updated_irr =
400 __fls(irr_val ^ prev_irr_val) + vec;
401 }
ad361091 402 }
810e6def 403 if (irr_val)
e7387b0e 404 *max_irr = __fls(irr_val) + vec;
a20ed54d 405 }
810e6def 406
e7387b0e
LA		 407 return ((max_updated_irr != -1) &&
408 (max_updated_irr == *max_irr));
a20ed54d 409}
705699a1
WV		 410EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
411
e7387b0e 412bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
705699a1
WV		 413{
414 struct kvm_lapic *apic = vcpu->arch.apic;
415
e7387b0e 416 return __kvm_apic_update_irr(pir, apic->regs, max_irr);
705699a1 417}
a20ed54d
YZ		 418EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
419
33e4c686 420static inline int apic_search_irr(struct kvm_lapic *apic)
97222cc8 421{
33e4c686 422 return find_highest_vector(apic->regs + APIC_IRR);
97222cc8
ED		 423}
424
425static inline int apic_find_highest_irr(struct kvm_lapic *apic)
426{
427 int result;
428
c7c9c56c
YZ		 429 /*
430 * Note that irr_pending is just a hint. It will be always
431 * true with virtual interrupt delivery enabled.
432 */
33e4c686
GN		 433 if (!apic->irr_pending)
434 return -1;
435
436 result = apic_search_irr(apic);
97222cc8
ED		 437 ASSERT(result == -1 || result >= 16);
438
439 return result;
440}
441
33e4c686
GN		 442static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
443{
56cc2406
WL		 444 struct kvm_vcpu *vcpu;
445
446 vcpu = apic->vcpu;
447
d62caabb 448 if (unlikely(vcpu->arch.apicv_active)) {
b95234c8 449 /* need to update RVI */
ee171d2f 450 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
b95234c8
PB		 451 kvm_x86_ops->hwapic_irr_update(vcpu,
452 apic_find_highest_irr(apic));
f210f757
NA		 453 } else {
454 apic->irr_pending = false;
ee171d2f 455 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
f210f757
NA		 456 if (apic_search_irr(apic) != -1)
457 apic->irr_pending = true;
56cc2406 458 }
33e4c686
GN		 459}
460
8680b94b
MT		 461static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
462{
56cc2406
WL		 463 struct kvm_vcpu *vcpu;
464
465 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
466 return;
467
468 vcpu = apic->vcpu;
fc57ac2c 469
8680b94b 470 /*
56cc2406
WL		 471 * With APIC virtualization enabled, all caching is disabled
472 * because the processor can modify ISR under the hood. Instead
473 * just set SVI.
8680b94b 474 */
d62caabb 475 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 476 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
56cc2406
WL		 477 else {
478 ++apic->isr_count;
479 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
480 /*
481 * ISR (in service register) bit is set when injecting an interrupt.
482 * The highest vector is injected. Thus the latest bit set matches
483 * the highest bit in ISR.
484 */
485 apic->highest_isr_cache = vec;
486 }
8680b94b
MT		 487}
488
fc57ac2c
PB		 489static inline int apic_find_highest_isr(struct kvm_lapic *apic)
490{
491 int result;
492
493 /*
494 * Note that isr_count is always 1, and highest_isr_cache
495 * is always -1, with APIC virtualization enabled.
496 */
497 if (!apic->isr_count)
498 return -1;
499 if (likely(apic->highest_isr_cache != -1))
500 return apic->highest_isr_cache;
501
502 result = find_highest_vector(apic->regs + APIC_ISR);
503 ASSERT(result == -1 || result >= 16);
504
505 return result;
506}
507
8680b94b
MT		 508static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
509{
fc57ac2c
PB		 510 struct kvm_vcpu *vcpu;
511 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
512 return;
513
514 vcpu = apic->vcpu;
515
516 /*
517 * We do get here for APIC virtualization enabled if the guest
518 * uses the Hyper-V APIC enlightenment. In this case we may need
519 * to trigger a new interrupt delivery by writing the SVI field;
520 * on the other hand isr_count and highest_isr_cache are unused
521 * and must be left alone.
522 */
d62caabb 523 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 524 kvm_x86_ops->hwapic_isr_update(vcpu,
fc57ac2c
PB		 525 apic_find_highest_isr(apic));
526 else {
8680b94b 527 --apic->isr_count;
fc57ac2c
PB		 528 BUG_ON(apic->isr_count < 0);
529 apic->highest_isr_cache = -1;
530 }
8680b94b
MT		 531}
532
6e5d865c
YS		 533int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
534{
33e4c686
GN		 535 /* This may race with setting of irr in __apic_accept_irq() and
536 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
537 * will cause vmexit immediately and the value will be recalculated
538 * on the next vmentry.
539 */
f8543d6a 540 return apic_find_highest_irr(vcpu->arch.apic);
6e5d865c 541}
76dfafd5 542EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
6e5d865c 543
6da7e3f6 544static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 545 int vector, int level, int trig_mode,
9e4aabe2 546 struct dest_map *dest_map);
6da7e3f6 547
b4f2225c 548int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
9e4aabe2 549 struct dest_map *dest_map)
97222cc8 550{
ad312c7c 551 struct kvm_lapic *apic = vcpu->arch.apic;
8be5453f 552
58c2dde1 553 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
b4f2225c 554 irq->level, irq->trig_mode, dest_map);
97222cc8
ED		 555}
556
4180bf1b 557int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
bdf7ffc8 558 unsigned long ipi_bitmap_high, u32 min,
4180bf1b
WL		 559 unsigned long icr, int op_64_bit)
560{
561 int i;
562 struct kvm_apic_map *map;
563 struct kvm_vcpu *vcpu;
564 struct kvm_lapic_irq irq = {0};
565 int cluster_size = op_64_bit ? 64 : 32;
566 int count = 0;
567
568 irq.vector = icr & APIC_VECTOR_MASK;
569 irq.delivery_mode = icr & APIC_MODE_MASK;
570 irq.level = (icr & APIC_INT_ASSERT) != 0;
571 irq.trig_mode = icr & APIC_INT_LEVELTRIG;
572
573 if (icr & APIC_DEST_MASK)
574 return -KVM_EINVAL;
575 if (icr & APIC_SHORT_MASK)
576 return -KVM_EINVAL;
577
578 rcu_read_lock();
579 map = rcu_dereference(kvm->arch.apic_map);
580
38ab012f
WL		 581 if (unlikely(!map)) {
582 count = -EOPNOTSUPP;
583 goto out;
584 }
585
bdf7ffc8
WL		 586 if (min > map->max_apic_id)
587 goto out;
4180bf1b 588 /* Bits above cluster_size are masked in the caller. */
bdf7ffc8
WL		 589 for_each_set_bit(i, &ipi_bitmap_low,
590 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
591 if (map->phys_map[min + i]) {
592 vcpu = map->phys_map[min + i]->vcpu;
593 count += kvm_apic_set_irq(vcpu, &irq, NULL);
594 }
4180bf1b
WL		 595 }
596
597 min += cluster_size;
bdf7ffc8
WL		 598
599 if (min > map->max_apic_id)
600 goto out;
601
602 for_each_set_bit(i, &ipi_bitmap_high,
603 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
604 if (map->phys_map[min + i]) {
605 vcpu = map->phys_map[min + i]->vcpu;
606 count += kvm_apic_set_irq(vcpu, &irq, NULL);
607 }
4180bf1b
WL		 608 }
609
bdf7ffc8 610out:
4180bf1b
WL		 611 rcu_read_unlock();
612 return count;
613}
614
ae7a2a3f
MT		 615static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
616{
4e335d9e
PB		 617
618 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
619 sizeof(val));
ae7a2a3f
MT		 620}
621
622static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
623{
4e335d9e
PB		 624
625 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
626 sizeof(*val));
ae7a2a3f
MT		 627}
628
629static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
630{
631 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
632}
633
634static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
635{
636 u8 val;
637 if (pv_eoi_get_user(vcpu, &val) < 0)
0d88800d 638 printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
96893977 639 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 640 return val & 0x1;
641}
642
643static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
644{
645 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
0d88800d 646 printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
96893977 647 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 648 return;
649 }
650 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
651}
652
653static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
654{
655 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
0d88800d 656 printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
96893977 657 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 658 return;
659 }
660 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
661}
662
b3c045d3
PB		 663static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
664{
3d92789f 665 int highest_irr;
fa59cc00 666 if (apic->vcpu->arch.apicv_active)
76dfafd5
PB		 667 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
668 else
669 highest_irr = apic_find_highest_irr(apic);
b3c045d3
PB		 670 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
671 return -1;
672 return highest_irr;
673}
674
675static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
97222cc8 676{
3842d135 677 u32 tpr, isrv, ppr, old_ppr;
97222cc8
ED		 678 int isr;
679
dfb95954
SS		 680 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
681 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
97222cc8
ED		 682 isr = apic_find_highest_isr(apic);
683 isrv = (isr != -1) ? isr : 0;
684
685 if ((tpr & 0xf0) >= (isrv & 0xf0))
686 ppr = tpr & 0xff;
687 else
688 ppr = isrv & 0xf0;
689
b3c045d3
PB		 690 *new_ppr = ppr;
691 if (old_ppr != ppr)
1e6e2755 692 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
b3c045d3
PB		 693
694 return ppr < old_ppr;
695}
696
697static void apic_update_ppr(struct kvm_lapic *apic)
698{
699 u32 ppr;
700
26fbbee5
PB		 701 if (__apic_update_ppr(apic, &ppr) &&
702 apic_has_interrupt_for_ppr(apic, ppr) != -1)
b3c045d3 703 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
97222cc8
ED		 704}
705
eb90f341
PB		 706void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
707{
708 apic_update_ppr(vcpu->arch.apic);
709}
710EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
711
97222cc8
ED		 712static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
713{
1e6e2755 714 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
97222cc8
ED		 715 apic_update_ppr(apic);
716}
717
03d2249e 718static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
394457a9 719{
b4535b58
RK		 720 return mda == (apic_x2apic_mode(apic) ?
721 X2APIC_BROADCAST : APIC_BROADCAST);
394457a9
NA		 722}
723
03d2249e 724static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 725{
03d2249e
RK		 726 if (kvm_apic_broadcast(apic, mda))
727 return true;
728
729 if (apic_x2apic_mode(apic))
6e500439 730 return mda == kvm_x2apic_id(apic);
03d2249e 731
5bd5db38
RK		 732 /*
733 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
734 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
735 * this allows unique addressing of VCPUs with APIC ID over 0xff.
736 * The 0xff condition is needed because writeable xAPIC ID.
737 */
738 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
739 return true;
740
b4535b58 741 return mda == kvm_xapic_id(apic);
97222cc8
ED		 742}
743
52c233a4 744static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 745{
0105d1a5
GN		 746 u32 logical_id;
747
394457a9 748 if (kvm_apic_broadcast(apic, mda))
9368b567 749 return true;
394457a9 750
dfb95954 751 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
97222cc8 752
9368b567 753 if (apic_x2apic_mode(apic))
8a395363
RK		 754 return ((logical_id >> 16) == (mda >> 16))
755 && (logical_id & mda & 0xffff) != 0;
97222cc8 756
9368b567 757 logical_id = GET_APIC_LOGICAL_ID(logical_id);
97222cc8 758
dfb95954 759 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
97222cc8 760 case APIC_DFR_FLAT:
9368b567 761 return (logical_id & mda) != 0;
97222cc8 762 case APIC_DFR_CLUSTER:
9368b567
RK		 763 return ((logical_id >> 4) == (mda >> 4))
764 && (logical_id & mda & 0xf) != 0;
97222cc8 765 default:
9368b567 766 return false;
97222cc8 767 }
97222cc8
ED		 768}
769
c519265f
RK		 770/* The KVM local APIC implementation has two quirks:
771 *
b4535b58
RK		 772 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
773 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
774 * KVM doesn't do that aliasing.
c519265f
RK		 775 *
776 * - in-kernel IOAPIC messages have to be delivered directly to
777 * x2APIC, because the kernel does not support interrupt remapping.
778 * In order to support broadcast without interrupt remapping, x2APIC
779 * rewrites the destination of non-IPI messages from APIC_BROADCAST
780 * to X2APIC_BROADCAST.
781 *
782 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
783 * important when userspace wants to use x2APIC-format MSIs, because
784 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
03d2249e 785 */
c519265f
RK		 786static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
787 struct kvm_lapic *source, struct kvm_lapic *target)
03d2249e
RK		 788{
789 bool ipi = source != NULL;
03d2249e 790
c519265f 791 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
b4535b58 792 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
03d2249e
RK		 793 return X2APIC_BROADCAST;
794
b4535b58 795 return dest_id;
03d2249e
RK		 796}
797
52c233a4 798bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
394457a9 799 int short_hand, unsigned int dest, int dest_mode)
97222cc8 800{
ad312c7c 801 struct kvm_lapic *target = vcpu->arch.apic;
c519265f 802 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
97222cc8 803
bd371396 804 ASSERT(target);
97222cc8
ED		 805 switch (short_hand) {
806 case APIC_DEST_NOSHORT:
3697f302 807 if (dest_mode == APIC_DEST_PHYSICAL)
03d2249e 808 return kvm_apic_match_physical_addr(target, mda);
343f94fe 809 else
03d2249e 810 return kvm_apic_match_logical_addr(target, mda);
97222cc8 811 case APIC_DEST_SELF:
9368b567 812 return target == source;
97222cc8 813 case APIC_DEST_ALLINC:
9368b567 814 return true;
97222cc8 815 case APIC_DEST_ALLBUT:
9368b567 816 return target != source;
97222cc8 817 default:
9368b567 818 return false;
97222cc8 819 }
97222cc8 820}
1e6e2755 821EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
97222cc8 822
52004014
FW		 823int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
824 const unsigned long *bitmap, u32 bitmap_size)
825{
826 u32 mod;
827 int i, idx = -1;
828
829 mod = vector % dest_vcpus;
830
831 for (i = 0; i <= mod; i++) {
832 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
833 BUG_ON(idx == bitmap_size);
834 }
835
836 return idx;
837}
838
4efd805f
RK		 839static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
840{
841 if (!kvm->arch.disabled_lapic_found) {
842 kvm->arch.disabled_lapic_found = true;
843 printk(KERN_INFO
844 "Disabled LAPIC found during irq injection\n");
845 }
846}
847
c519265f
RK		 848static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
849 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1e08ec4a 850{
c519265f
RK		 851 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
852 if ((irq->dest_id == APIC_BROADCAST &&
853 map->mode != KVM_APIC_MODE_X2APIC))
854 return true;
855 if (irq->dest_id == X2APIC_BROADCAST)
856 return true;
857 } else {
858 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
859 if (irq->dest_id == (x2apic_ipi ?
860 X2APIC_BROADCAST : APIC_BROADCAST))
861 return true;
862 }
1e08ec4a 863
c519265f
RK		 864 return false;
865}
1e08ec4a 866
64aa47bf
RK		 867/* Return true if the interrupt can be handled by using *bitmap as index mask
868 * for valid destinations in *dst array.
869 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
870 * Note: we may have zero kvm_lapic destinations when we return true, which
871 * means that the interrupt should be dropped. In this case, *bitmap would be
872 * zero and *dst undefined.
873 */
874static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
875 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
876 struct kvm_apic_map *map, struct kvm_lapic ***dst,
877 unsigned long *bitmap)
878{
879 int i, lowest;
1e08ec4a 880
64aa47bf
RK		 881 if (irq->shorthand == APIC_DEST_SELF && src) {
882 *dst = src;
883 *bitmap = 1;
884 return true;
885 } else if (irq->shorthand)
1e08ec4a
GN		 886 return false;
887
c519265f 888 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
9ea369b0
RK		 889 return false;
890
64aa47bf 891 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
0ca52e7b 892 if (irq->dest_id > map->max_apic_id) {
64aa47bf
RK		 893 *bitmap = 0;
894 } else {
1d487e9b
PB		 895 u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
896 *dst = &map->phys_map[dest_id];
64aa47bf
RK		 897 *bitmap = 1;
898 }
1e08ec4a 899 return true;
bea15428 900 }
698f9755 901
e45115b6
RK		 902 *bitmap = 0;
903 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
904 (u16 *)bitmap))
1e08ec4a 905 return false;
fa834e91 906
64aa47bf
RK		 907 if (!kvm_lowest_prio_delivery(irq))
908 return true;
3548a259 909
64aa47bf
RK		 910 if (!kvm_vector_hashing_enabled()) {
911 lowest = -1;
912 for_each_set_bit(i, bitmap, 16) {
913 if (!(*dst)[i])
914 continue;
915 if (lowest < 0)
916 lowest = i;
917 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
918 (*dst)[lowest]->vcpu) < 0)
919 lowest = i;
3548a259 920 }
64aa47bf
RK		 921 } else {
922 if (!*bitmap)
923 return true;
3548a259 924
64aa47bf
RK		 925 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
926 bitmap, 16);
45c3094a 927
64aa47bf
RK		 928 if (!(*dst)[lowest]) {
929 kvm_apic_disabled_lapic_found(kvm);
930 *bitmap = 0;
931 return true;
932 }
933 }
1e08ec4a 934
64aa47bf 935 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1e08ec4a 936
64aa47bf
RK		 937 return true;
938}
52004014 939
64aa47bf
RK		 940bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
941 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
942{
943 struct kvm_apic_map *map;
944 unsigned long bitmap;
945 struct kvm_lapic **dst = NULL;
946 int i;
947 bool ret;
52004014 948
64aa47bf 949 *r = -1;
52004014 950
64aa47bf
RK		 951 if (irq->shorthand == APIC_DEST_SELF) {
952 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
953 return true;
954 }
52004014 955
64aa47bf
RK		 956 rcu_read_lock();
957 map = rcu_dereference(kvm->arch.apic_map);
52004014 958
64aa47bf 959 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
0624fca9
PB		 960 if (ret) {
961 *r = 0;
64aa47bf
RK		 962 for_each_set_bit(i, &bitmap, 16) {
963 if (!dst[i])
964 continue;
64aa47bf 965 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1e08ec4a 966 }
0624fca9 967 }
1e08ec4a 968
1e08ec4a
GN		 969 rcu_read_unlock();
970 return ret;
971}
972
6228a0da
FW		 973/*
974 * This routine tries to handler interrupts in posted mode, here is how
975 * it deals with different cases:
976 * - For single-destination interrupts, handle it in posted mode
977 * - Else if vector hashing is enabled and it is a lowest-priority
978 * interrupt, handle it in posted mode and use the following mechanism
979 * to find the destinaiton vCPU.
980 * 1. For lowest-priority interrupts, store all the possible
981 * destination vCPUs in an array.
982 * 2. Use "guest vector % max number of destination vCPUs" to find
983 * the right destination vCPU in the array for the lowest-priority
984 * interrupt.
985 * - Otherwise, use remapped mode to inject the interrupt.
986 */
8feb4a04
FW		 987bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
988 struct kvm_vcpu **dest_vcpu)
989{
990 struct kvm_apic_map *map;
64aa47bf
RK		 991 unsigned long bitmap;
992 struct kvm_lapic **dst = NULL;
8feb4a04 993 bool ret = false;
8feb4a04
FW		 994
995 if (irq->shorthand)
996 return false;
997
998 rcu_read_lock();
999 map = rcu_dereference(kvm->arch.apic_map);
1000
64aa47bf
RK		 1001 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1002 hweight16(bitmap) == 1) {
1003 unsigned long i = find_first_bit(&bitmap, 16);
6228a0da 1004
64aa47bf
RK		 1005 if (dst[i]) {
1006 *dest_vcpu = dst[i]->vcpu;
1007 ret = true;
6228a0da 1008 }
8feb4a04
FW		 1009 }
1010
8feb4a04
FW		 1011 rcu_read_unlock();
1012 return ret;
1013}
1014
97222cc8
ED		 1015/*
1016 * Add a pending IRQ into lapic.
1017 * Return 1 if successfully added and 0 if discarded.
1018 */
1019static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 1020 int vector, int level, int trig_mode,
9e4aabe2 1021 struct dest_map *dest_map)
97222cc8 1022{
6da7e3f6 1023 int result = 0;
c5ec1534 1024 struct kvm_vcpu *vcpu = apic->vcpu;
97222cc8 1025
a183b638
PB		 1026 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1027 trig_mode, vector);
97222cc8 1028 switch (delivery_mode) {
97222cc8 1029 case APIC_DM_LOWEST:
e1035715 1030 vcpu->arch.apic_arb_prio++;
b2869f28 1031 /* fall through */
e1035715 1032 case APIC_DM_FIXED:
bdaffe1d
PB		 1033 if (unlikely(trig_mode && !level))
1034 break;
1035
97222cc8
ED		 1036 /* FIXME add logic for vcpu on reset */
1037 if (unlikely(!apic_enabled(apic)))
1038 break;
1039
11f5cc05
JK		 1040 result = 1;
1041
9daa5007 1042 if (dest_map) {
9e4aabe2 1043 __set_bit(vcpu->vcpu_id, dest_map->map);
9daa5007
JR		 1044 dest_map->vectors[vcpu->vcpu_id] = vector;
1045 }
a5d36f82 1046
bdaffe1d
PB		 1047 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1048 if (trig_mode)
ee171d2f
WY		 1049 kvm_lapic_set_vector(vector,
1050 apic->regs + APIC_TMR);
bdaffe1d 1051 else
ee171d2f
WY		 1052 kvm_lapic_clear_vector(vector,
1053 apic->regs + APIC_TMR);
bdaffe1d
PB		 1054 }
1055
d62caabb 1056 if (vcpu->arch.apicv_active)
5a71785d 1057 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
11f5cc05 1058 else {
1e6e2755 1059 kvm_lapic_set_irr(vector, apic);
5a71785d
YZ		 1060
1061 kvm_make_request(KVM_REQ_EVENT, vcpu);
1062 kvm_vcpu_kick(vcpu);
1063 }
97222cc8
ED		 1064 break;
1065
1066 case APIC_DM_REMRD:
24d2166b
R		 1067 result = 1;
1068 vcpu->arch.pv.pv_unhalted = 1;
1069 kvm_make_request(KVM_REQ_EVENT, vcpu);
1070 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1071 break;
1072
1073 case APIC_DM_SMI:
64d60670
PB		 1074 result = 1;
1075 kvm_make_request(KVM_REQ_SMI, vcpu);
1076 kvm_vcpu_kick(vcpu);
97222cc8 1077 break;
3419ffc8 1078
97222cc8 1079 case APIC_DM_NMI:
6da7e3f6 1080 result = 1;
3419ffc8 1081 kvm_inject_nmi(vcpu);
26df99c6 1082 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1083 break;
1084
1085 case APIC_DM_INIT:
a52315e1 1086 if (!trig_mode || level) {
6da7e3f6 1087 result = 1;
66450a21
JK		 1088 /* assumes that there are only KVM_APIC_INIT/SIPI */
1089 apic->pending_events = (1UL << KVM_APIC_INIT);
1090 /* make sure pending_events is visible before sending
1091 * the request */
1092 smp_wmb();
3842d135 1093 kvm_make_request(KVM_REQ_EVENT, vcpu);
c5ec1534 1094 kvm_vcpu_kick(vcpu);
c5ec1534 1095 }
97222cc8
ED		 1096 break;
1097
1098 case APIC_DM_STARTUP:
66450a21
JK		 1099 result = 1;
1100 apic->sipi_vector = vector;
1101 /* make sure sipi_vector is visible for the receiver */
1102 smp_wmb();
1103 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1104 kvm_make_request(KVM_REQ_EVENT, vcpu);
1105 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1106 break;
1107
23930f95
JK		 1108 case APIC_DM_EXTINT:
1109 /*
1110 * Should only be called by kvm_apic_local_deliver() with LVT0,
1111 * before NMI watchdog was enabled. Already handled by
1112 * kvm_apic_accept_pic_intr().
1113 */
1114 break;
1115
97222cc8
ED		 1116 default:
1117 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1118 delivery_mode);
1119 break;
1120 }
1121 return result;
1122}
1123
e1035715 1124int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
8be5453f 1125{
e1035715 1126 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
8be5453f
ZX		 1127}
1128
3bb345f3
PB		 1129static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1130{
6308630b 1131 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
3bb345f3
PB		 1132}
1133
c7c9c56c
YZ		 1134static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1135{
7543a635
SR		 1136 int trigger_mode;
1137
1138 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1139 if (!kvm_ioapic_handles_vector(apic, vector))
1140 return;
3bb345f3 1141
7543a635
SR		 1142 /* Request a KVM exit to inform the userspace IOAPIC. */
1143 if (irqchip_split(apic->vcpu->kvm)) {
1144 apic->vcpu->arch.pending_ioapic_eoi = vector;
1145 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1146 return;
c7c9c56c 1147 }
7543a635
SR		 1148
1149 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1150 trigger_mode = IOAPIC_LEVEL_TRIG;
1151 else
1152 trigger_mode = IOAPIC_EDGE_TRIG;
1153
1154 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
c7c9c56c
YZ		 1155}
1156
ae7a2a3f 1157static int apic_set_eoi(struct kvm_lapic *apic)
97222cc8
ED		 1158{
1159 int vector = apic_find_highest_isr(apic);
ae7a2a3f
MT		 1160
1161 trace_kvm_eoi(apic, vector);
1162
97222cc8
ED		 1163 /*
1164 * Not every write EOI will has corresponding ISR,
1165 * one example is when Kernel check timer on setup_IO_APIC
1166 */
1167 if (vector == -1)
ae7a2a3f 1168 return vector;
97222cc8 1169
8680b94b 1170 apic_clear_isr(vector, apic);
97222cc8
ED		 1171 apic_update_ppr(apic);
1172
5c919412
AS		 1173 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1174 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1175
c7c9c56c 1176 kvm_ioapic_send_eoi(apic, vector);
3842d135 1177 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
ae7a2a3f 1178 return vector;
97222cc8
ED		 1179}
1180
c7c9c56c
YZ		 1181/*
1182 * this interface assumes a trap-like exit, which has already finished
1183 * desired side effect including vISR and vPPR update.
1184 */
1185void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1186{
1187 struct kvm_lapic *apic = vcpu->arch.apic;
1188
1189 trace_kvm_eoi(apic, vector);
1190
1191 kvm_ioapic_send_eoi(apic, vector);
1192 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1193}
1194EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1195
97222cc8
ED		 1196static void apic_send_ipi(struct kvm_lapic *apic)
1197{
dfb95954
SS		 1198 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1199 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
58c2dde1 1200 struct kvm_lapic_irq irq;
97222cc8 1201
58c2dde1
GN		 1202 irq.vector = icr_low & APIC_VECTOR_MASK;
1203 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1204 irq.dest_mode = icr_low & APIC_DEST_MASK;
b7cb2231 1205 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
58c2dde1
GN		 1206 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1207 irq.shorthand = icr_low & APIC_SHORT_MASK;
93bbf0b8 1208 irq.msi_redir_hint = false;
0105d1a5
GN		 1209 if (apic_x2apic_mode(apic))
1210 irq.dest_id = icr_high;
1211 else
1212 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
97222cc8 1213
1000ff8d
GN		 1214 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1215
b4f2225c 1216 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
97222cc8
ED		 1217}
1218
1219static u32 apic_get_tmcct(struct kvm_lapic *apic)
1220{
8003c9ae 1221 ktime_t remaining, now;
b682b814 1222 s64 ns;
9da8f4e8 1223 u32 tmcct;
97222cc8
ED		 1224
1225 ASSERT(apic != NULL);
1226
9da8f4e8 1227 /* if initial count is 0, current count should also be 0 */
dfb95954 1228 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
b963a22e 1229 apic->lapic_timer.period == 0)
9da8f4e8
KP		 1230 return 0;
1231
5587859f 1232 now = ktime_get();
8003c9ae 1233 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
b682b814 1234 if (ktime_to_ns(remaining) < 0)
8b0e1953 1235 remaining = 0;
b682b814 1236
d3c7b77d
MT		 1237 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1238 tmcct = div64_u64(ns,
1239 (APIC_BUS_CYCLE_NS * apic->divide_count));
97222cc8
ED		 1240
1241 return tmcct;
1242}
1243
b209749f
AK		 1244static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1245{
1246 struct kvm_vcpu *vcpu = apic->vcpu;
1247 struct kvm_run *run = vcpu->run;
1248
a8eeb04a 1249 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
5fdbf976 1250 run->tpr_access.rip = kvm_rip_read(vcpu);
b209749f
AK		 1251 run->tpr_access.is_write = write;
1252}
1253
1254static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1255{
1256 if (apic->vcpu->arch.tpr_access_reporting)
1257 __report_tpr_access(apic, write);
1258}
1259
97222cc8
ED		 1260static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1261{
1262 u32 val = 0;
1263
1264 if (offset >= LAPIC_MMIO_LENGTH)
1265 return 0;
1266
1267 switch (offset) {
1268 case APIC_ARBPRI:
97222cc8
ED		 1269 break;
1270
1271 case APIC_TMCCT: /* Timer CCR */
a3e06bbe
LJ		 1272 if (apic_lvtt_tscdeadline(apic))
1273 return 0;
1274
97222cc8
ED		 1275 val = apic_get_tmcct(apic);
1276 break;
4a4541a4
AK		 1277 case APIC_PROCPRI:
1278 apic_update_ppr(apic);
dfb95954 1279 val = kvm_lapic_get_reg(apic, offset);
4a4541a4 1280 break;
b209749f
AK		 1281 case APIC_TASKPRI:
1282 report_tpr_access(apic, false);
1283 /* fall thru */
97222cc8 1284 default:
dfb95954 1285 val = kvm_lapic_get_reg(apic, offset);
97222cc8
ED		 1286 break;
1287 }
1288
1289 return val;
1290}
1291
d76685c4
GH		 1292static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1293{
1294 return container_of(dev, struct kvm_lapic, dev);
1295}
1296
01402cf8
PB		 1297#define APIC_REG_MASK(reg) (1ull << ((reg) >> 4))
1298#define APIC_REGS_MASK(first, count) \
1299 (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1300
1e6e2755 1301int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
0105d1a5 1302 void *data)
97222cc8 1303{
97222cc8
ED		 1304 unsigned char alignment = offset & 0xf;
1305 u32 result;
d5b0b5b1 1306 /* this bitmask has a bit cleared for each reserved register */
01402cf8
PB		 1307 u64 valid_reg_mask =
1308 APIC_REG_MASK(APIC_ID) |
1309 APIC_REG_MASK(APIC_LVR) |
1310 APIC_REG_MASK(APIC_TASKPRI) |
1311 APIC_REG_MASK(APIC_PROCPRI) |
1312 APIC_REG_MASK(APIC_LDR) |
1313 APIC_REG_MASK(APIC_DFR) |
1314 APIC_REG_MASK(APIC_SPIV) |
1315 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1316 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1317 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1318 APIC_REG_MASK(APIC_ESR) |
1319 APIC_REG_MASK(APIC_ICR) |
1320 APIC_REG_MASK(APIC_ICR2) |
1321 APIC_REG_MASK(APIC_LVTT) |
1322 APIC_REG_MASK(APIC_LVTTHMR) |
1323 APIC_REG_MASK(APIC_LVTPC) |
1324 APIC_REG_MASK(APIC_LVT0) |
1325 APIC_REG_MASK(APIC_LVT1) |
1326 APIC_REG_MASK(APIC_LVTERR) |
1327 APIC_REG_MASK(APIC_TMICT) |
1328 APIC_REG_MASK(APIC_TMCCT) |
1329 APIC_REG_MASK(APIC_TDCR);
1330
1331 /* ARBPRI is not valid on x2APIC */
1332 if (!apic_x2apic_mode(apic))
1333 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
0105d1a5 1334
0d88800d 1335 if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
0105d1a5 1336 return 1;
0105d1a5 1337
97222cc8
ED		 1338 result = __apic_read(apic, offset & ~0xf);
1339
229456fc
MT		 1340 trace_kvm_apic_read(offset, result);
1341
97222cc8
ED		 1342 switch (len) {
1343 case 1:
1344 case 2:
1345 case 4:
1346 memcpy(data, (char *)&result + alignment, len);
1347 break;
1348 default:
1349 printk(KERN_ERR "Local APIC read with len = %x, "
1350 "should be 1,2, or 4 instead\n", len);
1351 break;
1352 }
bda9020e 1353 return 0;
97222cc8 1354}
1e6e2755 1355EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
97222cc8 1356
0105d1a5
GN		 1357static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1358{
d1766202
VK		 1359 return addr >= apic->base_address &&
1360 addr < apic->base_address + LAPIC_MMIO_LENGTH;
0105d1a5
GN		 1361}
1362
e32edf4f 1363static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1364 gpa_t address, int len, void *data)
1365{
1366 struct kvm_lapic *apic = to_lapic(this);
1367 u32 offset = address - apic->base_address;
1368
1369 if (!apic_mmio_in_range(apic, address))
1370 return -EOPNOTSUPP;
1371
d1766202
VK		 1372 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1373 if (!kvm_check_has_quirk(vcpu->kvm,
1374 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1375 return -EOPNOTSUPP;
1376
1377 memset(data, 0xff, len);
1378 return 0;
1379 }
1380
1e6e2755 1381 kvm_lapic_reg_read(apic, offset, len, data);
0105d1a5
GN		 1382
1383 return 0;
1384}
1385
97222cc8
ED		 1386static void update_divide_count(struct kvm_lapic *apic)
1387{
1388 u32 tmp1, tmp2, tdcr;
1389
dfb95954 1390 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
97222cc8
ED		 1391 tmp1 = tdcr & 0xf;
1392 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
d3c7b77d 1393 apic->divide_count = 0x1 << (tmp2 & 0x7);
97222cc8
ED		 1394}
1395
ccbfa1d3
WL		 1396static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1397{
1398 /*
1399 * Do not allow the guest to program periodic timers with small
1400 * interval, since the hrtimers are not throttled by the host
1401 * scheduler.
1402 */
dedf9c5e 1403 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
ccbfa1d3
WL		 1404 s64 min_period = min_timer_period_us * 1000LL;
1405
1406 if (apic->lapic_timer.period < min_period) {
1407 pr_info_ratelimited(
1408 "kvm: vcpu %i: requested %lld ns "
1409 "lapic timer period limited to %lld ns\n",
1410 apic->vcpu->vcpu_id,
1411 apic->lapic_timer.period, min_period);
1412 apic->lapic_timer.period = min_period;
1413 }
1414 }
1415}
1416
b6ac0695
RK		 1417static void apic_update_lvtt(struct kvm_lapic *apic)
1418{
dfb95954 1419 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
b6ac0695
RK		 1420 apic->lapic_timer.timer_mode_mask;
1421
1422 if (apic->lapic_timer.timer_mode != timer_mode) {
c69518c8 1423 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
dedf9c5e 1424 APIC_LVT_TIMER_TSCDEADLINE)) {
dedf9c5e 1425 hrtimer_cancel(&apic->lapic_timer.timer);
44275932
RK		 1426 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1427 apic->lapic_timer.period = 0;
1428 apic->lapic_timer.tscdeadline = 0;
dedf9c5e 1429 }
b6ac0695 1430 apic->lapic_timer.timer_mode = timer_mode;
dedf9c5e 1431 limit_periodic_timer_frequency(apic);
b6ac0695
RK		 1432 }
1433}
1434
d0659d94
MT		 1435/*
1436 * On APICv, this test will cause a busy wait
1437 * during a higher-priority task.
1438 */
1439
1440static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1441{
1442 struct kvm_lapic *apic = vcpu->arch.apic;
dfb95954 1443 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
d0659d94
MT		 1444
1445 if (kvm_apic_hw_enabled(apic)) {
1446 int vec = reg & APIC_VECTOR_MASK;
f9339860 1447 void *bitmap = apic->regs + APIC_ISR;
d0659d94 1448
d62caabb 1449 if (vcpu->arch.apicv_active)
f9339860
MT		 1450 bitmap = apic->regs + APIC_IRR;
1451
1452 if (apic_test_vector(vec, bitmap))
1453 return true;
d0659d94
MT		 1454 }
1455 return false;
1456}
1457
b6aa57c6
SC		 1458static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1459{
1460 u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1461
1462 /*
1463 * If the guest TSC is running at a different ratio than the host, then
1464 * convert the delay to nanoseconds to achieve an accurate delay. Note
1465 * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1466 * always for VMX enabled hardware.
1467 */
1468 if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1469 __delay(min(guest_cycles,
1470 nsec_to_cycles(vcpu, timer_advance_ns)));
1471 } else {
1472 u64 delay_ns = guest_cycles * 1000000ULL;
1473 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1474 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1475 }
1476}
1477
84ea3aca 1478static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
ec0671d5 1479 s64 advance_expire_delta)
d0659d94
MT		 1480{
1481 struct kvm_lapic *apic = vcpu->arch.apic;
39497d76 1482 u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
84ea3aca
WL		 1483 u64 ns;
1484
1485 /* too early */
ec0671d5
WL		 1486 if (advance_expire_delta < 0) {
1487 ns = -advance_expire_delta * 1000000ULL;
84ea3aca
WL		 1488 do_div(ns, vcpu->arch.virtual_tsc_khz);
1489 timer_advance_ns -= min((u32)ns,
1490 timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
1491 } else {
1492 /* too late */
ec0671d5 1493 ns = advance_expire_delta * 1000000ULL;
84ea3aca
WL		 1494 do_div(ns, vcpu->arch.virtual_tsc_khz);
1495 timer_advance_ns += min((u32)ns,
1496 timer_advance_ns / LAPIC_TIMER_ADVANCE_ADJUST_STEP);
1497 }
1498
ec0671d5 1499 if (abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_DONE)
84ea3aca
WL		 1500 apic->lapic_timer.timer_advance_adjust_done = true;
1501 if (unlikely(timer_advance_ns > 5000)) {
548f7fb2
WL		 1502 timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
1503 apic->lapic_timer.timer_advance_adjust_done = false;
84ea3aca
WL		 1504 }
1505 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1506}
1507
0c5f81da 1508static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
84ea3aca
WL		 1509{
1510 struct kvm_lapic *apic = vcpu->arch.apic;
1511 u64 guest_tsc, tsc_deadline;
d0659d94 1512
d0659d94
MT		 1513 if (apic->lapic_timer.expired_tscdeadline == 0)
1514 return;
1515
d0659d94
MT		 1516 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1517 apic->lapic_timer.expired_tscdeadline = 0;
4ba76538 1518 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
ec0671d5 1519 apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
d0659d94 1520
d0659d94 1521 if (guest_tsc < tsc_deadline)
b6aa57c6 1522 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
3b8a5df6 1523
84ea3aca 1524 if (unlikely(!apic->lapic_timer.timer_advance_adjust_done))
ec0671d5 1525 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
5d87db71 1526}
0c5f81da
WL		 1527
1528void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1529{
1530 if (lapic_timer_int_injected(vcpu))
1531 __kvm_wait_lapic_expire(vcpu);
1532}
b6c4bc65 1533EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
5d87db71 1534
0c5f81da
WL		 1535static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1536{
1537 struct kvm_timer *ktimer = &apic->lapic_timer;
1538
1539 kvm_apic_local_deliver(apic, APIC_LVTT);
1540 if (apic_lvtt_tscdeadline(apic))
1541 ktimer->tscdeadline = 0;
1542 if (apic_lvtt_oneshot(apic)) {
1543 ktimer->tscdeadline = 0;
1544 ktimer->target_expiration = 0;
1545 }
1546}
1547
1548static void apic_timer_expired(struct kvm_lapic *apic)
1549{
1550 struct kvm_vcpu *vcpu = apic->vcpu;
0c5f81da
WL		 1551 struct kvm_timer *ktimer = &apic->lapic_timer;
1552
1553 if (atomic_read(&apic->lapic_timer.pending))
1554 return;
1555
1556 if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1557 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1558
1559 if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1560 if (apic->lapic_timer.timer_advance_ns)
1561 __kvm_wait_lapic_expire(vcpu);
1562 kvm_apic_inject_pending_timer_irqs(apic);
1563 return;
1564 }
1565
1566 atomic_inc(&apic->lapic_timer.pending);
1567 kvm_set_pending_timer(vcpu);
0c5f81da
WL		 1568}
1569
53f9eedf
YJ		 1570static void start_sw_tscdeadline(struct kvm_lapic *apic)
1571{
39497d76
SC		 1572 struct kvm_timer *ktimer = &apic->lapic_timer;
1573 u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
53f9eedf
YJ		 1574 u64 ns = 0;
1575 ktime_t expire;
1576 struct kvm_vcpu *vcpu = apic->vcpu;
1577 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1578 unsigned long flags;
1579 ktime_t now;
1580
1581 if (unlikely(!tscdeadline || !this_tsc_khz))
1582 return;
1583
1584 local_irq_save(flags);
1585
5587859f 1586 now = ktime_get();
53f9eedf 1587 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
c09d65d9
LA		 1588
1589 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1590 do_div(ns, this_tsc_khz);
1591
1592 if (likely(tscdeadline > guest_tsc) &&
39497d76 1593 likely(ns > apic->lapic_timer.timer_advance_ns)) {
53f9eedf 1594 expire = ktime_add_ns(now, ns);
39497d76 1595 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
4d151bf3 1596 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS);
53f9eedf
YJ		 1597 } else
1598 apic_timer_expired(apic);
1599
1600 local_irq_restore(flags);
1601}
1602
c301b909
WL		 1603static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1604{
1605 ktime_t now, remaining;
1606 u64 ns_remaining_old, ns_remaining_new;
1607
1608 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1609 * APIC_BUS_CYCLE_NS * apic->divide_count;
1610 limit_periodic_timer_frequency(apic);
1611
1612 now = ktime_get();
1613 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1614 if (ktime_to_ns(remaining) < 0)
1615 remaining = 0;
1616
1617 ns_remaining_old = ktime_to_ns(remaining);
1618 ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1619 apic->divide_count, old_divisor);
1620
1621 apic->lapic_timer.tscdeadline +=
1622 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1623 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1624 apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1625}
1626
8003c9ae 1627static bool set_target_expiration(struct kvm_lapic *apic)
7d7f7da2
WL		 1628{
1629 ktime_t now;
8003c9ae 1630 u64 tscl = rdtsc();
7d7f7da2 1631
5587859f 1632 now = ktime_get();
7d7f7da2 1633 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
8003c9ae 1634 * APIC_BUS_CYCLE_NS * apic->divide_count;
7d7f7da2 1635
5d74a699
RK		 1636 if (!apic->lapic_timer.period) {
1637 apic->lapic_timer.tscdeadline = 0;
8003c9ae 1638 return false;
7d7f7da2
WL		 1639 }
1640
ccbfa1d3 1641 limit_periodic_timer_frequency(apic);
7d7f7da2 1642
8003c9ae
WL		 1643 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1644 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1645 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1646
1647 return true;
1648}
1649
1650static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1651{
d8f2f498
DV		 1652 ktime_t now = ktime_get();
1653 u64 tscl = rdtsc();
1654 ktime_t delta;
1655
1656 /*
1657 * Synchronize both deadlines to the same time source or
1658 * differences in the periods (caused by differences in the
1659 * underlying clocks or numerical approximation errors) will
1660 * cause the two to drift apart over time as the errors
1661 * accumulate.
1662 */
8003c9ae
WL		 1663 apic->lapic_timer.target_expiration =
1664 ktime_add_ns(apic->lapic_timer.target_expiration,
1665 apic->lapic_timer.period);
d8f2f498
DV		 1666 delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1667 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1668 nsec_to_cycles(apic->vcpu, delta);
7d7f7da2
WL		 1669}
1670
ecf08dad
AB		 1671static void start_sw_period(struct kvm_lapic *apic)
1672{
1673 if (!apic->lapic_timer.period)
1674 return;
1675
1676 if (ktime_after(ktime_get(),
1677 apic->lapic_timer.target_expiration)) {
1678 apic_timer_expired(apic);
1679
1680 if (apic_lvtt_oneshot(apic))
1681 return;
1682
1683 advance_periodic_target_expiration(apic);
1684 }
1685
1686 hrtimer_start(&apic->lapic_timer.timer,
1687 apic->lapic_timer.target_expiration,
4d151bf3 1688 HRTIMER_MODE_ABS);
ecf08dad
AB		 1689}
1690
ce7a058a
YJ		 1691bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1692{
91005300
WL		 1693 if (!lapic_in_kernel(vcpu))
1694 return false;
1695
ce7a058a
YJ		 1696 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1697}
1698EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1699
7e810a38 1700static void cancel_hv_timer(struct kvm_lapic *apic)
bd97ad0e 1701{
1d518c68 1702 WARN_ON(preemptible());
a749e247 1703 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
bd97ad0e
WL		 1704 kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1705 apic->lapic_timer.hv_timer_in_use = false;
1706}
1707
a749e247 1708static bool start_hv_timer(struct kvm_lapic *apic)
196f20ca 1709{
35ee9e48 1710 struct kvm_timer *ktimer = &apic->lapic_timer;
f9927982
SC		 1711 struct kvm_vcpu *vcpu = apic->vcpu;
1712 bool expired;
196f20ca 1713
1d518c68 1714 WARN_ON(preemptible());
a749e247
PB		 1715 if (!kvm_x86_ops->set_hv_timer)
1716 return false;
1717
86bbc1e6
RK		 1718 if (!ktimer->tscdeadline)
1719 return false;
1720
f9927982 1721 if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
35ee9e48
PB		 1722 return false;
1723
1724 ktimer->hv_timer_in_use = true;
1725 hrtimer_cancel(&ktimer->timer);
196f20ca 1726
35ee9e48 1727 /*
f1ba5cfb
SC		 1728 * To simplify handling the periodic timer, leave the hv timer running
1729 * even if the deadline timer has expired, i.e. rely on the resulting
1730 * VM-Exit to recompute the periodic timer's target expiration.
35ee9e48 1731 */
f1ba5cfb
SC		 1732 if (!apic_lvtt_period(apic)) {
1733 /*
1734 * Cancel the hv timer if the sw timer fired while the hv timer
1735 * was being programmed, or if the hv timer itself expired.
1736 */
1737 if (atomic_read(&ktimer->pending)) {
1738 cancel_hv_timer(apic);
f9927982 1739 } else if (expired) {
c8533544 1740 apic_timer_expired(apic);
f1ba5cfb
SC		 1741 cancel_hv_timer(apic);
1742 }
c8533544 1743 }
a749e247 1744
f9927982 1745 trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
f1ba5cfb 1746
35ee9e48
PB		 1747 return true;
1748}
1749
a749e247 1750static void start_sw_timer(struct kvm_lapic *apic)
35ee9e48 1751{
a749e247 1752 struct kvm_timer *ktimer = &apic->lapic_timer;
1d518c68
WL		 1753
1754 WARN_ON(preemptible());
a749e247
PB		 1755 if (apic->lapic_timer.hv_timer_in_use)
1756 cancel_hv_timer(apic);
1757 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1758 return;
1759
1760 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1761 start_sw_period(apic);
1762 else if (apic_lvtt_tscdeadline(apic))
1763 start_sw_tscdeadline(apic);
1764 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1765}
35ee9e48 1766
a749e247
PB		 1767static void restart_apic_timer(struct kvm_lapic *apic)
1768{
1d518c68 1769 preempt_disable();
4ca88b3f
SC		 1770
1771 if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1772 goto out;
1773
a749e247
PB		 1774 if (!start_hv_timer(apic))
1775 start_sw_timer(apic);
4ca88b3f 1776out:
1d518c68 1777 preempt_enable();
196f20ca
WL		 1778}
1779
8003c9ae
WL		 1780void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1781{
1782 struct kvm_lapic *apic = vcpu->arch.apic;
1783
1d518c68
WL		 1784 preempt_disable();
1785 /* If the preempt notifier has already run, it also called apic_timer_expired */
1786 if (!apic->lapic_timer.hv_timer_in_use)
1787 goto out;
8003c9ae
WL		 1788 WARN_ON(swait_active(&vcpu->wq));
1789 cancel_hv_timer(apic);
1790 apic_timer_expired(apic);
1791
1792 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1793 advance_periodic_target_expiration(apic);
a749e247 1794 restart_apic_timer(apic);
8003c9ae 1795 }
1d518c68
WL		 1796out:
1797 preempt_enable();
8003c9ae
WL		 1798}
1799EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1800
ce7a058a
YJ		 1801void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1802{
a749e247 1803 restart_apic_timer(vcpu->arch.apic);
ce7a058a
YJ		 1804}
1805EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1806
1807void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1808{
1809 struct kvm_lapic *apic = vcpu->arch.apic;
1810
1d518c68 1811 preempt_disable();
ce7a058a 1812 /* Possibly the TSC deadline timer is not enabled yet */
a749e247
PB		 1813 if (apic->lapic_timer.hv_timer_in_use)
1814 start_sw_timer(apic);
1d518c68 1815 preempt_enable();
a749e247
PB		 1816}
1817EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
ce7a058a 1818
a749e247
PB		 1819void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1820{
1821 struct kvm_lapic *apic = vcpu->arch.apic;
ce7a058a 1822
a749e247
PB		 1823 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1824 restart_apic_timer(apic);
ce7a058a 1825}
ce7a058a 1826
97222cc8
ED		 1827static void start_apic_timer(struct kvm_lapic *apic)
1828{
d3c7b77d 1829 atomic_set(&apic->lapic_timer.pending, 0);
0b975a3c 1830
a749e247
PB		 1831 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1832 && !set_target_expiration(apic))
1833 return;
1834
1835 restart_apic_timer(apic);
97222cc8
ED		 1836}
1837
cc6e462c
JK		 1838static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1839{
59fd1323 1840 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
cc6e462c 1841
59fd1323
RK		 1842 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1843 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1844 if (lvt0_in_nmi_mode) {
42720138 1845 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
59fd1323
RK		 1846 } else
1847 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1848 }
cc6e462c
JK		 1849}
1850
1e6e2755 1851int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
97222cc8 1852{
0105d1a5 1853 int ret = 0;
97222cc8 1854
0105d1a5 1855 trace_kvm_apic_write(reg, val);
97222cc8 1856
0105d1a5 1857 switch (reg) {
97222cc8 1858 case APIC_ID: /* Local APIC ID */
0105d1a5 1859 if (!apic_x2apic_mode(apic))
a92e2543 1860 kvm_apic_set_xapic_id(apic, val >> 24);
0105d1a5
GN		 1861 else
1862 ret = 1;
97222cc8
ED		 1863 break;
1864
1865 case APIC_TASKPRI:
b209749f 1866 report_tpr_access(apic, true);
97222cc8
ED		 1867 apic_set_tpr(apic, val & 0xff);
1868 break;
1869
1870 case APIC_EOI:
1871 apic_set_eoi(apic);
1872 break;
1873
1874 case APIC_LDR:
0105d1a5 1875 if (!apic_x2apic_mode(apic))
1e08ec4a 1876 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
0105d1a5
GN		 1877 else
1878 ret = 1;
97222cc8
ED		 1879 break;
1880
1881 case APIC_DFR:
1e08ec4a 1882 if (!apic_x2apic_mode(apic)) {
1e6e2755 1883 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1e08ec4a
GN		 1884 recalculate_apic_map(apic->vcpu->kvm);
1885 } else
0105d1a5 1886 ret = 1;
97222cc8
ED		 1887 break;
1888
fc61b800
GN		 1889 case APIC_SPIV: {
1890 u32 mask = 0x3ff;
dfb95954 1891 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
fc61b800 1892 mask |= APIC_SPIV_DIRECTED_EOI;
f8c1ea10 1893 apic_set_spiv(apic, val & mask);
97222cc8
ED		 1894 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1895 int i;
1896 u32 lvt_val;
1897
1e6e2755 1898 for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
dfb95954 1899 lvt_val = kvm_lapic_get_reg(apic,
97222cc8 1900 APIC_LVTT + 0x10 * i);
1e6e2755 1901 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
97222cc8
ED		 1902 lvt_val | APIC_LVT_MASKED);
1903 }
b6ac0695 1904 apic_update_lvtt(apic);
d3c7b77d 1905 atomic_set(&apic->lapic_timer.pending, 0);
97222cc8
ED		 1906
1907 }
1908 break;
fc61b800 1909 }
97222cc8
ED		 1910 case APIC_ICR:
1911 /* No delay here, so we always clear the pending bit */
1e6e2755 1912 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
97222cc8
ED		 1913 apic_send_ipi(apic);
1914 break;
1915
1916 case APIC_ICR2:
0105d1a5
GN		 1917 if (!apic_x2apic_mode(apic))
1918 val &= 0xff000000;
1e6e2755 1919 kvm_lapic_set_reg(apic, APIC_ICR2, val);
97222cc8
ED		 1920 break;
1921
23930f95 1922 case APIC_LVT0:
cc6e462c 1923 apic_manage_nmi_watchdog(apic, val);
b2869f28 1924 /* fall through */
97222cc8
ED		 1925 case APIC_LVTTHMR:
1926 case APIC_LVTPC:
97222cc8
ED		 1927 case APIC_LVT1:
1928 case APIC_LVTERR:
1929 /* TODO: Check vector */
c48f1496 1930 if (!kvm_apic_sw_enabled(apic))
97222cc8
ED		 1931 val |= APIC_LVT_MASKED;
1932
0105d1a5 1933 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1e6e2755 1934 kvm_lapic_set_reg(apic, reg, val);
97222cc8
ED		 1935
1936 break;
1937
b6ac0695 1938 case APIC_LVTT:
c48f1496 1939 if (!kvm_apic_sw_enabled(apic))
a3e06bbe
LJ		 1940 val |= APIC_LVT_MASKED;
1941 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1e6e2755 1942 kvm_lapic_set_reg(apic, APIC_LVTT, val);
b6ac0695 1943 apic_update_lvtt(apic);
a3e06bbe
LJ		 1944 break;
1945
97222cc8 1946 case APIC_TMICT:
a3e06bbe
LJ		 1947 if (apic_lvtt_tscdeadline(apic))
1948 break;
1949
d3c7b77d 1950 hrtimer_cancel(&apic->lapic_timer.timer);
1e6e2755 1951 kvm_lapic_set_reg(apic, APIC_TMICT, val);
97222cc8 1952 start_apic_timer(apic);
0105d1a5 1953 break;
97222cc8 1954
c301b909
WL		 1955 case APIC_TDCR: {
1956 uint32_t old_divisor = apic->divide_count;
1957
1e6e2755 1958 kvm_lapic_set_reg(apic, APIC_TDCR, val);
97222cc8 1959 update_divide_count(apic);
c301b909
WL		 1960 if (apic->divide_count != old_divisor &&
1961 apic->lapic_timer.period) {
1962 hrtimer_cancel(&apic->lapic_timer.timer);
1963 update_target_expiration(apic, old_divisor);
1964 restart_apic_timer(apic);
1965 }
97222cc8 1966 break;
c301b909 1967 }
0105d1a5 1968 case APIC_ESR:
0d88800d 1969 if (apic_x2apic_mode(apic) && val != 0)
0105d1a5 1970 ret = 1;
0105d1a5
GN		 1971 break;
1972
1973 case APIC_SELF_IPI:
1974 if (apic_x2apic_mode(apic)) {
1e6e2755 1975 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
0105d1a5
GN		 1976 } else
1977 ret = 1;
1978 break;
97222cc8 1979 default:
0105d1a5 1980 ret = 1;
97222cc8
ED		 1981 break;
1982 }
0d88800d 1983
0105d1a5
GN		 1984 return ret;
1985}
1e6e2755 1986EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
0105d1a5 1987
e32edf4f 1988static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1989 gpa_t address, int len, const void *data)
1990{
1991 struct kvm_lapic *apic = to_lapic(this);
1992 unsigned int offset = address - apic->base_address;
1993 u32 val;
1994
1995 if (!apic_mmio_in_range(apic, address))
1996 return -EOPNOTSUPP;
1997
d1766202
VK		 1998 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1999 if (!kvm_check_has_quirk(vcpu->kvm,
2000 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2001 return -EOPNOTSUPP;
2002
2003 return 0;
2004 }
2005
0105d1a5
GN		 2006 /*
2007 * APIC register must be aligned on 128-bits boundary.
2008 * 32/64/128 bits registers must be accessed thru 32 bits.
2009 * Refer SDM 8.4.1
2010 */
0d88800d 2011 if (len != 4 || (offset & 0xf))
756975bb 2012 return 0;
0105d1a5
GN		 2013
2014 val = *(u32*)data;
2015
0d88800d 2016 kvm_lapic_reg_write(apic, offset & 0xff0, val);
0105d1a5 2017
bda9020e 2018 return 0;
97222cc8
ED		 2019}
2020
58fbbf26
KT		 2021void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2022{
1e6e2755 2023 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
58fbbf26
KT		 2024}
2025EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2026
83d4c286
YZ		 2027/* emulate APIC access in a trap manner */
2028void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2029{
2030 u32 val = 0;
2031
2032 /* hw has done the conditional check and inst decode */
2033 offset &= 0xff0;
2034
1e6e2755 2035 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
83d4c286
YZ		 2036
2037 /* TODO: optimize to just emulate side effect w/o one more write */
1e6e2755 2038 kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
83d4c286
YZ		 2039}
2040EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2041
d589444e 2042void kvm_free_lapic(struct kvm_vcpu *vcpu)
97222cc8 2043{
f8c1ea10
GN		 2044 struct kvm_lapic *apic = vcpu->arch.apic;
2045
ad312c7c 2046 if (!vcpu->arch.apic)
97222cc8
ED		 2047 return;
2048
f8c1ea10 2049 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 2050
c5cc421b
GN		 2051 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2052 static_key_slow_dec_deferred(&apic_hw_disabled);
2053
e462755c 2054 if (!apic->sw_enabled)
f8c1ea10 2055 static_key_slow_dec_deferred(&apic_sw_disabled);
97222cc8 2056
f8c1ea10
GN		 2057 if (apic->regs)
2058 free_page((unsigned long)apic->regs);
2059
2060 kfree(apic);
97222cc8
ED		 2061}
2062
2063/*
2064 *----------------------------------------------------------------------
2065 * LAPIC interface
2066 *----------------------------------------------------------------------
2067 */
a3e06bbe
LJ		 2068u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2069{
2070 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 2071
a10388e1
WL		 2072 if (!lapic_in_kernel(vcpu) ||
2073 !apic_lvtt_tscdeadline(apic))
a3e06bbe
LJ		 2074 return 0;
2075
2076 return apic->lapic_timer.tscdeadline;
2077}
2078
2079void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2080{
2081 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 2082
bce87cce 2083 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
54e9818f 2084 apic_lvtt_period(apic))
a3e06bbe
LJ		 2085 return;
2086
2087 hrtimer_cancel(&apic->lapic_timer.timer);
2088 apic->lapic_timer.tscdeadline = data;
2089 start_apic_timer(apic);
2090}
2091
97222cc8
ED		 2092void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2093{
ad312c7c 2094 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 2095
b93463aa 2096 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
dfb95954 2097 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
97222cc8
ED		 2098}
2099
2100u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2101{
97222cc8
ED		 2102 u64 tpr;
2103
dfb95954 2104 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
97222cc8
ED		 2105
2106 return (tpr & 0xf0) >> 4;
2107}
2108
2109void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2110{
8d14695f 2111 u64 old_value = vcpu->arch.apic_base;
ad312c7c 2112 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 2113
c7dd15b3 2114 if (!apic)
97222cc8 2115 value |= MSR_IA32_APICBASE_BSP;
c5af89b6 2116
e66d2ae7
JK		 2117 vcpu->arch.apic_base = value;
2118
c7dd15b3
JM		 2119 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2120 kvm_update_cpuid(vcpu);
2121
2122 if (!apic)
2123 return;
2124
c5cc421b 2125 /* update jump label if enable bit changes */
0dce7cd6 2126 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
49bd29ba
RK		 2127 if (value & MSR_IA32_APICBASE_ENABLE) {
2128 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
c5cc421b 2129 static_key_slow_dec_deferred(&apic_hw_disabled);
187ca84b 2130 } else {
c5cc421b 2131 static_key_slow_inc(&apic_hw_disabled.key);
187ca84b
WL		 2132 recalculate_apic_map(vcpu->kvm);
2133 }
c5cc421b
GN		 2134 }
2135
8d860bbe
JM		 2136 if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2137 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2138
2139 if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2140 kvm_x86_ops->set_virtual_apic_mode(vcpu);
8d14695f 2141
ad312c7c 2142 apic->base_address = apic->vcpu->arch.apic_base &
97222cc8
ED		 2143 MSR_IA32_APICBASE_BASE;
2144
db324fe6
NA		 2145 if ((value & MSR_IA32_APICBASE_ENABLE) &&
2146 apic->base_address != APIC_DEFAULT_PHYS_BASE)
2147 pr_warn_once("APIC base relocation is unsupported by KVM");
97222cc8
ED		 2148}
2149
d28bc9dd 2150void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
97222cc8 2151{
b7e31be3 2152 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8
ED		 2153 int i;
2154
b7e31be3
RK		 2155 if (!apic)
2156 return;
97222cc8 2157
97222cc8 2158 /* Stop the timer in case it's a reset to an active apic */
d3c7b77d 2159 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 2160
4d8e772b
RK		 2161 if (!init_event) {
2162 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
2163 MSR_IA32_APICBASE_ENABLE);
a92e2543 2164 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
4d8e772b 2165 }
fc61b800 2166 kvm_apic_set_version(apic->vcpu);
97222cc8 2167
1e6e2755
SS		 2168 for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2169 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
b6ac0695 2170 apic_update_lvtt(apic);
52b54190
JS		 2171 if (kvm_vcpu_is_reset_bsp(vcpu) &&
2172 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
1e6e2755 2173 kvm_lapic_set_reg(apic, APIC_LVT0,
90de4a18 2174 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
dfb95954 2175 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
97222cc8 2176
1e6e2755 2177 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
f8c1ea10 2178 apic_set_spiv(apic, 0xff);
1e6e2755 2179 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
c028dd6b
RK		 2180 if (!apic_x2apic_mode(apic))
2181 kvm_apic_set_ldr(apic, 0);
1e6e2755
SS		 2182 kvm_lapic_set_reg(apic, APIC_ESR, 0);
2183 kvm_lapic_set_reg(apic, APIC_ICR, 0);
2184 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2185 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2186 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
97222cc8 2187 for (i = 0; i < 8; i++) {
1e6e2755
SS		 2188 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2189 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2190 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
97222cc8 2191 }
d62caabb
AS		 2192 apic->irr_pending = vcpu->arch.apicv_active;
2193 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
8680b94b 2194 apic->highest_isr_cache = -1;
b33ac88b 2195 update_divide_count(apic);
d3c7b77d 2196 atomic_set(&apic->lapic_timer.pending, 0);
c5af89b6 2197 if (kvm_vcpu_is_bsp(vcpu))
5dbc8f3f
GN		 2198 kvm_lapic_set_base(vcpu,
2199 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
ae7a2a3f 2200 vcpu->arch.pv_eoi.msr_val = 0;
97222cc8 2201 apic_update_ppr(apic);
4191db26
JS		 2202 if (vcpu->arch.apicv_active) {
2203 kvm_x86_ops->apicv_post_state_restore(vcpu);
2204 kvm_x86_ops->hwapic_irr_update(vcpu, -1);
2205 kvm_x86_ops->hwapic_isr_update(vcpu, -1);
2206 }
97222cc8 2207
e1035715 2208 vcpu->arch.apic_arb_prio = 0;
41383771 2209 vcpu->arch.apic_attention = 0;
97222cc8
ED		 2210}
2211
97222cc8
ED		 2212/*
2213 *----------------------------------------------------------------------
2214 * timer interface
2215 *----------------------------------------------------------------------
2216 */
1b9778da 2217
2a6eac96 2218static bool lapic_is_periodic(struct kvm_lapic *apic)
97222cc8 2219{
d3c7b77d 2220 return apic_lvtt_period(apic);
97222cc8
ED		 2221}
2222
3d80840d
MT		 2223int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2224{
54e9818f 2225 struct kvm_lapic *apic = vcpu->arch.apic;
3d80840d 2226
1e3161b4 2227 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
54e9818f 2228 return atomic_read(&apic->lapic_timer.pending);
3d80840d
MT		 2229
2230 return 0;
2231}
2232
89342082 2233int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1b9778da 2234{
dfb95954 2235 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
23930f95 2236 int vector, mode, trig_mode;
23930f95 2237
c48f1496 2238 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
23930f95
JK		 2239 vector = reg & APIC_VECTOR_MASK;
2240 mode = reg & APIC_MODE_MASK;
2241 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
b4f2225c
YZ		 2242 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2243 NULL);
23930f95
JK		 2244 }
2245 return 0;
2246}
1b9778da 2247
8fdb2351 2248void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
23930f95 2249{
8fdb2351
JK		 2250 struct kvm_lapic *apic = vcpu->arch.apic;
2251
2252 if (apic)
2253 kvm_apic_local_deliver(apic, APIC_LVT0);
1b9778da
ED		 2254}
2255
d76685c4
GH		 2256static const struct kvm_io_device_ops apic_mmio_ops = {
2257 .read = apic_mmio_read,
2258 .write = apic_mmio_write,
d76685c4
GH		 2259};
2260
e9d90d47
AK		 2261static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2262{
2263 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2a6eac96 2264 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
e9d90d47 2265
5d87db71 2266 apic_timer_expired(apic);
e9d90d47 2267
2a6eac96 2268 if (lapic_is_periodic(apic)) {
8003c9ae 2269 advance_periodic_target_expiration(apic);
e9d90d47
AK		 2270 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2271 return HRTIMER_RESTART;
2272 } else
2273 return HRTIMER_NORESTART;
2274}
2275
c3941d9e 2276int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
97222cc8
ED		 2277{
2278 struct kvm_lapic *apic;
2279
2280 ASSERT(vcpu != NULL);
97222cc8 2281
254272ce 2282 apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
97222cc8
ED		 2283 if (!apic)
2284 goto nomem;
2285
ad312c7c 2286 vcpu->arch.apic = apic;
97222cc8 2287
254272ce 2288 apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
afc20184 2289 if (!apic->regs) {
97222cc8
ED		 2290 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2291 vcpu->vcpu_id);
d589444e 2292 goto nomem_free_apic;
97222cc8 2293 }
97222cc8
ED		 2294 apic->vcpu = vcpu;
2295
d3c7b77d 2296 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
4d151bf3 2297 HRTIMER_MODE_ABS);
e9d90d47 2298 apic->lapic_timer.timer.function = apic_timer_fn;
c3941d9e 2299 if (timer_advance_ns == -1) {
548f7fb2 2300 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_ADJUST_INIT;
c3941d9e
SC		 2301 apic->lapic_timer.timer_advance_adjust_done = false;
2302 } else {
2303 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2304 apic->lapic_timer.timer_advance_adjust_done = true;
2305 }
2306
d3c7b77d 2307
c5cc421b
GN		 2308 /*
2309 * APIC is created enabled. This will prevent kvm_lapic_set_base from
ee171d2f 2310 * thinking that APIC state has changed.
c5cc421b
GN		 2311 */
2312 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
f8c1ea10 2313 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
d76685c4 2314 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
97222cc8
ED		 2315
2316 return 0;
d589444e
RR		 2317nomem_free_apic:
2318 kfree(apic);
a251fb90 2319 vcpu->arch.apic = NULL;
97222cc8 2320nomem:
97222cc8
ED		 2321 return -ENOMEM;
2322}
97222cc8
ED		 2323
2324int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2325{
ad312c7c 2326 struct kvm_lapic *apic = vcpu->arch.apic;
b3c045d3 2327 u32 ppr;
97222cc8 2328
bb34e690 2329 if (!kvm_apic_hw_enabled(apic))
97222cc8
ED		 2330 return -1;
2331
b3c045d3
PB		 2332 __apic_update_ppr(apic, &ppr);
2333 return apic_has_interrupt_for_ppr(apic, ppr);
97222cc8
ED		 2334}
2335
40487c68
QH		 2336int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2337{
dfb95954 2338 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
40487c68
QH		 2339 int r = 0;
2340
c48f1496 2341 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
e7dca5c0
CL		 2342 r = 1;
2343 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2344 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2345 r = 1;
40487c68
QH		 2346 return r;
2347}
2348
1b9778da
ED		 2349void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2350{
ad312c7c 2351 struct kvm_lapic *apic = vcpu->arch.apic;
1b9778da 2352
54e9818f 2353 if (atomic_read(&apic->lapic_timer.pending) > 0) {
0c5f81da 2354 kvm_apic_inject_pending_timer_irqs(apic);
f1ed0450 2355 atomic_set(&apic->lapic_timer.pending, 0);
1b9778da
ED		 2356 }
2357}
2358
97222cc8
ED		 2359int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2360{
2361 int vector = kvm_apic_has_interrupt(vcpu);
ad312c7c 2362 struct kvm_lapic *apic = vcpu->arch.apic;
4d82d12b 2363 u32 ppr;
97222cc8
ED		 2364
2365 if (vector == -1)
2366 return -1;
2367
56cc2406
WL		 2368 /*
2369 * We get here even with APIC virtualization enabled, if doing
2370 * nested virtualization and L1 runs with the "acknowledge interrupt
2371 * on exit" mode. Then we cannot inject the interrupt via RVI,
2372 * because the process would deliver it through the IDT.
2373 */
2374
97222cc8 2375 apic_clear_irr(vector, apic);
5c919412 2376 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
4d82d12b
PB		 2377 /*
2378 * For auto-EOI interrupts, there might be another pending
2379 * interrupt above PPR, so check whether to raise another
2380 * KVM_REQ_EVENT.
2381 */
5c919412 2382 apic_update_ppr(apic);
4d82d12b
PB		 2383 } else {
2384 /*
2385 * For normal interrupts, PPR has been raised and there cannot
2386 * be a higher-priority pending interrupt---except if there was
2387 * a concurrent interrupt injection, but that would have
2388 * triggered KVM_REQ_EVENT already.
2389 */
2390 apic_set_isr(vector, apic);
2391 __apic_update_ppr(apic, &ppr);
5c919412
AS		 2392 }
2393
97222cc8
ED		 2394 return vector;
2395}
96ad2cc6 2396
a92e2543
RK		 2397static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2398 struct kvm_lapic_state *s, bool set)
2399{
2400 if (apic_x2apic_mode(vcpu->arch.apic)) {
2401 u32 *id = (u32 *)(s->regs + APIC_ID);
12806ba9 2402 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
a92e2543 2403
37131313
RK		 2404 if (vcpu->kvm->arch.x2apic_format) {
2405 if (*id != vcpu->vcpu_id)
2406 return -EINVAL;
2407 } else {
2408 if (set)
2409 *id >>= 24;
2410 else
2411 *id <<= 24;
2412 }
12806ba9
DDAG		 2413
2414 /* In x2APIC mode, the LDR is fixed and based on the id */
2415 if (set)
2416 *ldr = kvm_apic_calc_x2apic_ldr(*id);
a92e2543
RK		 2417 }
2418
2419 return 0;
2420}
2421
2422int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2423{
2424 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2425 return kvm_apic_state_fixup(vcpu, s, false);
2426}
2427
2428int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
96ad2cc6 2429{
ad312c7c 2430 struct kvm_lapic *apic = vcpu->arch.apic;
a92e2543
RK		 2431 int r;
2432
96ad2cc6 2433
5dbc8f3f 2434 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
64eb0620
GN		 2435 /* set SPIV separately to get count of SW disabled APICs right */
2436 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
a92e2543
RK		 2437
2438 r = kvm_apic_state_fixup(vcpu, s, true);
2439 if (r)
2440 return r;
0e96f31e 2441 memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
a92e2543
RK		 2442
2443 recalculate_apic_map(vcpu->kvm);
fc61b800
GN		 2444 kvm_apic_set_version(vcpu);
2445
96ad2cc6 2446 apic_update_ppr(apic);
d3c7b77d 2447 hrtimer_cancel(&apic->lapic_timer.timer);
b6ac0695 2448 apic_update_lvtt(apic);
dfb95954 2449 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
96ad2cc6
ED		 2450 update_divide_count(apic);
2451 start_apic_timer(apic);
6e24a6ef 2452 apic->irr_pending = true;
d62caabb 2453 apic->isr_count = vcpu->arch.apicv_active ?
c7c9c56c 2454 1 : count_vectors(apic->regs + APIC_ISR);
8680b94b 2455 apic->highest_isr_cache = -1;
d62caabb 2456 if (vcpu->arch.apicv_active) {
967235d3 2457 kvm_x86_ops->apicv_post_state_restore(vcpu);
4114c27d
WW		 2458 kvm_x86_ops->hwapic_irr_update(vcpu,
2459 apic_find_highest_irr(apic));
67c9dddc 2460 kvm_x86_ops->hwapic_isr_update(vcpu,
b4eef9b3 2461 apic_find_highest_isr(apic));
d62caabb 2462 }
3842d135 2463 kvm_make_request(KVM_REQ_EVENT, vcpu);
49df6397
SR		 2464 if (ioapic_in_kernel(vcpu->kvm))
2465 kvm_rtc_eoi_tracking_restore_one(vcpu);
0669a510
RK		 2466
2467 vcpu->arch.apic_arb_prio = 0;
a92e2543
RK		 2468
2469 return 0;
96ad2cc6 2470}
a3d7f85f 2471
2f52d58c 2472void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
a3d7f85f 2473{
a3d7f85f
ED		 2474 struct hrtimer *timer;
2475
0c5f81da
WL		 2476 if (!lapic_in_kernel(vcpu) ||
2477 kvm_can_post_timer_interrupt(vcpu))
a3d7f85f
ED		 2478 return;
2479
54e9818f 2480 timer = &vcpu->arch.apic->lapic_timer.timer;
a3d7f85f 2481 if (hrtimer_cancel(timer))
4d151bf3 2482 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
a3d7f85f 2483}
b93463aa 2484
ae7a2a3f
MT		 2485/*
2486 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2487 *
2488 * Detect whether guest triggered PV EOI since the
2489 * last entry. If yes, set EOI on guests's behalf.
2490 * Clear PV EOI in guest memory in any case.
2491 */
2492static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2493 struct kvm_lapic *apic)
2494{
2495 bool pending;
2496 int vector;
2497 /*
2498 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2499 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2500 *
2501 * KVM_APIC_PV_EOI_PENDING is unset:
2502 * -> host disabled PV EOI.
2503 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2504 * -> host enabled PV EOI, guest did not execute EOI yet.
2505 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2506 * -> host enabled PV EOI, guest executed EOI.
2507 */
2508 BUG_ON(!pv_eoi_enabled(vcpu));
2509 pending = pv_eoi_get_pending(vcpu);
2510 /*
2511 * Clear pending bit in any case: it will be set again on vmentry.
2512 * While this might not be ideal from performance point of view,
2513 * this makes sure pv eoi is only enabled when we know it's safe.
2514 */
2515 pv_eoi_clr_pending(vcpu);
2516 if (pending)
2517 return;
2518 vector = apic_set_eoi(apic);
2519 trace_kvm_pv_eoi(apic, vector);
2520}
2521
b93463aa
AK		 2522void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2523{
2524 u32 data;
b93463aa 2525
ae7a2a3f
MT		 2526 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2527 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2528
41383771 2529 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2530 return;
2531
4e335d9e
PB		 2532 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2533 sizeof(u32)))
603242a8 2534 return;
b93463aa
AK		 2535
2536 apic_set_tpr(vcpu->arch.apic, data & 0xff);
2537}
2538
ae7a2a3f
MT		 2539/*
2540 * apic_sync_pv_eoi_to_guest - called before vmentry
2541 *
2542 * Detect whether it's safe to enable PV EOI and
2543 * if yes do so.
2544 */
2545static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2546 struct kvm_lapic *apic)
2547{
2548 if (!pv_eoi_enabled(vcpu) ||
2549 /* IRR set or many bits in ISR: could be nested. */
2550 apic->irr_pending ||
2551 /* Cache not set: could be safe but we don't bother. */
2552 apic->highest_isr_cache == -1 ||
2553 /* Need EOI to update ioapic. */
3bb345f3 2554 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
ae7a2a3f
MT		 2555 /*
2556 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2557 * so we need not do anything here.
2558 */
2559 return;
2560 }
2561
2562 pv_eoi_set_pending(apic->vcpu);
2563}
2564
b93463aa
AK		 2565void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2566{
2567 u32 data, tpr;
2568 int max_irr, max_isr;
ae7a2a3f 2569 struct kvm_lapic *apic = vcpu->arch.apic;
b93463aa 2570
ae7a2a3f
MT		 2571 apic_sync_pv_eoi_to_guest(vcpu, apic);
2572
41383771 2573 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2574 return;
2575
dfb95954 2576 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
b93463aa
AK		 2577 max_irr = apic_find_highest_irr(apic);
2578 if (max_irr < 0)
2579 max_irr = 0;
2580 max_isr = apic_find_highest_isr(apic);
2581 if (max_isr < 0)
2582 max_isr = 0;
2583 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2584
4e335d9e
PB		 2585 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2586 sizeof(u32));
b93463aa
AK		 2587}
2588
fda4e2e8 2589int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
b93463aa 2590{
fda4e2e8 2591 if (vapic_addr) {
4e335d9e 2592 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
fda4e2e8
AH		 2593 &vcpu->arch.apic->vapic_cache,
2594 vapic_addr, sizeof(u32)))
2595 return -EINVAL;
41383771 2596 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8 2597 } else {
41383771 2598 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8
AH		 2599 }
2600
2601 vcpu->arch.apic->vapic_addr = vapic_addr;
2602 return 0;
b93463aa 2603}
0105d1a5
GN		 2604
2605int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2606{
2607 struct kvm_lapic *apic = vcpu->arch.apic;
2608 u32 reg = (msr - APIC_BASE_MSR) << 4;
2609
35754c98 2610 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2611 return 1;
2612
c69d3d9b
NA		 2613 if (reg == APIC_ICR2)
2614 return 1;
2615
0105d1a5 2616 /* if this is ICR write vector before command */
decdc283 2617 if (reg == APIC_ICR)
1e6e2755
SS		 2618 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2619 return kvm_lapic_reg_write(apic, reg, (u32)data);
0105d1a5
GN		 2620}
2621
2622int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2623{
2624 struct kvm_lapic *apic = vcpu->arch.apic;
2625 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2626
35754c98 2627 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2628 return 1;
2629
0d88800d 2630 if (reg == APIC_DFR || reg == APIC_ICR2)
c69d3d9b 2631 return 1;
c69d3d9b 2632
1e6e2755 2633 if (kvm_lapic_reg_read(apic, reg, 4, &low))
0105d1a5 2634 return 1;
decdc283 2635 if (reg == APIC_ICR)
1e6e2755 2636 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
0105d1a5
GN		 2637
2638 *data = (((u64)high) << 32) | low;
2639
2640 return 0;
2641}
10388a07
GN		 2642
2643int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2644{
2645 struct kvm_lapic *apic = vcpu->arch.apic;
2646
bce87cce 2647 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2648 return 1;
2649
2650 /* if this is ICR write vector before command */
2651 if (reg == APIC_ICR)
1e6e2755
SS		 2652 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2653 return kvm_lapic_reg_write(apic, reg, (u32)data);
10388a07
GN		 2654}
2655
2656int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2657{
2658 struct kvm_lapic *apic = vcpu->arch.apic;
2659 u32 low, high = 0;
2660
bce87cce 2661 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2662 return 1;
2663
1e6e2755 2664 if (kvm_lapic_reg_read(apic, reg, 4, &low))
10388a07
GN		 2665 return 1;
2666 if (reg == APIC_ICR)
1e6e2755 2667 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
10388a07
GN		 2668
2669 *data = (((u64)high) << 32) | low;
2670
2671 return 0;
2672}
ae7a2a3f 2673
72bbf935 2674int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
ae7a2a3f
MT		 2675{
2676 u64 addr = data & ~KVM_MSR_ENABLED;
a7c42bb6
VK		 2677 struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2678 unsigned long new_len;
2679
ae7a2a3f
MT		 2680 if (!IS_ALIGNED(addr, 4))
2681 return 1;
2682
2683 vcpu->arch.pv_eoi.msr_val = data;
2684 if (!pv_eoi_enabled(vcpu))
2685 return 0;
a7c42bb6
VK		 2686
2687 if (addr == ghc->gpa && len <= ghc->len)
2688 new_len = ghc->len;
2689 else
2690 new_len = len;
2691
2692 return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
ae7a2a3f 2693}
c5cc421b 2694
66450a21
JK		 2695void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2696{
2697 struct kvm_lapic *apic = vcpu->arch.apic;
2b4a273b 2698 u8 sipi_vector;
299018f4 2699 unsigned long pe;
66450a21 2700
bce87cce 2701 if (!lapic_in_kernel(vcpu) || !apic->pending_events)
66450a21
JK		 2702 return;
2703
cd7764fe
PB		 2704 /*
2705 * INITs are latched while in SMM. Because an SMM CPU cannot
2706 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2707 * and delay processing of INIT until the next RSM.
2708 */
2709 if (is_smm(vcpu)) {
2710 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2711 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2712 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2713 return;
2714 }
299018f4 2715
cd7764fe 2716 pe = xchg(&apic->pending_events, 0);
299018f4 2717 if (test_bit(KVM_APIC_INIT, &pe)) {
d28bc9dd 2718 kvm_vcpu_reset(vcpu, true);
66450a21
JK		 2719 if (kvm_vcpu_is_bsp(apic->vcpu))
2720 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2721 else
2722 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2723 }
299018f4 2724 if (test_bit(KVM_APIC_SIPI, &pe) &&
66450a21
JK		 2725 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2726 /* evaluate pending_events before reading the vector */
2727 smp_rmb();
2728 sipi_vector = apic->sipi_vector;
66450a21
JK		 2729 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2730 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2731 }
2732}
2733
c5cc421b
GN		 2734void kvm_lapic_init(void)
2735{
2736 /* do not patch jump label more than once per second */
2737 jump_label_rate_limit(&apic_hw_disabled, HZ);
f8c1ea10 2738 jump_label_rate_limit(&apic_sw_disabled, HZ);
c5cc421b 2739}
cef84c30
DM		 2740
2741void kvm_lapic_exit(void)
2742{
2743 static_key_deferred_flush(&apic_hw_disabled);
2744 static_key_deferred_flush(&apic_sw_disabled);
2745}
A mirror of Linus' kernel repository