]> git.ipfire.org Git - thirdparty/linux.git/blame - arch/x86/kvm/lapic.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tools/kvm_stat: add '-f help' to get the available event list
[thirdparty/linux.git] / arch / x86 / kvm / lapic.c
CommitLineData
97222cc8
ED		 1
2/*
3 * Local APIC virtualization
4 *
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright (C) 2007 Novell
7 * Copyright (C) 2007 Intel
9611c187 8 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
97222cc8
ED		 9 *
10 * Authors:
11 * Dor Laor <dor.laor@qumranet.com>
12 * Gregory Haskins <ghaskins@novell.com>
13 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
14 *
15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
16 *
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
19 */
20
edf88417 21#include <linux/kvm_host.h>
97222cc8
ED		 22#include <linux/kvm.h>
23#include <linux/mm.h>
24#include <linux/highmem.h>
25#include <linux/smp.h>
26#include <linux/hrtimer.h>
27#include <linux/io.h>
1767e931 28#include <linux/export.h>
6f6d6a1a 29#include <linux/math64.h>
5a0e3ad6 30#include <linux/slab.h>
97222cc8
ED		 31#include <asm/processor.h>
32#include <asm/msr.h>
33#include <asm/page.h>
34#include <asm/current.h>
35#include <asm/apicdef.h>
d0659d94 36#include <asm/delay.h>
60063497 37#include <linux/atomic.h>
c5cc421b 38#include <linux/jump_label.h>
5fdbf976 39#include "kvm_cache_regs.h"
97222cc8 40#include "irq.h"
229456fc 41#include "trace.h"
fc61b800 42#include "x86.h"
00b27a3e 43#include "cpuid.h"
5c919412 44#include "hyperv.h"
97222cc8 45
b682b814
MT		 46#ifndef CONFIG_X86_64
47#define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
48#else
49#define mod_64(x, y) ((x) % (y))
50#endif
51
97222cc8
ED		 52#define PRId64 "d"
53#define PRIx64 "llx"
54#define PRIu64 "u"
55#define PRIo64 "o"
56
57#define APIC_BUS_CYCLE_NS 1
58
59/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
60#define apic_debug(fmt, arg...)
61
97222cc8 62/* 14 is the version for Xeon and Pentium 8.4.8*/
1e6e2755 63#define APIC_VERSION (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
97222cc8
ED		 64#define LAPIC_MMIO_LENGTH (1 << 12)
65/* followed define is not in apicdef.h */
66#define APIC_SHORT_MASK 0xc0000
67#define APIC_DEST_NOSHORT 0x0
68#define APIC_DEST_MASK 0x800
69#define MAX_APIC_VECTOR 256
ecba9a52 70#define APIC_VECTORS_PER_REG 32
97222cc8 71
394457a9
NA		 72#define APIC_BROADCAST 0xFF
73#define X2APIC_BROADCAST 0xFFFFFFFFul
74
a0c9a822
MT		 75static inline int apic_test_vector(int vec, void *bitmap)
76{
77 return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
78}
79
10606919
YZ		 80bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
81{
82 struct kvm_lapic *apic = vcpu->arch.apic;
83
84 return apic_test_vector(vector, apic->regs + APIC_ISR) ||
85 apic_test_vector(vector, apic->regs + APIC_IRR);
86}
87
97222cc8
ED		 88static inline void apic_clear_vector(int vec, void *bitmap)
89{
90 clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
91}
92
8680b94b
MT		 93static inline int __apic_test_and_set_vector(int vec, void *bitmap)
94{
95 return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
96}
97
98static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
99{
100 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
101}
102
c5cc421b 103struct static_key_deferred apic_hw_disabled __read_mostly;
f8c1ea10
GN		 104struct static_key_deferred apic_sw_disabled __read_mostly;
105
97222cc8
ED		 106static inline int apic_enabled(struct kvm_lapic *apic)
107{
c48f1496 108 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
54e9818f
GN		 109}
110
97222cc8
ED		 111#define LVT_MASK \
112 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
113
114#define LINT_MASK \
115 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
116 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
117
6e500439
RK		 118static inline u8 kvm_xapic_id(struct kvm_lapic *apic)
119{
120 return kvm_lapic_get_reg(apic, APIC_ID) >> 24;
121}
122
123static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
124{
125 return apic->vcpu->vcpu_id;
126}
127
e45115b6
RK		 128static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
129 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
130 switch (map->mode) {
131 case KVM_APIC_MODE_X2APIC: {
132 u32 offset = (dest_id >> 16) * 16;
0ca52e7b 133 u32 max_apic_id = map->max_apic_id;
e45115b6
RK		 134
135 if (offset <= max_apic_id) {
136 u8 cluster_size = min(max_apic_id - offset + 1, 16U);
137
138 *cluster = &map->phys_map[offset];
139 *mask = dest_id & (0xffff >> (16 - cluster_size));
140 } else {
141 *mask = 0;
142 }
3b5a5ffa 143
e45115b6
RK		 144 return true;
145 }
146 case KVM_APIC_MODE_XAPIC_FLAT:
147 *cluster = map->xapic_flat_map;
148 *mask = dest_id & 0xff;
149 return true;
150 case KVM_APIC_MODE_XAPIC_CLUSTER:
444fdad8 151 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
e45115b6
RK		 152 *mask = dest_id & 0xf;
153 return true;
154 default:
155 /* Not optimized. */
156 return false;
157 }
3548a259
RK		 158}
159
af1bae54 160static void kvm_apic_map_free(struct rcu_head *rcu)
3b5a5ffa 161{
af1bae54 162 struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
3b5a5ffa 163
af1bae54 164 kvfree(map);
3b5a5ffa
RK		 165}
166
1e08ec4a
GN		 167static void recalculate_apic_map(struct kvm *kvm)
168{
169 struct kvm_apic_map *new, *old = NULL;
170 struct kvm_vcpu *vcpu;
171 int i;
6e500439 172 u32 max_id = 255; /* enough space for any xAPIC ID */
1e08ec4a
GN		 173
174 mutex_lock(&kvm->arch.apic_map_lock);
175
0ca52e7b
RK		 176 kvm_for_each_vcpu(i, vcpu, kvm)
177 if (kvm_apic_present(vcpu))
6e500439 178 max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
0ca52e7b 179
a7c3e901
MH		 180 new = kvzalloc(sizeof(struct kvm_apic_map) +
181 sizeof(struct kvm_lapic *) * ((u64)max_id + 1), GFP_KERNEL);
0ca52e7b 182
1e08ec4a
GN		 183 if (!new)
184 goto out;
185
0ca52e7b
RK		 186 new->max_apic_id = max_id;
187
173beedc
NA		 188 kvm_for_each_vcpu(i, vcpu, kvm) {
189 struct kvm_lapic *apic = vcpu->arch.apic;
e45115b6
RK		 190 struct kvm_lapic **cluster;
191 u16 mask;
5bd5db38
RK		 192 u32 ldr;
193 u8 xapic_id;
194 u32 x2apic_id;
1e08ec4a 195
df04d1d1
RK		 196 if (!kvm_apic_present(vcpu))
197 continue;
198
5bd5db38
RK		 199 xapic_id = kvm_xapic_id(apic);
200 x2apic_id = kvm_x2apic_id(apic);
201
202 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
203 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
204 x2apic_id <= new->max_apic_id)
205 new->phys_map[x2apic_id] = apic;
206 /*
207 * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
208 * prevent them from masking VCPUs with APIC ID <= 0xff.
209 */
210 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
211 new->phys_map[xapic_id] = apic;
3548a259 212
6e500439
RK		 213 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
214
3b5a5ffa
RK		 215 if (apic_x2apic_mode(apic)) {
216 new->mode |= KVM_APIC_MODE_X2APIC;
217 } else if (ldr) {
218 ldr = GET_APIC_LOGICAL_ID(ldr);
dfb95954 219 if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
3b5a5ffa
RK		 220 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
221 else
222 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
223 }
224
e45115b6 225 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
3548a259
RK		 226 continue;
227
e45115b6
RK		 228 if (mask)
229 cluster[ffs(mask) - 1] = apic;
1e08ec4a
GN		 230 }
231out:
232 old = rcu_dereference_protected(kvm->arch.apic_map,
233 lockdep_is_held(&kvm->arch.apic_map_lock));
234 rcu_assign_pointer(kvm->arch.apic_map, new);
235 mutex_unlock(&kvm->arch.apic_map_lock);
236
237 if (old)
af1bae54 238 call_rcu(&old->rcu, kvm_apic_map_free);
c7c9c56c 239
b053b2ae 240 kvm_make_scan_ioapic_request(kvm);
1e08ec4a
GN		 241}
242
1e1b6c26
NA		 243static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
244{
e462755c 245 bool enabled = val & APIC_SPIV_APIC_ENABLED;
1e1b6c26 246
1e6e2755 247 kvm_lapic_set_reg(apic, APIC_SPIV, val);
e462755c
RK		 248
249 if (enabled != apic->sw_enabled) {
250 apic->sw_enabled = enabled;
251 if (enabled) {
1e1b6c26
NA		 252 static_key_slow_dec_deferred(&apic_sw_disabled);
253 recalculate_apic_map(apic->vcpu->kvm);
254 } else
255 static_key_slow_inc(&apic_sw_disabled.key);
256 }
257}
258
a92e2543 259static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
1e08ec4a 260{
1e6e2755 261 kvm_lapic_set_reg(apic, APIC_ID, id << 24);
1e08ec4a
GN		 262 recalculate_apic_map(apic->vcpu->kvm);
263}
264
265static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
266{
1e6e2755 267 kvm_lapic_set_reg(apic, APIC_LDR, id);
1e08ec4a
GN		 268 recalculate_apic_map(apic->vcpu->kvm);
269}
270
a92e2543 271static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
257b9a5f
RK		 272{
273 u32 ldr = ((id >> 4) << 16) | (1 << (id & 0xf));
274
6e500439
RK		 275 WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
276
a92e2543 277 kvm_lapic_set_reg(apic, APIC_ID, id);
1e6e2755 278 kvm_lapic_set_reg(apic, APIC_LDR, ldr);
257b9a5f
RK		 279 recalculate_apic_map(apic->vcpu->kvm);
280}
281
97222cc8
ED		 282static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
283{
dfb95954 284 return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
97222cc8
ED		 285}
286
287static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
288{
dfb95954 289 return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
97222cc8
ED		 290}
291
a3e06bbe
LJ		 292static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
293{
f30ebc31 294 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
a3e06bbe
LJ		 295}
296
97222cc8
ED		 297static inline int apic_lvtt_period(struct kvm_lapic *apic)
298{
f30ebc31 299 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
a3e06bbe
LJ		 300}
301
302static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
303{
f30ebc31 304 return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
97222cc8
ED		 305}
306
cc6e462c
JK		 307static inline int apic_lvt_nmi_mode(u32 lvt_val)
308{
309 return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
310}
311
fc61b800
GN		 312void kvm_apic_set_version(struct kvm_vcpu *vcpu)
313{
314 struct kvm_lapic *apic = vcpu->arch.apic;
315 struct kvm_cpuid_entry2 *feat;
316 u32 v = APIC_VERSION;
317
bce87cce 318 if (!lapic_in_kernel(vcpu))
fc61b800
GN		 319 return;
320
321 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
322 if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))))
323 v |= APIC_LVR_DIRECTED_EOI;
1e6e2755 324 kvm_lapic_set_reg(apic, APIC_LVR, v);
fc61b800
GN		 325}
326
1e6e2755 327static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
a3e06bbe 328 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
97222cc8
ED		 329 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
330 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
331 LINT_MASK, LINT_MASK, /* LVT0-1 */
332 LVT_MASK /* LVTERR */
333};
334
335static int find_highest_vector(void *bitmap)
336{
ecba9a52
TY		 337 int vec;
338 u32 *reg;
97222cc8 339
ecba9a52
TY		 340 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
341 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
342 reg = bitmap + REG_POS(vec);
343 if (*reg)
810e6def 344 return __fls(*reg) + vec;
ecba9a52 345 }
97222cc8 346
ecba9a52 347 return -1;
97222cc8
ED		 348}
349
8680b94b
MT		 350static u8 count_vectors(void *bitmap)
351{
ecba9a52
TY		 352 int vec;
353 u32 *reg;
8680b94b 354 u8 count = 0;
ecba9a52
TY		 355
356 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
357 reg = bitmap + REG_POS(vec);
358 count += hweight32(*reg);
359 }
360
8680b94b
MT		 361 return count;
362}
363
810e6def 364int __kvm_apic_update_irr(u32 *pir, void *regs)
a20ed54d 365{
810e6def
PB		 366 u32 i, vec;
367 u32 pir_val, irr_val;
368 int max_irr = -1;
a20ed54d 369
810e6def 370 for (i = vec = 0; i <= 7; i++, vec += 32) {
ad361091 371 pir_val = READ_ONCE(pir[i]);
810e6def 372 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
ad361091 373 if (pir_val) {
810e6def
PB		 374 irr_val |= xchg(&pir[i], 0);
375 *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
ad361091 376 }
810e6def
PB		 377 if (irr_val)
378 max_irr = __fls(irr_val) + vec;
a20ed54d 379 }
810e6def
PB		 380
381 return max_irr;
a20ed54d 382}
705699a1
WV		 383EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
384
810e6def 385int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
705699a1
WV		 386{
387 struct kvm_lapic *apic = vcpu->arch.apic;
388
b95234c8 389 return __kvm_apic_update_irr(pir, apic->regs);
705699a1 390}
a20ed54d
YZ		 391EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
392
33e4c686 393static inline int apic_search_irr(struct kvm_lapic *apic)
97222cc8 394{
33e4c686 395 return find_highest_vector(apic->regs + APIC_IRR);
97222cc8
ED		 396}
397
398static inline int apic_find_highest_irr(struct kvm_lapic *apic)
399{
400 int result;
401
c7c9c56c
YZ		 402 /*
403 * Note that irr_pending is just a hint. It will be always
404 * true with virtual interrupt delivery enabled.
405 */
33e4c686
GN		 406 if (!apic->irr_pending)
407 return -1;
408
409 result = apic_search_irr(apic);
97222cc8
ED		 410 ASSERT(result == -1 || result >= 16);
411
412 return result;
413}
414
33e4c686
GN		 415static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
416{
56cc2406
WL		 417 struct kvm_vcpu *vcpu;
418
419 vcpu = apic->vcpu;
420
d62caabb 421 if (unlikely(vcpu->arch.apicv_active)) {
b95234c8 422 /* need to update RVI */
f210f757 423 apic_clear_vector(vec, apic->regs + APIC_IRR);
b95234c8
PB		 424 kvm_x86_ops->hwapic_irr_update(vcpu,
425 apic_find_highest_irr(apic));
f210f757
NA		 426 } else {
427 apic->irr_pending = false;
428 apic_clear_vector(vec, apic->regs + APIC_IRR);
429 if (apic_search_irr(apic) != -1)
430 apic->irr_pending = true;
56cc2406 431 }
33e4c686
GN		 432}
433
8680b94b
MT		 434static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
435{
56cc2406
WL		 436 struct kvm_vcpu *vcpu;
437
438 if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
439 return;
440
441 vcpu = apic->vcpu;
fc57ac2c 442
8680b94b 443 /*
56cc2406
WL		 444 * With APIC virtualization enabled, all caching is disabled
445 * because the processor can modify ISR under the hood. Instead
446 * just set SVI.
8680b94b 447 */
d62caabb 448 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 449 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
56cc2406
WL		 450 else {
451 ++apic->isr_count;
452 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
453 /*
454 * ISR (in service register) bit is set when injecting an interrupt.
455 * The highest vector is injected. Thus the latest bit set matches
456 * the highest bit in ISR.
457 */
458 apic->highest_isr_cache = vec;
459 }
8680b94b
MT		 460}
461
fc57ac2c
PB		 462static inline int apic_find_highest_isr(struct kvm_lapic *apic)
463{
464 int result;
465
466 /*
467 * Note that isr_count is always 1, and highest_isr_cache
468 * is always -1, with APIC virtualization enabled.
469 */
470 if (!apic->isr_count)
471 return -1;
472 if (likely(apic->highest_isr_cache != -1))
473 return apic->highest_isr_cache;
474
475 result = find_highest_vector(apic->regs + APIC_ISR);
476 ASSERT(result == -1 || result >= 16);
477
478 return result;
479}
480
8680b94b
MT		 481static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
482{
fc57ac2c
PB		 483 struct kvm_vcpu *vcpu;
484 if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
485 return;
486
487 vcpu = apic->vcpu;
488
489 /*
490 * We do get here for APIC virtualization enabled if the guest
491 * uses the Hyper-V APIC enlightenment. In this case we may need
492 * to trigger a new interrupt delivery by writing the SVI field;
493 * on the other hand isr_count and highest_isr_cache are unused
494 * and must be left alone.
495 */
d62caabb 496 if (unlikely(vcpu->arch.apicv_active))
67c9dddc 497 kvm_x86_ops->hwapic_isr_update(vcpu,
fc57ac2c
PB		 498 apic_find_highest_isr(apic));
499 else {
8680b94b 500 --apic->isr_count;
fc57ac2c
PB		 501 BUG_ON(apic->isr_count < 0);
502 apic->highest_isr_cache = -1;
503 }
8680b94b
MT		 504}
505
6e5d865c
YS		 506int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
507{
33e4c686
GN		 508 /* This may race with setting of irr in __apic_accept_irq() and
509 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
510 * will cause vmexit immediately and the value will be recalculated
511 * on the next vmentry.
512 */
f8543d6a 513 return apic_find_highest_irr(vcpu->arch.apic);
6e5d865c 514}
76dfafd5 515EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
6e5d865c 516
6da7e3f6 517static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 518 int vector, int level, int trig_mode,
9e4aabe2 519 struct dest_map *dest_map);
6da7e3f6 520
b4f2225c 521int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
9e4aabe2 522 struct dest_map *dest_map)
97222cc8 523{
ad312c7c 524 struct kvm_lapic *apic = vcpu->arch.apic;
8be5453f 525
58c2dde1 526 return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
b4f2225c 527 irq->level, irq->trig_mode, dest_map);
97222cc8
ED		 528}
529
ae7a2a3f
MT		 530static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
531{
4e335d9e
PB		 532
533 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
534 sizeof(val));
ae7a2a3f
MT		 535}
536
537static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
538{
4e335d9e
PB		 539
540 return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
541 sizeof(*val));
ae7a2a3f
MT		 542}
543
544static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
545{
546 return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
547}
548
549static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
550{
551 u8 val;
552 if (pv_eoi_get_user(vcpu, &val) < 0)
553 apic_debug("Can't read EOI MSR value: 0x%llx\n",
96893977 554 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 555 return val & 0x1;
556}
557
558static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
559{
560 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
561 apic_debug("Can't set EOI MSR value: 0x%llx\n",
96893977 562 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 563 return;
564 }
565 __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
566}
567
568static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
569{
570 if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
571 apic_debug("Can't clear EOI MSR value: 0x%llx\n",
96893977 572 (unsigned long long)vcpu->arch.pv_eoi.msr_val);
ae7a2a3f
MT		 573 return;
574 }
575 __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
576}
577
b3c045d3
PB		 578static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
579{
3d92789f 580 int highest_irr;
76dfafd5
PB		 581 if (kvm_x86_ops->sync_pir_to_irr && apic->vcpu->arch.apicv_active)
582 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
583 else
584 highest_irr = apic_find_highest_irr(apic);
b3c045d3
PB		 585 if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
586 return -1;
587 return highest_irr;
588}
589
590static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
97222cc8 591{
3842d135 592 u32 tpr, isrv, ppr, old_ppr;
97222cc8
ED		 593 int isr;
594
dfb95954
SS		 595 old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
596 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
97222cc8
ED		 597 isr = apic_find_highest_isr(apic);
598 isrv = (isr != -1) ? isr : 0;
599
600 if ((tpr & 0xf0) >= (isrv & 0xf0))
601 ppr = tpr & 0xff;
602 else
603 ppr = isrv & 0xf0;
604
605 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
606 apic, ppr, isr, isrv);
607
b3c045d3
PB		 608 *new_ppr = ppr;
609 if (old_ppr != ppr)
1e6e2755 610 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
b3c045d3
PB		 611
612 return ppr < old_ppr;
613}
614
615static void apic_update_ppr(struct kvm_lapic *apic)
616{
617 u32 ppr;
618
26fbbee5
PB		 619 if (__apic_update_ppr(apic, &ppr) &&
620 apic_has_interrupt_for_ppr(apic, ppr) != -1)
b3c045d3 621 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
97222cc8
ED		 622}
623
eb90f341
PB		 624void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
625{
626 apic_update_ppr(vcpu->arch.apic);
627}
628EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
629
97222cc8
ED		 630static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
631{
1e6e2755 632 kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
97222cc8
ED		 633 apic_update_ppr(apic);
634}
635
03d2249e 636static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
394457a9 637{
b4535b58
RK		 638 return mda == (apic_x2apic_mode(apic) ?
639 X2APIC_BROADCAST : APIC_BROADCAST);
394457a9
NA		 640}
641
03d2249e 642static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 643{
03d2249e
RK		 644 if (kvm_apic_broadcast(apic, mda))
645 return true;
646
647 if (apic_x2apic_mode(apic))
6e500439 648 return mda == kvm_x2apic_id(apic);
03d2249e 649
5bd5db38
RK		 650 /*
651 * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
652 * it were in x2APIC mode. Hotplugged VCPUs start in xAPIC mode and
653 * this allows unique addressing of VCPUs with APIC ID over 0xff.
654 * The 0xff condition is needed because writeable xAPIC ID.
655 */
656 if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
657 return true;
658
b4535b58 659 return mda == kvm_xapic_id(apic);
97222cc8
ED		 660}
661
52c233a4 662static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
97222cc8 663{
0105d1a5
GN		 664 u32 logical_id;
665
394457a9 666 if (kvm_apic_broadcast(apic, mda))
9368b567 667 return true;
394457a9 668
dfb95954 669 logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
97222cc8 670
9368b567 671 if (apic_x2apic_mode(apic))
8a395363
RK		 672 return ((logical_id >> 16) == (mda >> 16))
673 && (logical_id & mda & 0xffff) != 0;
97222cc8 674
9368b567 675 logical_id = GET_APIC_LOGICAL_ID(logical_id);
97222cc8 676
dfb95954 677 switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
97222cc8 678 case APIC_DFR_FLAT:
9368b567 679 return (logical_id & mda) != 0;
97222cc8 680 case APIC_DFR_CLUSTER:
9368b567
RK		 681 return ((logical_id >> 4) == (mda >> 4))
682 && (logical_id & mda & 0xf) != 0;
97222cc8 683 default:
7712de87 684 apic_debug("Bad DFR vcpu %d: %08x\n",
dfb95954 685 apic->vcpu->vcpu_id, kvm_lapic_get_reg(apic, APIC_DFR));
9368b567 686 return false;
97222cc8 687 }
97222cc8
ED		 688}
689
c519265f
RK		 690/* The KVM local APIC implementation has two quirks:
691 *
b4535b58
RK		 692 * - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
693 * in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
694 * KVM doesn't do that aliasing.
c519265f
RK		 695 *
696 * - in-kernel IOAPIC messages have to be delivered directly to
697 * x2APIC, because the kernel does not support interrupt remapping.
698 * In order to support broadcast without interrupt remapping, x2APIC
699 * rewrites the destination of non-IPI messages from APIC_BROADCAST
700 * to X2APIC_BROADCAST.
701 *
702 * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API. This is
703 * important when userspace wants to use x2APIC-format MSIs, because
704 * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
03d2249e 705 */
c519265f
RK		 706static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
707 struct kvm_lapic *source, struct kvm_lapic *target)
03d2249e
RK		 708{
709 bool ipi = source != NULL;
03d2249e 710
c519265f 711 if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
b4535b58 712 !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
03d2249e
RK		 713 return X2APIC_BROADCAST;
714
b4535b58 715 return dest_id;
03d2249e
RK		 716}
717
52c233a4 718bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
394457a9 719 int short_hand, unsigned int dest, int dest_mode)
97222cc8 720{
ad312c7c 721 struct kvm_lapic *target = vcpu->arch.apic;
c519265f 722 u32 mda = kvm_apic_mda(vcpu, dest, source, target);
97222cc8
ED		 723
724 apic_debug("target %p, source %p, dest 0x%x, "
343f94fe 725 "dest_mode 0x%x, short_hand 0x%x\n",
97222cc8
ED		 726 target, source, dest, dest_mode, short_hand);
727
bd371396 728 ASSERT(target);
97222cc8
ED		 729 switch (short_hand) {
730 case APIC_DEST_NOSHORT:
3697f302 731 if (dest_mode == APIC_DEST_PHYSICAL)
03d2249e 732 return kvm_apic_match_physical_addr(target, mda);
343f94fe 733 else
03d2249e 734 return kvm_apic_match_logical_addr(target, mda);
97222cc8 735 case APIC_DEST_SELF:
9368b567 736 return target == source;
97222cc8 737 case APIC_DEST_ALLINC:
9368b567 738 return true;
97222cc8 739 case APIC_DEST_ALLBUT:
9368b567 740 return target != source;
97222cc8 741 default:
7712de87
JK		 742 apic_debug("kvm: apic: Bad dest shorthand value %x\n",
743 short_hand);
9368b567 744 return false;
97222cc8 745 }
97222cc8 746}
1e6e2755 747EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
97222cc8 748
52004014
FW		 749int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
750 const unsigned long *bitmap, u32 bitmap_size)
751{
752 u32 mod;
753 int i, idx = -1;
754
755 mod = vector % dest_vcpus;
756
757 for (i = 0; i <= mod; i++) {
758 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
759 BUG_ON(idx == bitmap_size);
760 }
761
762 return idx;
763}
764
4efd805f
RK		 765static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
766{
767 if (!kvm->arch.disabled_lapic_found) {
768 kvm->arch.disabled_lapic_found = true;
769 printk(KERN_INFO
770 "Disabled LAPIC found during irq injection\n");
771 }
772}
773
c519265f
RK		 774static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
775 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
1e08ec4a 776{
c519265f
RK		 777 if (kvm->arch.x2apic_broadcast_quirk_disabled) {
778 if ((irq->dest_id == APIC_BROADCAST &&
779 map->mode != KVM_APIC_MODE_X2APIC))
780 return true;
781 if (irq->dest_id == X2APIC_BROADCAST)
782 return true;
783 } else {
784 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
785 if (irq->dest_id == (x2apic_ipi ?
786 X2APIC_BROADCAST : APIC_BROADCAST))
787 return true;
788 }
1e08ec4a 789
c519265f
RK		 790 return false;
791}
1e08ec4a 792
64aa47bf
RK		 793/* Return true if the interrupt can be handled by using *bitmap as index mask
794 * for valid destinations in *dst array.
795 * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
796 * Note: we may have zero kvm_lapic destinations when we return true, which
797 * means that the interrupt should be dropped. In this case, *bitmap would be
798 * zero and *dst undefined.
799 */
800static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
801 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
802 struct kvm_apic_map *map, struct kvm_lapic ***dst,
803 unsigned long *bitmap)
804{
805 int i, lowest;
1e08ec4a 806
64aa47bf
RK		 807 if (irq->shorthand == APIC_DEST_SELF && src) {
808 *dst = src;
809 *bitmap = 1;
810 return true;
811 } else if (irq->shorthand)
1e08ec4a
GN		 812 return false;
813
c519265f 814 if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
9ea369b0
RK		 815 return false;
816
64aa47bf 817 if (irq->dest_mode == APIC_DEST_PHYSICAL) {
0ca52e7b 818 if (irq->dest_id > map->max_apic_id) {
64aa47bf
RK		 819 *bitmap = 0;
820 } else {
821 *dst = &map->phys_map[irq->dest_id];
822 *bitmap = 1;
823 }
1e08ec4a 824 return true;
bea15428 825 }
698f9755 826
e45115b6
RK		 827 *bitmap = 0;
828 if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
829 (u16 *)bitmap))
1e08ec4a 830 return false;
fa834e91 831
64aa47bf
RK		 832 if (!kvm_lowest_prio_delivery(irq))
833 return true;
3548a259 834
64aa47bf
RK		 835 if (!kvm_vector_hashing_enabled()) {
836 lowest = -1;
837 for_each_set_bit(i, bitmap, 16) {
838 if (!(*dst)[i])
839 continue;
840 if (lowest < 0)
841 lowest = i;
842 else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
843 (*dst)[lowest]->vcpu) < 0)
844 lowest = i;
3548a259 845 }
64aa47bf
RK		 846 } else {
847 if (!*bitmap)
848 return true;
3548a259 849
64aa47bf
RK		 850 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
851 bitmap, 16);
45c3094a 852
64aa47bf
RK		 853 if (!(*dst)[lowest]) {
854 kvm_apic_disabled_lapic_found(kvm);
855 *bitmap = 0;
856 return true;
857 }
858 }
1e08ec4a 859
64aa47bf 860 *bitmap = (lowest >= 0) ? 1 << lowest : 0;
1e08ec4a 861
64aa47bf
RK		 862 return true;
863}
52004014 864
64aa47bf
RK		 865bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
866 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
867{
868 struct kvm_apic_map *map;
869 unsigned long bitmap;
870 struct kvm_lapic **dst = NULL;
871 int i;
872 bool ret;
52004014 873
64aa47bf 874 *r = -1;
52004014 875
64aa47bf
RK		 876 if (irq->shorthand == APIC_DEST_SELF) {
877 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
878 return true;
879 }
52004014 880
64aa47bf
RK		 881 rcu_read_lock();
882 map = rcu_dereference(kvm->arch.apic_map);
52004014 883
64aa47bf
RK		 884 ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
885 if (ret)
886 for_each_set_bit(i, &bitmap, 16) {
887 if (!dst[i])
888 continue;
889 if (*r < 0)
890 *r = 0;
891 *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1e08ec4a 892 }
1e08ec4a 893
1e08ec4a
GN		 894 rcu_read_unlock();
895 return ret;
896}
897
6228a0da
FW		 898/*
899 * This routine tries to handler interrupts in posted mode, here is how
900 * it deals with different cases:
901 * - For single-destination interrupts, handle it in posted mode
902 * - Else if vector hashing is enabled and it is a lowest-priority
903 * interrupt, handle it in posted mode and use the following mechanism
904 * to find the destinaiton vCPU.
905 * 1. For lowest-priority interrupts, store all the possible
906 * destination vCPUs in an array.
907 * 2. Use "guest vector % max number of destination vCPUs" to find
908 * the right destination vCPU in the array for the lowest-priority
909 * interrupt.
910 * - Otherwise, use remapped mode to inject the interrupt.
911 */
8feb4a04
FW		 912bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
913 struct kvm_vcpu **dest_vcpu)
914{
915 struct kvm_apic_map *map;
64aa47bf
RK		 916 unsigned long bitmap;
917 struct kvm_lapic **dst = NULL;
8feb4a04 918 bool ret = false;
8feb4a04
FW		 919
920 if (irq->shorthand)
921 return false;
922
923 rcu_read_lock();
924 map = rcu_dereference(kvm->arch.apic_map);
925
64aa47bf
RK		 926 if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
927 hweight16(bitmap) == 1) {
928 unsigned long i = find_first_bit(&bitmap, 16);
6228a0da 929
64aa47bf
RK		 930 if (dst[i]) {
931 *dest_vcpu = dst[i]->vcpu;
932 ret = true;
6228a0da 933 }
8feb4a04
FW		 934 }
935
8feb4a04
FW		 936 rcu_read_unlock();
937 return ret;
938}
939
97222cc8
ED		 940/*
941 * Add a pending IRQ into lapic.
942 * Return 1 if successfully added and 0 if discarded.
943 */
944static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
b4f2225c 945 int vector, int level, int trig_mode,
9e4aabe2 946 struct dest_map *dest_map)
97222cc8 947{
6da7e3f6 948 int result = 0;
c5ec1534 949 struct kvm_vcpu *vcpu = apic->vcpu;
97222cc8 950
a183b638
PB		 951 trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
952 trig_mode, vector);
97222cc8 953 switch (delivery_mode) {
97222cc8 954 case APIC_DM_LOWEST:
e1035715
GN		 955 vcpu->arch.apic_arb_prio++;
956 case APIC_DM_FIXED:
bdaffe1d
PB		 957 if (unlikely(trig_mode && !level))
958 break;
959
97222cc8
ED		 960 /* FIXME add logic for vcpu on reset */
961 if (unlikely(!apic_enabled(apic)))
962 break;
963
11f5cc05
JK		 964 result = 1;
965
9daa5007 966 if (dest_map) {
9e4aabe2 967 __set_bit(vcpu->vcpu_id, dest_map->map);
9daa5007
JR		 968 dest_map->vectors[vcpu->vcpu_id] = vector;
969 }
a5d36f82 970
bdaffe1d
PB		 971 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
972 if (trig_mode)
1e6e2755 973 kvm_lapic_set_vector(vector, apic->regs + APIC_TMR);
bdaffe1d
PB		 974 else
975 apic_clear_vector(vector, apic->regs + APIC_TMR);
976 }
977
d62caabb 978 if (vcpu->arch.apicv_active)
5a71785d 979 kvm_x86_ops->deliver_posted_interrupt(vcpu, vector);
11f5cc05 980 else {
1e6e2755 981 kvm_lapic_set_irr(vector, apic);
5a71785d
YZ		 982
983 kvm_make_request(KVM_REQ_EVENT, vcpu);
984 kvm_vcpu_kick(vcpu);
985 }
97222cc8
ED		 986 break;
987
988 case APIC_DM_REMRD:
24d2166b
R		 989 result = 1;
990 vcpu->arch.pv.pv_unhalted = 1;
991 kvm_make_request(KVM_REQ_EVENT, vcpu);
992 kvm_vcpu_kick(vcpu);
97222cc8
ED		 993 break;
994
995 case APIC_DM_SMI:
64d60670
PB		 996 result = 1;
997 kvm_make_request(KVM_REQ_SMI, vcpu);
998 kvm_vcpu_kick(vcpu);
97222cc8 999 break;
3419ffc8 1000
97222cc8 1001 case APIC_DM_NMI:
6da7e3f6 1002 result = 1;
3419ffc8 1003 kvm_inject_nmi(vcpu);
26df99c6 1004 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1005 break;
1006
1007 case APIC_DM_INIT:
a52315e1 1008 if (!trig_mode || level) {
6da7e3f6 1009 result = 1;
66450a21
JK		 1010 /* assumes that there are only KVM_APIC_INIT/SIPI */
1011 apic->pending_events = (1UL << KVM_APIC_INIT);
1012 /* make sure pending_events is visible before sending
1013 * the request */
1014 smp_wmb();
3842d135 1015 kvm_make_request(KVM_REQ_EVENT, vcpu);
c5ec1534
HQ		 1016 kvm_vcpu_kick(vcpu);
1017 } else {
1b10bf31
JK		 1018 apic_debug("Ignoring de-assert INIT to vcpu %d\n",
1019 vcpu->vcpu_id);
c5ec1534 1020 }
97222cc8
ED		 1021 break;
1022
1023 case APIC_DM_STARTUP:
1b10bf31
JK		 1024 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
1025 vcpu->vcpu_id, vector);
66450a21
JK		 1026 result = 1;
1027 apic->sipi_vector = vector;
1028 /* make sure sipi_vector is visible for the receiver */
1029 smp_wmb();
1030 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1031 kvm_make_request(KVM_REQ_EVENT, vcpu);
1032 kvm_vcpu_kick(vcpu);
97222cc8
ED		 1033 break;
1034
23930f95
JK		 1035 case APIC_DM_EXTINT:
1036 /*
1037 * Should only be called by kvm_apic_local_deliver() with LVT0,
1038 * before NMI watchdog was enabled. Already handled by
1039 * kvm_apic_accept_pic_intr().
1040 */
1041 break;
1042
97222cc8
ED		 1043 default:
1044 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1045 delivery_mode);
1046 break;
1047 }
1048 return result;
1049}
1050
e1035715 1051int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
8be5453f 1052{
e1035715 1053 return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
8be5453f
ZX		 1054}
1055
3bb345f3
PB		 1056static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1057{
6308630b 1058 return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
3bb345f3
PB		 1059}
1060
c7c9c56c
YZ		 1061static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1062{
7543a635
SR		 1063 int trigger_mode;
1064
1065 /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1066 if (!kvm_ioapic_handles_vector(apic, vector))
1067 return;
3bb345f3 1068
7543a635
SR		 1069 /* Request a KVM exit to inform the userspace IOAPIC. */
1070 if (irqchip_split(apic->vcpu->kvm)) {
1071 apic->vcpu->arch.pending_ioapic_eoi = vector;
1072 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1073 return;
c7c9c56c 1074 }
7543a635
SR		 1075
1076 if (apic_test_vector(vector, apic->regs + APIC_TMR))
1077 trigger_mode = IOAPIC_LEVEL_TRIG;
1078 else
1079 trigger_mode = IOAPIC_EDGE_TRIG;
1080
1081 kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
c7c9c56c
YZ		 1082}
1083
ae7a2a3f 1084static int apic_set_eoi(struct kvm_lapic *apic)
97222cc8
ED		 1085{
1086 int vector = apic_find_highest_isr(apic);
ae7a2a3f
MT		 1087
1088 trace_kvm_eoi(apic, vector);
1089
97222cc8
ED		 1090 /*
1091 * Not every write EOI will has corresponding ISR,
1092 * one example is when Kernel check timer on setup_IO_APIC
1093 */
1094 if (vector == -1)
ae7a2a3f 1095 return vector;
97222cc8 1096
8680b94b 1097 apic_clear_isr(vector, apic);
97222cc8
ED		 1098 apic_update_ppr(apic);
1099
5c919412
AS		 1100 if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1101 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1102
c7c9c56c 1103 kvm_ioapic_send_eoi(apic, vector);
3842d135 1104 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
ae7a2a3f 1105 return vector;
97222cc8
ED		 1106}
1107
c7c9c56c
YZ		 1108/*
1109 * this interface assumes a trap-like exit, which has already finished
1110 * desired side effect including vISR and vPPR update.
1111 */
1112void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1113{
1114 struct kvm_lapic *apic = vcpu->arch.apic;
1115
1116 trace_kvm_eoi(apic, vector);
1117
1118 kvm_ioapic_send_eoi(apic, vector);
1119 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1120}
1121EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1122
97222cc8
ED		 1123static void apic_send_ipi(struct kvm_lapic *apic)
1124{
dfb95954
SS		 1125 u32 icr_low = kvm_lapic_get_reg(apic, APIC_ICR);
1126 u32 icr_high = kvm_lapic_get_reg(apic, APIC_ICR2);
58c2dde1 1127 struct kvm_lapic_irq irq;
97222cc8 1128
58c2dde1
GN		 1129 irq.vector = icr_low & APIC_VECTOR_MASK;
1130 irq.delivery_mode = icr_low & APIC_MODE_MASK;
1131 irq.dest_mode = icr_low & APIC_DEST_MASK;
b7cb2231 1132 irq.level = (icr_low & APIC_INT_ASSERT) != 0;
58c2dde1
GN		 1133 irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1134 irq.shorthand = icr_low & APIC_SHORT_MASK;
93bbf0b8 1135 irq.msi_redir_hint = false;
0105d1a5
GN		 1136 if (apic_x2apic_mode(apic))
1137 irq.dest_id = icr_high;
1138 else
1139 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
97222cc8 1140
1000ff8d
GN		 1141 trace_kvm_apic_ipi(icr_low, irq.dest_id);
1142
97222cc8
ED		 1143 apic_debug("icr_high 0x%x, icr_low 0x%x, "
1144 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
93bbf0b8
JS		 1145 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x, "
1146 "msi_redir_hint 0x%x\n",
9b5843dd 1147 icr_high, icr_low, irq.shorthand, irq.dest_id,
58c2dde1 1148 irq.trig_mode, irq.level, irq.dest_mode, irq.delivery_mode,
93bbf0b8 1149 irq.vector, irq.msi_redir_hint);
58c2dde1 1150
b4f2225c 1151 kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
97222cc8
ED		 1152}
1153
1154static u32 apic_get_tmcct(struct kvm_lapic *apic)
1155{
8003c9ae 1156 ktime_t remaining, now;
b682b814 1157 s64 ns;
9da8f4e8 1158 u32 tmcct;
97222cc8
ED		 1159
1160 ASSERT(apic != NULL);
1161
9da8f4e8 1162 /* if initial count is 0, current count should also be 0 */
dfb95954 1163 if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
b963a22e 1164 apic->lapic_timer.period == 0)
9da8f4e8
KP		 1165 return 0;
1166
5587859f 1167 now = ktime_get();
8003c9ae 1168 remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
b682b814 1169 if (ktime_to_ns(remaining) < 0)
8b0e1953 1170 remaining = 0;
b682b814 1171
d3c7b77d
MT		 1172 ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1173 tmcct = div64_u64(ns,
1174 (APIC_BUS_CYCLE_NS * apic->divide_count));
97222cc8
ED		 1175
1176 return tmcct;
1177}
1178
b209749f
AK		 1179static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1180{
1181 struct kvm_vcpu *vcpu = apic->vcpu;
1182 struct kvm_run *run = vcpu->run;
1183
a8eeb04a 1184 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
5fdbf976 1185 run->tpr_access.rip = kvm_rip_read(vcpu);
b209749f
AK		 1186 run->tpr_access.is_write = write;
1187}
1188
1189static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1190{
1191 if (apic->vcpu->arch.tpr_access_reporting)
1192 __report_tpr_access(apic, write);
1193}
1194
97222cc8
ED		 1195static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1196{
1197 u32 val = 0;
1198
1199 if (offset >= LAPIC_MMIO_LENGTH)
1200 return 0;
1201
1202 switch (offset) {
1203 case APIC_ARBPRI:
7712de87 1204 apic_debug("Access APIC ARBPRI register which is for P6\n");
97222cc8
ED		 1205 break;
1206
1207 case APIC_TMCCT: /* Timer CCR */
a3e06bbe
LJ		 1208 if (apic_lvtt_tscdeadline(apic))
1209 return 0;
1210
97222cc8
ED		 1211 val = apic_get_tmcct(apic);
1212 break;
4a4541a4
AK		 1213 case APIC_PROCPRI:
1214 apic_update_ppr(apic);
dfb95954 1215 val = kvm_lapic_get_reg(apic, offset);
4a4541a4 1216 break;
b209749f
AK		 1217 case APIC_TASKPRI:
1218 report_tpr_access(apic, false);
1219 /* fall thru */
97222cc8 1220 default:
dfb95954 1221 val = kvm_lapic_get_reg(apic, offset);
97222cc8
ED		 1222 break;
1223 }
1224
1225 return val;
1226}
1227
d76685c4
GH		 1228static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1229{
1230 return container_of(dev, struct kvm_lapic, dev);
1231}
1232
1e6e2755 1233int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
0105d1a5 1234 void *data)
97222cc8 1235{
97222cc8
ED		 1236 unsigned char alignment = offset & 0xf;
1237 u32 result;
d5b0b5b1 1238 /* this bitmask has a bit cleared for each reserved register */
0105d1a5 1239 static const u64 rmask = 0x43ff01ffffffe70cULL;
97222cc8
ED		 1240
1241 if ((alignment + len) > 4) {
4088bb3c
GN		 1242 apic_debug("KVM_APIC_READ: alignment error %x %d\n",
1243 offset, len);
0105d1a5 1244 return 1;
97222cc8 1245 }
0105d1a5
GN		 1246
1247 if (offset > 0x3f0 || !(rmask & (1ULL << (offset >> 4)))) {
4088bb3c
GN		 1248 apic_debug("KVM_APIC_READ: read reserved register %x\n",
1249 offset);
0105d1a5
GN		 1250 return 1;
1251 }
1252
97222cc8
ED		 1253 result = __apic_read(apic, offset & ~0xf);
1254
229456fc
MT		 1255 trace_kvm_apic_read(offset, result);
1256
97222cc8
ED		 1257 switch (len) {
1258 case 1:
1259 case 2:
1260 case 4:
1261 memcpy(data, (char *)&result + alignment, len);
1262 break;
1263 default:
1264 printk(KERN_ERR "Local APIC read with len = %x, "
1265 "should be 1,2, or 4 instead\n", len);
1266 break;
1267 }
bda9020e 1268 return 0;
97222cc8 1269}
1e6e2755 1270EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
97222cc8 1271
0105d1a5
GN		 1272static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1273{
c48f1496 1274 return kvm_apic_hw_enabled(apic) &&
0105d1a5
GN		 1275 addr >= apic->base_address &&
1276 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1277}
1278
e32edf4f 1279static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1280 gpa_t address, int len, void *data)
1281{
1282 struct kvm_lapic *apic = to_lapic(this);
1283 u32 offset = address - apic->base_address;
1284
1285 if (!apic_mmio_in_range(apic, address))
1286 return -EOPNOTSUPP;
1287
1e6e2755 1288 kvm_lapic_reg_read(apic, offset, len, data);
0105d1a5
GN		 1289
1290 return 0;
1291}
1292
97222cc8
ED		 1293static void update_divide_count(struct kvm_lapic *apic)
1294{
1295 u32 tmp1, tmp2, tdcr;
1296
dfb95954 1297 tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
97222cc8
ED		 1298 tmp1 = tdcr & 0xf;
1299 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
d3c7b77d 1300 apic->divide_count = 0x1 << (tmp2 & 0x7);
97222cc8
ED		 1301
1302 apic_debug("timer divide count is 0x%x\n",
9b5843dd 1303 apic->divide_count);
97222cc8
ED		 1304}
1305
b6ac0695
RK		 1306static void apic_update_lvtt(struct kvm_lapic *apic)
1307{
dfb95954 1308 u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
b6ac0695
RK		 1309 apic->lapic_timer.timer_mode_mask;
1310
1311 if (apic->lapic_timer.timer_mode != timer_mode) {
1312 apic->lapic_timer.timer_mode = timer_mode;
1313 hrtimer_cancel(&apic->lapic_timer.timer);
1314 }
1315}
1316
5d87db71
RK		 1317static void apic_timer_expired(struct kvm_lapic *apic)
1318{
1319 struct kvm_vcpu *vcpu = apic->vcpu;
8577370f 1320 struct swait_queue_head *q = &vcpu->wq;
d0659d94 1321 struct kvm_timer *ktimer = &apic->lapic_timer;
5d87db71 1322
5d87db71
RK		 1323 if (atomic_read(&apic->lapic_timer.pending))
1324 return;
1325
1326 atomic_inc(&apic->lapic_timer.pending);
bab5bb39 1327 kvm_set_pending_timer(vcpu);
5d87db71 1328
8577370f
MT		 1329 if (swait_active(q))
1330 swake_up(q);
d0659d94
MT		 1331
1332 if (apic_lvtt_tscdeadline(apic))
1333 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1334}
1335
1336/*
1337 * On APICv, this test will cause a busy wait
1338 * during a higher-priority task.
1339 */
1340
1341static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1342{
1343 struct kvm_lapic *apic = vcpu->arch.apic;
dfb95954 1344 u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
d0659d94
MT		 1345
1346 if (kvm_apic_hw_enabled(apic)) {
1347 int vec = reg & APIC_VECTOR_MASK;
f9339860 1348 void *bitmap = apic->regs + APIC_ISR;
d0659d94 1349
d62caabb 1350 if (vcpu->arch.apicv_active)
f9339860
MT		 1351 bitmap = apic->regs + APIC_IRR;
1352
1353 if (apic_test_vector(vec, bitmap))
1354 return true;
d0659d94
MT		 1355 }
1356 return false;
1357}
1358
1359void wait_lapic_expire(struct kvm_vcpu *vcpu)
1360{
1361 struct kvm_lapic *apic = vcpu->arch.apic;
1362 u64 guest_tsc, tsc_deadline;
1363
bce87cce 1364 if (!lapic_in_kernel(vcpu))
d0659d94
MT		 1365 return;
1366
1367 if (apic->lapic_timer.expired_tscdeadline == 0)
1368 return;
1369
1370 if (!lapic_timer_int_injected(vcpu))
1371 return;
1372
1373 tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1374 apic->lapic_timer.expired_tscdeadline = 0;
4ba76538 1375 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
6c19b753 1376 trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline);
d0659d94
MT		 1377
1378 /* __delay is delay_tsc whenever the hardware has TSC, thus always. */
1379 if (guest_tsc < tsc_deadline)
b606f189
MT		 1380 __delay(min(tsc_deadline - guest_tsc,
1381 nsec_to_cycles(vcpu, lapic_timer_advance_ns)));
5d87db71
RK		 1382}
1383
53f9eedf
YJ		 1384static void start_sw_tscdeadline(struct kvm_lapic *apic)
1385{
1386 u64 guest_tsc, tscdeadline = apic->lapic_timer.tscdeadline;
1387 u64 ns = 0;
1388 ktime_t expire;
1389 struct kvm_vcpu *vcpu = apic->vcpu;
1390 unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1391 unsigned long flags;
1392 ktime_t now;
1393
1394 if (unlikely(!tscdeadline || !this_tsc_khz))
1395 return;
1396
1397 local_irq_save(flags);
1398
5587859f 1399 now = ktime_get();
53f9eedf
YJ		 1400 guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1401 if (likely(tscdeadline > guest_tsc)) {
1402 ns = (tscdeadline - guest_tsc) * 1000000ULL;
1403 do_div(ns, this_tsc_khz);
1404 expire = ktime_add_ns(now, ns);
1405 expire = ktime_sub_ns(expire, lapic_timer_advance_ns);
1406 hrtimer_start(&apic->lapic_timer.timer,
1407 expire, HRTIMER_MODE_ABS_PINNED);
1408 } else
1409 apic_timer_expired(apic);
1410
1411 local_irq_restore(flags);
1412}
1413
7d7f7da2 1414static void start_sw_period(struct kvm_lapic *apic)
8003c9ae
WL		 1415{
1416 if (!apic->lapic_timer.period)
1417 return;
1418
1419 if (apic_lvtt_oneshot(apic) &&
5587859f 1420 ktime_after(ktime_get(),
8003c9ae
WL		 1421 apic->lapic_timer.target_expiration)) {
1422 apic_timer_expired(apic);
1423 return;
1424 }
1425
1426 hrtimer_start(&apic->lapic_timer.timer,
1427 apic->lapic_timer.target_expiration,
1428 HRTIMER_MODE_ABS_PINNED);
1429}
1430
1431static bool set_target_expiration(struct kvm_lapic *apic)
7d7f7da2
WL		 1432{
1433 ktime_t now;
8003c9ae 1434 u64 tscl = rdtsc();
7d7f7da2 1435
5587859f 1436 now = ktime_get();
7d7f7da2 1437 apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
8003c9ae 1438 * APIC_BUS_CYCLE_NS * apic->divide_count;
7d7f7da2
WL		 1439
1440 if (!apic->lapic_timer.period)
8003c9ae
WL		 1441 return false;
1442
7d7f7da2
WL		 1443 /*
1444 * Do not allow the guest to program periodic timers with small
1445 * interval, since the hrtimers are not throttled by the host
1446 * scheduler.
1447 */
1448 if (apic_lvtt_period(apic)) {
1449 s64 min_period = min_timer_period_us * 1000LL;
1450
1451 if (apic->lapic_timer.period < min_period) {
1452 pr_info_ratelimited(
1453 "kvm: vcpu %i: requested %lld ns "
1454 "lapic timer period limited to %lld ns\n",
1455 apic->vcpu->vcpu_id,
1456 apic->lapic_timer.period, min_period);
1457 apic->lapic_timer.period = min_period;
1458 }
1459 }
1460
7d7f7da2
WL		 1461 apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016"
1462 PRIx64 ", "
1463 "timer initial count 0x%x, period %lldns, "
1464 "expire @ 0x%016" PRIx64 ".\n", __func__,
1465 APIC_BUS_CYCLE_NS, ktime_to_ns(now),
1466 kvm_lapic_get_reg(apic, APIC_TMICT),
1467 apic->lapic_timer.period,
1468 ktime_to_ns(ktime_add_ns(now,
1469 apic->lapic_timer.period)));
8003c9ae
WL		 1470
1471 apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1472 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1473 apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1474
1475 return true;
1476}
1477
1478static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1479{
1480 apic->lapic_timer.tscdeadline +=
1481 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1482 apic->lapic_timer.target_expiration =
1483 ktime_add_ns(apic->lapic_timer.target_expiration,
1484 apic->lapic_timer.period);
7d7f7da2
WL		 1485}
1486
ce7a058a
YJ		 1487bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1488{
91005300
WL		 1489 if (!lapic_in_kernel(vcpu))
1490 return false;
1491
ce7a058a
YJ		 1492 return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1493}
1494EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1495
7e810a38 1496static void cancel_hv_timer(struct kvm_lapic *apic)
bd97ad0e 1497{
a749e247 1498 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
5acc1ca4 1499 preempt_disable();
bd97ad0e
WL		 1500 kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1501 apic->lapic_timer.hv_timer_in_use = false;
5acc1ca4 1502 preempt_enable();
bd97ad0e
WL		 1503}
1504
a749e247 1505static bool start_hv_timer(struct kvm_lapic *apic)
196f20ca 1506{
35ee9e48
PB		 1507 struct kvm_timer *ktimer = &apic->lapic_timer;
1508 int r;
196f20ca 1509
a749e247
PB		 1510 if (!kvm_x86_ops->set_hv_timer)
1511 return false;
1512
35ee9e48
PB		 1513 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1514 return false;
1515
1516 r = kvm_x86_ops->set_hv_timer(apic->vcpu, ktimer->tscdeadline);
1517 if (r < 0)
1518 return false;
1519
1520 ktimer->hv_timer_in_use = true;
1521 hrtimer_cancel(&ktimer->timer);
196f20ca 1522
35ee9e48
PB		 1523 /*
1524 * Also recheck ktimer->pending, in case the sw timer triggered in
1525 * the window. For periodic timer, leave the hv timer running for
1526 * simplicity, and the deadline will be recomputed on the next vmexit.
1527 */
c8533544
WL		 1528 if (!apic_lvtt_period(apic) && (r || atomic_read(&ktimer->pending))) {
1529 if (r)
1530 apic_timer_expired(apic);
35ee9e48 1531 return false;
c8533544 1532 }
a749e247
PB		 1533
1534 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, true);
35ee9e48
PB		 1535 return true;
1536}
1537
a749e247 1538static void start_sw_timer(struct kvm_lapic *apic)
35ee9e48 1539{
a749e247
PB		 1540 struct kvm_timer *ktimer = &apic->lapic_timer;
1541 if (apic->lapic_timer.hv_timer_in_use)
1542 cancel_hv_timer(apic);
1543 if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1544 return;
1545
1546 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1547 start_sw_period(apic);
1548 else if (apic_lvtt_tscdeadline(apic))
1549 start_sw_tscdeadline(apic);
1550 trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1551}
35ee9e48 1552
a749e247
PB		 1553static void restart_apic_timer(struct kvm_lapic *apic)
1554{
1555 if (!start_hv_timer(apic))
1556 start_sw_timer(apic);
196f20ca
WL		 1557}
1558
8003c9ae
WL		 1559void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1560{
1561 struct kvm_lapic *apic = vcpu->arch.apic;
1562
1563 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1564 WARN_ON(swait_active(&vcpu->wq));
1565 cancel_hv_timer(apic);
1566 apic_timer_expired(apic);
1567
1568 if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1569 advance_periodic_target_expiration(apic);
a749e247 1570 restart_apic_timer(apic);
8003c9ae
WL		 1571 }
1572}
1573EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1574
ce7a058a
YJ		 1575void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1576{
a749e247 1577 restart_apic_timer(vcpu->arch.apic);
ce7a058a
YJ		 1578}
1579EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1580
1581void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1582{
1583 struct kvm_lapic *apic = vcpu->arch.apic;
1584
1585 /* Possibly the TSC deadline timer is not enabled yet */
a749e247
PB		 1586 if (apic->lapic_timer.hv_timer_in_use)
1587 start_sw_timer(apic);
1588}
1589EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
ce7a058a 1590
a749e247
PB		 1591void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1592{
1593 struct kvm_lapic *apic = vcpu->arch.apic;
ce7a058a 1594
a749e247
PB		 1595 WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1596 restart_apic_timer(apic);
ce7a058a 1597}
ce7a058a 1598
97222cc8
ED		 1599static void start_apic_timer(struct kvm_lapic *apic)
1600{
d3c7b77d 1601 atomic_set(&apic->lapic_timer.pending, 0);
0b975a3c 1602
a749e247
PB		 1603 if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1604 && !set_target_expiration(apic))
1605 return;
1606
1607 restart_apic_timer(apic);
97222cc8
ED		 1608}
1609
cc6e462c
JK		 1610static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1611{
59fd1323 1612 bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
cc6e462c 1613
59fd1323
RK		 1614 if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1615 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1616 if (lvt0_in_nmi_mode) {
cc6e462c
JK		 1617 apic_debug("Receive NMI setting on APIC_LVT0 "
1618 "for cpu %d\n", apic->vcpu->vcpu_id);
42720138 1619 atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
59fd1323
RK		 1620 } else
1621 atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1622 }
cc6e462c
JK		 1623}
1624
1e6e2755 1625int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
97222cc8 1626{
0105d1a5 1627 int ret = 0;
97222cc8 1628
0105d1a5 1629 trace_kvm_apic_write(reg, val);
97222cc8 1630
0105d1a5 1631 switch (reg) {
97222cc8 1632 case APIC_ID: /* Local APIC ID */
0105d1a5 1633 if (!apic_x2apic_mode(apic))
a92e2543 1634 kvm_apic_set_xapic_id(apic, val >> 24);
0105d1a5
GN		 1635 else
1636 ret = 1;
97222cc8
ED		 1637 break;
1638
1639 case APIC_TASKPRI:
b209749f 1640 report_tpr_access(apic, true);
97222cc8
ED		 1641 apic_set_tpr(apic, val & 0xff);
1642 break;
1643
1644 case APIC_EOI:
1645 apic_set_eoi(apic);
1646 break;
1647
1648 case APIC_LDR:
0105d1a5 1649 if (!apic_x2apic_mode(apic))
1e08ec4a 1650 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
0105d1a5
GN		 1651 else
1652 ret = 1;
97222cc8
ED		 1653 break;
1654
1655 case APIC_DFR:
1e08ec4a 1656 if (!apic_x2apic_mode(apic)) {
1e6e2755 1657 kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1e08ec4a
GN		 1658 recalculate_apic_map(apic->vcpu->kvm);
1659 } else
0105d1a5 1660 ret = 1;
97222cc8
ED		 1661 break;
1662
fc61b800
GN		 1663 case APIC_SPIV: {
1664 u32 mask = 0x3ff;
dfb95954 1665 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
fc61b800 1666 mask |= APIC_SPIV_DIRECTED_EOI;
f8c1ea10 1667 apic_set_spiv(apic, val & mask);
97222cc8
ED		 1668 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1669 int i;
1670 u32 lvt_val;
1671
1e6e2755 1672 for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
dfb95954 1673 lvt_val = kvm_lapic_get_reg(apic,
97222cc8 1674 APIC_LVTT + 0x10 * i);
1e6e2755 1675 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
97222cc8
ED		 1676 lvt_val | APIC_LVT_MASKED);
1677 }
b6ac0695 1678 apic_update_lvtt(apic);
d3c7b77d 1679 atomic_set(&apic->lapic_timer.pending, 0);
97222cc8
ED		 1680
1681 }
1682 break;
fc61b800 1683 }
97222cc8
ED		 1684 case APIC_ICR:
1685 /* No delay here, so we always clear the pending bit */
1e6e2755 1686 kvm_lapic_set_reg(apic, APIC_ICR, val & ~(1 << 12));
97222cc8
ED		 1687 apic_send_ipi(apic);
1688 break;
1689
1690 case APIC_ICR2:
0105d1a5
GN		 1691 if (!apic_x2apic_mode(apic))
1692 val &= 0xff000000;
1e6e2755 1693 kvm_lapic_set_reg(apic, APIC_ICR2, val);
97222cc8
ED		 1694 break;
1695
23930f95 1696 case APIC_LVT0:
cc6e462c 1697 apic_manage_nmi_watchdog(apic, val);
97222cc8
ED		 1698 case APIC_LVTTHMR:
1699 case APIC_LVTPC:
97222cc8
ED		 1700 case APIC_LVT1:
1701 case APIC_LVTERR:
1702 /* TODO: Check vector */
c48f1496 1703 if (!kvm_apic_sw_enabled(apic))
97222cc8
ED		 1704 val |= APIC_LVT_MASKED;
1705
0105d1a5 1706 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
1e6e2755 1707 kvm_lapic_set_reg(apic, reg, val);
97222cc8
ED		 1708
1709 break;
1710
b6ac0695 1711 case APIC_LVTT:
c48f1496 1712 if (!kvm_apic_sw_enabled(apic))
a3e06bbe
LJ		 1713 val |= APIC_LVT_MASKED;
1714 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1e6e2755 1715 kvm_lapic_set_reg(apic, APIC_LVTT, val);
b6ac0695 1716 apic_update_lvtt(apic);
a3e06bbe
LJ		 1717 break;
1718
97222cc8 1719 case APIC_TMICT:
a3e06bbe
LJ		 1720 if (apic_lvtt_tscdeadline(apic))
1721 break;
1722
d3c7b77d 1723 hrtimer_cancel(&apic->lapic_timer.timer);
1e6e2755 1724 kvm_lapic_set_reg(apic, APIC_TMICT, val);
97222cc8 1725 start_apic_timer(apic);
0105d1a5 1726 break;
97222cc8
ED		 1727
1728 case APIC_TDCR:
1729 if (val & 4)
7712de87 1730 apic_debug("KVM_WRITE:TDCR %x\n", val);
1e6e2755 1731 kvm_lapic_set_reg(apic, APIC_TDCR, val);
97222cc8
ED		 1732 update_divide_count(apic);
1733 break;
1734
0105d1a5
GN		 1735 case APIC_ESR:
1736 if (apic_x2apic_mode(apic) && val != 0) {
7712de87 1737 apic_debug("KVM_WRITE:ESR not zero %x\n", val);
0105d1a5
GN		 1738 ret = 1;
1739 }
1740 break;
1741
1742 case APIC_SELF_IPI:
1743 if (apic_x2apic_mode(apic)) {
1e6e2755 1744 kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
0105d1a5
GN		 1745 } else
1746 ret = 1;
1747 break;
97222cc8 1748 default:
0105d1a5 1749 ret = 1;
97222cc8
ED		 1750 break;
1751 }
0105d1a5
GN		 1752 if (ret)
1753 apic_debug("Local APIC Write to read-only register %x\n", reg);
1754 return ret;
1755}
1e6e2755 1756EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
0105d1a5 1757
e32edf4f 1758static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
0105d1a5
GN		 1759 gpa_t address, int len, const void *data)
1760{
1761 struct kvm_lapic *apic = to_lapic(this);
1762 unsigned int offset = address - apic->base_address;
1763 u32 val;
1764
1765 if (!apic_mmio_in_range(apic, address))
1766 return -EOPNOTSUPP;
1767
1768 /*
1769 * APIC register must be aligned on 128-bits boundary.
1770 * 32/64/128 bits registers must be accessed thru 32 bits.
1771 * Refer SDM 8.4.1
1772 */
1773 if (len != 4 || (offset & 0xf)) {
1774 /* Don't shout loud, $infamous_os would cause only noise. */
1775 apic_debug("apic write: bad size=%d %lx\n", len, (long)address);
756975bb 1776 return 0;
0105d1a5
GN		 1777 }
1778
1779 val = *(u32*)data;
1780
1781 /* too common printing */
1782 if (offset != APIC_EOI)
1783 apic_debug("%s: offset 0x%x with length 0x%x, and value is "
1784 "0x%x\n", __func__, offset, len, val);
1785
1e6e2755 1786 kvm_lapic_reg_write(apic, offset & 0xff0, val);
0105d1a5 1787
bda9020e 1788 return 0;
97222cc8
ED		 1789}
1790
58fbbf26
KT		 1791void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
1792{
1e6e2755 1793 kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
58fbbf26
KT		 1794}
1795EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
1796
83d4c286
YZ		 1797/* emulate APIC access in a trap manner */
1798void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
1799{
1800 u32 val = 0;
1801
1802 /* hw has done the conditional check and inst decode */
1803 offset &= 0xff0;
1804
1e6e2755 1805 kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
83d4c286
YZ		 1806
1807 /* TODO: optimize to just emulate side effect w/o one more write */
1e6e2755 1808 kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
83d4c286
YZ		 1809}
1810EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
1811
d589444e 1812void kvm_free_lapic(struct kvm_vcpu *vcpu)
97222cc8 1813{
f8c1ea10
GN		 1814 struct kvm_lapic *apic = vcpu->arch.apic;
1815
ad312c7c 1816 if (!vcpu->arch.apic)
97222cc8
ED		 1817 return;
1818
f8c1ea10 1819 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1820
c5cc421b
GN		 1821 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
1822 static_key_slow_dec_deferred(&apic_hw_disabled);
1823
e462755c 1824 if (!apic->sw_enabled)
f8c1ea10 1825 static_key_slow_dec_deferred(&apic_sw_disabled);
97222cc8 1826
f8c1ea10
GN		 1827 if (apic->regs)
1828 free_page((unsigned long)apic->regs);
1829
1830 kfree(apic);
97222cc8
ED		 1831}
1832
1833/*
1834 *----------------------------------------------------------------------
1835 * LAPIC interface
1836 *----------------------------------------------------------------------
1837 */
a3e06bbe
LJ		 1838u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1839{
1840 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1841
a10388e1
WL		 1842 if (!lapic_in_kernel(vcpu) ||
1843 !apic_lvtt_tscdeadline(apic))
a3e06bbe
LJ		 1844 return 0;
1845
1846 return apic->lapic_timer.tscdeadline;
1847}
1848
1849void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
1850{
1851 struct kvm_lapic *apic = vcpu->arch.apic;
a3e06bbe 1852
bce87cce 1853 if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
54e9818f 1854 apic_lvtt_period(apic))
a3e06bbe
LJ		 1855 return;
1856
1857 hrtimer_cancel(&apic->lapic_timer.timer);
1858 apic->lapic_timer.tscdeadline = data;
1859 start_apic_timer(apic);
1860}
1861
97222cc8
ED		 1862void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
1863{
ad312c7c 1864 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1865
b93463aa 1866 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
dfb95954 1867 | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
97222cc8
ED		 1868}
1869
1870u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
1871{
97222cc8
ED		 1872 u64 tpr;
1873
dfb95954 1874 tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
97222cc8
ED		 1875
1876 return (tpr & 0xf0) >> 4;
1877}
1878
1879void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1880{
8d14695f 1881 u64 old_value = vcpu->arch.apic_base;
ad312c7c 1882 struct kvm_lapic *apic = vcpu->arch.apic;
97222cc8 1883
c7dd15b3 1884 if (!apic)
97222cc8 1885 value |= MSR_IA32_APICBASE_BSP;
c5af89b6 1886
e66d2ae7
JK		 1887 vcpu->arch.apic_base = value;
1888
c7dd15b3
JM		 1889 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
1890 kvm_update_cpuid(vcpu);
1891
1892 if (!apic)
1893 return;
1894
c5cc421b 1895 /* update jump label if enable bit changes */
0dce7cd6 1896 if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
49bd29ba
RK		 1897 if (value & MSR_IA32_APICBASE_ENABLE) {
1898 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
c5cc421b 1899 static_key_slow_dec_deferred(&apic_hw_disabled);
187ca84b 1900 } else {
c5cc421b 1901 static_key_slow_inc(&apic_hw_disabled.key);
187ca84b
WL		 1902 recalculate_apic_map(vcpu->kvm);
1903 }
c5cc421b
GN		 1904 }
1905
8d14695f
YZ		 1906 if ((old_value ^ value) & X2APIC_ENABLE) {
1907 if (value & X2APIC_ENABLE) {
257b9a5f 1908 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
8d14695f
YZ		 1909 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, true);
1910 } else
1911 kvm_x86_ops->set_virtual_x2apic_mode(vcpu, false);
0105d1a5 1912 }
8d14695f 1913
ad312c7c 1914 apic->base_address = apic->vcpu->arch.apic_base &
97222cc8
ED		 1915 MSR_IA32_APICBASE_BASE;
1916
db324fe6
NA		 1917 if ((value & MSR_IA32_APICBASE_ENABLE) &&
1918 apic->base_address != APIC_DEFAULT_PHYS_BASE)
1919 pr_warn_once("APIC base relocation is unsupported by KVM");
1920
97222cc8
ED		 1921 /* with FSB delivery interrupt, we can restart APIC functionality */
1922 apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is "
ad312c7c 1923 "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1924
1925}
1926
d28bc9dd 1927void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
97222cc8
ED		 1928{
1929 struct kvm_lapic *apic;
1930 int i;
1931
b8688d51 1932 apic_debug("%s\n", __func__);
97222cc8
ED		 1933
1934 ASSERT(vcpu);
ad312c7c 1935 apic = vcpu->arch.apic;
97222cc8
ED		 1936 ASSERT(apic != NULL);
1937
1938 /* Stop the timer in case it's a reset to an active apic */
d3c7b77d 1939 hrtimer_cancel(&apic->lapic_timer.timer);
97222cc8 1940
4d8e772b
RK		 1941 if (!init_event) {
1942 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
1943 MSR_IA32_APICBASE_ENABLE);
a92e2543 1944 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
4d8e772b 1945 }
fc61b800 1946 kvm_apic_set_version(apic->vcpu);
97222cc8 1947
1e6e2755
SS		 1948 for (i = 0; i < KVM_APIC_LVT_NUM; i++)
1949 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
b6ac0695 1950 apic_update_lvtt(apic);
52b54190
JS		 1951 if (kvm_vcpu_is_reset_bsp(vcpu) &&
1952 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
1e6e2755 1953 kvm_lapic_set_reg(apic, APIC_LVT0,
90de4a18 1954 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
dfb95954 1955 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
97222cc8 1956
1e6e2755 1957 kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
f8c1ea10 1958 apic_set_spiv(apic, 0xff);
1e6e2755 1959 kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
c028dd6b
RK		 1960 if (!apic_x2apic_mode(apic))
1961 kvm_apic_set_ldr(apic, 0);
1e6e2755
SS		 1962 kvm_lapic_set_reg(apic, APIC_ESR, 0);
1963 kvm_lapic_set_reg(apic, APIC_ICR, 0);
1964 kvm_lapic_set_reg(apic, APIC_ICR2, 0);
1965 kvm_lapic_set_reg(apic, APIC_TDCR, 0);
1966 kvm_lapic_set_reg(apic, APIC_TMICT, 0);
97222cc8 1967 for (i = 0; i < 8; i++) {
1e6e2755
SS		 1968 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
1969 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
1970 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
97222cc8 1971 }
d62caabb
AS		 1972 apic->irr_pending = vcpu->arch.apicv_active;
1973 apic->isr_count = vcpu->arch.apicv_active ? 1 : 0;
8680b94b 1974 apic->highest_isr_cache = -1;
b33ac88b 1975 update_divide_count(apic);
d3c7b77d 1976 atomic_set(&apic->lapic_timer.pending, 0);
c5af89b6 1977 if (kvm_vcpu_is_bsp(vcpu))
5dbc8f3f
GN		 1978 kvm_lapic_set_base(vcpu,
1979 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
ae7a2a3f 1980 vcpu->arch.pv_eoi.msr_val = 0;
97222cc8
ED		 1981 apic_update_ppr(apic);
1982
e1035715 1983 vcpu->arch.apic_arb_prio = 0;
41383771 1984 vcpu->arch.apic_attention = 0;
e1035715 1985
6e500439 1986 apic_debug("%s: vcpu=%p, id=0x%x, base_msr="
b8688d51 1987 "0x%016" PRIx64 ", base_address=0x%0lx.\n", __func__,
6e500439 1988 vcpu, kvm_lapic_get_reg(apic, APIC_ID),
ad312c7c 1989 vcpu->arch.apic_base, apic->base_address);
97222cc8
ED		 1990}
1991
97222cc8
ED		 1992/*
1993 *----------------------------------------------------------------------
1994 * timer interface
1995 *----------------------------------------------------------------------
1996 */
1b9778da 1997
2a6eac96 1998static bool lapic_is_periodic(struct kvm_lapic *apic)
97222cc8 1999{
d3c7b77d 2000 return apic_lvtt_period(apic);
97222cc8
ED		 2001}
2002
3d80840d
MT		 2003int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2004{
54e9818f 2005 struct kvm_lapic *apic = vcpu->arch.apic;
3d80840d 2006
1e3161b4 2007 if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
54e9818f 2008 return atomic_read(&apic->lapic_timer.pending);
3d80840d
MT		 2009
2010 return 0;
2011}
2012
89342082 2013int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1b9778da 2014{
dfb95954 2015 u32 reg = kvm_lapic_get_reg(apic, lvt_type);
23930f95 2016 int vector, mode, trig_mode;
23930f95 2017
c48f1496 2018 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
23930f95
JK		 2019 vector = reg & APIC_VECTOR_MASK;
2020 mode = reg & APIC_MODE_MASK;
2021 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
b4f2225c
YZ		 2022 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2023 NULL);
23930f95
JK		 2024 }
2025 return 0;
2026}
1b9778da 2027
8fdb2351 2028void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
23930f95 2029{
8fdb2351
JK		 2030 struct kvm_lapic *apic = vcpu->arch.apic;
2031
2032 if (apic)
2033 kvm_apic_local_deliver(apic, APIC_LVT0);
1b9778da
ED		 2034}
2035
d76685c4
GH		 2036static const struct kvm_io_device_ops apic_mmio_ops = {
2037 .read = apic_mmio_read,
2038 .write = apic_mmio_write,
d76685c4
GH		 2039};
2040
e9d90d47
AK		 2041static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2042{
2043 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2a6eac96 2044 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
e9d90d47 2045
5d87db71 2046 apic_timer_expired(apic);
e9d90d47 2047
2a6eac96 2048 if (lapic_is_periodic(apic)) {
8003c9ae 2049 advance_periodic_target_expiration(apic);
e9d90d47
AK		 2050 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2051 return HRTIMER_RESTART;
2052 } else
2053 return HRTIMER_NORESTART;
2054}
2055
97222cc8
ED		 2056int kvm_create_lapic(struct kvm_vcpu *vcpu)
2057{
2058 struct kvm_lapic *apic;
2059
2060 ASSERT(vcpu != NULL);
2061 apic_debug("apic_init %d\n", vcpu->vcpu_id);
2062
2063 apic = kzalloc(sizeof(*apic), GFP_KERNEL);
2064 if (!apic)
2065 goto nomem;
2066
ad312c7c 2067 vcpu->arch.apic = apic;
97222cc8 2068
afc20184
TY		 2069 apic->regs = (void *)get_zeroed_page(GFP_KERNEL);
2070 if (!apic->regs) {
97222cc8
ED		 2071 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2072 vcpu->vcpu_id);
d589444e 2073 goto nomem_free_apic;
97222cc8 2074 }
97222cc8
ED		 2075 apic->vcpu = vcpu;
2076
d3c7b77d 2077 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
61abdbe0 2078 HRTIMER_MODE_ABS_PINNED);
e9d90d47 2079 apic->lapic_timer.timer.function = apic_timer_fn;
d3c7b77d 2080
c5cc421b
GN		 2081 /*
2082 * APIC is created enabled. This will prevent kvm_lapic_set_base from
2083 * thinking that APIC satet has changed.
2084 */
2085 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
f8c1ea10 2086 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
d28bc9dd 2087 kvm_lapic_reset(vcpu, false);
d76685c4 2088 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
97222cc8
ED		 2089
2090 return 0;
d589444e
RR		 2091nomem_free_apic:
2092 kfree(apic);
97222cc8 2093nomem:
97222cc8
ED		 2094 return -ENOMEM;
2095}
97222cc8
ED		 2096
2097int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2098{
ad312c7c 2099 struct kvm_lapic *apic = vcpu->arch.apic;
b3c045d3 2100 u32 ppr;
97222cc8 2101
f8543d6a 2102 if (!apic_enabled(apic))
97222cc8
ED		 2103 return -1;
2104
b3c045d3
PB		 2105 __apic_update_ppr(apic, &ppr);
2106 return apic_has_interrupt_for_ppr(apic, ppr);
97222cc8
ED		 2107}
2108
40487c68
QH		 2109int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2110{
dfb95954 2111 u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
40487c68
QH		 2112 int r = 0;
2113
c48f1496 2114 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
e7dca5c0
CL		 2115 r = 1;
2116 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2117 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2118 r = 1;
40487c68
QH		 2119 return r;
2120}
2121
1b9778da
ED		 2122void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2123{
ad312c7c 2124 struct kvm_lapic *apic = vcpu->arch.apic;
1b9778da 2125
54e9818f 2126 if (atomic_read(&apic->lapic_timer.pending) > 0) {
f1ed0450 2127 kvm_apic_local_deliver(apic, APIC_LVTT);
fae0ba21
NA		 2128 if (apic_lvtt_tscdeadline(apic))
2129 apic->lapic_timer.tscdeadline = 0;
8003c9ae
WL		 2130 if (apic_lvtt_oneshot(apic)) {
2131 apic->lapic_timer.tscdeadline = 0;
8b0e1953 2132 apic->lapic_timer.target_expiration = 0;
8003c9ae 2133 }
f1ed0450 2134 atomic_set(&apic->lapic_timer.pending, 0);
1b9778da
ED		 2135 }
2136}
2137
97222cc8
ED		 2138int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2139{
2140 int vector = kvm_apic_has_interrupt(vcpu);
ad312c7c 2141 struct kvm_lapic *apic = vcpu->arch.apic;
4d82d12b 2142 u32 ppr;
97222cc8
ED		 2143
2144 if (vector == -1)
2145 return -1;
2146
56cc2406
WL		 2147 /*
2148 * We get here even with APIC virtualization enabled, if doing
2149 * nested virtualization and L1 runs with the "acknowledge interrupt
2150 * on exit" mode. Then we cannot inject the interrupt via RVI,
2151 * because the process would deliver it through the IDT.
2152 */
2153
97222cc8 2154 apic_clear_irr(vector, apic);
5c919412 2155 if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
4d82d12b
PB		 2156 /*
2157 * For auto-EOI interrupts, there might be another pending
2158 * interrupt above PPR, so check whether to raise another
2159 * KVM_REQ_EVENT.
2160 */
5c919412 2161 apic_update_ppr(apic);
4d82d12b
PB		 2162 } else {
2163 /*
2164 * For normal interrupts, PPR has been raised and there cannot
2165 * be a higher-priority pending interrupt---except if there was
2166 * a concurrent interrupt injection, but that would have
2167 * triggered KVM_REQ_EVENT already.
2168 */
2169 apic_set_isr(vector, apic);
2170 __apic_update_ppr(apic, &ppr);
5c919412
AS		 2171 }
2172
97222cc8
ED		 2173 return vector;
2174}
96ad2cc6 2175
a92e2543
RK		 2176static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2177 struct kvm_lapic_state *s, bool set)
2178{
2179 if (apic_x2apic_mode(vcpu->arch.apic)) {
2180 u32 *id = (u32 *)(s->regs + APIC_ID);
2181
37131313
RK		 2182 if (vcpu->kvm->arch.x2apic_format) {
2183 if (*id != vcpu->vcpu_id)
2184 return -EINVAL;
2185 } else {
2186 if (set)
2187 *id >>= 24;
2188 else
2189 *id <<= 24;
2190 }
a92e2543
RK		 2191 }
2192
2193 return 0;
2194}
2195
2196int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2197{
2198 memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2199 return kvm_apic_state_fixup(vcpu, s, false);
2200}
2201
2202int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
96ad2cc6 2203{
ad312c7c 2204 struct kvm_lapic *apic = vcpu->arch.apic;
a92e2543
RK		 2205 int r;
2206
96ad2cc6 2207
5dbc8f3f 2208 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
64eb0620
GN		 2209 /* set SPIV separately to get count of SW disabled APICs right */
2210 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
a92e2543
RK		 2211
2212 r = kvm_apic_state_fixup(vcpu, s, true);
2213 if (r)
2214 return r;
64eb0620 2215 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
a92e2543
RK		 2216
2217 recalculate_apic_map(vcpu->kvm);
fc61b800
GN		 2218 kvm_apic_set_version(vcpu);
2219
96ad2cc6 2220 apic_update_ppr(apic);
d3c7b77d 2221 hrtimer_cancel(&apic->lapic_timer.timer);
b6ac0695 2222 apic_update_lvtt(apic);
dfb95954 2223 apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
96ad2cc6
ED		 2224 update_divide_count(apic);
2225 start_apic_timer(apic);
6e24a6ef 2226 apic->irr_pending = true;
d62caabb 2227 apic->isr_count = vcpu->arch.apicv_active ?
c7c9c56c 2228 1 : count_vectors(apic->regs + APIC_ISR);
8680b94b 2229 apic->highest_isr_cache = -1;
d62caabb 2230 if (vcpu->arch.apicv_active) {
967235d3 2231 kvm_x86_ops->apicv_post_state_restore(vcpu);
4114c27d
WW		 2232 kvm_x86_ops->hwapic_irr_update(vcpu,
2233 apic_find_highest_irr(apic));
67c9dddc 2234 kvm_x86_ops->hwapic_isr_update(vcpu,
b4eef9b3 2235 apic_find_highest_isr(apic));
d62caabb 2236 }
3842d135 2237 kvm_make_request(KVM_REQ_EVENT, vcpu);
49df6397
SR		 2238 if (ioapic_in_kernel(vcpu->kvm))
2239 kvm_rtc_eoi_tracking_restore_one(vcpu);
0669a510
RK		 2240
2241 vcpu->arch.apic_arb_prio = 0;
a92e2543
RK		 2242
2243 return 0;
96ad2cc6 2244}
a3d7f85f 2245
2f52d58c 2246void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
a3d7f85f 2247{
a3d7f85f
ED		 2248 struct hrtimer *timer;
2249
bce87cce 2250 if (!lapic_in_kernel(vcpu))
a3d7f85f
ED		 2251 return;
2252
54e9818f 2253 timer = &vcpu->arch.apic->lapic_timer.timer;
a3d7f85f 2254 if (hrtimer_cancel(timer))
61abdbe0 2255 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
a3d7f85f 2256}
b93463aa 2257
ae7a2a3f
MT		 2258/*
2259 * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2260 *
2261 * Detect whether guest triggered PV EOI since the
2262 * last entry. If yes, set EOI on guests's behalf.
2263 * Clear PV EOI in guest memory in any case.
2264 */
2265static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2266 struct kvm_lapic *apic)
2267{
2268 bool pending;
2269 int vector;
2270 /*
2271 * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2272 * and KVM_PV_EOI_ENABLED in guest memory as follows:
2273 *
2274 * KVM_APIC_PV_EOI_PENDING is unset:
2275 * -> host disabled PV EOI.
2276 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2277 * -> host enabled PV EOI, guest did not execute EOI yet.
2278 * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2279 * -> host enabled PV EOI, guest executed EOI.
2280 */
2281 BUG_ON(!pv_eoi_enabled(vcpu));
2282 pending = pv_eoi_get_pending(vcpu);
2283 /*
2284 * Clear pending bit in any case: it will be set again on vmentry.
2285 * While this might not be ideal from performance point of view,
2286 * this makes sure pv eoi is only enabled when we know it's safe.
2287 */
2288 pv_eoi_clr_pending(vcpu);
2289 if (pending)
2290 return;
2291 vector = apic_set_eoi(apic);
2292 trace_kvm_pv_eoi(apic, vector);
2293}
2294
b93463aa
AK		 2295void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2296{
2297 u32 data;
b93463aa 2298
ae7a2a3f
MT		 2299 if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2300 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2301
41383771 2302 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2303 return;
2304
4e335d9e
PB		 2305 if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2306 sizeof(u32)))
603242a8 2307 return;
b93463aa
AK		 2308
2309 apic_set_tpr(vcpu->arch.apic, data & 0xff);
2310}
2311
ae7a2a3f
MT		 2312/*
2313 * apic_sync_pv_eoi_to_guest - called before vmentry
2314 *
2315 * Detect whether it's safe to enable PV EOI and
2316 * if yes do so.
2317 */
2318static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2319 struct kvm_lapic *apic)
2320{
2321 if (!pv_eoi_enabled(vcpu) ||
2322 /* IRR set or many bits in ISR: could be nested. */
2323 apic->irr_pending ||
2324 /* Cache not set: could be safe but we don't bother. */
2325 apic->highest_isr_cache == -1 ||
2326 /* Need EOI to update ioapic. */
3bb345f3 2327 kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
ae7a2a3f
MT		 2328 /*
2329 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2330 * so we need not do anything here.
2331 */
2332 return;
2333 }
2334
2335 pv_eoi_set_pending(apic->vcpu);
2336}
2337
b93463aa
AK		 2338void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2339{
2340 u32 data, tpr;
2341 int max_irr, max_isr;
ae7a2a3f 2342 struct kvm_lapic *apic = vcpu->arch.apic;
b93463aa 2343
ae7a2a3f
MT		 2344 apic_sync_pv_eoi_to_guest(vcpu, apic);
2345
41383771 2346 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
b93463aa
AK		 2347 return;
2348
dfb95954 2349 tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
b93463aa
AK		 2350 max_irr = apic_find_highest_irr(apic);
2351 if (max_irr < 0)
2352 max_irr = 0;
2353 max_isr = apic_find_highest_isr(apic);
2354 if (max_isr < 0)
2355 max_isr = 0;
2356 data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2357
4e335d9e
PB		 2358 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2359 sizeof(u32));
b93463aa
AK		 2360}
2361
fda4e2e8 2362int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
b93463aa 2363{
fda4e2e8 2364 if (vapic_addr) {
4e335d9e 2365 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
fda4e2e8
AH		 2366 &vcpu->arch.apic->vapic_cache,
2367 vapic_addr, sizeof(u32)))
2368 return -EINVAL;
41383771 2369 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8 2370 } else {
41383771 2371 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
fda4e2e8
AH		 2372 }
2373
2374 vcpu->arch.apic->vapic_addr = vapic_addr;
2375 return 0;
b93463aa 2376}
0105d1a5
GN		 2377
2378int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2379{
2380 struct kvm_lapic *apic = vcpu->arch.apic;
2381 u32 reg = (msr - APIC_BASE_MSR) << 4;
2382
35754c98 2383 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2384 return 1;
2385
c69d3d9b
NA		 2386 if (reg == APIC_ICR2)
2387 return 1;
2388
0105d1a5 2389 /* if this is ICR write vector before command */
decdc283 2390 if (reg == APIC_ICR)
1e6e2755
SS		 2391 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2392 return kvm_lapic_reg_write(apic, reg, (u32)data);
0105d1a5
GN		 2393}
2394
2395int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2396{
2397 struct kvm_lapic *apic = vcpu->arch.apic;
2398 u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2399
35754c98 2400 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
0105d1a5
GN		 2401 return 1;
2402
c69d3d9b
NA		 2403 if (reg == APIC_DFR || reg == APIC_ICR2) {
2404 apic_debug("KVM_APIC_READ: read x2apic reserved register %x\n",
2405 reg);
2406 return 1;
2407 }
2408
1e6e2755 2409 if (kvm_lapic_reg_read(apic, reg, 4, &low))
0105d1a5 2410 return 1;
decdc283 2411 if (reg == APIC_ICR)
1e6e2755 2412 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
0105d1a5
GN		 2413
2414 *data = (((u64)high) << 32) | low;
2415
2416 return 0;
2417}
10388a07
GN		 2418
2419int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2420{
2421 struct kvm_lapic *apic = vcpu->arch.apic;
2422
bce87cce 2423 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2424 return 1;
2425
2426 /* if this is ICR write vector before command */
2427 if (reg == APIC_ICR)
1e6e2755
SS		 2428 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2429 return kvm_lapic_reg_write(apic, reg, (u32)data);
10388a07
GN		 2430}
2431
2432int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2433{
2434 struct kvm_lapic *apic = vcpu->arch.apic;
2435 u32 low, high = 0;
2436
bce87cce 2437 if (!lapic_in_kernel(vcpu))
10388a07
GN		 2438 return 1;
2439
1e6e2755 2440 if (kvm_lapic_reg_read(apic, reg, 4, &low))
10388a07
GN		 2441 return 1;
2442 if (reg == APIC_ICR)
1e6e2755 2443 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
10388a07
GN		 2444
2445 *data = (((u64)high) << 32) | low;
2446
2447 return 0;
2448}
ae7a2a3f
MT		 2449
2450int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
2451{
2452 u64 addr = data & ~KVM_MSR_ENABLED;
2453 if (!IS_ALIGNED(addr, 4))
2454 return 1;
2455
2456 vcpu->arch.pv_eoi.msr_val = data;
2457 if (!pv_eoi_enabled(vcpu))
2458 return 0;
4e335d9e 2459 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
8f964525 2460 addr, sizeof(u8));
ae7a2a3f 2461}
c5cc421b 2462
66450a21
JK		 2463void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2464{
2465 struct kvm_lapic *apic = vcpu->arch.apic;
2b4a273b 2466 u8 sipi_vector;
299018f4 2467 unsigned long pe;
66450a21 2468
bce87cce 2469 if (!lapic_in_kernel(vcpu) || !apic->pending_events)
66450a21
JK		 2470 return;
2471
cd7764fe
PB		 2472 /*
2473 * INITs are latched while in SMM. Because an SMM CPU cannot
2474 * be in KVM_MP_STATE_INIT_RECEIVED state, just eat SIPIs
2475 * and delay processing of INIT until the next RSM.
2476 */
2477 if (is_smm(vcpu)) {
2478 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2479 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2480 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2481 return;
2482 }
299018f4 2483
cd7764fe 2484 pe = xchg(&apic->pending_events, 0);
299018f4 2485 if (test_bit(KVM_APIC_INIT, &pe)) {
d28bc9dd
NA		 2486 kvm_lapic_reset(vcpu, true);
2487 kvm_vcpu_reset(vcpu, true);
66450a21
JK		 2488 if (kvm_vcpu_is_bsp(apic->vcpu))
2489 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2490 else
2491 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2492 }
299018f4 2493 if (test_bit(KVM_APIC_SIPI, &pe) &&
66450a21
JK		 2494 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2495 /* evaluate pending_events before reading the vector */
2496 smp_rmb();
2497 sipi_vector = apic->sipi_vector;
98eff52a 2498 apic_debug("vcpu %d received sipi with vector # %x\n",
66450a21
JK		 2499 vcpu->vcpu_id, sipi_vector);
2500 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2501 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2502 }
2503}
2504
c5cc421b
GN		 2505void kvm_lapic_init(void)
2506{
2507 /* do not patch jump label more than once per second */
2508 jump_label_rate_limit(&apic_hw_disabled, HZ);
f8c1ea10 2509 jump_label_rate_limit(&apic_sw_disabled, HZ);
c5cc421b 2510}
cef84c30
DM		 2511
2512void kvm_lapic_exit(void)
2513{
2514 static_key_deferred_flush(&apic_hw_disabled);
2515 static_key_deferred_flush(&apic_sw_disabled);
2516}
A mirror of Linus' kernel repository