2 * Copyright (C) 2015 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/arm-smccc.h>
19 #include <linux/types.h>
20 #include <linux/jump_label.h>
21 #include <uapi/linux/psci.h>
23 #include <kvm/arm_psci.h>
25 #include <asm/cpufeature.h>
26 #include <asm/kprobes.h>
27 #include <asm/kvm_asm.h>
28 #include <asm/kvm_emulate.h>
29 #include <asm/kvm_host.h>
30 #include <asm/kvm_hyp.h>
31 #include <asm/kvm_mmu.h>
32 #include <asm/fpsimd.h>
33 #include <asm/debug-monitors.h>
34 #include <asm/processor.h>
35 #include <asm/thread_info.h>
37 /* Check whether the FP regs were dirtied while in the host-side run loop: */
38 static bool __hyp_text
update_fp_enabled(struct kvm_vcpu
*vcpu
)
40 if (vcpu
->arch
.host_thread_info
->flags
& _TIF_FOREIGN_FPSTATE
)
41 vcpu
->arch
.flags
&= ~(KVM_ARM64_FP_ENABLED
|
44 return !!(vcpu
->arch
.flags
& KVM_ARM64_FP_ENABLED
);
47 /* Save the 32-bit only FPSIMD system register state */
48 static void __hyp_text
__fpsimd_save_fpexc32(struct kvm_vcpu
*vcpu
)
50 if (!vcpu_el1_is_32bit(vcpu
))
53 vcpu
->arch
.ctxt
.sys_regs
[FPEXC32_EL2
] = read_sysreg(fpexc32_el2
);
56 static void __hyp_text
__activate_traps_fpsimd32(struct kvm_vcpu
*vcpu
)
59 * We are about to set CPTR_EL2.TFP to trap all floating point
60 * register accesses to EL2, however, the ARM ARM clearly states that
61 * traps are only taken to EL2 if the operation would not otherwise
62 * trap to EL1. Therefore, always make sure that for 32-bit guests,
63 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
64 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
65 * it will cause an exception.
67 if (vcpu_el1_is_32bit(vcpu
) && system_supports_fpsimd()) {
68 write_sysreg(1 << 30, fpexc32_el2
);
73 static void __hyp_text
__activate_traps_common(struct kvm_vcpu
*vcpu
)
75 /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */
76 write_sysreg(1 << 15, hstr_el2
);
79 * Make sure we trap PMU access from EL0 to EL2. Also sanitize
80 * PMSELR_EL0 to make sure it never contains the cycle
81 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
82 * EL1 instead of being trapped to EL2.
84 write_sysreg(0, pmselr_el0
);
85 write_sysreg(ARMV8_PMU_USERENR_MASK
, pmuserenr_el0
);
86 write_sysreg(vcpu
->arch
.mdcr_el2
, mdcr_el2
);
89 static void __hyp_text
__deactivate_traps_common(void)
91 write_sysreg(0, hstr_el2
);
92 write_sysreg(0, pmuserenr_el0
);
95 static void activate_traps_vhe(struct kvm_vcpu
*vcpu
)
99 val
= read_sysreg(cpacr_el1
);
100 val
|= CPACR_EL1_TTA
;
101 val
&= ~CPACR_EL1_ZEN
;
102 if (!update_fp_enabled(vcpu
)) {
103 val
&= ~CPACR_EL1_FPEN
;
104 __activate_traps_fpsimd32(vcpu
);
107 write_sysreg(val
, cpacr_el1
);
109 write_sysreg(kvm_get_hyp_vector(), vbar_el1
);
111 NOKPROBE_SYMBOL(activate_traps_vhe
);
113 static void __hyp_text
__activate_traps_nvhe(struct kvm_vcpu
*vcpu
)
117 __activate_traps_common(vcpu
);
119 val
= CPTR_EL2_DEFAULT
;
120 val
|= CPTR_EL2_TTA
| CPTR_EL2_TZ
;
121 if (!update_fp_enabled(vcpu
)) {
123 __activate_traps_fpsimd32(vcpu
);
126 write_sysreg(val
, cptr_el2
);
129 static void __hyp_text
__activate_traps(struct kvm_vcpu
*vcpu
)
131 u64 hcr
= vcpu
->arch
.hcr_el2
;
133 write_sysreg(hcr
, hcr_el2
);
135 if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN
) && (hcr
& HCR_VSE
))
136 write_sysreg_s(vcpu
->arch
.vsesr_el2
, SYS_VSESR_EL2
);
139 activate_traps_vhe(vcpu
);
141 __activate_traps_nvhe(vcpu
);
144 static void deactivate_traps_vhe(void)
146 extern char vectors
[]; /* kernel exception vectors */
147 write_sysreg(HCR_HOST_VHE_FLAGS
, hcr_el2
);
150 * ARM erratum 1165522 requires the actual execution of the above
151 * before we can switch to the EL2/EL0 translation regime used by
154 asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522
));
156 write_sysreg(CPACR_EL1_DEFAULT
, cpacr_el1
);
157 write_sysreg(vectors
, vbar_el1
);
159 NOKPROBE_SYMBOL(deactivate_traps_vhe
);
161 static void __hyp_text
__deactivate_traps_nvhe(void)
163 u64 mdcr_el2
= read_sysreg(mdcr_el2
);
165 __deactivate_traps_common();
167 mdcr_el2
&= MDCR_EL2_HPMN_MASK
;
168 mdcr_el2
|= MDCR_EL2_E2PB_MASK
<< MDCR_EL2_E2PB_SHIFT
;
170 write_sysreg(mdcr_el2
, mdcr_el2
);
171 write_sysreg(HCR_HOST_NVHE_FLAGS
, hcr_el2
);
172 write_sysreg(CPTR_EL2_DEFAULT
, cptr_el2
);
175 static void __hyp_text
__deactivate_traps(struct kvm_vcpu
*vcpu
)
178 * If we pended a virtual abort, preserve it until it gets
179 * cleared. See D1.14.3 (Virtual Interrupts) for details, but
180 * the crucial bit is "On taking a vSError interrupt,
181 * HCR_EL2.VSE is cleared to 0."
183 if (vcpu
->arch
.hcr_el2
& HCR_VSE
)
184 vcpu
->arch
.hcr_el2
= read_sysreg(hcr_el2
);
187 deactivate_traps_vhe();
189 __deactivate_traps_nvhe();
192 void activate_traps_vhe_load(struct kvm_vcpu
*vcpu
)
194 __activate_traps_common(vcpu
);
197 void deactivate_traps_vhe_put(void)
199 u64 mdcr_el2
= read_sysreg(mdcr_el2
);
201 mdcr_el2
&= MDCR_EL2_HPMN_MASK
|
202 MDCR_EL2_E2PB_MASK
<< MDCR_EL2_E2PB_SHIFT
|
205 write_sysreg(mdcr_el2
, mdcr_el2
);
207 __deactivate_traps_common();
210 static void __hyp_text
__activate_vm(struct kvm
*kvm
)
212 __load_guest_stage2(kvm
);
215 static void __hyp_text
__deactivate_vm(struct kvm_vcpu
*vcpu
)
217 write_sysreg(0, vttbr_el2
);
220 /* Save VGICv3 state on non-VHE systems */
221 static void __hyp_text
__hyp_vgic_save_state(struct kvm_vcpu
*vcpu
)
223 if (static_branch_unlikely(&kvm_vgic_global_state
.gicv3_cpuif
)) {
224 __vgic_v3_save_state(vcpu
);
225 __vgic_v3_deactivate_traps(vcpu
);
229 /* Restore VGICv3 state on non_VEH systems */
230 static void __hyp_text
__hyp_vgic_restore_state(struct kvm_vcpu
*vcpu
)
232 if (static_branch_unlikely(&kvm_vgic_global_state
.gicv3_cpuif
)) {
233 __vgic_v3_activate_traps(vcpu
);
234 __vgic_v3_restore_state(vcpu
);
238 static bool __hyp_text
__true_value(void)
243 static bool __hyp_text
__false_value(void)
248 static hyp_alternate_select(__check_arm_834220
,
249 __false_value
, __true_value
,
250 ARM64_WORKAROUND_834220
);
252 static bool __hyp_text
__translate_far_to_hpfar(u64 far
, u64
*hpfar
)
257 * Resolve the IPA the hard way using the guest VA.
259 * Stage-1 translation already validated the memory access
260 * rights. As such, we can use the EL1 translation regime, and
261 * don't have to distinguish between EL0 and EL1 access.
263 * We do need to save/restore PAR_EL1 though, as we haven't
264 * saved the guest context yet, and we may return early...
266 par
= read_sysreg(par_el1
);
267 asm volatile("at s1e1r, %0" : : "r" (far
));
270 tmp
= read_sysreg(par_el1
);
271 write_sysreg(par
, par_el1
);
273 if (unlikely(tmp
& 1))
274 return false; /* Translation failed, back to guest */
276 /* Convert PAR to HPFAR format */
277 *hpfar
= PAR_TO_HPFAR(tmp
);
281 static bool __hyp_text
__populate_fault_info(struct kvm_vcpu
*vcpu
)
287 esr
= vcpu
->arch
.fault
.esr_el2
;
288 ec
= ESR_ELx_EC(esr
);
290 if (ec
!= ESR_ELx_EC_DABT_LOW
&& ec
!= ESR_ELx_EC_IABT_LOW
)
293 far
= read_sysreg_el2(far
);
296 * The HPFAR can be invalid if the stage 2 fault did not
297 * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
298 * bit is clear) and one of the two following cases are true:
299 * 1. The fault was due to a permission fault
300 * 2. The processor carries errata 834220
302 * Therefore, for all non S1PTW faults where we either have a
303 * permission fault or the errata workaround is enabled, we
304 * resolve the IPA using the AT instruction.
306 if (!(esr
& ESR_ELx_S1PTW
) &&
307 (__check_arm_834220()() || (esr
& ESR_ELx_FSC_TYPE
) == FSC_PERM
)) {
308 if (!__translate_far_to_hpfar(far
, &hpfar
))
311 hpfar
= read_sysreg(hpfar_el2
);
314 vcpu
->arch
.fault
.far_el2
= far
;
315 vcpu
->arch
.fault
.hpfar_el2
= hpfar
;
319 static bool __hyp_text
__hyp_switch_fpsimd(struct kvm_vcpu
*vcpu
)
321 struct user_fpsimd_state
*host_fpsimd
= vcpu
->arch
.host_fpsimd_state
;
324 write_sysreg(read_sysreg(cpacr_el1
) | CPACR_EL1_FPEN
,
327 write_sysreg(read_sysreg(cptr_el2
) & ~(u64
)CPTR_EL2_TFP
,
332 if (vcpu
->arch
.flags
& KVM_ARM64_FP_HOST
) {
334 * In the SVE case, VHE is assumed: it is enforced by
335 * Kconfig and kvm_arch_init().
337 if (system_supports_sve() &&
338 (vcpu
->arch
.flags
& KVM_ARM64_HOST_SVE_IN_USE
)) {
339 struct thread_struct
*thread
= container_of(
341 struct thread_struct
, uw
.fpsimd_state
);
343 sve_save_state(sve_pffr(thread
), &host_fpsimd
->fpsr
);
345 __fpsimd_save_state(host_fpsimd
);
348 vcpu
->arch
.flags
&= ~KVM_ARM64_FP_HOST
;
351 __fpsimd_restore_state(&vcpu
->arch
.ctxt
.gp_regs
.fp_regs
);
353 /* Skip restoring fpexc32 for AArch64 guests */
354 if (!(read_sysreg(hcr_el2
) & HCR_RW
))
355 write_sysreg(vcpu
->arch
.ctxt
.sys_regs
[FPEXC32_EL2
],
358 vcpu
->arch
.flags
|= KVM_ARM64_FP_ENABLED
;
364 * Return true when we were able to fixup the guest exit and should return to
365 * the guest, false when we should restore the host state and return to the
368 static bool __hyp_text
fixup_guest_exit(struct kvm_vcpu
*vcpu
, u64
*exit_code
)
370 if (ARM_EXCEPTION_CODE(*exit_code
) != ARM_EXCEPTION_IRQ
)
371 vcpu
->arch
.fault
.esr_el2
= read_sysreg_el2(esr
);
374 * We're using the raw exception code in order to only process
375 * the trap if no SError is pending. We will come back to the
376 * same PC once the SError has been injected, and replay the
377 * trapping instruction.
379 if (*exit_code
!= ARM_EXCEPTION_TRAP
)
383 * We trap the first access to the FP/SIMD to save the host context
384 * and restore the guest context lazily.
385 * If FP/SIMD is not implemented, handle the trap and inject an
386 * undefined instruction exception to the guest.
388 if (system_supports_fpsimd() &&
389 kvm_vcpu_trap_get_class(vcpu
) == ESR_ELx_EC_FP_ASIMD
)
390 return __hyp_switch_fpsimd(vcpu
);
392 if (!__populate_fault_info(vcpu
))
395 if (static_branch_unlikely(&vgic_v2_cpuif_trap
)) {
398 valid
= kvm_vcpu_trap_get_class(vcpu
) == ESR_ELx_EC_DABT_LOW
&&
399 kvm_vcpu_trap_get_fault_type(vcpu
) == FSC_FAULT
&&
400 kvm_vcpu_dabt_isvalid(vcpu
) &&
401 !kvm_vcpu_dabt_isextabt(vcpu
) &&
402 !kvm_vcpu_dabt_iss1tw(vcpu
);
405 int ret
= __vgic_v2_perform_cpuif_access(vcpu
);
410 /* Promote an illegal access to an SError.*/
412 *exit_code
= ARM_EXCEPTION_EL1_SERROR
;
418 if (static_branch_unlikely(&vgic_v3_cpuif_trap
) &&
419 (kvm_vcpu_trap_get_class(vcpu
) == ESR_ELx_EC_SYS64
||
420 kvm_vcpu_trap_get_class(vcpu
) == ESR_ELx_EC_CP15_32
)) {
421 int ret
= __vgic_v3_perform_cpuif_access(vcpu
);
428 /* Return to the host kernel and handle the exit */
432 static inline bool __hyp_text
__needs_ssbd_off(struct kvm_vcpu
*vcpu
)
434 if (!cpus_have_const_cap(ARM64_SSBD
))
437 return !(vcpu
->arch
.workaround_flags
& VCPU_WORKAROUND_2_FLAG
);
440 static void __hyp_text
__set_guest_arch_workaround_state(struct kvm_vcpu
*vcpu
)
442 #ifdef CONFIG_ARM64_SSBD
444 * The host runs with the workaround always present. If the
445 * guest wants it disabled, so be it...
447 if (__needs_ssbd_off(vcpu
) &&
448 __hyp_this_cpu_read(arm64_ssbd_callback_required
))
449 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2
, 0, NULL
);
453 static void __hyp_text
__set_host_arch_workaround_state(struct kvm_vcpu
*vcpu
)
455 #ifdef CONFIG_ARM64_SSBD
457 * If the guest has disabled the workaround, bring it back on.
459 if (__needs_ssbd_off(vcpu
) &&
460 __hyp_this_cpu_read(arm64_ssbd_callback_required
))
461 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2
, 1, NULL
);
465 /* Switch to the guest for VHE systems running in EL2 */
466 int kvm_vcpu_run_vhe(struct kvm_vcpu
*vcpu
)
468 struct kvm_cpu_context
*host_ctxt
;
469 struct kvm_cpu_context
*guest_ctxt
;
472 host_ctxt
= vcpu
->arch
.host_cpu_context
;
473 host_ctxt
->__hyp_running_vcpu
= vcpu
;
474 guest_ctxt
= &vcpu
->arch
.ctxt
;
476 sysreg_save_host_state_vhe(host_ctxt
);
479 * ARM erratum 1165522 requires us to configure both stage 1 and
480 * stage 2 translation for the guest context before we clear
483 * We have already configured the guest's stage 1 translation in
484 * kvm_vcpu_load_sysregs above. We must now call __activate_vm
485 * before __activate_traps, because __activate_vm configures
486 * stage 2 translation, and __activate_traps clear HCR_EL2.TGE
487 * (among other things).
489 __activate_vm(vcpu
->kvm
);
490 __activate_traps(vcpu
);
492 sysreg_restore_guest_state_vhe(guest_ctxt
);
493 __debug_switch_to_guest(vcpu
);
495 __set_guest_arch_workaround_state(vcpu
);
498 /* Jump in the fire! */
499 exit_code
= __guest_enter(vcpu
, host_ctxt
);
501 /* And we're baaack! */
502 } while (fixup_guest_exit(vcpu
, &exit_code
));
504 __set_host_arch_workaround_state(vcpu
);
506 sysreg_save_guest_state_vhe(guest_ctxt
);
508 __deactivate_traps(vcpu
);
510 sysreg_restore_host_state_vhe(host_ctxt
);
512 if (vcpu
->arch
.flags
& KVM_ARM64_FP_ENABLED
)
513 __fpsimd_save_fpexc32(vcpu
);
515 __debug_switch_to_host(vcpu
);
519 NOKPROBE_SYMBOL(kvm_vcpu_run_vhe
);
521 /* Switch to the guest for legacy non-VHE systems */
522 int __hyp_text
__kvm_vcpu_run_nvhe(struct kvm_vcpu
*vcpu
)
524 struct kvm_cpu_context
*host_ctxt
;
525 struct kvm_cpu_context
*guest_ctxt
;
528 vcpu
= kern_hyp_va(vcpu
);
530 host_ctxt
= kern_hyp_va(vcpu
->arch
.host_cpu_context
);
531 host_ctxt
->__hyp_running_vcpu
= vcpu
;
532 guest_ctxt
= &vcpu
->arch
.ctxt
;
534 __sysreg_save_state_nvhe(host_ctxt
);
536 __activate_vm(kern_hyp_va(vcpu
->kvm
));
537 __activate_traps(vcpu
);
539 __hyp_vgic_restore_state(vcpu
);
540 __timer_enable_traps(vcpu
);
543 * We must restore the 32-bit state before the sysregs, thanks
544 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
546 __sysreg32_restore_state(vcpu
);
547 __sysreg_restore_state_nvhe(guest_ctxt
);
548 __debug_switch_to_guest(vcpu
);
550 __set_guest_arch_workaround_state(vcpu
);
553 /* Jump in the fire! */
554 exit_code
= __guest_enter(vcpu
, host_ctxt
);
556 /* And we're baaack! */
557 } while (fixup_guest_exit(vcpu
, &exit_code
));
559 __set_host_arch_workaround_state(vcpu
);
561 __sysreg_save_state_nvhe(guest_ctxt
);
562 __sysreg32_save_state(vcpu
);
563 __timer_disable_traps(vcpu
);
564 __hyp_vgic_save_state(vcpu
);
566 __deactivate_traps(vcpu
);
567 __deactivate_vm(vcpu
);
569 __sysreg_restore_state_nvhe(host_ctxt
);
571 if (vcpu
->arch
.flags
& KVM_ARM64_FP_ENABLED
)
572 __fpsimd_save_fpexc32(vcpu
);
575 * This must come after restoring the host sysregs, since a non-VHE
576 * system may enable SPE here and make use of the TTBRs.
578 __debug_switch_to_host(vcpu
);
583 static const char __hyp_panic_string
[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
585 static void __hyp_text
__hyp_call_panic_nvhe(u64 spsr
, u64 elr
, u64 par
,
586 struct kvm_cpu_context
*__host_ctxt
)
588 struct kvm_vcpu
*vcpu
;
589 unsigned long str_va
;
591 vcpu
= __host_ctxt
->__hyp_running_vcpu
;
593 if (read_sysreg(vttbr_el2
)) {
594 __timer_disable_traps(vcpu
);
595 __deactivate_traps(vcpu
);
596 __deactivate_vm(vcpu
);
597 __sysreg_restore_state_nvhe(__host_ctxt
);
601 * Force the panic string to be loaded from the literal pool,
602 * making sure it is a kernel address and not a PC-relative
605 asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va
));
607 __hyp_do_panic(str_va
,
609 read_sysreg(esr_el2
), read_sysreg_el2(far
),
610 read_sysreg(hpfar_el2
), par
, vcpu
);
613 static void __hyp_call_panic_vhe(u64 spsr
, u64 elr
, u64 par
,
614 struct kvm_cpu_context
*host_ctxt
)
616 struct kvm_vcpu
*vcpu
;
617 vcpu
= host_ctxt
->__hyp_running_vcpu
;
619 __deactivate_traps(vcpu
);
620 sysreg_restore_host_state_vhe(host_ctxt
);
622 panic(__hyp_panic_string
,
624 read_sysreg_el2(esr
), read_sysreg_el2(far
),
625 read_sysreg(hpfar_el2
), par
, vcpu
);
627 NOKPROBE_SYMBOL(__hyp_call_panic_vhe
);
629 void __hyp_text __noreturn
hyp_panic(struct kvm_cpu_context
*host_ctxt
)
631 u64 spsr
= read_sysreg_el2(spsr
);
632 u64 elr
= read_sysreg_el2(elr
);
633 u64 par
= read_sysreg(par_el1
);
636 __hyp_call_panic_nvhe(spsr
, elr
, par
, host_ctxt
);
638 __hyp_call_panic_vhe(spsr
, elr
, par
, host_ctxt
);