KVM: arm64: Correctly cap ZCR_EL2 provided by a guest hypervisor

ZCR_EL2 can be updated by a VHE guest hypervisor either using ZCR_EL2
(which traps) or ZCR_EL1 (which does not trap). KVM handles both in
different way:

- on ZCR_EL2 trap, ZCR_EL2.LEN is immediately capped at the VM's own
  VL limit. This has the potential to break existing SW that relies
  on the full LEN field to be stateful.

- on ZCR_EL1 access, we do absolutely nothing.

On restoring the SVE context for an L2 guest, we directly restore the
guest hypervisor's view of ZCR_EL2 into the physical ZCR_EL2. If the
guest's view of the register was updated using the ZCR_EL2 accessor,
the value has already been sanitised (with the caveat mentioned above).

But if the guest used ZCR_EL1, the raw value is written into the HW,
and the L2 guest can now access VLs that it shouldn't.

Fix all the above by moving the VL capping to the restore points,
ensuring that:

- the HW is always programmed with a capped value, irrespective of
  the accessor being used,

- the ZCR_EL2.LEN field is always completely stateful, irrespective
  of the accessor being used.

Additionally, move ZCR_EL2 to be a sanitised register, ensuring that
only the LEN field is actually stateful. This requires some creative
construction of the RES0 mask, as the sysreg generation script does
not yet generate RAZ/WI fields.

Fixes: b3d29a823099 ("KVM: arm64: nv: Handle ZCR_EL2 traps")
Signed-off-by: Mark Brown <broonie@kernel.org>
Cc: stable@vger.kernel.org
Link: https://patch.msgid.link/20260529-kvm-arm64-fix-zcr-len-nv-v2-1-86cad51992bd@kernel.org
[maz: rewrote commit message, tidy up access_zcr_el2()]
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 65eead8362e0b49d7ca4f36165802d2a964d1fd4..a49042bfa801ff8ce3fe54edd5d93c0273eee2de 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -511,7 +511,6 @@ enum vcpu_sysreg {
        ACTLR_EL2,      /* Auxiliary Control Register (EL2) */
        CPTR_EL2,       /* Architectural Feature Trap Register (EL2) */
        HACR_EL2,       /* Hypervisor Auxiliary Control Register */
-       ZCR_EL2,        /* SVE Control Register (EL2) */
        TTBR0_EL2,      /* Translation Table Base Register 0 (EL2) */
        TTBR1_EL2,      /* Translation Table Base Register 1 (EL2) */
        TCR_EL2,        /* Translation Control Register (EL2) */
@@ -543,6 +542,7 @@ enum vcpu_sysreg {
        SCTLR2_EL2,     /* System Control Register 2 (EL2) */
        MDCR_EL2,       /* Monitor Debug Configuration Register (EL2) */
        CNTHCTL_EL2,    /* Counter-timer Hypervisor Control register */
+       ZCR_EL2,        /* SVE Control Register (EL2) */
 
        /* Any VNCR-capable reg goes after this point */
        MARKER(__VNCR_START__),

diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index bf0eb5e434274af0842178c84208907a3c9795d7..320cd45d49c5aeaa3ba15b22a8a21c7969b0bda9 100644 (file)
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -462,11 +462,13 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
 
 static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
 {
+       u64 zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+
        /*
         * The vCPU's saved SVE state layout always matches the max VL of the
         * vCPU. Start off with the max VL so we can load the SVE state.
         */
-       sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
+       sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2);
        __sve_restore_state(vcpu_sve_pffr(vcpu),
                            &vcpu->arch.ctxt.fp_regs.fpsr,
                            true);
@@ -476,8 +478,10 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
         * nested guest, as the guest hypervisor could select a smaller VL. Slap
         * that into hardware before wrapping up.
         */
-       if (is_nested_ctxt(vcpu))
-               sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
+       if (is_nested_ctxt(vcpu)) {
+               zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2));
+               sve_cond_update_zcr_vq(zcr_el2, SYS_ZCR_EL2);
+       }
 
        write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
 }
@@ -501,11 +505,11 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
                return;
 
        if (vcpu_has_sve(vcpu)) {
+               zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+
                /* A guest hypervisor may restrict the effective max VL. */
                if (is_nested_ctxt(vcpu))
-                       zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
-               else
-                       zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;
+                       zcr_el2 = min(zcr_el2, __vcpu_sys_reg(vcpu, ZCR_EL2));
 
                write_sysreg_el2(zcr_el2, SYS_ZCR);
 

diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 883b6c1008fbb96a43f3d62c983f5c5293a6e800..38f672e9408787be6a774122bb27e9f82dc01f77 100644 (file)
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -1834,6 +1834,11 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
        resx.res1 = VNCR_EL2_RES1;
        set_sysreg_masks(kvm, VNCR_EL2, resx);
 
+       /* ZCR_EL2 - bits 8:4 are RAZ/WI so treat them as RES0 */
+       resx.res0 = ZCR_ELx_RES0 | GENMASK_ULL(8, 4);
+       resx.res1 = ZCR_ELx_RES1;
+       set_sysreg_masks(kvm, ZCR_EL2, resx);
+
 out:
        for (enum vcpu_sysreg sr = __SANITISED_REG_START__; sr < NR_SYS_REGS; sr++)
                __vcpu_rmw_sys_reg(vcpu, sr, |=, 0);

diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 148fc3400ea81559ea630312db4a02bb7a39ba7e..fa5c93c7a1352fa9ffd1b32ab01ddf3475a8d1c7 100644 (file)
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -2862,21 +2862,16 @@ static bool access_zcr_el2(struct kvm_vcpu *vcpu,
                           struct sys_reg_params *p,
                           const struct sys_reg_desc *r)
 {
-       unsigned int vq;
-
        if (guest_hyp_sve_traps_enabled(vcpu)) {
                kvm_inject_nested_sve_trap(vcpu);
                return false;
        }
 
-       if (!p->is_write) {
+       if (!p->is_write)
                p->regval = __vcpu_sys_reg(vcpu, ZCR_EL2);
-               return true;
-       }
+       else
+               __vcpu_assign_sys_reg(vcpu, ZCR_EL2, p->regval);
 
-       vq = SYS_FIELD_GET(ZCR_ELx, LEN, p->regval) + 1;
-       vq = min(vq, vcpu_sve_max_vq(vcpu));
-       __vcpu_assign_sys_reg(vcpu, ZCR_EL2, vq - 1);
        return true;
 }