RISC-V: KVM: Flush VS-stage TLB after VCPU migration for Andes cores

Most implementations cache the combined result of two-stage translation,
but some, like Andes cores, use split TLBs that store VS-stage and
G-stage entries separately.

On such systems, when a VCPU migrates to another CPU, an additional
HFENCE.VVMA is required to avoid using stale VS-stage entries, which
could otherwise cause guest faults.

Introduce a static key to identify CPUs with split two-stage TLBs.
When enabled, KVM issues an extra HFENCE.VVMA on VCPU migration to
prevent stale VS-stage mappings.

Signed-off-by: Hui Min Mina Chou <minachou@andestech.com>
Signed-off-by: Ben Zong-You Xie <ben717@andestech.com>
Reviewed-by: Radim Krčmář <rkrcmar@ventanamicro.com>
Reviewed-by: Nutty Liu <nutty.liu@hotmail.com>
Link: https://lore.kernel.org/r/20251117084555.157642-1-minachou@andestech.com
Signed-off-by: Anup Patel <anup@brainfault.org>

diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index e30548a4ab60817037a40df952ef9804e6538617..24585304c02b14b410e3df5c77d41a6db6f6b273 100644 (file)
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -330,4 +330,7 @@ bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu);
 
 void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu);
 
+/* Flags representing implementation specific details */
+DECLARE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
+
 #endif /* __RISCV_KVM_HOST_H__ */

diff --git a/arch/riscv/include/asm/kvm_tlb.h b/arch/riscv/include/asm/kvm_tlb.h
index 38a2f933ad3abe67244f10885f82941ab9869c63..a0e7099bcb858b213d22baab0775ac0f616b7966 100644 (file)
--- a/arch/riscv/include/asm/kvm_tlb.h
+++ b/arch/riscv/include/asm/kvm_tlb.h
@@ -49,6 +49,7 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid,
                                     unsigned long gva, unsigned long gvsz,
                                     unsigned long order);
 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid);
+void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu);
 
 void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu);
 

diff --git a/arch/riscv/include/asm/kvm_vmid.h b/arch/riscv/include/asm/kvm_vmid.h
index ab98e1434fb7959d629fce0f31418c249f0b8076..db61b0525a8d0fb4f09cd931a3d61b88ee38ee38 100644 (file)
--- a/arch/riscv/include/asm/kvm_vmid.h
+++ b/arch/riscv/include/asm/kvm_vmid.h
@@ -22,6 +22,5 @@ unsigned long kvm_riscv_gstage_vmid_bits(void);
 int kvm_riscv_gstage_vmid_init(struct kvm *kvm);
 bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);
 void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);
-void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu);
 
 #endif

diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c
index 77dc1655b442380a9c31daf70dce622df32e9f0d..45536af521f055e03b32ebbe8aac8c78ba3dcddf 100644 (file)
--- a/arch/riscv/kvm/main.c
+++ b/arch/riscv/kvm/main.c
@@ -15,6 +15,18 @@
 #include <asm/kvm_nacl.h>
 #include <asm/sbi.h>
 
+DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
+
+static void kvm_riscv_setup_vendor_features(void)
+{
+       /* Andes AX66: split two-stage TLBs */
+       if (riscv_cached_mvendorid(0) == ANDES_VENDOR_ID &&
+           (riscv_cached_marchid(0) & 0xFFFF) == 0x8A66) {
+               static_branch_enable(&kvm_riscv_vsstage_tlb_no_gpa);
+               kvm_info("VS-stage TLB does not cache guest physical address and VMID\n");
+       }
+}
+
 long kvm_arch_dev_ioctl(struct file *filp,
                        unsigned int ioctl, unsigned long arg)
 {
@@ -160,6 +172,8 @@ static int __init riscv_kvm_init(void)
                kvm_info("AIA available with %d guest external interrupts\n",
                         kvm_riscv_aia_nr_hgei);
 
+       kvm_riscv_setup_vendor_features();
+
        kvm_register_perf_callbacks(NULL);
 
        rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);

diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c
index 3c5a70a2b9271761d2a2d2bee6db3524cf8d4a3c..ff1aeac4eb8eb58104e008101f3e581f9c0e6e2d 100644 (file)
--- a/arch/riscv/kvm/tlb.c
+++ b/arch/riscv/kvm/tlb.c
@@ -158,6 +158,36 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid)
        csr_write(CSR_HGATP, hgatp);
 }
 
+void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
+{
+       unsigned long vmid;
+
+       if (!kvm_riscv_gstage_vmid_bits() ||
+           vcpu->arch.last_exit_cpu == vcpu->cpu)
+               return;
+
+       /*
+        * On RISC-V platforms with hardware VMID support, we share same
+        * VMID for all VCPUs of a particular Guest/VM. This means we might
+        * have stale G-stage TLB entries on the current Host CPU due to
+        * some other VCPU of the same Guest which ran previously on the
+        * current Host CPU.
+        *
+        * To cleanup stale TLB entries, we simply flush all G-stage TLB
+        * entries by VMID whenever underlying Host CPU changes for a VCPU.
+        */
+
+       vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
+       kvm_riscv_local_hfence_gvma_vmid_all(vmid);
+
+       /*
+        * Flush VS-stage TLB entries for implementation where VS-stage
+        * TLB does not cahce guest physical address and VMID.
+        */
+       if (static_branch_unlikely(&kvm_riscv_vsstage_tlb_no_gpa))
+               kvm_riscv_local_hfence_vvma_all(vmid);
+}
+
 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
 {
        kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_RCVD);

diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 5ce35aba60696cf5a1d736b31e08cf7da77e7792..9f07a3177a28ee46d46803bafe0de93d0eda3971 100644 (file)
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -968,7 +968,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                 * Note: This should be done after G-stage VMID has been
                 * updated using kvm_riscv_gstage_vmid_ver_changed()
                 */
-               kvm_riscv_gstage_vmid_sanitize(vcpu);
+               kvm_riscv_local_tlb_sanitize(vcpu);
 
                trace_kvm_entry(vcpu);
 

diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c
index abb1c2bf25423444e6e7d336141830773f2d4696..cf34d448289d79cb6b1f3832d97ee6997f27de5c 100644 (file)
--- a/arch/riscv/kvm/vmid.c
+++ b/arch/riscv/kvm/vmid.c
@@ -122,26 +122,3 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
        kvm_for_each_vcpu(i, v, vcpu->kvm)
                kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
 }
-
-void kvm_riscv_gstage_vmid_sanitize(struct kvm_vcpu *vcpu)
-{
-       unsigned long vmid;
-
-       if (!kvm_riscv_gstage_vmid_bits() ||
-           vcpu->arch.last_exit_cpu == vcpu->cpu)
-               return;
-
-       /*
-        * On RISC-V platforms with hardware VMID support, we share same
-        * VMID for all VCPUs of a particular Guest/VM. This means we might
-        * have stale G-stage TLB entries on the current Host CPU due to
-        * some other VCPU of the same Guest which ran previously on the
-        * current Host CPU.
-        *
-        * To cleanup stale TLB entries, we simply flush all G-stage TLB
-        * entries by VMID whenever underlying Host CPU changes for a VCPU.
-        */
-
-       vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
-       kvm_riscv_local_hfence_gvma_vmid_all(vmid);
-}