KVM: arm64: Return -EFAULT from VCPU_RUN on access to a poisoned pte

If a protected vCPU faults on an IPA which appears to be mapped, query
the hypervisor to determine whether or not the faulting pte has been
poisoned by a forceful reclaim. If the pte has been poisoned, return
-EFAULT back to userspace rather than retrying the instruction forever.

Tested-by: Fuad Tabba <tabba@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Will Deacon <will@kernel.org>
Link: https://patch.msgid.link/20260330144841.26181-28-will@kernel.org
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 04a230e906a76c4bedb3bc6440d10c4035721953..6c79f7504d80f5c65ce84e7f77f73e8da8dae3b2 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -90,6 +90,7 @@ enum __kvm_host_smccc_func {
        __KVM_HOST_SMCCC_FUNC___pkvm_unreserve_vm,
        __KVM_HOST_SMCCC_FUNC___pkvm_init_vm,
        __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu,
+       __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_in_poison_fault,
        __KVM_HOST_SMCCC_FUNC___pkvm_force_reclaim_guest_page,
        __KVM_HOST_SMCCC_FUNC___pkvm_reclaim_dying_guest_page,
        __KVM_HOST_SMCCC_FUNC___pkvm_start_teardown_vm,

diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
index acc0311036004934f54e29d309427848ce804fee..8bc9a2489298d4db311a21bf1ed705462befc597 100644 (file)
--- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h
@@ -40,6 +40,7 @@ int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages);
 int __pkvm_host_share_ffa(u64 pfn, u64 nr_pages);
 int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages);
 int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu);
+int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu);
 int __pkvm_host_force_reclaim_page_guest(phys_addr_t phys);
 int __pkvm_host_reclaim_page_guest(u64 gfn, struct pkvm_hyp_vm *vm);
 int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu *vcpu,

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 456c832077173e88f1a51e0027c1b2bd119c1d58..90e3b14fe287962541d8aab173a206e0271d7443 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -573,6 +573,15 @@ static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt)
        cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva);
 }
 
+static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt)
+{
+       int ret;
+       struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
+
+       ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL;
+       cpu_reg(host_ctxt, 1) = ret;
+}
+
 static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt)
 {
        DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
@@ -641,6 +650,7 @@ static const hcall_t host_hcall[] = {
        HANDLE_FUNC(__pkvm_unreserve_vm),
        HANDLE_FUNC(__pkvm_init_vm),
        HANDLE_FUNC(__pkvm_init_vcpu),
+       HANDLE_FUNC(__pkvm_vcpu_in_poison_fault),
        HANDLE_FUNC(__pkvm_force_reclaim_guest_page),
        HANDLE_FUNC(__pkvm_reclaim_dying_guest_page),
        HANDLE_FUNC(__pkvm_start_teardown_vm),

diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 73bdbd4a508ee81cac3a7d73cb47e643efd9941d..31ca3db26fb5aab95b224b8f1653aa68e6ed894d 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -890,6 +890,49 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep,
        return 0;
 }
 
+int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu)
+{
+       struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
+       kvm_pte_t pte;
+       s8 level;
+       u64 ipa;
+       int ret;
+
+       switch (kvm_vcpu_trap_get_class(&hyp_vcpu->vcpu)) {
+       case ESR_ELx_EC_DABT_LOW:
+       case ESR_ELx_EC_IABT_LOW:
+               if (kvm_vcpu_trap_is_translation_fault(&hyp_vcpu->vcpu))
+                       break;
+               fallthrough;
+       default:
+               return -EINVAL;
+       }
+
+       /*
+        * The host has the faulting IPA when it calls us from the guest
+        * fault handler but we retrieve it ourselves from the FAR so as
+        * to avoid exposing an "oracle" that could reveal data access
+        * patterns of the guest after initial donation of its pages.
+        */
+       ipa = kvm_vcpu_get_fault_ipa(&hyp_vcpu->vcpu);
+       ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu));
+
+       guest_lock_component(vm);
+       ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
+       if (ret)
+               goto unlock;
+
+       if (level != KVM_PGTABLE_LAST_LEVEL) {
+               ret = -EINVAL;
+               goto unlock;
+       }
+
+       ret = guest_pte_is_poisoned(pte);
+unlock:
+       guest_unlock_component(vm);
+       return ret;
+}
+
 int __pkvm_host_share_hyp(u64 pfn)
 {
        u64 phys = hyp_pfn_to_phys(pfn);

diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 10edd496593628b62f368b48949a8ac0ca12c9e3..7f35df29e98489ad39d9c5e6c421d01b3a5db696 100644 (file)
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -423,10 +423,13 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
                        return -EINVAL;
 
                /*
-                * We raced with another vCPU.
+                * We either raced with another vCPU or the guest PTE
+                * has been poisoned by an erroneous host access.
                 */
-               if (mapping)
-                       return -EAGAIN;
+               if (mapping) {
+                       ret = kvm_call_hyp_nvhe(__pkvm_vcpu_in_poison_fault);
+                       return ret ? -EFAULT : -EAGAIN;
+               }
 
                ret = kvm_call_hyp_nvhe(__pkvm_host_donate_guest, pfn, gfn);
        } else {