KVM: arm64: Fix __pkvm_init_switch_pgd call ABI

Fix the mismatch between the (incorrect) C signature, C call site, and
asm implementation by aligning all three on an API passing the
parameters (pgd and SP) separately, instead of as a bundled struct.

Remove the now unnecessary memory accesses while the MMU is off from the
asm, which simplifies the C caller (as it does not need to convert a VA
struct pointer to PA) and makes the code slightly more robust by
offsetting the struct fields from C and properly expressing the call to
the C compiler (e.g. type checker and kCFI).

Fixes: f320bc742bc2 ("KVM: arm64: Prepare the creation of s1 mappings at EL2")
Signed-off-by: Pierre-Clément Tosi <ptosi@google.com>
Acked-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20240610063244.2828978-3-ptosi@google.com
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>

diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index b05bceca33850d454c450ae84c9c973c872a1ef7..c838309e4ec47e395d78127a8ee6bad8390c4411 100644 (file)
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -124,8 +124,8 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
 #endif
 
 #ifdef __KVM_NVHE_HYPERVISOR__
-void __pkvm_init_switch_pgd(phys_addr_t phys, unsigned long size,
-                           phys_addr_t pgd, void *sp, void *cont_fn);
+void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
+               void (*fn)(void));
 int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
                unsigned long *per_cpu_base, u32 hyp_va_bits);
 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 2994878d68ea7ecc752cd88558f954678ded3a76..3a2836a52e85107e99fef9076ad8af2853f87b81 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -265,33 +265,35 @@ alternative_else_nop_endif
 
 SYM_CODE_END(__kvm_handle_stub_hvc)
 
+/*
+ * void __pkvm_init_switch_pgd(phys_addr_t pgd, unsigned long sp,
+ *                             void (*fn)(void));
+ */
 SYM_FUNC_START(__pkvm_init_switch_pgd)
        /* Turn the MMU off */
        pre_disable_mmu_workaround
-       mrs     x2, sctlr_el2
-       bic     x3, x2, #SCTLR_ELx_M
-       msr     sctlr_el2, x3
+       mrs     x3, sctlr_el2
+       bic     x4, x3, #SCTLR_ELx_M
+       msr     sctlr_el2, x4
        isb
 
        tlbi    alle2
 
        /* Install the new pgtables */
-       ldr     x3, [x0, #NVHE_INIT_PGD_PA]
-       phys_to_ttbr x4, x3
+       phys_to_ttbr x5, x0
 alternative_if ARM64_HAS_CNP
-       orr     x4, x4, #TTBR_CNP_BIT
+       orr     x5, x5, #TTBR_CNP_BIT
 alternative_else_nop_endif
-       msr     ttbr0_el2, x4
+       msr     ttbr0_el2, x5
 
        /* Set the new stack pointer */
-       ldr     x0, [x0, #NVHE_INIT_STACK_HYP_VA]
-       mov     sp, x0
+       mov     sp, x1
 
        /* And turn the MMU back on! */
        dsb     nsh
        isb
-       set_sctlr_el2   x2
-       ret     x1
+       set_sctlr_el2   x3
+       ret     x2
 SYM_FUNC_END(__pkvm_init_switch_pgd)
 
        .popsection

diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index f4350ba07b0b0c37440af0783d313108e1cc0a92..174007f3faddd9ab151c087e506f506e1d910e3e 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -339,7 +339,7 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
 {
        struct kvm_nvhe_init_params *params;
        void *virt = hyp_phys_to_virt(phys);
-       void (*fn)(phys_addr_t params_pa, void *finalize_fn_va);
+       typeof(__pkvm_init_switch_pgd) *fn;
        int ret;
 
        BUG_ON(kvm_check_pvm_sysreg_table());
@@ -363,7 +363,7 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
        /* Jump in the idmap page to switch to the new page-tables */
        params = this_cpu_ptr(&kvm_init_params);
        fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd);
-       fn(__hyp_pa(params), __pkvm_init_finalise);
+       fn(params->pgd_pa, params->stack_hyp_va, __pkvm_init_finalise);
 
        unreachable();
 }