arch/x86/kvm/vmx/vmenter.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/linkage.h>
   3 #include <asm/asm.h>
   4 #include <asm/bitsperlong.h>
   5 #include <asm/kvm_vcpu_regs.h>
   6 #include <asm/nospec-branch.h>
   7
   8 #define WORD_SIZE (BITS_PER_LONG / 8)
   9
  10 #define VCPU_RAX        __VCPU_REGS_RAX * WORD_SIZE
  11 #define VCPU_RCX        __VCPU_REGS_RCX * WORD_SIZE
  12 #define VCPU_RDX        __VCPU_REGS_RDX * WORD_SIZE
  13 #define VCPU_RBX        __VCPU_REGS_RBX * WORD_SIZE
  14 /* Intentionally omit RSP as it's context switched by hardware */
  15 #define VCPU_RBP        __VCPU_REGS_RBP * WORD_SIZE
  16 #define VCPU_RSI        __VCPU_REGS_RSI * WORD_SIZE
  17 #define VCPU_RDI        __VCPU_REGS_RDI * WORD_SIZE
  18
  19 #ifdef CONFIG_X86_64
  20 #define VCPU_R8         __VCPU_REGS_R8  * WORD_SIZE
  21 #define VCPU_R9         __VCPU_REGS_R9  * WORD_SIZE
  22 #define VCPU_R10        __VCPU_REGS_R10 * WORD_SIZE
  23 #define VCPU_R11        __VCPU_REGS_R11 * WORD_SIZE
  24 #define VCPU_R12        __VCPU_REGS_R12 * WORD_SIZE
  25 #define VCPU_R13        __VCPU_REGS_R13 * WORD_SIZE
  26 #define VCPU_R14        __VCPU_REGS_R14 * WORD_SIZE
  27 #define VCPU_R15        __VCPU_REGS_R15 * WORD_SIZE
  28 #endif
  29
  30         .text
  31
  32 /**
  33  * vmx_vmenter - VM-Enter the current loaded VMCS
  34  *
  35  * %RFLAGS.ZF:  !VMCS.LAUNCHED, i.e. controls VMLAUNCH vs. VMRESUME
  36  *
  37  * Returns:
  38  *      %RFLAGS.CF is set on VM-Fail Invalid
  39  *      %RFLAGS.ZF is set on VM-Fail Valid
  40  *      %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
  41  *
  42  * Note that VMRESUME/VMLAUNCH fall-through and return directly if
  43  * they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
  44  * to vmx_vmexit.
  45  */
  46 SYM_FUNC_START(vmx_vmenter)
  47         /* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
  48         je 2f
  49
  50 1:      vmresume
  51         ret
  52
  53 2:      vmlaunch
  54         ret
  55
  56 3:      cmpb $0, kvm_rebooting
  57         je 4f
  58         ret
  59 4:      ud2
  60
  61         _ASM_EXTABLE(1b, 3b)
  62         _ASM_EXTABLE(2b, 3b)
  63
  64 SYM_FUNC_END(vmx_vmenter)
  65
  66 /**
  67  * vmx_vmexit - Handle a VMX VM-Exit
  68  *
  69  * Returns:
  70  *      %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
  71  *
  72  * This is vmx_vmenter's partner in crime.  On a VM-Exit, control will jump
  73  * here after hardware loads the host's state, i.e. this is the destination
  74  * referred to by VMCS.HOST_RIP.
  75  */
  76 SYM_FUNC_START(vmx_vmexit)
  77 #ifdef CONFIG_RETPOLINE
  78         ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
  79         /* Preserve guest's RAX, it's used to stuff the RSB. */
  80         push %_ASM_AX
  81
  82         /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
  83         FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
  84
  85         pop %_ASM_AX
  86 .Lvmexit_skip_rsb:
  87 #endif
  88         ret
  89 SYM_FUNC_END(vmx_vmexit)
  90
  91 /**
  92  * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
  93  * @vmx:        struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
  94  * @regs:       unsigned long * (to guest registers)
  95  * @launched:   %true if the VMCS has been launched
  96  *
  97  * Returns:
  98  *      0 on VM-Exit, 1 on VM-Fail
  99  */
 100 SYM_FUNC_START(__vmx_vcpu_run)
 101         push %_ASM_BP
 102         mov  %_ASM_SP, %_ASM_BP
 103 #ifdef CONFIG_X86_64
 104         push %r15
 105         push %r14
 106         push %r13
 107         push %r12
 108 #else
 109         push %edi
 110         push %esi
 111 #endif
 112         push %_ASM_BX
 113
 114         /*
 115          * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
 116          * @regs is needed after VM-Exit to save the guest's register values.
 117          */
 118         push %_ASM_ARG2
 119
 120         /* Copy @launched to BL, _ASM_ARG3 is volatile. */
 121         mov %_ASM_ARG3B, %bl
 122
 123         /* Adjust RSP to account for the CALL to vmx_vmenter(). */
 124         lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
 125         call vmx_update_host_rsp
 126
 127         /* Load @regs to RAX. */
 128         mov (%_ASM_SP), %_ASM_AX
 129
 130         /* Check if vmlaunch or vmresume is needed */
 131         cmpb $0, %bl
 132
 133         /* Load guest registers.  Don't clobber flags. */
 134         mov VCPU_RCX(%_ASM_AX), %_ASM_CX
 135         mov VCPU_RDX(%_ASM_AX), %_ASM_DX
 136         mov VCPU_RBX(%_ASM_AX), %_ASM_BX
 137         mov VCPU_RBP(%_ASM_AX), %_ASM_BP
 138         mov VCPU_RSI(%_ASM_AX), %_ASM_SI
 139         mov VCPU_RDI(%_ASM_AX), %_ASM_DI
 140 #ifdef CONFIG_X86_64
 141         mov VCPU_R8 (%_ASM_AX),  %r8
 142         mov VCPU_R9 (%_ASM_AX),  %r9
 143         mov VCPU_R10(%_ASM_AX), %r10
 144         mov VCPU_R11(%_ASM_AX), %r11
 145         mov VCPU_R12(%_ASM_AX), %r12
 146         mov VCPU_R13(%_ASM_AX), %r13
 147         mov VCPU_R14(%_ASM_AX), %r14
 148         mov VCPU_R15(%_ASM_AX), %r15
 149 #endif
 150         /* Load guest RAX.  This kills the @regs pointer! */
 151         mov VCPU_RAX(%_ASM_AX), %_ASM_AX
 152
 153         /* Enter guest mode */
 154         call vmx_vmenter
 155
 156         /* Jump on VM-Fail. */
 157         jbe 2f
 158
 159         /* Temporarily save guest's RAX. */
 160         push %_ASM_AX
 161
 162         /* Reload @regs to RAX. */
 163         mov WORD_SIZE(%_ASM_SP), %_ASM_AX
 164
 165         /* Save all guest registers, including RAX from the stack */
 166         __ASM_SIZE(pop) VCPU_RAX(%_ASM_AX)
 167         mov %_ASM_CX,   VCPU_RCX(%_ASM_AX)
 168         mov %_ASM_DX,   VCPU_RDX(%_ASM_AX)
 169         mov %_ASM_BX,   VCPU_RBX(%_ASM_AX)
 170         mov %_ASM_BP,   VCPU_RBP(%_ASM_AX)
 171         mov %_ASM_SI,   VCPU_RSI(%_ASM_AX)
 172         mov %_ASM_DI,   VCPU_RDI(%_ASM_AX)
 173 #ifdef CONFIG_X86_64
 174         mov %r8,  VCPU_R8 (%_ASM_AX)
 175         mov %r9,  VCPU_R9 (%_ASM_AX)
 176         mov %r10, VCPU_R10(%_ASM_AX)
 177         mov %r11, VCPU_R11(%_ASM_AX)
 178         mov %r12, VCPU_R12(%_ASM_AX)
 179         mov %r13, VCPU_R13(%_ASM_AX)
 180         mov %r14, VCPU_R14(%_ASM_AX)
 181         mov %r15, VCPU_R15(%_ASM_AX)
 182 #endif
 183
 184         /* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */
 185         xor %eax, %eax
 186
 187         /*
 188          * Clear all general purpose registers except RSP and RAX to prevent
 189          * speculative use of the guest's values, even those that are reloaded
 190          * via the stack.  In theory, an L1 cache miss when restoring registers
 191          * could lead to speculative execution with the guest's values.
 192          * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially
 193          * free.  RSP and RAX are exempt as RSP is restored by hardware during
 194          * VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail.
 195          */
 196 1:      xor %ecx, %ecx
 197         xor %edx, %edx
 198         xor %ebx, %ebx
 199         xor %ebp, %ebp
 200         xor %esi, %esi
 201         xor %edi, %edi
 202 #ifdef CONFIG_X86_64
 203         xor %r8d,  %r8d
 204         xor %r9d,  %r9d
 205         xor %r10d, %r10d
 206         xor %r11d, %r11d
 207         xor %r12d, %r12d
 208         xor %r13d, %r13d
 209         xor %r14d, %r14d
 210         xor %r15d, %r15d
 211 #endif
 212
 213         /* "POP" @regs. */
 214         add $WORD_SIZE, %_ASM_SP
 215         pop %_ASM_BX
 216
 217 #ifdef CONFIG_X86_64
 218         pop %r12
 219         pop %r13
 220         pop %r14
 221         pop %r15
 222 #else
 223         pop %esi
 224         pop %edi
 225 #endif
 226         pop %_ASM_BP
 227         ret
 228
 229         /* VM-Fail.  Out-of-line to avoid a taken Jcc after VM-Exit. */
 230 2:      mov $1, %eax
 231         jmp 1b
 232 SYM_FUNC_END(__vmx_vcpu_run)
 233
 234 /**
 235  * vmread_error_trampoline - Trampoline from inline asm to vmread_error()
 236  * @field:      VMCS field encoding that failed
 237  * @fault:      %true if the VMREAD faulted, %false if it failed
 238
 239  * Save and restore volatile registers across a call to vmread_error().  Note,
 240  * all parameters are passed on the stack.
 241  */
 242 SYM_FUNC_START(vmread_error_trampoline)
 243         push %_ASM_BP
 244         mov  %_ASM_SP, %_ASM_BP
 245
 246         push %_ASM_AX
 247         push %_ASM_CX
 248         push %_ASM_DX
 249 #ifdef CONFIG_X86_64
 250         push %rdi
 251         push %rsi
 252         push %r8
 253         push %r9
 254         push %r10
 255         push %r11
 256 #endif
 257 #ifdef CONFIG_X86_64
 258         /* Load @field and @fault to arg1 and arg2 respectively. */
 259         mov 3*WORD_SIZE(%rbp), %_ASM_ARG2
 260         mov 2*WORD_SIZE(%rbp), %_ASM_ARG1
 261 #else
 262         /* Parameters are passed on the stack for 32-bit (see asmlinkage). */
 263         push 3*WORD_SIZE(%ebp)
 264         push 2*WORD_SIZE(%ebp)
 265 #endif
 266
 267         call vmread_error
 268
 269 #ifndef CONFIG_X86_64
 270         add $8, %esp
 271 #endif
 272
 273         /* Zero out @fault, which will be popped into the result register. */
 274         _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP)
 275
 276 #ifdef CONFIG_X86_64
 277         pop %r11
 278         pop %r10
 279         pop %r9
 280         pop %r8
 281         pop %rsi
 282         pop %rdi
 283 #endif
 284         pop %_ASM_DX
 285         pop %_ASM_CX
 286         pop %_ASM_AX
 287         pop %_ASM_BP
 288
 289         ret
 290 SYM_FUNC_END(vmread_error_trampoline)