KVM: Introduce vcpu->wants_to_run

Introduce vcpu->wants_to_run to indicate when a vCPU is in its core run
loop, i.e. when the vCPU is running the KVM_RUN ioctl and immediate_exit
was not set.

Replace all references to vcpu->run->immediate_exit with
!vcpu->wants_to_run to avoid TOCTOU races with userspace. For example, a
malicious userspace could invoked KVM_RUN with immediate_exit=true and
then after KVM reads it to set wants_to_run=false, flip it to false.
This would result in the vCPU running in KVM_RUN with
wants_to_run=false. This wouldn't cause any real bugs today but is a
dangerous landmine.

Signed-off-by: David Matlack <dmatlack@google.com>
Link: https://lore.kernel.org/r/20240503181734.1467938-2-dmatlack@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 9996a989b52e8742566ff6a8fa978981e2a91d49..f9a418e9ea6a6c2e08f9548c165cd38ffa9327f7 100644 (file)
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1099,7 +1099,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
        vcpu_load(vcpu);
 
-       if (run->immediate_exit) {
+       if (!vcpu->wants_to_run) {
                ret = -EINTR;
                goto out;
        }

diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c
index 9e8030d4512902656a17597a186566118bdc6891..f27d2bf14dc058166df1678ef1b98bcab32b4474 100644 (file)
--- a/arch/loongarch/kvm/vcpu.c
+++ b/arch/loongarch/kvm/vcpu.c
@@ -1266,7 +1266,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                        kvm_complete_iocsr_read(vcpu, run);
        }
 
-       if (run->immediate_exit)
+       if (!vcpu->wants_to_run)
                return r;
 
        /* Clear exit_reason */

diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index 231ac052b506b9a69768e2f06be969fabddd0e45..f1a99962027a4fdd3fe42ebd183de71d07a4f665 100644 (file)
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -436,7 +436,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                vcpu->mmio_needed = 0;
        }
 
-       if (vcpu->run->immediate_exit)
+       if (!vcpu->wants_to_run)
                goto out;
 
        lose_fpu(1);

diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index d32abe7fe6ab79ea81a4583126b036a24e01d4b0..961aadc71de223c53f70e70d114a5b6caf2ac697 100644 (file)
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -1852,7 +1852,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
        kvm_sigset_activate(vcpu);
 
-       if (run->immediate_exit)
+       if (!vcpu->wants_to_run)
                r = -EINTR;
        else
                r = kvmppc_vcpu_run(vcpu);

diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 17e21df36cc1ec3501435a5cf57dd4bfac24fdd8..2123df6ee409eca6c56c14126e044a5f072537fd 100644 (file)
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -760,7 +760,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                return ret;
        }
 
-       if (run->immediate_exit) {
+       if (!vcpu->wants_to_run) {
                kvm_vcpu_srcu_read_unlock(vcpu);
                return -EINTR;
        }

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 4641083ee1009193bb1535bedd898464dfd07f22..5da80897608515f6a47f3c8a6fb87cd491c3e850 100644 (file)
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -5026,7 +5026,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
        if (vcpu->kvm->arch.pv.dumping)
                return -EINVAL;
 
-       if (kvm_run->immediate_exit)
+       if (!vcpu->wants_to_run)
                return -EINTR;
 
        if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 994aa281b07d532eda5afd7928aa025823d63d93..de581487e3c9bb1c0f12226560c70f62e6906a08 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11407,7 +11407,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
        kvm_vcpu_srcu_read_lock(vcpu);
        if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
-               if (kvm_run->immediate_exit) {
+               if (!vcpu->wants_to_run) {
                        r = -EINTR;
                        goto out;
                }
@@ -11485,7 +11485,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
                WARN_ON_ONCE(vcpu->mmio_needed);
        }
 
-       if (kvm_run->immediate_exit) {
+       if (!vcpu->wants_to_run) {
                r = -EINTR;
                goto out;
        }

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 7b9d2633a9315a35dfe271f9cdd76ea088345294..d72ced3e74d1cc14928853a11b07f0e57b41361c 100644 (file)
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -378,6 +378,7 @@ struct kvm_vcpu {
                bool dy_eligible;
        } spin_loop;
 #endif
+       bool wants_to_run;
        bool preempted;
        bool ready;
        bool scheduled_out;

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index fed80bfbe99bbbc96007d88096fb09f5c6102768..66bfdfa1180527b8ac6eddde5ac3790fd03c4336 100644 (file)
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -4435,7 +4435,10 @@ static long kvm_vcpu_ioctl(struct file *filp,
                                synchronize_rcu();
                        put_pid(oldpid);
                }
+               vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit);
                r = kvm_arch_vcpu_ioctl_run(vcpu);
+               vcpu->wants_to_run = false;
+
                trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
                break;
        }