KVM: x86/mmu: Fold set_external_spte_present() into its sole caller

Fold set_external_spte_present() into __tdp_mmu_set_spte_atomic() in
anticipation of propagating *all* changes (like atomic zap) triggered by
tdp_mmu_set_spte_atomic() to the external PTEs.

No functional change intended.

Signed-off-by: Yan Zhao <yan.y.zhao@intel.com>
Link: https://patch.msgid.link/20260509075520.4177-1-yan.y.zhao@intel.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 0dc1b0597f8aceaa2ece2833dbd40cc1fb61031b..128089571e34caab51ac6b24148b7e102ad17b75 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -495,33 +495,6 @@ static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared)
        call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);
 }
 
-static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep,
-                                                gfn_t gfn, u64 *old_spte,
-                                                u64 new_spte, int level)
-{
-       bool was_present = is_shadow_present_pte(*old_spte);
-       int ret;
-
-       KVM_BUG_ON(was_present, kvm);
-
-       lockdep_assert_held(&kvm->mmu_lock);
-       /*
-        * We need to lock out other updates to the SPTE until the external
-        * page table has been modified. Use FROZEN_SPTE similar to
-        * the zapping case.
-        */
-       if (!try_cmpxchg64(rcu_dereference(sptep), old_spte, FROZEN_SPTE))
-               return -EBUSY;
-
-       ret = kvm_x86_call(set_external_spte)(kvm, gfn, level, new_spte);
-
-       if (ret)
-               __kvm_tdp_mmu_write_spte(sptep, *old_spte);
-       else
-               __kvm_tdp_mmu_write_spte(sptep, new_spte);
-       return ret;
-}
-
 /**
  * handle_changed_spte - handle bookkeeping associated with an SPTE change
  * @kvm: kvm instance
@@ -626,6 +599,8 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm,
                                                         struct tdp_iter *iter,
                                                         u64 new_spte)
 {
+       u64 *raw_sptep = rcu_dereference(iter->sptep);
+
        /*
         * The caller is responsible for ensuring the old SPTE is not a FROZEN
         * SPTE.  KVM should never attempt to zap or manipulate a FROZEN SPTE,
@@ -635,8 +610,13 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm,
        WARN_ON_ONCE(iter->yielded || is_frozen_spte(iter->old_spte));
 
        if (is_mirror_sptep(iter->sptep) && !is_frozen_spte(new_spte)) {
+               bool was_present = is_shadow_present_pte(iter->old_spte);
                int ret;
 
+               KVM_BUG_ON(was_present, kvm);
+
+               lockdep_assert_held(&kvm->mmu_lock);
+
                /*
                 * Users of atomic zapping don't operate on mirror roots,
                 * so don't handle it and bug the VM if it's seen.
@@ -644,25 +624,35 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm,
                if (KVM_BUG_ON(!is_shadow_present_pte(new_spte), kvm))
                        return -EBUSY;
 
-               ret = set_external_spte_present(kvm, iter->sptep, iter->gfn,
-                                               &iter->old_spte, new_spte, iter->level);
-               if (ret)
-                       return ret;
-       } else {
-               u64 *sptep = rcu_dereference(iter->sptep);
-
                /*
-                * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs
-                * and does not hold the mmu_lock.  On failure, i.e. if a
-                * different logical CPU modified the SPTE, try_cmpxchg64()
-                * updates iter->old_spte with the current value, so the caller
-                * operates on fresh data, e.g. if it retries
-                * tdp_mmu_set_spte_atomic()
+                * We need to lock out other updates to the SPTE until the external
+                * page table has been modified. Use FROZEN_SPTE similar to
+                * the zapping case.
                 */
-               if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))
+               if (!try_cmpxchg64(raw_sptep, &iter->old_spte, FROZEN_SPTE))
                        return -EBUSY;
+
+               ret = kvm_x86_call(set_external_spte)(kvm, iter->gfn, iter->level,
+                                                     new_spte);
+
+               if (ret)
+                       __kvm_tdp_mmu_write_spte(iter->sptep, iter->old_spte);
+               else
+                       __kvm_tdp_mmu_write_spte(iter->sptep, new_spte);
+
+               return ret;
        }
 
+       /*
+        * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and
+        * does not hold the mmu_lock.  On failure, i.e. if a different logical
+        * CPU modified the SPTE, try_cmpxchg64() updates iter->old_spte with
+        * the current value, so the caller operates on fresh data, e.g. if it
+        * retries tdp_mmu_set_spte_atomic().
+        */
+       if (!try_cmpxchg64(raw_sptep, &iter->old_spte, new_spte))
+               return -EBUSY;
+
        return 0;
 }