+++ /dev/null
-From e9ee1a47b9290e949ebb77c91e363bbcc9ed48a9 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 27 Oct 2023 11:21:52 -0700
-Subject: KVM: Add a dedicated mmu_notifier flag for reclaiming freed memory
-
-From: Sean Christopherson <seanjc@google.com>
-
-[ Upstream commit cec29eef0a815386d520d61c2cbe16d537931639 ]
-
-Handle AMD SEV's kvm_arch_guest_memory_reclaimed() hook by having
-__kvm_handle_hva_range() return whether or not an overlapping memslot
-was found, i.e. mmu_lock was acquired. Using the .on_unlock() hook
-works, but kvm_arch_guest_memory_reclaimed() needs to run after dropping
-mmu_lock, which makes .on_lock() and .on_unlock() asymmetrical.
-
-Use a small struct to return the tuple of the notifier-specific return,
-plus whether or not overlap was found. Because the iteration helpers are
-__always_inlined, practically speaking, the struct will never actually be
-returned from a function call (not to mention the size of the struct will
-be two bytes in practice).
-
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
-Reviewed-by: Fuad Tabba <tabba@google.com>
-Tested-by: Fuad Tabba <tabba@google.com>
-Message-Id: <20231027182217.3615211-11-seanjc@google.com>
-Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-Stable-dep-of: c3f3edf73a8f ("KVM: Stop processing *all* memslots when "null" mmu_notifier handler is found")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 53 +++++++++++++++++++++++++++++++--------------
- 1 file changed, 37 insertions(+), 16 deletions(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 9cc57b23ec81f..b3b3edcc537de 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -561,6 +561,19 @@ struct kvm_mmu_notifier_range {
- bool may_block;
- };
-
-+/*
-+ * The inner-most helper returns a tuple containing the return value from the
-+ * arch- and action-specific handler, plus a flag indicating whether or not at
-+ * least one memslot was found, i.e. if the handler found guest memory.
-+ *
-+ * Note, most notifiers are averse to booleans, so even though KVM tracks the
-+ * return from arch code as a bool, outer helpers will cast it to an int. :-(
-+ */
-+typedef struct kvm_mmu_notifier_return {
-+ bool ret;
-+ bool found_memslot;
-+} kvm_mn_ret_t;
-+
- /*
- * Use a dedicated stub instead of NULL to indicate that there is no callback
- * function/handler. The compiler technically can't guarantee that a real
-@@ -582,22 +595,25 @@ static const union kvm_mmu_notifier_arg KVM_MMU_NOTIFIER_NO_ARG;
- node; \
- node = interval_tree_iter_next(node, start, last)) \
-
--static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
-- const struct kvm_mmu_notifier_range *range)
-+static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
-+ const struct kvm_mmu_notifier_range *range)
- {
-- bool ret = false, locked = false;
-+ struct kvm_mmu_notifier_return r = {
-+ .ret = false,
-+ .found_memslot = false,
-+ };
- struct kvm_gfn_range gfn_range;
- struct kvm_memory_slot *slot;
- struct kvm_memslots *slots;
- int i, idx;
-
- if (WARN_ON_ONCE(range->end <= range->start))
-- return 0;
-+ return r;
-
- /* A null handler is allowed if and only if on_lock() is provided. */
- if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
- IS_KVM_NULL_FN(range->handler)))
-- return 0;
-+ return r;
-
- idx = srcu_read_lock(&kvm->srcu);
-
-@@ -631,8 +647,8 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
- gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
- gfn_range.slot = slot;
-
-- if (!locked) {
-- locked = true;
-+ if (!r.found_memslot) {
-+ r.found_memslot = true;
- KVM_MMU_LOCK(kvm);
- if (!IS_KVM_NULL_FN(range->on_lock))
- range->on_lock(kvm);
-@@ -640,14 +656,14 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
- if (IS_KVM_NULL_FN(range->handler))
- break;
- }
-- ret |= range->handler(kvm, &gfn_range);
-+ r.ret |= range->handler(kvm, &gfn_range);
- }
- }
-
-- if (range->flush_on_ret && ret)
-+ if (range->flush_on_ret && r.ret)
- kvm_flush_remote_tlbs(kvm);
-
-- if (locked) {
-+ if (r.found_memslot) {
- KVM_MMU_UNLOCK(kvm);
- if (!IS_KVM_NULL_FN(range->on_unlock))
- range->on_unlock(kvm);
-@@ -655,8 +671,7 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
-
- srcu_read_unlock(&kvm->srcu, idx);
-
-- /* The notifiers are averse to booleans. :-( */
-- return (int)ret;
-+ return r;
- }
-
- static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
-@@ -677,7 +692,7 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
- .may_block = false,
- };
-
-- return __kvm_handle_hva_range(kvm, &range);
-+ return __kvm_handle_hva_range(kvm, &range).ret;
- }
-
- static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
-@@ -696,7 +711,7 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn
- .may_block = false,
- };
-
-- return __kvm_handle_hva_range(kvm, &range);
-+ return __kvm_handle_hva_range(kvm, &range).ret;
- }
-
- static bool kvm_change_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
-@@ -798,7 +813,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- .end = range->end,
- .handler = kvm_mmu_unmap_gfn_range,
- .on_lock = kvm_mmu_invalidate_begin,
-- .on_unlock = kvm_arch_guest_memory_reclaimed,
-+ .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = true,
- .may_block = mmu_notifier_range_blockable(range),
- };
-@@ -830,7 +845,13 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end,
- hva_range.may_block);
-
-- __kvm_handle_hva_range(kvm, &hva_range);
-+ /*
-+ * If one or more memslots were found and thus zapped, notify arch code
-+ * that guest memory has been reclaimed. This needs to be done *after*
-+ * dropping mmu_lock, as x86's reclaim path is slooooow.
-+ */
-+ if (__kvm_handle_hva_range(kvm, &hva_range).found_memslot)
-+ kvm_arch_guest_memory_reclaimed(kvm);
-
- return 0;
- }
---
-2.43.0
-
+++ /dev/null
-From 5566397d2a56c5c8dacbcd391db95a84994fac00 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 27 Oct 2023 11:21:44 -0700
-Subject: KVM: Assert that mmu_invalidate_in_progress *never* goes negative
-
-From: Sean Christopherson <seanjc@google.com>
-
-[ Upstream commit c0db19232c1ed6bd7fcb825c28b014c52732c19e ]
-
-Move the assertion on the in-progress invalidation count from the primary
-MMU's notifier path to KVM's common notification path, i.e. assert that
-the count doesn't go negative even when the invalidation is coming from
-KVM itself.
-
-Opportunistically convert the assertion to a KVM_BUG_ON(), i.e. kill only
-the affected VM, not the entire kernel. A corrupted count is fatal to the
-VM, e.g. the non-zero (negative) count will cause mmu_invalidate_retry()
-to block any and all attempts to install new mappings. But it's far from
-guaranteed that an end() without a start() is fatal or even problematic to
-anything other than the target VM, e.g. the underlying bug could simply be
-a duplicate call to end(). And it's much more likely that a missed
-invalidation, i.e. a potential use-after-free, would manifest as no
-notification whatsoever, not an end() without a start().
-
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
-Reviewed-by: Fuad Tabba <tabba@google.com>
-Tested-by: Fuad Tabba <tabba@google.com>
-Message-Id: <20231027182217.3615211-3-seanjc@google.com>
-Reviewed-by: Kai Huang <kai.huang@intel.com>
-Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-Stable-dep-of: c3f3edf73a8f ("KVM: Stop processing *all* memslots when "null" mmu_notifier handler is found")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 3 +--
- 1 file changed, 1 insertion(+), 2 deletions(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 0524933856d42..5a97e6c7d9c20 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -833,6 +833,7 @@ void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
- * in conjunction with the smp_rmb in mmu_invalidate_retry().
- */
- kvm->mmu_invalidate_in_progress--;
-+ KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
- }
-
- static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
-@@ -863,8 +864,6 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
- */
- if (wake)
- rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
--
-- BUG_ON(kvm->mmu_invalidate_in_progress < 0);
- }
-
- static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
---
-2.43.0
-
+++ /dev/null
-From 5ba477e9b1493fda1c2979d1bf27b52bd04d8aaf Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 27 Oct 2023 11:21:53 -0700
-Subject: KVM: Drop .on_unlock() mmu_notifier hook
-
-From: Sean Christopherson <seanjc@google.com>
-
-[ Upstream commit 193bbfaacc84f9ee9c281ec0a8dd2ec8e4821e57 ]
-
-Drop the .on_unlock() mmu_notifer hook now that it's no longer used for
-notifying arch code that memory has been reclaimed. Adding .on_unlock()
-and invoking it *after* dropping mmu_lock was a terrible idea, as doing so
-resulted in .on_lock() and .on_unlock() having divergent and asymmetric
-behavior, and set future developers up for failure, i.e. all but asked for
-bugs where KVM relied on using .on_unlock() to try to run a callback while
-holding mmu_lock.
-
-Opportunistically add a lockdep assertion in kvm_mmu_invalidate_end() to
-guard against future bugs of this nature.
-
-Reported-by: Isaku Yamahata <isaku.yamahata@intel.com>
-Link: https://lore.kernel.org/all/20230802203119.GB2021422@ls.amr.corp.intel.com
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
-Reviewed-by: Fuad Tabba <tabba@google.com>
-Tested-by: Fuad Tabba <tabba@google.com>
-Message-Id: <20231027182217.3615211-12-seanjc@google.com>
-Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-Stable-dep-of: c3f3edf73a8f ("KVM: Stop processing *all* memslots when "null" mmu_notifier handler is found")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 11 +----------
- 1 file changed, 1 insertion(+), 10 deletions(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index b3b3edcc537de..5ac350ba4e996 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -544,7 +544,6 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
- typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
-
- typedef void (*on_lock_fn_t)(struct kvm *kvm);
--typedef void (*on_unlock_fn_t)(struct kvm *kvm);
-
- struct kvm_mmu_notifier_range {
- /*
-@@ -556,7 +555,6 @@ struct kvm_mmu_notifier_range {
- union kvm_mmu_notifier_arg arg;
- gfn_handler_t handler;
- on_lock_fn_t on_lock;
-- on_unlock_fn_t on_unlock;
- bool flush_on_ret;
- bool may_block;
- };
-@@ -663,11 +661,8 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
- if (range->flush_on_ret && r.ret)
- kvm_flush_remote_tlbs(kvm);
-
-- if (r.found_memslot) {
-+ if (r.found_memslot)
- KVM_MMU_UNLOCK(kvm);
-- if (!IS_KVM_NULL_FN(range->on_unlock))
-- range->on_unlock(kvm);
-- }
-
- srcu_read_unlock(&kvm->srcu, idx);
-
-@@ -687,7 +682,6 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
- .arg = arg,
- .handler = handler,
- .on_lock = (void *)kvm_null_fn,
-- .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = true,
- .may_block = false,
- };
-@@ -706,7 +700,6 @@ static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn
- .end = end,
- .handler = handler,
- .on_lock = (void *)kvm_null_fn,
-- .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = false,
- .may_block = false,
- };
-@@ -813,7 +806,6 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- .end = range->end,
- .handler = kvm_mmu_unmap_gfn_range,
- .on_lock = kvm_mmu_invalidate_begin,
-- .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = true,
- .may_block = mmu_notifier_range_blockable(range),
- };
-@@ -891,7 +883,6 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
- .end = range->end,
- .handler = (void *)kvm_null_fn,
- .on_lock = kvm_mmu_invalidate_end,
-- .on_unlock = (void *)kvm_null_fn,
- .flush_on_ret = false,
- .may_block = mmu_notifier_range_blockable(range),
- };
---
-2.43.0
-
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
-@@ -3813,12 +3813,13 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *m
+@@ -3772,12 +3772,13 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *m
{
struct kvm *kvm = me->kvm;
struct kvm_vcpu *vcpu;
kvm_vcpu_set_in_spin_loop(me, true);
/*
* We boost the priority of a VCPU that is runnable but not
-@@ -3849,7 +3850,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *m
+@@ -3808,7 +3809,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *m
yielded = kvm_vcpu_yield_to(vcpu);
if (yielded > 0) {
+++ /dev/null
-From cdf74c69ece21da50e61ab85d3c0ab026d7a5b5f Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Wed, 12 Jun 2024 09:41:51 -0500
-Subject: KVM: Stop processing *all* memslots when "null" mmu_notifier handler
- is found
-
-From: Babu Moger <babu.moger@amd.com>
-
-[ Upstream commit c3f3edf73a8f854f8766a69d2734198a58762e33 ]
-
-Bail from outer address space loop, not just the inner memslot loop, when
-a "null" handler is encountered by __kvm_handle_hva_range(), which is the
-intended behavior. On x86, which has multiple address spaces thanks to
-SMM emulation, breaking from just the memslot loop results in undefined
-behavior due to assigning the non-existent return value from kvm_null_fn()
-to a bool.
-
-In practice, the bug is benign as kvm_mmu_notifier_invalidate_range_end()
-is the only caller that passes handler=kvm_null_fn, and it doesn't set
-flush_on_ret, i.e. assigning garbage to r.ret is ultimately ignored. And
-for most configuration the compiler elides the entire sequence, i.e. there
-is no undefined behavior at runtime.
-
- ------------[ cut here ]------------
- UBSAN: invalid-load in arch/x86/kvm/../../../virt/kvm/kvm_main.c:655:10
- load of value 160 is not a valid value for type '_Bool'
- CPU: 370 PID: 8246 Comm: CPU 0/KVM Not tainted 6.8.2-amdsos-build58-ubuntu-22.04+ #1
- Hardware name: AMD Corporation Sh54p/Sh54p, BIOS WPC4429N 04/25/2024
- Call Trace:
- <TASK>
- dump_stack_lvl+0x48/0x60
- ubsan_epilogue+0x5/0x30
- __ubsan_handle_load_invalid_value+0x79/0x80
- kvm_mmu_notifier_invalidate_range_end.cold+0x18/0x4f [kvm]
- __mmu_notifier_invalidate_range_end+0x63/0xe0
- __split_huge_pmd+0x367/0xfc0
- do_huge_pmd_wp_page+0x1cc/0x380
- __handle_mm_fault+0x8ee/0xe50
- handle_mm_fault+0xe4/0x4a0
- __get_user_pages+0x190/0x840
- get_user_pages_unlocked+0xe0/0x590
- hva_to_pfn+0x114/0x550 [kvm]
- kvm_faultin_pfn+0xed/0x5b0 [kvm]
- kvm_tdp_page_fault+0x123/0x170 [kvm]
- kvm_mmu_page_fault+0x244/0xaa0 [kvm]
- vcpu_enter_guest+0x592/0x1070 [kvm]
- kvm_arch_vcpu_ioctl_run+0x145/0x8a0 [kvm]
- kvm_vcpu_ioctl+0x288/0x6d0 [kvm]
- __x64_sys_ioctl+0x8f/0xd0
- do_syscall_64+0x77/0x120
- entry_SYSCALL_64_after_hwframe+0x6e/0x76
- </TASK>
- ---[ end trace ]---
-
-Fixes: 071064f14d87 ("KVM: Don't take mmu_lock for range invalidation unless necessary")
-Signed-off-by: Babu Moger <babu.moger@amd.com>
-Link: https://lore.kernel.org/r/b8723d39903b64c241c50f5513f804390c7b5eec.1718203311.git.babu.moger@amd.com
-[sean: massage changelog]
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 3 ++-
- 1 file changed, 2 insertions(+), 1 deletion(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 5ac350ba4e996..61c48e88c9796 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -652,7 +652,7 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
- range->on_lock(kvm);
-
- if (IS_KVM_NULL_FN(range->handler))
-- break;
-+ goto mmu_unlock;
- }
- r.ret |= range->handler(kvm, &gfn_range);
- }
-@@ -661,6 +661,7 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
- if (range->flush_on_ret && r.ret)
- kvm_flush_remote_tlbs(kvm);
-
-+mmu_unlock:
- if (r.found_memslot)
- KVM_MMU_UNLOCK(kvm);
-
---
-2.43.0
-
+++ /dev/null
-From 75255b23d060e7de1c17cb37cc2dcb7fb9226f79 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 27 Oct 2023 11:21:43 -0700
-Subject: KVM: Tweak kvm_hva_range and hva_handler_t to allow reusing for gfn
- ranges
-
-From: Sean Christopherson <seanjc@google.com>
-
-[ Upstream commit e97b39c5c4362dc1cbc37a563ddac313b96c84f3 ]
-
-Rework and rename "struct kvm_hva_range" into "kvm_mmu_notifier_range" so
-that the structure can be used to handle notifications that operate on gfn
-context, i.e. that aren't tied to a host virtual address. Rename the
-handler typedef too (arguably it should always have been gfn_handler_t).
-
-Practically speaking, this is a nop for 64-bit kernels as the only
-meaningful change is to store start+end as u64s instead of unsigned longs.
-
-Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
-Reviewed-by: Xiaoyao Li <xiaoyao.li@intel.com>
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Reviewed-by: Fuad Tabba <tabba@google.com>
-Tested-by: Fuad Tabba <tabba@google.com>
-Message-Id: <20231027182217.3615211-2-seanjc@google.com>
-Reviewed-by: Kai Huang <kai.huang@intel.com>
-Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-Stable-dep-of: c3f3edf73a8f ("KVM: Stop processing *all* memslots when "null" mmu_notifier handler is found")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 34 +++++++++++++++++++---------------
- 1 file changed, 19 insertions(+), 15 deletions(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 486800a7024b3..0524933856d42 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -541,18 +541,22 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
- return container_of(mn, struct kvm, mmu_notifier);
- }
-
--typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
-+typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
-
- typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
- unsigned long end);
-
- typedef void (*on_unlock_fn_t)(struct kvm *kvm);
-
--struct kvm_hva_range {
-- unsigned long start;
-- unsigned long end;
-+struct kvm_mmu_notifier_range {
-+ /*
-+ * 64-bit addresses, as KVM notifiers can operate on host virtual
-+ * addresses (unsigned long) and guest physical addresses (64-bit).
-+ */
-+ u64 start;
-+ u64 end;
- union kvm_mmu_notifier_arg arg;
-- hva_handler_t handler;
-+ gfn_handler_t handler;
- on_lock_fn_t on_lock;
- on_unlock_fn_t on_unlock;
- bool flush_on_ret;
-@@ -581,7 +585,7 @@ static const union kvm_mmu_notifier_arg KVM_MMU_NOTIFIER_NO_ARG;
- node = interval_tree_iter_next(node, start, last)) \
-
- static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
-- const struct kvm_hva_range *range)
-+ const struct kvm_mmu_notifier_range *range)
- {
- bool ret = false, locked = false;
- struct kvm_gfn_range gfn_range;
-@@ -608,9 +612,9 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
- unsigned long hva_start, hva_end;
-
- slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
-- hva_start = max(range->start, slot->userspace_addr);
-- hva_end = min(range->end, slot->userspace_addr +
-- (slot->npages << PAGE_SHIFT));
-+ hva_start = max_t(unsigned long, range->start, slot->userspace_addr);
-+ hva_end = min_t(unsigned long, range->end,
-+ slot->userspace_addr + (slot->npages << PAGE_SHIFT));
-
- /*
- * To optimize for the likely case where the address
-@@ -660,10 +664,10 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
- unsigned long start,
- unsigned long end,
- union kvm_mmu_notifier_arg arg,
-- hva_handler_t handler)
-+ gfn_handler_t handler)
- {
- struct kvm *kvm = mmu_notifier_to_kvm(mn);
-- const struct kvm_hva_range range = {
-+ const struct kvm_mmu_notifier_range range = {
- .start = start,
- .end = end,
- .arg = arg,
-@@ -680,10 +684,10 @@ static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
- static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
- unsigned long start,
- unsigned long end,
-- hva_handler_t handler)
-+ gfn_handler_t handler)
- {
- struct kvm *kvm = mmu_notifier_to_kvm(mn);
-- const struct kvm_hva_range range = {
-+ const struct kvm_mmu_notifier_range range = {
- .start = start,
- .end = end,
- .handler = handler,
-@@ -771,7 +775,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
- {
- struct kvm *kvm = mmu_notifier_to_kvm(mn);
-- const struct kvm_hva_range hva_range = {
-+ const struct kvm_mmu_notifier_range hva_range = {
- .start = range->start,
- .end = range->end,
- .handler = kvm_unmap_gfn_range,
-@@ -835,7 +839,7 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
- {
- struct kvm *kvm = mmu_notifier_to_kvm(mn);
-- const struct kvm_hva_range hva_range = {
-+ const struct kvm_mmu_notifier_range hva_range = {
- .start = range->start,
- .end = range->end,
- .handler = (void *)kvm_null_fn,
---
-2.43.0
-
+++ /dev/null
-From 68a14ccc3fb35047cc4900c8ddd4b6f959e25b77 Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Fri, 27 Oct 2023 11:21:45 -0700
-Subject: KVM: Use gfn instead of hva for mmu_notifier_retry
-
-From: Chao Peng <chao.p.peng@linux.intel.com>
-
-[ Upstream commit 8569992d64b8f750e34b7858eac5d7daaf0f80fd ]
-
-Currently in mmu_notifier invalidate path, hva range is recorded and then
-checked against by mmu_invalidate_retry_hva() in the page fault handling
-path. However, for the soon-to-be-introduced private memory, a page fault
-may not have a hva associated, checking gfn(gpa) makes more sense.
-
-For existing hva based shared memory, gfn is expected to also work. The
-only downside is when aliasing multiple gfns to a single hva, the
-current algorithm of checking multiple ranges could result in a much
-larger range being rejected. Such aliasing should be uncommon, so the
-impact is expected small.
-
-Suggested-by: Sean Christopherson <seanjc@google.com>
-Cc: Xu Yilun <yilun.xu@intel.com>
-Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
-Reviewed-by: Fuad Tabba <tabba@google.com>
-Tested-by: Fuad Tabba <tabba@google.com>
-[sean: convert vmx_set_apic_access_page_addr() to gfn-based API]
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
-Reviewed-by: Xu Yilun <yilun.xu@linux.intel.com>
-Message-Id: <20231027182217.3615211-4-seanjc@google.com>
-Reviewed-by: Kai Huang <kai.huang@intel.com>
-Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
-Stable-dep-of: c3f3edf73a8f ("KVM: Stop processing *all* memslots when "null" mmu_notifier handler is found")
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- arch/x86/kvm/mmu/mmu.c | 10 ++++++----
- arch/x86/kvm/vmx/vmx.c | 11 +++++-----
- include/linux/kvm_host.h | 33 +++++++++++++++++++-----------
- virt/kvm/kvm_main.c | 43 +++++++++++++++++++++++++++++++---------
- 4 files changed, 66 insertions(+), 31 deletions(-)
-
-diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
-index 294775b7383b4..e32b7072324e5 100644
---- a/arch/x86/kvm/mmu/mmu.c
-+++ b/arch/x86/kvm/mmu/mmu.c
-@@ -3056,7 +3056,7 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
- *
- * There are several ways to safely use this helper:
- *
-- * - Check mmu_invalidate_retry_hva() after grabbing the mapping level, before
-+ * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
- * consuming it. In this case, mmu_lock doesn't need to be held during the
- * lookup, but it does need to be held while checking the MMU notifier.
- *
-@@ -4358,7 +4358,7 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
- return true;
-
- return fault->slot &&
-- mmu_invalidate_retry_hva(vcpu->kvm, fault->mmu_seq, fault->hva);
-+ mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);
- }
-
- static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
-@@ -6245,7 +6245,9 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
-
- write_lock(&kvm->mmu_lock);
-
-- kvm_mmu_invalidate_begin(kvm, 0, -1ul);
-+ kvm_mmu_invalidate_begin(kvm);
-+
-+ kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
-
- flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
-
-@@ -6255,7 +6257,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
- if (flush)
- kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
-
-- kvm_mmu_invalidate_end(kvm, 0, -1ul);
-+ kvm_mmu_invalidate_end(kvm);
-
- write_unlock(&kvm->mmu_lock);
- }
-diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
-index dae499e2da84e..bb6b0f15ceb91 100644
---- a/arch/x86/kvm/vmx/vmx.c
-+++ b/arch/x86/kvm/vmx/vmx.c
-@@ -6766,10 +6766,10 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
- return;
-
- /*
-- * Grab the memslot so that the hva lookup for the mmu_notifier retry
-- * is guaranteed to use the same memslot as the pfn lookup, i.e. rely
-- * on the pfn lookup's validation of the memslot to ensure a valid hva
-- * is used for the retry check.
-+ * Explicitly grab the memslot using KVM's internal slot ID to ensure
-+ * KVM doesn't unintentionally grab a userspace memslot. It _should_
-+ * be impossible for userspace to create a memslot for the APIC when
-+ * APICv is enabled, but paranoia won't hurt in this case.
- */
- slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT);
- if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
-@@ -6794,8 +6794,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
- return;
-
- read_lock(&vcpu->kvm->mmu_lock);
-- if (mmu_invalidate_retry_hva(kvm, mmu_seq,
-- gfn_to_hva_memslot(slot, gfn))) {
-+ if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) {
- kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
- read_unlock(&vcpu->kvm->mmu_lock);
- goto out;
-diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
-index fb6c6109fdcad..11d0916883460 100644
---- a/include/linux/kvm_host.h
-+++ b/include/linux/kvm_host.h
-@@ -787,8 +787,8 @@ struct kvm {
- struct mmu_notifier mmu_notifier;
- unsigned long mmu_invalidate_seq;
- long mmu_invalidate_in_progress;
-- unsigned long mmu_invalidate_range_start;
-- unsigned long mmu_invalidate_range_end;
-+ gfn_t mmu_invalidate_range_start;
-+ gfn_t mmu_invalidate_range_end;
- #endif
- struct list_head devices;
- u64 manual_dirty_log_protect;
-@@ -1392,10 +1392,9 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc);
- void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
- #endif
-
--void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
-- unsigned long end);
--void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
-- unsigned long end);
-+void kvm_mmu_invalidate_begin(struct kvm *kvm);
-+void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end);
-+void kvm_mmu_invalidate_end(struct kvm *kvm);
-
- long kvm_arch_dev_ioctl(struct file *filp,
- unsigned int ioctl, unsigned long arg);
-@@ -1970,9 +1969,9 @@ static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)
- return 0;
- }
-
--static inline int mmu_invalidate_retry_hva(struct kvm *kvm,
-+static inline int mmu_invalidate_retry_gfn(struct kvm *kvm,
- unsigned long mmu_seq,
-- unsigned long hva)
-+ gfn_t gfn)
- {
- lockdep_assert_held(&kvm->mmu_lock);
- /*
-@@ -1981,10 +1980,20 @@ static inline int mmu_invalidate_retry_hva(struct kvm *kvm,
- * that might be being invalidated. Note that it may include some false
- * positives, due to shortcuts when handing concurrent invalidations.
- */
-- if (unlikely(kvm->mmu_invalidate_in_progress) &&
-- hva >= kvm->mmu_invalidate_range_start &&
-- hva < kvm->mmu_invalidate_range_end)
-- return 1;
-+ if (unlikely(kvm->mmu_invalidate_in_progress)) {
-+ /*
-+ * Dropping mmu_lock after bumping mmu_invalidate_in_progress
-+ * but before updating the range is a KVM bug.
-+ */
-+ if (WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA ||
-+ kvm->mmu_invalidate_range_end == INVALID_GPA))
-+ return 1;
-+
-+ if (gfn >= kvm->mmu_invalidate_range_start &&
-+ gfn < kvm->mmu_invalidate_range_end)
-+ return 1;
-+ }
-+
- if (kvm->mmu_invalidate_seq != mmu_seq)
- return 1;
- return 0;
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 5a97e6c7d9c20..9cc57b23ec81f 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -543,9 +543,7 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
-
- typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
-
--typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
-- unsigned long end);
--
-+typedef void (*on_lock_fn_t)(struct kvm *kvm);
- typedef void (*on_unlock_fn_t)(struct kvm *kvm);
-
- struct kvm_mmu_notifier_range {
-@@ -637,7 +635,8 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
- locked = true;
- KVM_MMU_LOCK(kvm);
- if (!IS_KVM_NULL_FN(range->on_lock))
-- range->on_lock(kvm, range->start, range->end);
-+ range->on_lock(kvm);
-+
- if (IS_KVM_NULL_FN(range->handler))
- break;
- }
-@@ -742,16 +741,29 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
- kvm_handle_hva_range(mn, address, address + 1, arg, kvm_change_spte_gfn);
- }
-
--void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
-- unsigned long end)
-+void kvm_mmu_invalidate_begin(struct kvm *kvm)
- {
-+ lockdep_assert_held_write(&kvm->mmu_lock);
- /*
- * The count increase must become visible at unlock time as no
- * spte can be established without taking the mmu_lock and
- * count is also read inside the mmu_lock critical section.
- */
- kvm->mmu_invalidate_in_progress++;
-+
- if (likely(kvm->mmu_invalidate_in_progress == 1)) {
-+ kvm->mmu_invalidate_range_start = INVALID_GPA;
-+ kvm->mmu_invalidate_range_end = INVALID_GPA;
-+ }
-+}
-+
-+void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
-+{
-+ lockdep_assert_held_write(&kvm->mmu_lock);
-+
-+ WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);
-+
-+ if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
- kvm->mmu_invalidate_range_start = start;
- kvm->mmu_invalidate_range_end = end;
- } else {
-@@ -771,6 +783,12 @@ void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
- }
- }
-
-+static bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
-+{
-+ kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
-+ return kvm_unmap_gfn_range(kvm, range);
-+}
-+
- static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- const struct mmu_notifier_range *range)
- {
-@@ -778,7 +796,7 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- const struct kvm_mmu_notifier_range hva_range = {
- .start = range->start,
- .end = range->end,
-- .handler = kvm_unmap_gfn_range,
-+ .handler = kvm_mmu_unmap_gfn_range,
- .on_lock = kvm_mmu_invalidate_begin,
- .on_unlock = kvm_arch_guest_memory_reclaimed,
- .flush_on_ret = true,
-@@ -817,9 +835,10 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
- return 0;
- }
-
--void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
-- unsigned long end)
-+void kvm_mmu_invalidate_end(struct kvm *kvm)
- {
-+ lockdep_assert_held_write(&kvm->mmu_lock);
-+
- /*
- * This sequence increase will notify the kvm page fault that
- * the page that is going to be mapped in the spte could have
-@@ -834,6 +853,12 @@ void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
- */
- kvm->mmu_invalidate_in_progress--;
- KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
-+
-+ /*
-+ * Assert that at least one range was added between start() and end().
-+ * Not adding a range isn't fatal, but it is a KVM bug.
-+ */
-+ WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
- }
-
- static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
---
-2.43.0
-
arm64-dts-imx93-11x11-evk-remove-the-no-sdio-propert.patch
arm64-dts-freescale-imx8mm-verdin-enable-hysteresis-.patch
acpica-revert-acpica-avoid-info-mapping-multiple-bar.patch
-kvm-tweak-kvm_hva_range-and-hva_handler_t-to-allow-r.patch
-kvm-assert-that-mmu_invalidate_in_progress-never-goe.patch
-kvm-use-gfn-instead-of-hva-for-mmu_notifier_retry.patch
-kvm-add-a-dedicated-mmu_notifier-flag-for-reclaiming.patch
-kvm-drop-.on_unlock-mmu_notifier-hook.patch
-kvm-stop-processing-all-memslots-when-null-mmu_notif.patch
spi-spi-imx-imx51-revert-burst-length-calculation-ba.patch
io_uring-rsrc-fix-incorrect-assignment-of-iter-nr_se.patch
firmware-psci-fix-return-value-from-psci_system_susp.patch
+++ /dev/null
-From 47a29c14df8610cf93fe93f05cf3becaf0d629da Mon Sep 17 00:00:00 2001
-From: Sasha Levin <sashal@kernel.org>
-Date: Wed, 12 Jun 2024 09:41:51 -0500
-Subject: KVM: Stop processing *all* memslots when "null" mmu_notifier handler
- is found
-
-From: Babu Moger <babu.moger@amd.com>
-
-[ Upstream commit c3f3edf73a8f854f8766a69d2734198a58762e33 ]
-
-Bail from outer address space loop, not just the inner memslot loop, when
-a "null" handler is encountered by __kvm_handle_hva_range(), which is the
-intended behavior. On x86, which has multiple address spaces thanks to
-SMM emulation, breaking from just the memslot loop results in undefined
-behavior due to assigning the non-existent return value from kvm_null_fn()
-to a bool.
-
-In practice, the bug is benign as kvm_mmu_notifier_invalidate_range_end()
-is the only caller that passes handler=kvm_null_fn, and it doesn't set
-flush_on_ret, i.e. assigning garbage to r.ret is ultimately ignored. And
-for most configuration the compiler elides the entire sequence, i.e. there
-is no undefined behavior at runtime.
-
- ------------[ cut here ]------------
- UBSAN: invalid-load in arch/x86/kvm/../../../virt/kvm/kvm_main.c:655:10
- load of value 160 is not a valid value for type '_Bool'
- CPU: 370 PID: 8246 Comm: CPU 0/KVM Not tainted 6.8.2-amdsos-build58-ubuntu-22.04+ #1
- Hardware name: AMD Corporation Sh54p/Sh54p, BIOS WPC4429N 04/25/2024
- Call Trace:
- <TASK>
- dump_stack_lvl+0x48/0x60
- ubsan_epilogue+0x5/0x30
- __ubsan_handle_load_invalid_value+0x79/0x80
- kvm_mmu_notifier_invalidate_range_end.cold+0x18/0x4f [kvm]
- __mmu_notifier_invalidate_range_end+0x63/0xe0
- __split_huge_pmd+0x367/0xfc0
- do_huge_pmd_wp_page+0x1cc/0x380
- __handle_mm_fault+0x8ee/0xe50
- handle_mm_fault+0xe4/0x4a0
- __get_user_pages+0x190/0x840
- get_user_pages_unlocked+0xe0/0x590
- hva_to_pfn+0x114/0x550 [kvm]
- kvm_faultin_pfn+0xed/0x5b0 [kvm]
- kvm_tdp_page_fault+0x123/0x170 [kvm]
- kvm_mmu_page_fault+0x244/0xaa0 [kvm]
- vcpu_enter_guest+0x592/0x1070 [kvm]
- kvm_arch_vcpu_ioctl_run+0x145/0x8a0 [kvm]
- kvm_vcpu_ioctl+0x288/0x6d0 [kvm]
- __x64_sys_ioctl+0x8f/0xd0
- do_syscall_64+0x77/0x120
- entry_SYSCALL_64_after_hwframe+0x6e/0x76
- </TASK>
- ---[ end trace ]---
-
-Fixes: 071064f14d87 ("KVM: Don't take mmu_lock for range invalidation unless necessary")
-Signed-off-by: Babu Moger <babu.moger@amd.com>
-Link: https://lore.kernel.org/r/b8723d39903b64c241c50f5513f804390c7b5eec.1718203311.git.babu.moger@amd.com
-[sean: massage changelog]
-Signed-off-by: Sean Christopherson <seanjc@google.com>
-Signed-off-by: Sasha Levin <sashal@kernel.org>
----
- virt/kvm/kvm_main.c | 3 ++-
- 1 file changed, 2 insertions(+), 1 deletion(-)
-
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index ff0a20565f908..3b7fa9a62c62f 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -650,7 +650,7 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
- range->on_lock(kvm);
-
- if (IS_KVM_NULL_FN(range->handler))
-- break;
-+ goto mmu_unlock;
- }
- r.ret |= range->handler(kvm, &gfn_range);
- }
-@@ -659,6 +659,7 @@ static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
- if (range->flush_on_ret && r.ret)
- kvm_flush_remote_tlbs(kvm);
-
-+mmu_unlock:
- if (r.found_memslot)
- KVM_MMU_UNLOCK(kvm);
-
---
-2.43.0
-
nfsd-fix-oops-when-reading-pool_stats-before-server-.patch
acpica-revert-acpica-avoid-info-mapping-multiple-bar.patch
acpi-scan-ignore-camera-graph-port-nodes-on-all-dell.patch
-kvm-stop-processing-all-memslots-when-null-mmu_notif.patch
spi-fix-spi-slave-probe-failure.patch
x86-resctrl-don-t-try-to-free-nonexistent-rmids.patch
spi-spi-imx-imx51-revert-burst-length-calculation-ba.patch
projects / thirdparty / kernel / stable-queue.git / commitdiff
