From: Greg Kroah-Hartman Date: Thu, 13 Mar 2025 16:09:19 +0000 (+0100) Subject: 6.12-stable patches X-Git-Tag: v6.6.84~64 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5c81212fa06f3de9a9ae86915239fa0c1281ebbe;p=thirdparty%2Fkernel%2Fstable-queue.git 6.12-stable patches added patches: mm-fix-kernel-bug-when-userfaultfd_move-encounters-swapcache.patch mm-slab-kvfree_rcu-switch-to-wq_mem_reclaim-wq.patch series userfaultfd-fix-pte-unmapping-stack-allocated-pte-copies.patch --- diff --git a/queue-6.12/mm-fix-kernel-bug-when-userfaultfd_move-encounters-swapcache.patch b/queue-6.12/mm-fix-kernel-bug-when-userfaultfd_move-encounters-swapcache.patch new file mode 100644 index 0000000000..0b0cbb1e3a --- /dev/null +++ b/queue-6.12/mm-fix-kernel-bug-when-userfaultfd_move-encounters-swapcache.patch @@ -0,0 +1,276 @@ +From c50f8e6053b0503375c2975bf47f182445aebb4c Mon Sep 17 00:00:00 2001 +From: Barry Song +Date: Wed, 26 Feb 2025 13:14:00 +1300 +Subject: mm: fix kernel BUG when userfaultfd_move encounters swapcache + +From: Barry Song + +commit c50f8e6053b0503375c2975bf47f182445aebb4c upstream. + +userfaultfd_move() checks whether the PTE entry is present or a +swap entry. + +- If the PTE entry is present, move_present_pte() handles folio + migration by setting: + + src_folio->index = linear_page_index(dst_vma, dst_addr); + +- If the PTE entry is a swap entry, move_swap_pte() simply copies + the PTE to the new dst_addr. + +This approach is incorrect because, even if the PTE is a swap entry, +it can still reference a folio that remains in the swap cache. + +This creates a race window between steps 2 and 4. + 1. add_to_swap: The folio is added to the swapcache. + 2. try_to_unmap: PTEs are converted to swap entries. + 3. pageout: The folio is written back. + 4. Swapcache is cleared. +If userfaultfd_move() occurs in the window between steps 2 and 4, +after the swap PTE has been moved to the destination, accessing the +destination triggers do_swap_page(), which may locate the folio in +the swapcache. However, since the folio's index has not been updated +to match the destination VMA, do_swap_page() will detect a mismatch. + +This can result in two critical issues depending on the system +configuration. + +If KSM is disabled, both small and large folios can trigger a BUG +during the add_rmap operation due to: + + page_pgoff(folio, page) != linear_page_index(vma, address) + +[ 13.336953] page: refcount:6 mapcount:1 mapping:00000000f43db19c index:0xffffaf150 pfn:0x4667c +[ 13.337520] head: order:2 mapcount:1 entire_mapcount:0 nr_pages_mapped:1 pincount:0 +[ 13.337716] memcg:ffff00000405f000 +[ 13.337849] anon flags: 0x3fffc0000020459(locked|uptodate|dirty|owner_priv_1|head|swapbacked|node=0|zone=0|lastcpupid=0xffff) +[ 13.338630] raw: 03fffc0000020459 ffff80008507b538 ffff80008507b538 ffff000006260361 +[ 13.338831] raw: 0000000ffffaf150 0000000000004000 0000000600000000 ffff00000405f000 +[ 13.339031] head: 03fffc0000020459 ffff80008507b538 ffff80008507b538 ffff000006260361 +[ 13.339204] head: 0000000ffffaf150 0000000000004000 0000000600000000 ffff00000405f000 +[ 13.339375] head: 03fffc0000000202 fffffdffc0199f01 ffffffff00000000 0000000000000001 +[ 13.339546] head: 0000000000000004 0000000000000000 00000000ffffffff 0000000000000000 +[ 13.339736] page dumped because: VM_BUG_ON_PAGE(page_pgoff(folio, page) != linear_page_index(vma, address)) +[ 13.340190] ------------[ cut here ]------------ +[ 13.340316] kernel BUG at mm/rmap.c:1380! +[ 13.340683] Internal error: Oops - BUG: 00000000f2000800 [#1] PREEMPT SMP +[ 13.340969] Modules linked in: +[ 13.341257] CPU: 1 UID: 0 PID: 107 Comm: a.out Not tainted 6.14.0-rc3-gcf42737e247a-dirty #299 +[ 13.341470] Hardware name: linux,dummy-virt (DT) +[ 13.341671] pstate: 60000005 (nZCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) +[ 13.341815] pc : __page_check_anon_rmap+0xa0/0xb0 +[ 13.341920] lr : __page_check_anon_rmap+0xa0/0xb0 +[ 13.342018] sp : ffff80008752bb20 +[ 13.342093] x29: ffff80008752bb20 x28: fffffdffc0199f00 x27: 0000000000000001 +[ 13.342404] x26: 0000000000000000 x25: 0000000000000001 x24: 0000000000000001 +[ 13.342575] x23: 0000ffffaf0d0000 x22: 0000ffffaf0d0000 x21: fffffdffc0199f00 +[ 13.342731] x20: fffffdffc0199f00 x19: ffff000006210700 x18: 00000000ffffffff +[ 13.342881] x17: 6c203d2120296567 x16: 6170202c6f696c6f x15: 662866666f67705f +[ 13.343033] x14: 6567617028454741 x13: 2929737365726464 x12: ffff800083728ab0 +[ 13.343183] x11: ffff800082996bf8 x10: 0000000000000fd7 x9 : ffff80008011bc40 +[ 13.343351] x8 : 0000000000017fe8 x7 : 00000000fffff000 x6 : ffff8000829eebf8 +[ 13.343498] x5 : c0000000fffff000 x4 : 0000000000000000 x3 : 0000000000000000 +[ 13.343645] x2 : 0000000000000000 x1 : ffff0000062db980 x0 : 000000000000005f +[ 13.343876] Call trace: +[ 13.344045] __page_check_anon_rmap+0xa0/0xb0 (P) +[ 13.344234] folio_add_anon_rmap_ptes+0x22c/0x320 +[ 13.344333] do_swap_page+0x1060/0x1400 +[ 13.344417] __handle_mm_fault+0x61c/0xbc8 +[ 13.344504] handle_mm_fault+0xd8/0x2e8 +[ 13.344586] do_page_fault+0x20c/0x770 +[ 13.344673] do_translation_fault+0xb4/0xf0 +[ 13.344759] do_mem_abort+0x48/0xa0 +[ 13.344842] el0_da+0x58/0x130 +[ 13.344914] el0t_64_sync_handler+0xc4/0x138 +[ 13.345002] el0t_64_sync+0x1ac/0x1b0 +[ 13.345208] Code: aa1503e0 f000f801 910f6021 97ff5779 (d4210000) +[ 13.345504] ---[ end trace 0000000000000000 ]--- +[ 13.345715] note: a.out[107] exited with irqs disabled +[ 13.345954] note: a.out[107] exited with preempt_count 2 + +If KSM is enabled, Peter Xu also discovered that do_swap_page() may +trigger an unexpected CoW operation for small folios because +ksm_might_need_to_copy() allocates a new folio when the folio index +does not match linear_page_index(vma, addr). + +This patch also checks the swapcache when handling swap entries. If a +match is found in the swapcache, it processes it similarly to a present +PTE. +However, there are some differences. For example, the folio is no longer +exclusive because folio_try_share_anon_rmap_pte() is performed during +unmapping. +Furthermore, in the case of swapcache, the folio has already been +unmapped, eliminating the risk of concurrent rmap walks and removing the +need to acquire src_folio's anon_vma or lock. + +Note that for large folios, in the swapcache handling path, we directly +return -EBUSY since split_folio() will return -EBUSY regardless if +the folio is under writeback or unmapped. This is not an urgent issue, +so a follow-up patch may address it separately. + +[v-songbaohua@oppo.com: minor cleanup according to Peter Xu] + Link: https://lkml.kernel.org/r/20250226024411.47092-1-21cnbao@gmail.com +Link: https://lkml.kernel.org/r/20250226001400.9129-1-21cnbao@gmail.com +Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") +Signed-off-by: Barry Song +Acked-by: Peter Xu +Reviewed-by: Suren Baghdasaryan +Cc: Andrea Arcangeli +Cc: Al Viro +Cc: Axel Rasmussen +Cc: Brian Geffon +Cc: Christian Brauner +Cc: David Hildenbrand +Cc: Hugh Dickins +Cc: Jann Horn +Cc: Kalesh Singh +Cc: Liam R. Howlett +Cc: Lokesh Gidra +Cc: Matthew Wilcox (Oracle) +Cc: Michal Hocko +Cc: Mike Rapoport (IBM) +Cc: Nicolas Geoffray +Cc: Ryan Roberts +Cc: Shuah Khan +Cc: ZhangPeng +Cc: Tangquan Zheng +Cc: +Signed-off-by: Andrew Morton +[ surenb: resolved merged conflict caused by the difference in + move_swap_pte() arguments ] +Signed-off-by: Suren Baghdasaryan +Signed-off-by: Greg Kroah-Hartman +--- + mm/userfaultfd.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++------- + 1 file changed, 66 insertions(+), 9 deletions(-) + +--- a/mm/userfaultfd.c ++++ b/mm/userfaultfd.c +@@ -18,6 +18,7 @@ + #include + #include + #include "internal.h" ++#include "swap.h" + + static __always_inline + bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end) +@@ -1067,15 +1068,13 @@ out: + return err; + } + +-static int move_swap_pte(struct mm_struct *mm, ++static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, + pte_t orig_dst_pte, pte_t orig_src_pte, +- spinlock_t *dst_ptl, spinlock_t *src_ptl) ++ spinlock_t *dst_ptl, spinlock_t *src_ptl, ++ struct folio *src_folio) + { +- if (!pte_swp_exclusive(orig_src_pte)) +- return -EBUSY; +- + double_pt_lock(dst_ptl, src_ptl); + + if (!pte_same(ptep_get(src_pte), orig_src_pte) || +@@ -1084,6 +1083,16 @@ static int move_swap_pte(struct mm_struc + return -EAGAIN; + } + ++ /* ++ * The src_folio resides in the swapcache, requiring an update to its ++ * index and mapping to align with the dst_vma, where a swap-in may ++ * occur and hit the swapcache after moving the PTE. ++ */ ++ if (src_folio) { ++ folio_move_anon_rmap(src_folio, dst_vma); ++ src_folio->index = linear_page_index(dst_vma, dst_addr); ++ } ++ + orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte); + set_pte_at(mm, dst_addr, dst_pte, orig_src_pte); + double_pt_unlock(dst_ptl, src_ptl); +@@ -1130,6 +1139,7 @@ static int move_pages_pte(struct mm_stru + __u64 mode) + { + swp_entry_t entry; ++ struct swap_info_struct *si = NULL; + pte_t orig_src_pte, orig_dst_pte; + pte_t src_folio_pte; + spinlock_t *src_ptl, *dst_ptl; +@@ -1312,6 +1322,8 @@ retry: + orig_dst_pte, orig_src_pte, + dst_ptl, src_ptl, src_folio); + } else { ++ struct folio *folio = NULL; ++ + entry = pte_to_swp_entry(orig_src_pte); + if (non_swap_entry(entry)) { + if (is_migration_entry(entry)) { +@@ -1325,10 +1337,53 @@ retry: + goto out; + } + +- err = move_swap_pte(mm, dst_addr, src_addr, +- dst_pte, src_pte, +- orig_dst_pte, orig_src_pte, +- dst_ptl, src_ptl); ++ if (!pte_swp_exclusive(orig_src_pte)) { ++ err = -EBUSY; ++ goto out; ++ } ++ ++ si = get_swap_device(entry); ++ if (unlikely(!si)) { ++ err = -EAGAIN; ++ goto out; ++ } ++ /* ++ * Verify the existence of the swapcache. If present, the folio's ++ * index and mapping must be updated even when the PTE is a swap ++ * entry. The anon_vma lock is not taken during this process since ++ * the folio has already been unmapped, and the swap entry is ++ * exclusive, preventing rmap walks. ++ * ++ * For large folios, return -EBUSY immediately, as split_folio() ++ * also returns -EBUSY when attempting to split unmapped large ++ * folios in the swapcache. This issue needs to be resolved ++ * separately to allow proper handling. ++ */ ++ if (!src_folio) ++ folio = filemap_get_folio(swap_address_space(entry), ++ swap_cache_index(entry)); ++ if (!IS_ERR_OR_NULL(folio)) { ++ if (folio_test_large(folio)) { ++ err = -EBUSY; ++ folio_put(folio); ++ goto out; ++ } ++ src_folio = folio; ++ src_folio_pte = orig_src_pte; ++ if (!folio_trylock(src_folio)) { ++ pte_unmap(&orig_src_pte); ++ pte_unmap(&orig_dst_pte); ++ src_pte = dst_pte = NULL; ++ put_swap_device(si); ++ si = NULL; ++ /* now we can block and wait */ ++ folio_lock(src_folio); ++ goto retry; ++ } ++ } ++ err = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, ++ orig_dst_pte, orig_src_pte, ++ dst_ptl, src_ptl, src_folio); + } + + out: +@@ -1345,6 +1400,8 @@ out: + if (src_pte) + pte_unmap(src_pte); + mmu_notifier_invalidate_range_end(&range); ++ if (si) ++ put_swap_device(si); + + return err; + } diff --git a/queue-6.12/mm-slab-kvfree_rcu-switch-to-wq_mem_reclaim-wq.patch b/queue-6.12/mm-slab-kvfree_rcu-switch-to-wq_mem_reclaim-wq.patch new file mode 100644 index 0000000000..28fc48b868 --- /dev/null +++ b/queue-6.12/mm-slab-kvfree_rcu-switch-to-wq_mem_reclaim-wq.patch @@ -0,0 +1,142 @@ +From dfd3df31c9db752234d7d2e09bef2aeabb643ce4 Mon Sep 17 00:00:00 2001 +From: "Uladzislau Rezki (Sony)" +Date: Fri, 28 Feb 2025 13:13:56 +0100 +Subject: mm/slab/kvfree_rcu: Switch to WQ_MEM_RECLAIM wq + +From: Uladzislau Rezki (Sony) + +commit dfd3df31c9db752234d7d2e09bef2aeabb643ce4 upstream. + +Currently kvfree_rcu() APIs use a system workqueue which is +"system_unbound_wq" to driver RCU machinery to reclaim a memory. + +Recently, it has been noted that the following kernel warning can +be observed: + + +workqueue: WQ_MEM_RECLAIM nvme-wq:nvme_scan_work is flushing !WQ_MEM_RECLAIM events_unbound:kfree_rcu_work + WARNING: CPU: 21 PID: 330 at kernel/workqueue.c:3719 check_flush_dependency+0x112/0x120 + Modules linked in: intel_uncore_frequency(E) intel_uncore_frequency_common(E) skx_edac(E) ... + CPU: 21 UID: 0 PID: 330 Comm: kworker/u144:6 Tainted: G E 6.13.2-0_g925d379822da #1 + Hardware name: Wiwynn Twin Lakes MP/Twin Lakes Passive MP, BIOS YMM20 02/01/2023 + Workqueue: nvme-wq nvme_scan_work + RIP: 0010:check_flush_dependency+0x112/0x120 + Code: 05 9a 40 14 02 01 48 81 c6 c0 00 00 00 48 8b 50 18 48 81 c7 c0 00 00 00 48 89 f9 48 ... + RSP: 0018:ffffc90000df7bd8 EFLAGS: 00010082 + RAX: 000000000000006a RBX: ffffffff81622390 RCX: 0000000000000027 + RDX: 00000000fffeffff RSI: 000000000057ffa8 RDI: ffff88907f960c88 + RBP: 0000000000000000 R08: ffffffff83068e50 R09: 000000000002fffd + R10: 0000000000000004 R11: 0000000000000000 R12: ffff8881001a4400 + R13: 0000000000000000 R14: ffff88907f420fb8 R15: 0000000000000000 + FS: 0000000000000000(0000) GS:ffff88907f940000(0000) knlGS:0000000000000000 + CR2: 00007f60c3001000 CR3: 000000107d010005 CR4: 00000000007726f0 + PKRU: 55555554 + Call Trace: + + ? __warn+0xa4/0x140 + ? check_flush_dependency+0x112/0x120 + ? report_bug+0xe1/0x140 + ? check_flush_dependency+0x112/0x120 + ? handle_bug+0x5e/0x90 + ? exc_invalid_op+0x16/0x40 + ? asm_exc_invalid_op+0x16/0x20 + ? timer_recalc_next_expiry+0x190/0x190 + ? check_flush_dependency+0x112/0x120 + ? check_flush_dependency+0x112/0x120 + __flush_work.llvm.1643880146586177030+0x174/0x2c0 + flush_rcu_work+0x28/0x30 + kvfree_rcu_barrier+0x12f/0x160 + kmem_cache_destroy+0x18/0x120 + bioset_exit+0x10c/0x150 + disk_release.llvm.6740012984264378178+0x61/0xd0 + device_release+0x4f/0x90 + kobject_put+0x95/0x180 + nvme_put_ns+0x23/0xc0 + nvme_remove_invalid_namespaces+0xb3/0xd0 + nvme_scan_work+0x342/0x490 + process_scheduled_works+0x1a2/0x370 + worker_thread+0x2ff/0x390 + ? pwq_release_workfn+0x1e0/0x1e0 + kthread+0xb1/0xe0 + ? __kthread_parkme+0x70/0x70 + ret_from_fork+0x30/0x40 + ? __kthread_parkme+0x70/0x70 + ret_from_fork_asm+0x11/0x20 + + ---[ end trace 0000000000000000 ]--- + + +To address this switch to use of independent WQ_MEM_RECLAIM +workqueue, so the rules are not violated from workqueue framework +point of view. + +Apart of that, since kvfree_rcu() does reclaim memory it is worth +to go with WQ_MEM_RECLAIM type of wq because it is designed for +this purpose. + +Fixes: 6c6c47b063b5 ("mm, slab: call kvfree_rcu_barrier() from kmem_cache_destroy()"), +Reported-by: Keith Busch +Closes: https://lore.kernel.org/all/Z7iqJtCjHKfo8Kho@kbusch-mbp/ +Cc: stable@vger.kernel.org +Signed-off-by: Uladzislau Rezki (Sony) +Reviewed-by: Joel Fernandes +Signed-off-by: Vlastimil Babka +Signed-off-by: Uladzislau Rezki (Sony) +Signed-off-by: Greg Kroah-Hartman +--- + kernel/rcu/tree.c | 14 ++++++++++---- + 1 file changed, 10 insertions(+), 4 deletions(-) + +--- a/kernel/rcu/tree.c ++++ b/kernel/rcu/tree.c +@@ -3191,6 +3191,8 @@ void call_rcu(struct rcu_head *head, rcu + } + EXPORT_SYMBOL_GPL(call_rcu); + ++static struct workqueue_struct *rcu_reclaim_wq; ++ + /* Maximum number of jiffies to wait before draining a batch. */ + #define KFREE_DRAIN_JIFFIES (5 * HZ) + #define KFREE_N_BATCHES 2 +@@ -3519,10 +3521,10 @@ __schedule_delayed_monitor_work(struct k + if (delayed_work_pending(&krcp->monitor_work)) { + delay_left = krcp->monitor_work.timer.expires - jiffies; + if (delay < delay_left) +- mod_delayed_work(system_unbound_wq, &krcp->monitor_work, delay); ++ mod_delayed_work(rcu_reclaim_wq, &krcp->monitor_work, delay); + return; + } +- queue_delayed_work(system_unbound_wq, &krcp->monitor_work, delay); ++ queue_delayed_work(rcu_reclaim_wq, &krcp->monitor_work, delay); + } + + static void +@@ -3620,7 +3622,7 @@ kvfree_rcu_queue_batch(struct kfree_rcu_ + // "free channels", the batch can handle. Break + // the loop since it is done with this CPU thus + // queuing an RCU work is _always_ success here. +- queued = queue_rcu_work(system_unbound_wq, &krwp->rcu_work); ++ queued = queue_rcu_work(rcu_reclaim_wq, &krwp->rcu_work); + WARN_ON_ONCE(!queued); + break; + } +@@ -3708,7 +3710,7 @@ run_page_cache_worker(struct kfree_rcu_c + if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING && + !atomic_xchg(&krcp->work_in_progress, 1)) { + if (atomic_read(&krcp->backoff_page_cache_fill)) { +- queue_delayed_work(system_unbound_wq, ++ queue_delayed_work(rcu_reclaim_wq, + &krcp->page_cache_work, + msecs_to_jiffies(rcu_delay_page_cache_fill_msec)); + } else { +@@ -5662,6 +5664,10 @@ static void __init kfree_rcu_batch_init( + int i, j; + struct shrinker *kfree_rcu_shrinker; + ++ rcu_reclaim_wq = alloc_workqueue("kvfree_rcu_reclaim", ++ WQ_UNBOUND | WQ_MEM_RECLAIM, 0); ++ WARN_ON(!rcu_reclaim_wq); ++ + /* Clamp it to [0:100] seconds interval. */ + if (rcu_delay_page_cache_fill_msec < 0 || + rcu_delay_page_cache_fill_msec > 100 * MSEC_PER_SEC) { diff --git a/queue-6.12/series b/queue-6.12/series new file mode 100644 index 0000000000..8ceb56d461 --- /dev/null +++ b/queue-6.12/series @@ -0,0 +1,3 @@ +mm-slab-kvfree_rcu-switch-to-wq_mem_reclaim-wq.patch +mm-fix-kernel-bug-when-userfaultfd_move-encounters-swapcache.patch +userfaultfd-fix-pte-unmapping-stack-allocated-pte-copies.patch diff --git a/queue-6.12/userfaultfd-fix-pte-unmapping-stack-allocated-pte-copies.patch b/queue-6.12/userfaultfd-fix-pte-unmapping-stack-allocated-pte-copies.patch new file mode 100644 index 0000000000..eabea4b030 --- /dev/null +++ b/queue-6.12/userfaultfd-fix-pte-unmapping-stack-allocated-pte-copies.patch @@ -0,0 +1,95 @@ +From 927e926d72d9155fde3264459fe9bfd7b5e40d28 Mon Sep 17 00:00:00 2001 +From: Suren Baghdasaryan +Date: Wed, 26 Feb 2025 10:55:09 -0800 +Subject: userfaultfd: fix PTE unmapping stack-allocated PTE copies + +From: Suren Baghdasaryan + +commit 927e926d72d9155fde3264459fe9bfd7b5e40d28 upstream. + +Current implementation of move_pages_pte() copies source and destination +PTEs in order to detect concurrent changes to PTEs involved in the move. +However these copies are also used to unmap the PTEs, which will fail if +CONFIG_HIGHPTE is enabled because the copies are allocated on the stack. +Fix this by using the actual PTEs which were kmap()ed. + +Link: https://lkml.kernel.org/r/20250226185510.2732648-3-surenb@google.com +Fixes: adef440691ba ("userfaultfd: UFFDIO_MOVE uABI") +Signed-off-by: Suren Baghdasaryan +Reported-by: Peter Xu +Reviewed-by: Peter Xu +Cc: Andrea Arcangeli +Cc: Barry Song <21cnbao@gmail.com> +Cc: Barry Song +Cc: David Hildenbrand +Cc: Hugh Dickins +Cc: Jann Horn +Cc: Kalesh Singh +Cc: Liam R. Howlett +Cc: Lokesh Gidra +Cc: Lorenzo Stoakes +Cc: Matthew Wilcow (Oracle) +Cc: +Signed-off-by: Andrew Morton +Signed-off-by: Greg Kroah-Hartman +--- + mm/userfaultfd.c | 20 ++++++++++---------- + 1 file changed, 10 insertions(+), 10 deletions(-) + +--- a/mm/userfaultfd.c ++++ b/mm/userfaultfd.c +@@ -1265,8 +1265,8 @@ retry: + spin_unlock(src_ptl); + + if (!locked) { +- pte_unmap(&orig_src_pte); +- pte_unmap(&orig_dst_pte); ++ pte_unmap(src_pte); ++ pte_unmap(dst_pte); + src_pte = dst_pte = NULL; + /* now we can block and wait */ + folio_lock(src_folio); +@@ -1282,8 +1282,8 @@ retry: + /* at this point we have src_folio locked */ + if (folio_test_large(src_folio)) { + /* split_folio() can block */ +- pte_unmap(&orig_src_pte); +- pte_unmap(&orig_dst_pte); ++ pte_unmap(src_pte); ++ pte_unmap(dst_pte); + src_pte = dst_pte = NULL; + err = split_folio(src_folio); + if (err) +@@ -1308,8 +1308,8 @@ retry: + goto out; + } + if (!anon_vma_trylock_write(src_anon_vma)) { +- pte_unmap(&orig_src_pte); +- pte_unmap(&orig_dst_pte); ++ pte_unmap(src_pte); ++ pte_unmap(dst_pte); + src_pte = dst_pte = NULL; + /* now we can block and wait */ + anon_vma_lock_write(src_anon_vma); +@@ -1327,8 +1327,8 @@ retry: + entry = pte_to_swp_entry(orig_src_pte); + if (non_swap_entry(entry)) { + if (is_migration_entry(entry)) { +- pte_unmap(&orig_src_pte); +- pte_unmap(&orig_dst_pte); ++ pte_unmap(src_pte); ++ pte_unmap(dst_pte); + src_pte = dst_pte = NULL; + migration_entry_wait(mm, src_pmd, src_addr); + err = -EAGAIN; +@@ -1371,8 +1371,8 @@ retry: + src_folio = folio; + src_folio_pte = orig_src_pte; + if (!folio_trylock(src_folio)) { +- pte_unmap(&orig_src_pte); +- pte_unmap(&orig_dst_pte); ++ pte_unmap(src_pte); ++ pte_unmap(dst_pte); + src_pte = dst_pte = NULL; + put_swap_device(si); + si = NULL;