From b9d71e2ed5a3ccd28a5e7ce5e6ba22bbea6e1c4c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 17 Feb 2026 13:28:37 +0100 Subject: [PATCH] 6.6-stable patches added patches: mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch mm-hugetlb-fix-hugetlb_pmd_shared.patch mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch --- ...tlb_page_range-to-use-pt_share_count.patch | 111 +++ ...nsharing-pmd-tables-using-mmu_gather.patch | 762 ++++++++++++++++++ .../mm-hugetlb-fix-hugetlb_pmd_shared.patch | 90 +++ ...comments-related-to-huge_pmd_unshare.patch | 87 ++ queue-6.6/series | 4 + 5 files changed, 1054 insertions(+) create mode 100644 queue-6.6/mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch create mode 100644 queue-6.6/mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch create mode 100644 queue-6.6/mm-hugetlb-fix-hugetlb_pmd_shared.patch create mode 100644 queue-6.6/mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch diff --git a/queue-6.6/mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch b/queue-6.6/mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch new file mode 100644 index 0000000000..1842541bd8 --- /dev/null +++ b/queue-6.6/mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch @@ -0,0 +1,111 @@ +From 14967a9c7d247841b0312c48dcf8cd29e55a4cc8 Mon Sep 17 00:00:00 2001 +From: Jane Chu +Date: Mon, 15 Sep 2025 18:45:20 -0600 +Subject: mm/hugetlb: fix copy_hugetlb_page_range() to use ->pt_share_count + +From: Jane Chu + +commit 14967a9c7d247841b0312c48dcf8cd29e55a4cc8 upstream. + +commit 59d9094df3d79 ("mm: hugetlb: independent PMD page table shared +count") introduced ->pt_share_count dedicated to hugetlb PMD share count +tracking, but omitted fixing copy_hugetlb_page_range(), leaving the +function relying on page_count() for tracking that no longer works. + +When lazy page table copy for hugetlb is disabled, that is, revert commit +bcd51a3c679d ("hugetlb: lazy page table copies in fork()") fork()'ing with +hugetlb PMD sharing quickly lockup - + +[ 239.446559] watchdog: BUG: soft lockup - CPU#75 stuck for 27s! +[ 239.446611] RIP: 0010:native_queued_spin_lock_slowpath+0x7e/0x2e0 +[ 239.446631] Call Trace: +[ 239.446633] +[ 239.446636] _raw_spin_lock+0x3f/0x60 +[ 239.446639] copy_hugetlb_page_range+0x258/0xb50 +[ 239.446645] copy_page_range+0x22b/0x2c0 +[ 239.446651] dup_mmap+0x3e2/0x770 +[ 239.446654] dup_mm.constprop.0+0x5e/0x230 +[ 239.446657] copy_process+0xd17/0x1760 +[ 239.446660] kernel_clone+0xc0/0x3e0 +[ 239.446661] __do_sys_clone+0x65/0xa0 +[ 239.446664] do_syscall_64+0x82/0x930 +[ 239.446668] ? count_memcg_events+0xd2/0x190 +[ 239.446671] ? syscall_trace_enter+0x14e/0x1f0 +[ 239.446676] ? syscall_exit_work+0x118/0x150 +[ 239.446677] ? arch_exit_to_user_mode_prepare.constprop.0+0x9/0xb0 +[ 239.446681] ? clear_bhb_loop+0x30/0x80 +[ 239.446684] ? clear_bhb_loop+0x30/0x80 +[ 239.446686] entry_SYSCALL_64_after_hwframe+0x76/0x7e + +There are two options to resolve the potential latent issue: + 1. warn against PMD sharing in copy_hugetlb_page_range(), + 2. fix it. +This patch opts for the second option. +While at it, simplify the comment, the details are not actually relevant +anymore. + +Link: https://lkml.kernel.org/r/20250916004520.1604530-1-jane.chu@oracle.com +Fixes: 59d9094df3d7 ("mm: hugetlb: independent PMD page table shared count") +Signed-off-by: Jane Chu +Reviewed-by: Harry Yoo +Acked-by: Oscar Salvador +Acked-by: David Hildenbrand +Cc: Jann Horn +Cc: Liu Shixin +Cc: Muchun Song +Signed-off-by: Andrew Morton +Signed-off-by: David Hildenbrand (Arm) +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/mm_types.h | 5 +++++ + mm/hugetlb.c | 15 +++++---------- + 2 files changed, 10 insertions(+), 10 deletions(-) + +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -492,6 +492,11 @@ static inline int ptdesc_pmd_pts_count(s + { + return atomic_read(&ptdesc->pt_share_count); + } ++ ++static inline bool ptdesc_pmd_is_shared(struct ptdesc *ptdesc) ++{ ++ return !!ptdesc_pmd_pts_count(ptdesc); ++} + #else + static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) + { +--- a/mm/hugetlb.c ++++ b/mm/hugetlb.c +@@ -5090,18 +5090,13 @@ int copy_hugetlb_page_range(struct mm_st + break; + } + +- /* +- * If the pagetables are shared don't copy or take references. +- * +- * dst_pte == src_pte is the common case of src/dest sharing. +- * However, src could have 'unshared' and dst shares with +- * another vma. So page_count of ptep page is checked instead +- * to reliably determine whether pte is shared. +- */ +- if (page_count(virt_to_page(dst_pte)) > 1) { ++#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING ++ /* If the pagetables are shared, there is nothing to do */ ++ if (ptdesc_pmd_is_shared(virt_to_ptdesc(dst_pte))) { + addr |= last_addr_mask; + continue; + } ++#endif + + dst_ptl = huge_pte_lock(h, dst, dst_pte); + src_ptl = huge_pte_lockptr(h, src, src_pte); +@@ -7077,7 +7072,7 @@ int huge_pmd_unshare(struct mm_struct *m + hugetlb_vma_assert_locked(vma); + if (sz != PMD_SIZE) + return 0; +- if (!ptdesc_pmd_pts_count(virt_to_ptdesc(ptep))) ++ if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep))) + return 0; + + pud_clear(pud); diff --git a/queue-6.6/mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch b/queue-6.6/mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch new file mode 100644 index 0000000000..67420db831 --- /dev/null +++ b/queue-6.6/mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch @@ -0,0 +1,762 @@ +From 8ce720d5bd91e9dc16db3604aa4b1bf76770a9a1 Mon Sep 17 00:00:00 2001 +From: "David Hildenbrand (Red Hat)" +Date: Tue, 23 Dec 2025 22:40:37 +0100 +Subject: mm/hugetlb: fix excessive IPI broadcasts when unsharing PMD tables using mmu_gather + +From: David Hildenbrand (Red Hat) + +commit 8ce720d5bd91e9dc16db3604aa4b1bf76770a9a1 upstream. + +As reported, ever since commit 1013af4f585f ("mm/hugetlb: fix +huge_pmd_unshare() vs GUP-fast race") we can end up in some situations +where we perform so many IPI broadcasts when unsharing hugetlb PMD page +tables that it severely regresses some workloads. + +In particular, when we fork()+exit(), or when we munmap() a large +area backed by many shared PMD tables, we perform one IPI broadcast per +unshared PMD table. + +There are two optimizations to be had: + +(1) When we process (unshare) multiple such PMD tables, such as during + exit(), it is sufficient to send a single IPI broadcast (as long as + we respect locking rules) instead of one per PMD table. + + Locking prevents that any of these PMD tables could get reused before + we drop the lock. + +(2) When we are not the last sharer (> 2 users including us), there is + no need to send the IPI broadcast. The shared PMD tables cannot + become exclusive (fully unshared) before an IPI will be broadcasted + by the last sharer. + + Concurrent GUP-fast could walk into a PMD table just before we + unshared it. It could then succeed in grabbing a page from the + shared page table even after munmap() etc succeeded (and supressed + an IPI). But there is not difference compared to GUP-fast just + sleeping for a while after grabbing the page and re-enabling IRQs. + + Most importantly, GUP-fast will never walk into page tables that are + no-longer shared, because the last sharer will issue an IPI + broadcast. + + (if ever required, checking whether the PUD changed in GUP-fast + after grabbing the page like we do in the PTE case could handle + this) + +So let's rework PMD sharing TLB flushing + IPI sync to use the mmu_gather +infrastructure so we can implement these optimizations and demystify the +code at least a bit. Extend the mmu_gather infrastructure to be able to +deal with our special hugetlb PMD table sharing implementation. + +To make initialization of the mmu_gather easier when working on a single +VMA (in particular, when dealing with hugetlb), provide +tlb_gather_mmu_vma(). + +We'll consolidate the handling for (full) unsharing of PMD tables in +tlb_unshare_pmd_ptdesc() and tlb_flush_unshared_tables(), and track +in "struct mmu_gather" whether we had (full) unsharing of PMD tables. + +Because locking is very special (concurrent unsharing+reuse must be +prevented), we disallow deferring flushing to tlb_finish_mmu() and instead +require an explicit earlier call to tlb_flush_unshared_tables(). + +From hugetlb code, we call huge_pmd_unshare_flush() where we make sure +that the expected lock protecting us from concurrent unsharing+reuse is +still held. + +Check with a VM_WARN_ON_ONCE() in tlb_finish_mmu() that +tlb_flush_unshared_tables() was properly called earlier. + +Document it all properly. + +Notes about tlb_remove_table_sync_one() interaction with unsharing: + +There are two fairly tricky things: + +(1) tlb_remove_table_sync_one() is a NOP on architectures without + CONFIG_MMU_GATHER_RCU_TABLE_FREE. + + Here, the assumption is that the previous TLB flush would send an + IPI to all relevant CPUs. Careful: some architectures like x86 only + send IPIs to all relevant CPUs when tlb->freed_tables is set. + + The relevant architectures should be selecting + MMU_GATHER_RCU_TABLE_FREE, but x86 might not do that in stable + kernels and it might have been problematic before this patch. + + Also, the arch flushing behavior (independent of IPIs) is different + when tlb->freed_tables is set. Do we have to enlighten them to also + take care of tlb->unshared_tables? So far we didn't care, so + hopefully we are fine. Of course, we could be setting + tlb->freed_tables as well, but that might then unnecessarily flush + too much, because the semantics of tlb->freed_tables are a bit + fuzzy. + + This patch changes nothing in this regard. + +(2) tlb_remove_table_sync_one() is not a NOP on architectures with + CONFIG_MMU_GATHER_RCU_TABLE_FREE that actually don't need a sync. + + Take x86 as an example: in the common case (!pv, !X86_FEATURE_INVLPGB) + we still issue IPIs during TLB flushes and don't actually need the + second tlb_remove_table_sync_one(). + + This optimized can be implemented on top of this, by checking e.g., in + tlb_remove_table_sync_one() whether we really need IPIs. But as + described in (1), it really must honor tlb->freed_tables then to + send IPIs to all relevant CPUs. + +Notes on TLB flushing changes: + +(1) Flushing for non-shared PMD tables + + We're converting from flush_hugetlb_tlb_range() to + tlb_remove_huge_tlb_entry(). Given that we properly initialize the + MMU gather in tlb_gather_mmu_vma() to be hugetlb aware, similar to + __unmap_hugepage_range(), that should be fine. + +(2) Flushing for shared PMD tables + + We're converting from various things (flush_hugetlb_tlb_range(), + tlb_flush_pmd_range(), flush_tlb_range()) to tlb_flush_pmd_range(). + + tlb_flush_pmd_range() achieves the same that + tlb_remove_huge_tlb_entry() would achieve in these scenarios. + Note that tlb_remove_huge_tlb_entry() also calls + __tlb_remove_tlb_entry(), however that is only implemented on + powerpc, which does not support PMD table sharing. + + Similar to (1), tlb_gather_mmu_vma() should make sure that TLB + flushing keeps on working as expected. + +Further, note that the ptdesc_pmd_pts_dec() in huge_pmd_share() is not a +concern, as we are holding the i_mmap_lock the whole time, preventing +concurrent unsharing. That ptdesc_pmd_pts_dec() usage will be removed +separately as a cleanup later. + +There are plenty more cleanups to be had, but they have to wait until +this is fixed. + +[david@kernel.org: fix kerneldoc] + Link: https://lkml.kernel.org/r/f223dd74-331c-412d-93fc-69e360a5006c@kernel.org +Link: https://lkml.kernel.org/r/20251223214037.580860-5-david@kernel.org +Fixes: 1013af4f585f ("mm/hugetlb: fix huge_pmd_unshare() vs GUP-fast race") +Signed-off-by: David Hildenbrand (Red Hat) +Reported-by: Uschakow, Stanislav" +Closes: https://lore.kernel.org/all/4d3878531c76479d9f8ca9789dc6485d@amazon.de/ +Tested-by: Laurence Oberman +Acked-by: Harry Yoo +Reviewed-by: Lorenzo Stoakes +Cc: Lance Yang +Cc: Liu Shixin +Cc: Oscar Salvador +Cc: Rik van Riel +Cc: +Signed-off-by: Andrew Morton +Signed-off-by: David Hildenbrand (Arm) +Signed-off-by: Greg Kroah-Hartman +--- + include/asm-generic/tlb.h | 77 ++++++++++++++++++++++++++++ + include/linux/hugetlb.h | 15 +++-- + include/linux/mm_types.h | 1 + mm/hugetlb.c | 123 ++++++++++++++++++++++++++-------------------- + mm/mmu_gather.c | 33 ++++++++++++ + mm/rmap.c | 25 ++++++--- + 6 files changed, 208 insertions(+), 66 deletions(-) + +--- a/include/asm-generic/tlb.h ++++ b/include/asm-generic/tlb.h +@@ -46,7 +46,8 @@ + * + * The mmu_gather API consists of: + * +- * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_finish_mmu() ++ * - tlb_gather_mmu() / tlb_gather_mmu_fullmm() / tlb_gather_mmu_vma() / ++ * tlb_finish_mmu() + * + * start and finish a mmu_gather + * +@@ -337,6 +338,20 @@ struct mmu_gather { + unsigned int vma_huge : 1; + unsigned int vma_pfn : 1; + ++ /* ++ * Did we unshare (unmap) any shared page tables? For now only ++ * used for hugetlb PMD table sharing. ++ */ ++ unsigned int unshared_tables : 1; ++ ++ /* ++ * Did we unshare any page tables such that they are now exclusive ++ * and could get reused+modified by the new owner? When setting this ++ * flag, "unshared_tables" will be set as well. For now only used ++ * for hugetlb PMD table sharing. ++ */ ++ unsigned int fully_unshared_tables : 1; ++ + unsigned int batch_count; + + #ifndef CONFIG_MMU_GATHER_NO_GATHER +@@ -373,6 +388,7 @@ static inline void __tlb_reset_range(str + tlb->cleared_pmds = 0; + tlb->cleared_puds = 0; + tlb->cleared_p4ds = 0; ++ tlb->unshared_tables = 0; + /* + * Do not reset mmu_gather::vma_* fields here, we do not + * call into tlb_start_vma() again to set them if there is an +@@ -452,7 +468,7 @@ static inline void tlb_flush_mmu_tlbonly + * these bits. + */ + if (!(tlb->freed_tables || tlb->cleared_ptes || tlb->cleared_pmds || +- tlb->cleared_puds || tlb->cleared_p4ds)) ++ tlb->cleared_puds || tlb->cleared_p4ds || tlb->unshared_tables)) + return; + + tlb_flush(tlb); +@@ -718,6 +734,63 @@ static inline bool huge_pmd_needs_flush( + } + #endif + ++#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING ++static inline void tlb_unshare_pmd_ptdesc(struct mmu_gather *tlb, struct ptdesc *pt, ++ unsigned long addr) ++{ ++ /* ++ * The caller must make sure that concurrent unsharing + exclusive ++ * reuse is impossible until tlb_flush_unshared_tables() was called. ++ */ ++ VM_WARN_ON_ONCE(!ptdesc_pmd_is_shared(pt)); ++ ptdesc_pmd_pts_dec(pt); ++ ++ /* Clearing a PUD pointing at a PMD table with PMD leaves. */ ++ tlb_flush_pmd_range(tlb, addr & PUD_MASK, PUD_SIZE); ++ ++ /* ++ * If the page table is now exclusively owned, we fully unshared ++ * a page table. ++ */ ++ if (!ptdesc_pmd_is_shared(pt)) ++ tlb->fully_unshared_tables = true; ++ tlb->unshared_tables = true; ++} ++ ++static inline void tlb_flush_unshared_tables(struct mmu_gather *tlb) ++{ ++ /* ++ * As soon as the caller drops locks to allow for reuse of ++ * previously-shared tables, these tables could get modified and ++ * even reused outside of hugetlb context, so we have to make sure that ++ * any page table walkers (incl. TLB, GUP-fast) are aware of that ++ * change. ++ * ++ * Even if we are not fully unsharing a PMD table, we must ++ * flush the TLB for the unsharer now. ++ */ ++ if (tlb->unshared_tables) ++ tlb_flush_mmu_tlbonly(tlb); ++ ++ /* ++ * Similarly, we must make sure that concurrent GUP-fast will not ++ * walk previously-shared page tables that are getting modified+reused ++ * elsewhere. So broadcast an IPI to wait for any concurrent GUP-fast. ++ * ++ * We only perform this when we are the last sharer of a page table, ++ * as the IPI will reach all CPUs: any GUP-fast. ++ * ++ * Note that on configs where tlb_remove_table_sync_one() is a NOP, ++ * the expectation is that the tlb_flush_mmu_tlbonly() would have issued ++ * required IPIs already for us. ++ */ ++ if (tlb->fully_unshared_tables) { ++ tlb_remove_table_sync_one(); ++ tlb->fully_unshared_tables = false; ++ } ++} ++#endif /* CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ ++ + #endif /* CONFIG_MMU */ + + #endif /* _ASM_GENERIC__TLB_H */ +--- a/include/linux/hugetlb.h ++++ b/include/linux/hugetlb.h +@@ -241,8 +241,9 @@ pte_t *huge_pte_alloc(struct mm_struct * + pte_t *huge_pte_offset(struct mm_struct *mm, + unsigned long addr, unsigned long sz); + unsigned long hugetlb_mask_last_page(struct hstate *h); +-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, +- unsigned long addr, pte_t *ptep); ++int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep); ++void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma); + void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, + unsigned long *start, unsigned long *end); + +@@ -304,13 +305,17 @@ static inline struct address_space *huge + return NULL; + } + +-static inline int huge_pmd_unshare(struct mm_struct *mm, +- struct vm_area_struct *vma, +- unsigned long addr, pte_t *ptep) ++static inline int huge_pmd_unshare(struct mmu_gather *tlb, ++ struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) + { + return 0; + } + ++static inline void huge_pmd_unshare_flush(struct mmu_gather *tlb, ++ struct vm_area_struct *vma) ++{ ++} ++ + static inline void adjust_range_if_pmd_sharing_possible( + struct vm_area_struct *vma, + unsigned long *start, unsigned long *end) +--- a/include/linux/mm_types.h ++++ b/include/linux/mm_types.h +@@ -1177,6 +1177,7 @@ static inline unsigned int mm_cid_size(v + struct mmu_gather; + extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); + extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); ++void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma); + extern void tlb_finish_mmu(struct mmu_gather *tlb); + + struct vm_fault; +--- a/mm/hugetlb.c ++++ b/mm/hugetlb.c +@@ -5269,7 +5269,7 @@ int move_hugetlb_page_tables(struct vm_a + unsigned long last_addr_mask; + pte_t *src_pte, *dst_pte; + struct mmu_notifier_range range; +- bool shared_pmd = false; ++ struct mmu_gather tlb; + + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, old_addr, + old_end); +@@ -5279,6 +5279,7 @@ int move_hugetlb_page_tables(struct vm_a + * range. + */ + flush_cache_range(vma, range.start, range.end); ++ tlb_gather_mmu_vma(&tlb, vma); + + mmu_notifier_invalidate_range_start(&range); + last_addr_mask = hugetlb_mask_last_page(h); +@@ -5295,8 +5296,7 @@ int move_hugetlb_page_tables(struct vm_a + if (huge_pte_none(huge_ptep_get(src_pte))) + continue; + +- if (huge_pmd_unshare(mm, vma, old_addr, src_pte)) { +- shared_pmd = true; ++ if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { + old_addr |= last_addr_mask; + new_addr |= last_addr_mask; + continue; +@@ -5307,15 +5307,16 @@ int move_hugetlb_page_tables(struct vm_a + break; + + move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte, sz); ++ tlb_remove_huge_tlb_entry(h, &tlb, src_pte, old_addr); + } + +- if (shared_pmd) +- flush_hugetlb_tlb_range(vma, range.start, range.end); +- else +- flush_hugetlb_tlb_range(vma, old_end - len, old_end); ++ tlb_flush_mmu_tlbonly(&tlb); ++ huge_pmd_unshare_flush(&tlb, vma); ++ + mmu_notifier_invalidate_range_end(&range); + i_mmap_unlock_write(mapping); + hugetlb_vma_unlock_write(vma); ++ tlb_finish_mmu(&tlb); + + return len + old_addr - old_end; + } +@@ -5333,7 +5334,6 @@ void __unmap_hugepage_range(struct mmu_g + struct hstate *h = hstate_vma(vma); + unsigned long sz = huge_page_size(h); + unsigned long last_addr_mask; +- bool force_flush = false; + + WARN_ON(!is_vm_hugetlb_page(vma)); + BUG_ON(start & ~huge_page_mask(h)); +@@ -5356,10 +5356,8 @@ void __unmap_hugepage_range(struct mmu_g + } + + ptl = huge_pte_lock(h, mm, ptep); +- if (huge_pmd_unshare(mm, vma, address, ptep)) { ++ if (huge_pmd_unshare(tlb, vma, address, ptep)) { + spin_unlock(ptl); +- tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE); +- force_flush = true; + address |= last_addr_mask; + continue; + } +@@ -5434,14 +5432,7 @@ void __unmap_hugepage_range(struct mmu_g + } + tlb_end_vma(tlb, vma); + +- /* +- * There is nothing protecting a previously-shared page table that we +- * unshared through huge_pmd_unshare() from getting freed after we +- * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() +- * succeeded, flush the range corresponding to the pud. +- */ +- if (force_flush) +- tlb_flush_mmu_tlbonly(tlb); ++ huge_pmd_unshare_flush(tlb, vma); + } + + void __hugetlb_zap_begin(struct vm_area_struct *vma, +@@ -6573,11 +6564,11 @@ long hugetlb_change_protection(struct vm + pte_t pte; + struct hstate *h = hstate_vma(vma); + long pages = 0, psize = huge_page_size(h); +- bool shared_pmd = false; + struct mmu_notifier_range range; + unsigned long last_addr_mask; + bool uffd_wp = cp_flags & MM_CP_UFFD_WP; + bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; ++ struct mmu_gather tlb; + + /* + * In the case of shared PMDs, the area to flush could be beyond +@@ -6590,6 +6581,7 @@ long hugetlb_change_protection(struct vm + + BUG_ON(address >= end); + flush_cache_range(vma, range.start, range.end); ++ tlb_gather_mmu_vma(&tlb, vma); + + mmu_notifier_invalidate_range_start(&range); + hugetlb_vma_lock_write(vma); +@@ -6614,7 +6606,7 @@ long hugetlb_change_protection(struct vm + } + } + ptl = huge_pte_lock(h, mm, ptep); +- if (huge_pmd_unshare(mm, vma, address, ptep)) { ++ if (huge_pmd_unshare(&tlb, vma, address, ptep)) { + /* + * When uffd-wp is enabled on the vma, unshare + * shouldn't happen at all. Warn about it if it +@@ -6623,7 +6615,6 @@ long hugetlb_change_protection(struct vm + WARN_ON_ONCE(uffd_wp || uffd_wp_resolve); + pages++; + spin_unlock(ptl); +- shared_pmd = true; + address |= last_addr_mask; + continue; + } +@@ -6675,6 +6666,7 @@ long hugetlb_change_protection(struct vm + pte = huge_pte_clear_uffd_wp(pte); + huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte); + pages++; ++ tlb_remove_huge_tlb_entry(h, &tlb, ptep, address); + } else { + /* None pte */ + if (unlikely(uffd_wp)) +@@ -6687,16 +6679,9 @@ long hugetlb_change_protection(struct vm + + cond_resched(); + } +- /* +- * There is nothing protecting a previously-shared page table that we +- * unshared through huge_pmd_unshare() from getting freed after we +- * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() +- * succeeded, flush the range corresponding to the pud. +- */ +- if (shared_pmd) +- flush_hugetlb_tlb_range(vma, range.start, range.end); +- else +- flush_hugetlb_tlb_range(vma, start, end); ++ ++ tlb_flush_mmu_tlbonly(&tlb); ++ huge_pmd_unshare_flush(&tlb, vma); + /* + * No need to call mmu_notifier_arch_invalidate_secondary_tlbs() we are + * downgrading page table protection not changing it to point to a new +@@ -6707,6 +6692,7 @@ long hugetlb_change_protection(struct vm + i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); + mmu_notifier_invalidate_range_end(&range); ++ tlb_finish_mmu(&tlb); + + return pages > 0 ? (pages << h->order) : pages; + } +@@ -7044,18 +7030,27 @@ out: + return pte; + } + +-/* +- * unmap huge page backed by shared pte. ++/** ++ * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users ++ * @tlb: the current mmu_gather. ++ * @vma: the vma covering the pmd table. ++ * @addr: the address we are trying to unshare. ++ * @ptep: pointer into the (pmd) page table. ++ * ++ * Called with the page table lock held, the i_mmap_rwsem held in write mode ++ * and the hugetlb vma lock held in write mode. + * +- * Called with page table lock held. ++ * Note: The caller must call huge_pmd_unshare_flush() before dropping the ++ * i_mmap_rwsem. + * +- * returns: 1 successfully unmapped a shared pte page +- * 0 the underlying pte page is not shared, or it is the last user ++ * Returns: 1 if it was a shared PMD table and it got unmapped, or 0 if it ++ * was not a shared PMD table. + */ +-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, +- unsigned long addr, pte_t *ptep) ++int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep) + { + unsigned long sz = huge_page_size(hstate_vma(vma)); ++ struct mm_struct *mm = vma->vm_mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_offset(pgd, addr); + pud_t *pud = pud_offset(p4d, addr); +@@ -7068,18 +7063,36 @@ int huge_pmd_unshare(struct mm_struct *m + return 0; + + pud_clear(pud); +- /* +- * Once our caller drops the rmap lock, some other process might be +- * using this page table as a normal, non-hugetlb page table. +- * Wait for pending gup_fast() in other threads to finish before letting +- * that happen. +- */ +- tlb_remove_table_sync_one(); +- ptdesc_pmd_pts_dec(virt_to_ptdesc(ptep)); ++ ++ tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr); ++ + mm_dec_nr_pmds(mm); + return 1; + } + ++/* ++ * huge_pmd_unshare_flush - Complete a sequence of huge_pmd_unshare() calls ++ * @tlb: the current mmu_gather. ++ * @vma: the vma covering the pmd table. ++ * ++ * Perform necessary TLB flushes or IPI broadcasts to synchronize PMD table ++ * unsharing with concurrent page table walkers. ++ * ++ * This function must be called after a sequence of huge_pmd_unshare() ++ * calls while still holding the i_mmap_rwsem. ++ */ ++void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) ++{ ++ /* ++ * We must synchronize page table unsharing such that nobody will ++ * try reusing a previously-shared page table while it might still ++ * be in use by previous sharers (TLB, GUP_fast). ++ */ ++ i_mmap_assert_write_locked(vma->vm_file->f_mapping); ++ ++ tlb_flush_unshared_tables(tlb); ++} ++ + #else /* !CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING */ + + pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, +@@ -7088,12 +7101,16 @@ pte_t *huge_pmd_share(struct mm_struct * + return NULL; + } + +-int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, +- unsigned long addr, pte_t *ptep) ++int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma, ++ unsigned long addr, pte_t *ptep) + { + return 0; + } + ++void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma) ++{ ++} ++ + void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, + unsigned long *start, unsigned long *end) + { +@@ -7326,6 +7343,7 @@ static void hugetlb_unshare_pmds(struct + unsigned long sz = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + struct mmu_notifier_range range; ++ struct mmu_gather tlb; + unsigned long address; + spinlock_t *ptl; + pte_t *ptep; +@@ -7337,6 +7355,8 @@ static void hugetlb_unshare_pmds(struct + return; + + flush_cache_range(vma, start, end); ++ tlb_gather_mmu_vma(&tlb, vma); ++ + /* + * No need to call adjust_range_if_pmd_sharing_possible(), because + * we have already done the PUD_SIZE alignment. +@@ -7355,10 +7375,10 @@ static void hugetlb_unshare_pmds(struct + if (!ptep) + continue; + ptl = huge_pte_lock(h, mm, ptep); +- huge_pmd_unshare(mm, vma, address, ptep); ++ huge_pmd_unshare(&tlb, vma, address, ptep); + spin_unlock(ptl); + } +- flush_hugetlb_tlb_range(vma, start, end); ++ huge_pmd_unshare_flush(&tlb, vma); + if (take_locks) { + i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); +@@ -7368,6 +7388,7 @@ static void hugetlb_unshare_pmds(struct + * Documentation/mm/mmu_notifier.rst. + */ + mmu_notifier_invalidate_range_end(&range); ++ tlb_finish_mmu(&tlb); + } + + /* +--- a/mm/mmu_gather.c ++++ b/mm/mmu_gather.c +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -321,6 +322,7 @@ static void __tlb_gather_mmu(struct mmu_ + tlb->page_size = 0; + #endif + ++ tlb->fully_unshared_tables = 0; + __tlb_reset_range(tlb); + inc_tlb_flush_pending(tlb->mm); + } +@@ -355,6 +357,31 @@ void tlb_gather_mmu_fullmm(struct mmu_ga + } + + /** ++ * tlb_gather_mmu_vma - initialize an mmu_gather structure for operating on a ++ * single VMA ++ * @tlb: the mmu_gather structure to initialize ++ * @vma: the vm_area_struct ++ * ++ * Called to initialize an (on-stack) mmu_gather structure for operating on ++ * a single VMA. In contrast to tlb_gather_mmu(), calling this function will ++ * not require another call to tlb_start_vma(). In contrast to tlb_start_vma(), ++ * this function will *not* call flush_cache_range(). ++ * ++ * For hugetlb VMAs, this function will also initialize the mmu_gather ++ * page_size accordingly, not requiring a separate call to ++ * tlb_change_page_size(). ++ * ++ */ ++void tlb_gather_mmu_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) ++{ ++ tlb_gather_mmu(tlb, vma->vm_mm); ++ tlb_update_vma_flags(tlb, vma); ++ if (is_vm_hugetlb_page(vma)) ++ /* All entries have the same size. */ ++ tlb_change_page_size(tlb, huge_page_size(hstate_vma(vma))); ++} ++ ++/** + * tlb_finish_mmu - finish an mmu_gather structure + * @tlb: the mmu_gather structure to finish + * +@@ -364,6 +391,12 @@ void tlb_gather_mmu_fullmm(struct mmu_ga + void tlb_finish_mmu(struct mmu_gather *tlb) + { + /* ++ * We expect an earlier huge_pmd_unshare_flush() call to sort this out, ++ * due to complicated locking requirements with page table unsharing. ++ */ ++ VM_WARN_ON_ONCE(tlb->fully_unshared_tables); ++ ++ /* + * If there are parallel threads are doing PTE changes on same range + * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB + * flush by batching, one thread may end up seeing inconsistent PTEs +--- a/mm/rmap.c ++++ b/mm/rmap.c +@@ -76,7 +76,7 @@ + #include + #include + +-#include ++#include + + #define CREATE_TRACE_POINTS + #include +@@ -1568,16 +1568,20 @@ static bool try_to_unmap_one(struct foli + * if unsuccessful. + */ + if (!anon) { ++ struct mmu_gather tlb; ++ + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + page_vma_mapped_walk_done(&pvmw); + ret = false; + break; + } +- if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { ++ ++ tlb_gather_mmu_vma(&tlb, vma); ++ if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); +- flush_tlb_range(vma, +- range.start, range.end); ++ huge_pmd_unshare_flush(&tlb, vma); ++ tlb_finish_mmu(&tlb); + /* + * The PMD table was unmapped, + * consequently unmapping the folio. +@@ -1586,6 +1590,7 @@ static bool try_to_unmap_one(struct foli + break; + } + hugetlb_vma_unlock_write(vma); ++ tlb_finish_mmu(&tlb); + } + pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); + } else { +@@ -1927,17 +1932,20 @@ static bool try_to_migrate_one(struct fo + * fail if unsuccessful. + */ + if (!anon) { ++ struct mmu_gather tlb; ++ + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + page_vma_mapped_walk_done(&pvmw); + ret = false; + break; + } +- if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { +- hugetlb_vma_unlock_write(vma); +- flush_tlb_range(vma, +- range.start, range.end); + ++ tlb_gather_mmu_vma(&tlb, vma); ++ if (huge_pmd_unshare(&tlb, vma, address, pvmw.pte)) { ++ hugetlb_vma_unlock_write(vma); ++ huge_pmd_unshare_flush(&tlb, vma); ++ tlb_finish_mmu(&tlb); + /* + * The PMD table was unmapped, + * consequently unmapping the folio. +@@ -1946,6 +1954,7 @@ static bool try_to_migrate_one(struct fo + break; + } + hugetlb_vma_unlock_write(vma); ++ tlb_finish_mmu(&tlb); + } + /* Nuke the hugetlb page table entry */ + pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); diff --git a/queue-6.6/mm-hugetlb-fix-hugetlb_pmd_shared.patch b/queue-6.6/mm-hugetlb-fix-hugetlb_pmd_shared.patch new file mode 100644 index 0000000000..2dab6d1380 --- /dev/null +++ b/queue-6.6/mm-hugetlb-fix-hugetlb_pmd_shared.patch @@ -0,0 +1,90 @@ +From ca1a47cd3f5f4c46ca188b1c9a27af87d1ab2216 Mon Sep 17 00:00:00 2001 +From: "David Hildenbrand (Red Hat)" +Date: Tue, 23 Dec 2025 22:40:34 +0100 +Subject: mm/hugetlb: fix hugetlb_pmd_shared() + +From: David Hildenbrand (Red Hat) + +commit ca1a47cd3f5f4c46ca188b1c9a27af87d1ab2216 upstream. + +Patch series "mm/hugetlb: fixes for PMD table sharing (incl. using +mmu_gather)", v3. + +One functional fix, one performance regression fix, and two related +comment fixes. + +I cleaned up my prototype I recently shared [1] for the performance fix, +deferring most of the cleanups I had in the prototype to a later point. +While doing that I identified the other things. + +The goal of this patch set is to be backported to stable trees "fairly" +easily. At least patch #1 and #4. + +Patch #1 fixes hugetlb_pmd_shared() not detecting any sharing +Patch #2 + #3 are simple comment fixes that patch #4 interacts with. +Patch #4 is a fix for the reported performance regression due to excessive +IPI broadcasts during fork()+exit(). + +The last patch is all about TLB flushes, IPIs and mmu_gather. +Read: complicated + +There are plenty of cleanups in the future to be had + one reasonable +optimization on x86. But that's all out of scope for this series. + +Runtime tested, with a focus on fixing the performance regression using +the original reproducer [2] on x86. + + +This patch (of 4): + +We switched from (wrongly) using the page count to an independent shared +count. Now, shared page tables have a refcount of 1 (excluding +speculative references) and instead use ptdesc->pt_share_count to identify +sharing. + +We didn't convert hugetlb_pmd_shared(), so right now, we would never +detect a shared PMD table as such, because sharing/unsharing no longer +touches the refcount of a PMD table. + +Page migration, like mbind() or migrate_pages() would allow for migrating +folios mapped into such shared PMD tables, even though the folios are not +exclusive. In smaps we would account them as "private" although they are +"shared", and we would be wrongly setting the PM_MMAP_EXCLUSIVE in the +pagemap interface. + +Fix it by properly using ptdesc_pmd_is_shared() in hugetlb_pmd_shared(). + +Link: https://lkml.kernel.org/r/20251223214037.580860-1-david@kernel.org +Link: https://lkml.kernel.org/r/20251223214037.580860-2-david@kernel.org +Link: https://lore.kernel.org/all/8cab934d-4a56-44aa-b641-bfd7e23bd673@kernel.org/ [1] +Link: https://lore.kernel.org/all/8cab934d-4a56-44aa-b641-bfd7e23bd673@kernel.org/ [2] +Fixes: 59d9094df3d7 ("mm: hugetlb: independent PMD page table shared count") +Signed-off-by: David Hildenbrand (Red Hat) +Reviewed-by: Rik van Riel +Reviewed-by: Lance Yang +Tested-by: Lance Yang +Reviewed-by: Harry Yoo +Tested-by: Laurence Oberman +Reviewed-by: Lorenzo Stoakes +Acked-by: Oscar Salvador +Cc: Liu Shixin +Cc: Uschakow, Stanislav" +Cc: +Signed-off-by: Andrew Morton +Signed-off-by: David Hildenbrand (Arm) +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/hugetlb.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +--- a/include/linux/hugetlb.h ++++ b/include/linux/hugetlb.h +@@ -1247,7 +1247,7 @@ static inline __init void hugetlb_cma_re + #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING + static inline bool hugetlb_pmd_shared(pte_t *pte) + { +- return page_count(virt_to_page(pte)) > 1; ++ return ptdesc_pmd_is_shared(virt_to_ptdesc(pte)); + } + #else + static inline bool hugetlb_pmd_shared(pte_t *pte) diff --git a/queue-6.6/mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch b/queue-6.6/mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch new file mode 100644 index 0000000000..1f279ef49a --- /dev/null +++ b/queue-6.6/mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch @@ -0,0 +1,87 @@ +From 3937027caecb4f8251e82dd857ba1d749bb5a428 Mon Sep 17 00:00:00 2001 +From: "David Hildenbrand (Red Hat)" +Date: Tue, 23 Dec 2025 22:40:35 +0100 +Subject: mm/hugetlb: fix two comments related to huge_pmd_unshare() + +From: David Hildenbrand (Red Hat) + +commit 3937027caecb4f8251e82dd857ba1d749bb5a428 upstream. + +Ever since we stopped using the page count to detect shared PMD page +tables, these comments are outdated. + +The only reason we have to flush the TLB early is because once we drop the +i_mmap_rwsem, the previously shared page table could get freed (to then +get reallocated and used for other purpose). So we really have to flush +the TLB before that could happen. + +So let's simplify the comments a bit. + +The "If we unshared PMDs, the TLB flush was not recorded in mmu_gather." +part introduced as in commit a4a118f2eead ("hugetlbfs: flush TLBs +correctly after huge_pmd_unshare") was confusing: sure it is recorded in +the mmu_gather, otherwise tlb_flush_mmu_tlbonly() wouldn't do anything. +So let's drop that comment while at it as well. + +We'll centralize these comments in a single helper as we rework the code +next. + +Link: https://lkml.kernel.org/r/20251223214037.580860-3-david@kernel.org +Fixes: 59d9094df3d7 ("mm: hugetlb: independent PMD page table shared count") +Signed-off-by: David Hildenbrand (Red Hat) +Reviewed-by: Rik van Riel +Tested-by: Laurence Oberman +Reviewed-by: Lorenzo Stoakes +Acked-by: Oscar Salvador +Reviewed-by: Harry Yoo +Cc: Liu Shixin +Cc: Lance Yang +Cc: "Uschakow, Stanislav" +Cc: +Signed-off-by: Andrew Morton +Signed-off-by: David Hildenbrand (Arm) +Signed-off-by: Greg Kroah-Hartman +--- + mm/hugetlb.c | 24 ++++++++---------------- + 1 file changed, 8 insertions(+), 16 deletions(-) + +--- a/mm/hugetlb.c ++++ b/mm/hugetlb.c +@@ -5435,17 +5435,10 @@ void __unmap_hugepage_range(struct mmu_g + tlb_end_vma(tlb, vma); + + /* +- * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We +- * could defer the flush until now, since by holding i_mmap_rwsem we +- * guaranteed that the last refernece would not be dropped. But we must +- * do the flushing before we return, as otherwise i_mmap_rwsem will be +- * dropped and the last reference to the shared PMDs page might be +- * dropped as well. +- * +- * In theory we could defer the freeing of the PMD pages as well, but +- * huge_pmd_unshare() relies on the exact page_count for the PMD page to +- * detect sharing, so we cannot defer the release of the page either. +- * Instead, do flush now. ++ * There is nothing protecting a previously-shared page table that we ++ * unshared through huge_pmd_unshare() from getting freed after we ++ * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() ++ * succeeded, flush the range corresponding to the pud. + */ + if (force_flush) + tlb_flush_mmu_tlbonly(tlb); +@@ -6695,11 +6688,10 @@ long hugetlb_change_protection(struct vm + cond_resched(); + } + /* +- * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare +- * may have cleared our pud entry and done put_page on the page table: +- * once we release i_mmap_rwsem, another task can do the final put_page +- * and that page table be reused and filled with junk. If we actually +- * did unshare a page of pmds, flush the range corresponding to the pud. ++ * There is nothing protecting a previously-shared page table that we ++ * unshared through huge_pmd_unshare() from getting freed after we ++ * release i_mmap_rwsem, so flush the TLB now. If huge_pmd_unshare() ++ * succeeded, flush the range corresponding to the pud. + */ + if (shared_pmd) + flush_hugetlb_tlb_range(vma, range.start, range.end); diff --git a/queue-6.6/series b/queue-6.6/series index 4b00bff6d8..09f4f0212f 100644 --- a/queue-6.6/series +++ b/queue-6.6/series @@ -22,3 +22,7 @@ gpiolib-acpi-fix-gpio-count-with-string-references.patch loongarch-add-writecombine-shadow-mapping-in-kasan.patch loongarch-rework-kasan-initialization-for-ptw-enabled-systems.patch revert-wireguard-device-enable-threaded-napi.patch +mm-hugetlb-fix-copy_hugetlb_page_range-to-use-pt_share_count.patch +mm-hugetlb-fix-hugetlb_pmd_shared.patch +mm-hugetlb-fix-two-comments-related-to-huge_pmd_unshare.patch +mm-hugetlb-fix-excessive-ipi-broadcasts-when-unsharing-pmd-tables-using-mmu_gather.patch -- 2.47.3