From ac6be8f6e8548c96da1c34bc85eecbe451749b99 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 15 Dec 2022 07:47:32 +0100 Subject: [PATCH] 4.14-stable patches added patches: mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch --- ...-gup-fast-interaction-by-sending-ipi.patch | 109 ++++++++++++++++++ ...tifiers-in-shmem-file-collapse-paths.patch | 67 +++++++++++ queue-4.14/series | 2 + 3 files changed, 178 insertions(+) create mode 100644 queue-4.14/mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch create mode 100644 queue-4.14/mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch diff --git a/queue-4.14/mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch b/queue-4.14/mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch new file mode 100644 index 00000000000..751d1957af9 --- /dev/null +++ b/queue-4.14/mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch @@ -0,0 +1,109 @@ +From 2ba99c5e08812494bc57f319fb562f527d9bacd8 Mon Sep 17 00:00:00 2001 +From: Jann Horn +Date: Fri, 25 Nov 2022 22:37:13 +0100 +Subject: mm/khugepaged: fix GUP-fast interaction by sending IPI + +From: Jann Horn + +commit 2ba99c5e08812494bc57f319fb562f527d9bacd8 upstream. + +Since commit 70cbc3cc78a99 ("mm: gup: fix the fast GUP race against THP +collapse"), the lockless_pages_from_mm() fastpath rechecks the pmd_t to +ensure that the page table was not removed by khugepaged in between. + +However, lockless_pages_from_mm() still requires that the page table is +not concurrently freed. Fix it by sending IPIs (if the architecture uses +semi-RCU-style page table freeing) before freeing/reusing page tables. + +Link: https://lkml.kernel.org/r/20221129154730.2274278-2-jannh@google.com +Link: https://lkml.kernel.org/r/20221128180252.1684965-2-jannh@google.com +Link: https://lkml.kernel.org/r/20221125213714.4115729-2-jannh@google.com +Fixes: ba76149f47d8 ("thp: khugepaged") +Signed-off-by: Jann Horn +Reviewed-by: Yang Shi +Acked-by: David Hildenbrand +Cc: John Hubbard +Cc: Peter Xu +Cc: +Signed-off-by: Andrew Morton +[manual backport: two of the three places in khugepaged that can free +ptes were refactored into a common helper between 5.15 and 6.0; +TLB flushing was refactored between 5.4 and 5.10; +TLB flushing was refactored between 4.19 and 5.4; +pmd collapse for PTE-mapped THP was only added in 5.4; +ugly hack for s390 in <=4.19] +Signed-off-by: Jann Horn +Signed-off-by: Greg Kroah-Hartman +--- + include/asm-generic/tlb.h | 6 ++++++ + mm/khugepaged.c | 15 +++++++++++++++ + mm/memory.c | 5 +++++ + 3 files changed, 26 insertions(+) + +--- a/include/asm-generic/tlb.h ++++ b/include/asm-generic/tlb.h +@@ -60,6 +60,12 @@ struct mmu_table_batch { + extern void tlb_table_flush(struct mmu_gather *tlb); + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); + ++void tlb_remove_table_sync_one(void); ++ ++#else ++ ++static inline void tlb_remove_table_sync_one(void) { } ++ + #endif + + /* +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -23,6 +23,19 @@ + #include + #include "internal.h" + ++/* gross hack for <=4.19 stable */ ++#ifdef CONFIG_S390 ++static void tlb_remove_table_smp_sync(void *arg) ++{ ++ /* Simply deliver the interrupt */ ++} ++ ++static void tlb_remove_table_sync_one(void) ++{ ++ smp_call_function(tlb_remove_table_smp_sync, NULL, 1); ++} ++#endif ++ + enum scan_result { + SCAN_FAIL, + SCAN_SUCCEED, +@@ -1046,6 +1059,7 @@ static void collapse_huge_page(struct mm + _pmd = pmdp_collapse_flush(vma, address, pmd); + spin_unlock(pmd_ptl); + mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); ++ tlb_remove_table_sync_one(); + + spin_lock(pte_ptl); + isolated = __collapse_huge_page_isolate(vma, address, pte); +@@ -1295,6 +1309,7 @@ static void retract_page_tables(struct a + _pmd = pmdp_collapse_flush(vma, addr, pmd); + spin_unlock(ptl); + atomic_long_dec(&mm->nr_ptes); ++ tlb_remove_table_sync_one(); + pte_free(mm, pmd_pgtable(_pmd)); + } + up_write(&mm->mmap_sem); +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -373,6 +373,11 @@ static void tlb_remove_table_smp_sync(vo + /* Simply deliver the interrupt */ + } + ++void tlb_remove_table_sync_one(void) ++{ ++ smp_call_function(tlb_remove_table_smp_sync, NULL, 1); ++} ++ + static void tlb_remove_table_one(void *table) + { + /* diff --git a/queue-4.14/mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch b/queue-4.14/mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch new file mode 100644 index 00000000000..bec985bd4aa --- /dev/null +++ b/queue-4.14/mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch @@ -0,0 +1,67 @@ +From f268f6cf875f3220afc77bdd0bf1bb136eb54db9 Mon Sep 17 00:00:00 2001 +From: Jann Horn +Date: Fri, 25 Nov 2022 22:37:14 +0100 +Subject: mm/khugepaged: invoke MMU notifiers in shmem/file collapse paths + +From: Jann Horn + +commit f268f6cf875f3220afc77bdd0bf1bb136eb54db9 upstream. + +Any codepath that zaps page table entries must invoke MMU notifiers to +ensure that secondary MMUs (like KVM) don't keep accessing pages which +aren't mapped anymore. Secondary MMUs don't hold their own references to +pages that are mirrored over, so failing to notify them can lead to page +use-after-free. + +I'm marking this as addressing an issue introduced in commit f3f0e1d2150b +("khugepaged: add support of collapse for tmpfs/shmem pages"), but most of +the security impact of this only came in commit 27e1f8273113 ("khugepaged: +enable collapse pmd for pte-mapped THP"), which actually omitted flushes +for the removal of present PTEs, not just for the removal of empty page +tables. + +Link: https://lkml.kernel.org/r/20221129154730.2274278-3-jannh@google.com +Link: https://lkml.kernel.org/r/20221128180252.1684965-3-jannh@google.com +Link: https://lkml.kernel.org/r/20221125213714.4115729-3-jannh@google.com +Fixes: f3f0e1d2150b ("khugepaged: add support of collapse for tmpfs/shmem pages") +Signed-off-by: Jann Horn +Acked-by: David Hildenbrand +Reviewed-by: Yang Shi +Cc: John Hubbard +Cc: Peter Xu +Cc: +Signed-off-by: Andrew Morton +[manual backport: this code was refactored from two copies into a common +helper between 5.15 and 6.0; +pmd collapse for PTE-mapped THP was only added in 5.4; +MMU notifier API changed between 4.19 and 5.4] +Signed-off-by: Jann Horn +Signed-off-by: Greg Kroah-Hartman +--- + mm/khugepaged.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +--- a/mm/khugepaged.c ++++ b/mm/khugepaged.c +@@ -1304,13 +1304,20 @@ static void retract_page_tables(struct a + */ + if (down_write_trylock(&mm->mmap_sem)) { + if (!khugepaged_test_exit(mm)) { +- spinlock_t *ptl = pmd_lock(mm, pmd); ++ spinlock_t *ptl; ++ unsigned long end = addr + HPAGE_PMD_SIZE; ++ ++ mmu_notifier_invalidate_range_start(mm, addr, ++ end); ++ ptl = pmd_lock(mm, pmd); + /* assume page table is clear */ + _pmd = pmdp_collapse_flush(vma, addr, pmd); + spin_unlock(ptl); + atomic_long_dec(&mm->nr_ptes); + tlb_remove_table_sync_one(); + pte_free(mm, pmd_pgtable(_pmd)); ++ mmu_notifier_invalidate_range_end(mm, addr, ++ end); + } + up_write(&mm->mmap_sem); + } diff --git a/queue-4.14/series b/queue-4.14/series index d067f6253d7..1f58dbb7943 100644 --- a/queue-4.14/series +++ b/queue-4.14/series @@ -1,2 +1,4 @@ libtraceevent-fix-build-with-binutils-2.35.patch once-add-do_once_slow-for-sleepable-contexts.patch +mm-khugepaged-fix-gup-fast-interaction-by-sending-ipi.patch +mm-khugepaged-invoke-mmu-notifiers-in-shmem-file-collapse-paths.patch -- 2.47.3