--- /dev/null
+From 70cbc3cc78a997d8247b50389d37c4e1736019da Mon Sep 17 00:00:00 2001
+From: Yang Shi <shy828301@gmail.com>
+Date: Wed, 7 Sep 2022 11:01:43 -0700
+Subject: mm: gup: fix the fast GUP race against THP collapse
+
+From: Yang Shi <shy828301@gmail.com>
+
+commit 70cbc3cc78a997d8247b50389d37c4e1736019da upstream.
+
+Since general RCU GUP fast was introduced in commit 2667f50e8b81 ("mm:
+introduce a general RCU get_user_pages_fast()"), a TLB flush is no longer
+sufficient to handle concurrent GUP-fast in all cases, it only handles
+traditional IPI-based GUP-fast correctly. On architectures that send an
+IPI broadcast on TLB flush, it works as expected. But on the
+architectures that do not use IPI to broadcast TLB flush, it may have the
+below race:
+
+ CPU A CPU B
+THP collapse fast GUP
+ gup_pmd_range() <-- see valid pmd
+ gup_pte_range() <-- work on pte
+pmdp_collapse_flush() <-- clear pmd and flush
+__collapse_huge_page_isolate()
+ check page pinned <-- before GUP bump refcount
+ pin the page
+ check PTE <-- no change
+__collapse_huge_page_copy()
+ copy data to huge page
+ ptep_clear()
+install huge pmd for the huge page
+ return the stale page
+discard the stale page
+
+The race can be fixed by checking whether PMD is changed or not after
+taking the page pin in fast GUP, just like what it does for PTE. If the
+PMD is changed it means there may be parallel THP collapse, so GUP should
+back off.
+
+Also update the stale comment about serializing against fast GUP in
+khugepaged.
+
+Link: https://lkml.kernel.org/r/20220907180144.555485-1-shy828301@gmail.com
+Fixes: 2667f50e8b81 ("mm: introduce a general RCU get_user_pages_fast()")
+Acked-by: David Hildenbrand <david@redhat.com>
+Acked-by: Peter Xu <peterx@redhat.com>
+Signed-off-by: Yang Shi <shy828301@gmail.com>
+Reviewed-by: John Hubbard <jhubbard@nvidia.com>
+Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
+Cc: Hugh Dickins <hughd@google.com>
+Cc: Jason Gunthorpe <jgg@nvidia.com>
+Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
+Cc: Michael Ellerman <mpe@ellerman.id.au>
+Cc: Nicholas Piggin <npiggin@gmail.com>
+Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
+Cc: <stable@vger.kernel.org>
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ mm/gup.c | 34 ++++++++++++++++++++++++++++------
+ mm/khugepaged.c | 10 ++++++----
+ 2 files changed, 34 insertions(+), 10 deletions(-)
+
+--- a/mm/gup.c
++++ b/mm/gup.c
+@@ -2278,8 +2278,28 @@ static void __maybe_unused undo_dev_page
+ }
+
+ #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
+-static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
+- unsigned int flags, struct page **pages, int *nr)
++/*
++ * Fast-gup relies on pte change detection to avoid concurrent pgtable
++ * operations.
++ *
++ * To pin the page, fast-gup needs to do below in order:
++ * (1) pin the page (by prefetching pte), then (2) check pte not changed.
++ *
++ * For the rest of pgtable operations where pgtable updates can be racy
++ * with fast-gup, we need to do (1) clear pte, then (2) check whether page
++ * is pinned.
++ *
++ * Above will work for all pte-level operations, including THP split.
++ *
++ * For THP collapse, it's a bit more complicated because fast-gup may be
++ * walking a pgtable page that is being freed (pte is still valid but pmd
++ * can be cleared already). To avoid race in such condition, we need to
++ * also check pmd here to make sure pmd doesn't change (corresponds to
++ * pmdp_collapse_flush() in the THP collapse code path).
++ */
++static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
++ unsigned long end, unsigned int flags,
++ struct page **pages, int *nr)
+ {
+ struct dev_pagemap *pgmap = NULL;
+ int nr_start = *nr, ret = 0;
+@@ -2325,7 +2345,8 @@ static int gup_pte_range(pmd_t pmd, unsi
+ goto pte_unmap;
+ }
+
+- if (unlikely(pte_val(pte) != pte_val(*ptep))) {
++ if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
++ unlikely(pte_val(pte) != pte_val(*ptep))) {
+ gup_put_folio(folio, 1, flags);
+ goto pte_unmap;
+ }
+@@ -2372,8 +2393,9 @@ pte_unmap:
+ * get_user_pages_fast_only implementation that can pin pages. Thus it's still
+ * useful to have gup_huge_pmd even if we can't operate on ptes.
+ */
+-static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
+- unsigned int flags, struct page **pages, int *nr)
++static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
++ unsigned long end, unsigned int flags,
++ struct page **pages, int *nr)
+ {
+ return 0;
+ }
+@@ -2697,7 +2719,7 @@ static int gup_pmd_range(pud_t *pudp, pu
+ if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
+ PMD_SHIFT, next, flags, pages, nr))
+ return 0;
+- } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
++ } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
+ return 0;
+ } while (pmdp++, addr = next, addr != end);
+
+--- a/mm/khugepaged.c
++++ b/mm/khugepaged.c
+@@ -1121,10 +1121,12 @@ static void collapse_huge_page(struct mm
+
+ pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
+ /*
+- * After this gup_fast can't run anymore. This also removes
+- * any huge TLB entry from the CPU so we won't allow
+- * huge and small TLB entries for the same virtual address
+- * to avoid the risk of CPU bugs in that area.
++ * This removes any huge TLB entry from the CPU so we won't allow
++ * huge and small TLB entries for the same virtual address to
++ * avoid the risk of CPU bugs in that area.
++ *
++ * Parallel fast GUP is fine since fast GUP will back off when
++ * it detects PMD is changed.
+ */
+ _pmd = pmdp_collapse_flush(vma, address, pmd);
+ spin_unlock(pmd_ptl);
--- /dev/null
+From 3d36424b3b5850bd92f3e89b953a430d7cfc88ef Mon Sep 17 00:00:00 2001
+From: Mel Gorman <mgorman@techsingularity.net>
+Date: Wed, 24 Aug 2022 12:14:50 +0100
+Subject: mm/page_alloc: fix race condition between build_all_zonelists and page allocation
+
+From: Mel Gorman <mgorman@techsingularity.net>
+
+commit 3d36424b3b5850bd92f3e89b953a430d7cfc88ef upstream.
+
+Patrick Daly reported the following problem;
+
+ NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK] - before offline operation
+ [0] - ZONE_MOVABLE
+ [1] - ZONE_NORMAL
+ [2] - NULL
+
+ For a GFP_KERNEL allocation, alloc_pages_slowpath() will save the
+ offset of ZONE_NORMAL in ac->preferred_zoneref. If a concurrent
+ memory_offline operation removes the last page from ZONE_MOVABLE,
+ build_all_zonelists() & build_zonerefs_node() will update
+ node_zonelists as shown below. Only populated zones are added.
+
+ NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK] - after offline operation
+ [0] - ZONE_NORMAL
+ [1] - NULL
+ [2] - NULL
+
+The race is simple -- page allocation could be in progress when a memory
+hot-remove operation triggers a zonelist rebuild that removes zones. The
+allocation request will still have a valid ac->preferred_zoneref that is
+now pointing to NULL and triggers an OOM kill.
+
+This problem probably always existed but may be slightly easier to trigger
+due to 6aa303defb74 ("mm, vmscan: only allocate and reclaim from zones
+with pages managed by the buddy allocator") which distinguishes between
+zones that are completely unpopulated versus zones that have valid pages
+not managed by the buddy allocator (e.g. reserved, memblock, ballooning
+etc). Memory hotplug had multiple stages with timing considerations
+around managed/present page updates, the zonelist rebuild and the zone
+span updates. As David Hildenbrand puts it
+
+ memory offlining adjusts managed+present pages of the zone
+ essentially in one go. If after the adjustments, the zone is no
+ longer populated (present==0), we rebuild the zone lists.
+
+ Once that's done, we try shrinking the zone (start+spanned
+ pages) -- which results in zone_start_pfn == 0 if there are no
+ more pages. That happens *after* rebuilding the zonelists via
+ remove_pfn_range_from_zone().
+
+The only requirement to fix the race is that a page allocation request
+identifies when a zonelist rebuild has happened since the allocation
+request started and no page has yet been allocated. Use a seqlock_t to
+track zonelist updates with a lockless read-side of the zonelist and
+protecting the rebuild and update of the counter with a spinlock.
+
+[akpm@linux-foundation.org: make zonelist_update_seq static]
+Link: https://lkml.kernel.org/r/20220824110900.vh674ltxmzb3proq@techsingularity.net
+Fixes: 6aa303defb74 ("mm, vmscan: only allocate and reclaim from zones with pages managed by the buddy allocator")
+Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
+Reported-by: Patrick Daly <quic_pdaly@quicinc.com>
+Acked-by: Michal Hocko <mhocko@suse.com>
+Reviewed-by: David Hildenbrand <david@redhat.com>
+Cc: <stable@vger.kernel.org> [4.9+]
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ mm/page_alloc.c | 53 +++++++++++++++++++++++++++++++++++++++++++----------
+ 1 file changed, 43 insertions(+), 10 deletions(-)
+
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@ -4623,6 +4623,30 @@ void fs_reclaim_release(gfp_t gfp_mask)
+ EXPORT_SYMBOL_GPL(fs_reclaim_release);
+ #endif
+
++/*
++ * Zonelists may change due to hotplug during allocation. Detect when zonelists
++ * have been rebuilt so allocation retries. Reader side does not lock and
++ * retries the allocation if zonelist changes. Writer side is protected by the
++ * embedded spin_lock.
++ */
++static DEFINE_SEQLOCK(zonelist_update_seq);
++
++static unsigned int zonelist_iter_begin(void)
++{
++ if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
++ return read_seqbegin(&zonelist_update_seq);
++
++ return 0;
++}
++
++static unsigned int check_retry_zonelist(unsigned int seq)
++{
++ if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
++ return read_seqretry(&zonelist_update_seq, seq);
++
++ return seq;
++}
++
+ /* Perform direct synchronous page reclaim */
+ static unsigned long
+ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
+@@ -4916,6 +4940,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, u
+ int compaction_retries;
+ int no_progress_loops;
+ unsigned int cpuset_mems_cookie;
++ unsigned int zonelist_iter_cookie;
+ int reserve_flags;
+
+ /*
+@@ -4926,11 +4951,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, u
+ (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
+ gfp_mask &= ~__GFP_ATOMIC;
+
+-retry_cpuset:
++restart:
+ compaction_retries = 0;
+ no_progress_loops = 0;
+ compact_priority = DEF_COMPACT_PRIORITY;
+ cpuset_mems_cookie = read_mems_allowed_begin();
++ zonelist_iter_cookie = zonelist_iter_begin();
+
+ /*
+ * The fast path uses conservative alloc_flags to succeed only until
+@@ -5102,9 +5128,13 @@ retry:
+ goto retry;
+
+
+- /* Deal with possible cpuset update races before we start OOM killing */
+- if (check_retry_cpuset(cpuset_mems_cookie, ac))
+- goto retry_cpuset;
++ /*
++ * Deal with possible cpuset update races or zonelist updates to avoid
++ * a unnecessary OOM kill.
++ */
++ if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
++ check_retry_zonelist(zonelist_iter_cookie))
++ goto restart;
+
+ /* Reclaim has failed us, start killing things */
+ page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
+@@ -5124,9 +5154,13 @@ retry:
+ }
+
+ nopage:
+- /* Deal with possible cpuset update races before we fail */
+- if (check_retry_cpuset(cpuset_mems_cookie, ac))
+- goto retry_cpuset;
++ /*
++ * Deal with possible cpuset update races or zonelist updates to avoid
++ * a unnecessary OOM kill.
++ */
++ if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
++ check_retry_zonelist(zonelist_iter_cookie))
++ goto restart;
+
+ /*
+ * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
+@@ -6421,9 +6455,8 @@ static void __build_all_zonelists(void *
+ int nid;
+ int __maybe_unused cpu;
+ pg_data_t *self = data;
+- static DEFINE_SPINLOCK(lock);
+
+- spin_lock(&lock);
++ write_seqlock(&zonelist_update_seq);
+
+ #ifdef CONFIG_NUMA
+ memset(node_load, 0, sizeof(node_load));
+@@ -6460,7 +6493,7 @@ static void __build_all_zonelists(void *
+ #endif
+ }
+
+- spin_unlock(&lock);
++ write_sequnlock(&zonelist_update_seq);
+ }
+
+ static noinline void __init
--- /dev/null
+From 80e2b584f3abfc31c3fe5573007f0d1d10810fde Mon Sep 17 00:00:00 2001
+From: Zi Yan <ziy@nvidia.com>
+Date: Tue, 13 Sep 2022 22:39:13 -0400
+Subject: mm/page_isolation: fix isolate_single_pageblock() isolation behavior
+
+From: Zi Yan <ziy@nvidia.com>
+
+commit 80e2b584f3abfc31c3fe5573007f0d1d10810fde upstream.
+
+set_migratetype_isolate() does not allow isolating MIGRATE_CMA pageblocks
+unless it is used for CMA allocation. isolate_single_pageblock() did not
+have the same behavior when it is used together with
+set_migratetype_isolate() in start_isolate_page_range(). This allows
+alloc_contig_range() with migratetype other than MIGRATE_CMA, like
+MIGRATE_MOVABLE (used by alloc_contig_pages()), to isolate first and last
+pageblock but fail the rest. The failure leads to changing migratetype of
+the first and last pageblock to MIGRATE_MOVABLE from MIGRATE_CMA,
+corrupting the CMA region. This can happen during gigantic page
+allocations.
+
+Like Doug said here:
+https://lore.kernel.org/linux-mm/a3363a52-883b-dcd1-b77f-f2bb378d6f2d@gmail.com/T/#u,
+for gigantic page allocations, the user would notice no difference,
+since the allocation on CMA region will fail as well as it did before.
+But it might hurt the performance of device drivers that use CMA, since
+CMA region size decreases.
+
+Fix it by passing migratetype into isolate_single_pageblock(), so that
+set_migratetype_isolate() used by isolate_single_pageblock() will prevent
+the isolation happening.
+
+Link: https://lkml.kernel.org/r/20220914023913.1855924-1-zi.yan@sent.com
+Fixes: b2c9e2fbba32 ("mm: make alloc_contig_range work at pageblock granularity")
+Signed-off-by: Zi Yan <ziy@nvidia.com>
+Reported-by: Doug Berger <opendmb@gmail.com>
+Cc: David Hildenbrand <david@redhat.com>
+Cc: Doug Berger <opendmb@gmail.com>
+Cc: Mike Kravetz <mike.kravetz@oracle.com>
+Cc: <stable@vger.kernel.org>
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ mm/page_isolation.c | 25 ++++++++++++++-----------
+ 1 file changed, 14 insertions(+), 11 deletions(-)
+
+--- a/mm/page_isolation.c
++++ b/mm/page_isolation.c
+@@ -288,6 +288,7 @@ __first_valid_page(unsigned long pfn, un
+ * @isolate_before: isolate the pageblock before the boundary_pfn
+ * @skip_isolation: the flag to skip the pageblock isolation in second
+ * isolate_single_pageblock()
++ * @migratetype: migrate type to set in error recovery.
+ *
+ * Free and in-use pages can be as big as MAX_ORDER-1 and contain more than one
+ * pageblock. When not all pageblocks within a page are isolated at the same
+@@ -302,9 +303,9 @@ __first_valid_page(unsigned long pfn, un
+ * the in-use page then splitting the free page.
+ */
+ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
+- gfp_t gfp_flags, bool isolate_before, bool skip_isolation)
++ gfp_t gfp_flags, bool isolate_before, bool skip_isolation,
++ int migratetype)
+ {
+- unsigned char saved_mt;
+ unsigned long start_pfn;
+ unsigned long isolate_pageblock;
+ unsigned long pfn;
+@@ -328,13 +329,13 @@ static int isolate_single_pageblock(unsi
+ start_pfn = max(ALIGN_DOWN(isolate_pageblock, MAX_ORDER_NR_PAGES),
+ zone->zone_start_pfn);
+
+- saved_mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock));
++ if (skip_isolation) {
++ int mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock));
+
+- if (skip_isolation)
+- VM_BUG_ON(!is_migrate_isolate(saved_mt));
+- else {
+- ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt, flags,
+- isolate_pageblock, isolate_pageblock + pageblock_nr_pages);
++ VM_BUG_ON(!is_migrate_isolate(mt));
++ } else {
++ ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype,
++ flags, isolate_pageblock, isolate_pageblock + pageblock_nr_pages);
+
+ if (ret)
+ return ret;
+@@ -475,7 +476,7 @@ static int isolate_single_pageblock(unsi
+ failed:
+ /* restore the original migratetype */
+ if (!skip_isolation)
+- unset_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt);
++ unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype);
+ return -EBUSY;
+ }
+
+@@ -537,7 +538,8 @@ int start_isolate_page_range(unsigned lo
+ bool skip_isolation = false;
+
+ /* isolate [isolate_start, isolate_start + pageblock_nr_pages) pageblock */
+- ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false, skip_isolation);
++ ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false,
++ skip_isolation, migratetype);
+ if (ret)
+ return ret;
+
+@@ -545,7 +547,8 @@ int start_isolate_page_range(unsigned lo
+ skip_isolation = true;
+
+ /* isolate [isolate_end - pageblock_nr_pages, isolate_end) pageblock */
+- ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true, skip_isolation);
++ ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true,
++ skip_isolation, migratetype);
+ if (ret) {
+ unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype);
+ return ret;
--- /dev/null
+From dac22531bbd4af2426c4e29e05594415ccfa365d Mon Sep 17 00:00:00 2001
+From: Maurizio Lombardi <mlombard@redhat.com>
+Date: Fri, 15 Jul 2022 14:50:13 +0200
+Subject: mm: prevent page_frag_alloc() from corrupting the memory
+
+From: Maurizio Lombardi <mlombard@redhat.com>
+
+commit dac22531bbd4af2426c4e29e05594415ccfa365d upstream.
+
+A number of drivers call page_frag_alloc() with a fragment's size >
+PAGE_SIZE.
+
+In low memory conditions, __page_frag_cache_refill() may fail the order
+3 cache allocation and fall back to order 0; In this case, the cache
+will be smaller than the fragment, causing memory corruptions.
+
+Prevent this from happening by checking if the newly allocated cache is
+large enough for the fragment; if not, the allocation will fail and
+page_frag_alloc() will return NULL.
+
+Link: https://lkml.kernel.org/r/20220715125013.247085-1-mlombard@redhat.com
+Fixes: b63ae8ca096d ("mm/net: Rename and move page fragment handling from net/ to mm/")
+Signed-off-by: Maurizio Lombardi <mlombard@redhat.com>
+Reviewed-by: Alexander Duyck <alexanderduyck@fb.com>
+Cc: Chen Lin <chen45464546@163.com>
+Cc: Jakub Kicinski <kuba@kernel.org>
+Cc: <stable@vger.kernel.org>
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ mm/page_alloc.c | 12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@ -5651,6 +5651,18 @@ refill:
+ /* reset page count bias and offset to start of new frag */
+ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
+ offset = size - fragsz;
++ if (unlikely(offset < 0)) {
++ /*
++ * The caller is trying to allocate a fragment
++ * with fragsz > PAGE_SIZE but the cache isn't big
++ * enough to satisfy the request, this may
++ * happen in low memory conditions.
++ * We don't release the cache page because
++ * it could make memory pressure worse
++ * so we simply return NULL here.
++ */
++ return NULL;
++ }
+ }
+
+ nc->pagecnt_bias--;
mptcp-fix-unreleased-socket-in-accept-queue.patch
mmc-moxart-fix-4-bit-bus-width-and-remove-8-bit-bus-width.patch
mmc-hsq-fix-data-stomping-during-mmc-recovery.patch
+mm-gup-fix-the-fast-gup-race-against-thp-collapse.patch
+mm-page_alloc-fix-race-condition-between-build_all_zonelists-and-page-allocation.patch
+mm-prevent-page_frag_alloc-from-corrupting-the-memory.patch
+mm-page_isolation-fix-isolate_single_pageblock-isolation-behavior.patch
projects / thirdparty / kernel / stable-queue.git / commitdiff
