--- /dev/null
+From 08f70b6345667b36b24a756767a275a43ebb1225 Mon Sep 17 00:00:00 2001
+From: Sasha Levin <sashal@kernel.org>
+Date: Tue, 6 Aug 2024 12:30:36 -0700
+Subject: mm: gup: stop abusing try_grab_folio
+
+From: Yang Shi <yang@os.amperecomputing.com>
+
+commit f442fa6141379a20b48ae3efabee827a3d260787 upstream
+
+A kernel warning was reported when pinning folio in CMA memory when
+launching SEV virtual machine. The splat looks like:
+
+[ 464.325306] WARNING: CPU: 13 PID: 6734 at mm/gup.c:1313 __get_user_pages+0x423/0x520
+[ 464.325464] CPU: 13 PID: 6734 Comm: qemu-kvm Kdump: loaded Not tainted 6.6.33+ #6
+[ 464.325477] RIP: 0010:__get_user_pages+0x423/0x520
+[ 464.325515] Call Trace:
+[ 464.325520] <TASK>
+[ 464.325523] ? __get_user_pages+0x423/0x520
+[ 464.325528] ? __warn+0x81/0x130
+[ 464.325536] ? __get_user_pages+0x423/0x520
+[ 464.325541] ? report_bug+0x171/0x1a0
+[ 464.325549] ? handle_bug+0x3c/0x70
+[ 464.325554] ? exc_invalid_op+0x17/0x70
+[ 464.325558] ? asm_exc_invalid_op+0x1a/0x20
+[ 464.325567] ? __get_user_pages+0x423/0x520
+[ 464.325575] __gup_longterm_locked+0x212/0x7a0
+[ 464.325583] internal_get_user_pages_fast+0xfb/0x190
+[ 464.325590] pin_user_pages_fast+0x47/0x60
+[ 464.325598] sev_pin_memory+0xca/0x170 [kvm_amd]
+[ 464.325616] sev_mem_enc_register_region+0x81/0x130 [kvm_amd]
+
+Per the analysis done by yangge, when starting the SEV virtual machine, it
+will call pin_user_pages_fast(..., FOLL_LONGTERM, ...) to pin the memory.
+But the page is in CMA area, so fast GUP will fail then fallback to the
+slow path due to the longterm pinnalbe check in try_grab_folio().
+
+The slow path will try to pin the pages then migrate them out of CMA area.
+But the slow path also uses try_grab_folio() to pin the page, it will
+also fail due to the same check then the above warning is triggered.
+
+In addition, the try_grab_folio() is supposed to be used in fast path and
+it elevates folio refcount by using add ref unless zero. We are guaranteed
+to have at least one stable reference in slow path, so the simple atomic add
+could be used. The performance difference should be trivial, but the
+misuse may be confusing and misleading.
+
+Redefined try_grab_folio() to try_grab_folio_fast(), and try_grab_page()
+to try_grab_folio(), and use them in the proper paths. This solves both
+the abuse and the kernel warning.
+
+The proper naming makes their usecase more clear and should prevent from
+abusing in the future.
+
+peterx said:
+
+: The user will see the pin fails, for gpu-slow it further triggers the WARN
+: right below that failure (as in the original report):
+:
+: folio = try_grab_folio(page, page_increm - 1,
+: foll_flags);
+: if (WARN_ON_ONCE(!folio)) { <------------------------ here
+: /*
+: * Release the 1st page ref if the
+: * folio is problematic, fail hard.
+: */
+: gup_put_folio(page_folio(page), 1,
+: foll_flags);
+: ret = -EFAULT;
+: goto out;
+: }
+
+[1] https://lore.kernel.org/linux-mm/1719478388-31917-1-git-send-email-yangge1116@126.com/
+
+[shy828301@gmail.com: fix implicit declaration of function try_grab_folio_fast]
+ Link: https://lkml.kernel.org/r/CAHbLzkowMSso-4Nufc9hcMehQsK9PNz3OSu-+eniU-2Mm-xjhA@mail.gmail.com
+Link: https://lkml.kernel.org/r/20240628191458.2605553-1-yang@os.amperecomputing.com
+Fixes: 57edfcfd3419 ("mm/gup: accelerate thp gup even for "pages != NULL"")
+Signed-off-by: Yang Shi <yang@os.amperecomputing.com>
+Reported-by: yangge <yangge1116@126.com>
+Cc: Christoph Hellwig <hch@infradead.org>
+Cc: David Hildenbrand <david@redhat.com>
+Cc: Peter Xu <peterx@redhat.com>
+Cc: <stable@vger.kernel.org> [6.6+]
+Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
+Signed-off-by: Sasha Levin <sashal@kernel.org>
+---
+ mm/gup.c | 251 ++++++++++++++++++++++++-----------------------
+ mm/huge_memory.c | 4 +-
+ mm/hugetlb.c | 2 +-
+ mm/internal.h | 4 +-
+ 4 files changed, 134 insertions(+), 127 deletions(-)
+
+diff --git a/mm/gup.c b/mm/gup.c
+index f50fe2219a13b..fdd75384160d8 100644
+--- a/mm/gup.c
++++ b/mm/gup.c
+@@ -97,95 +97,6 @@ static inline struct folio *try_get_folio(struct page *page, int refs)
+ return folio;
+ }
+
+-/**
+- * try_grab_folio() - Attempt to get or pin a folio.
+- * @page: pointer to page to be grabbed
+- * @refs: the value to (effectively) add to the folio's refcount
+- * @flags: gup flags: these are the FOLL_* flag values.
+- *
+- * "grab" names in this file mean, "look at flags to decide whether to use
+- * FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
+- *
+- * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
+- * same time. (That's true throughout the get_user_pages*() and
+- * pin_user_pages*() APIs.) Cases:
+- *
+- * FOLL_GET: folio's refcount will be incremented by @refs.
+- *
+- * FOLL_PIN on large folios: folio's refcount will be incremented by
+- * @refs, and its pincount will be incremented by @refs.
+- *
+- * FOLL_PIN on single-page folios: folio's refcount will be incremented by
+- * @refs * GUP_PIN_COUNTING_BIAS.
+- *
+- * Return: The folio containing @page (with refcount appropriately
+- * incremented) for success, or NULL upon failure. If neither FOLL_GET
+- * nor FOLL_PIN was set, that's considered failure, and furthermore,
+- * a likely bug in the caller, so a warning is also emitted.
+- */
+-struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags)
+-{
+- struct folio *folio;
+-
+- if (WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == 0))
+- return NULL;
+-
+- if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))
+- return NULL;
+-
+- if (flags & FOLL_GET)
+- return try_get_folio(page, refs);
+-
+- /* FOLL_PIN is set */
+-
+- /*
+- * Don't take a pin on the zero page - it's not going anywhere
+- * and it is used in a *lot* of places.
+- */
+- if (is_zero_page(page))
+- return page_folio(page);
+-
+- folio = try_get_folio(page, refs);
+- if (!folio)
+- return NULL;
+-
+- /*
+- * Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
+- * right zone, so fail and let the caller fall back to the slow
+- * path.
+- */
+- if (unlikely((flags & FOLL_LONGTERM) &&
+- !folio_is_longterm_pinnable(folio))) {
+- if (!put_devmap_managed_page_refs(&folio->page, refs))
+- folio_put_refs(folio, refs);
+- return NULL;
+- }
+-
+- /*
+- * When pinning a large folio, use an exact count to track it.
+- *
+- * However, be sure to *also* increment the normal folio
+- * refcount field at least once, so that the folio really
+- * is pinned. That's why the refcount from the earlier
+- * try_get_folio() is left intact.
+- */
+- if (folio_test_large(folio))
+- atomic_add(refs, &folio->_pincount);
+- else
+- folio_ref_add(folio,
+- refs * (GUP_PIN_COUNTING_BIAS - 1));
+- /*
+- * Adjust the pincount before re-checking the PTE for changes.
+- * This is essentially a smp_mb() and is paired with a memory
+- * barrier in page_try_share_anon_rmap().
+- */
+- smp_mb__after_atomic();
+-
+- node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
+-
+- return folio;
+-}
+-
+ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
+ {
+ if (flags & FOLL_PIN) {
+@@ -203,58 +114,59 @@ static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)
+ }
+
+ /**
+- * try_grab_page() - elevate a page's refcount by a flag-dependent amount
+- * @page: pointer to page to be grabbed
+- * @flags: gup flags: these are the FOLL_* flag values.
++ * try_grab_folio() - add a folio's refcount by a flag-dependent amount
++ * @folio: pointer to folio to be grabbed
++ * @refs: the value to (effectively) add to the folio's refcount
++ * @flags: gup flags: these are the FOLL_* flag values
+ *
+ * This might not do anything at all, depending on the flags argument.
+ *
+ * "grab" names in this file mean, "look at flags to decide whether to use
+- * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
++ * FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
+ *
+ * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
+- * time. Cases: please see the try_grab_folio() documentation, with
+- * "refs=1".
++ * time.
+ *
+ * Return: 0 for success, or if no action was required (if neither FOLL_PIN
+ * nor FOLL_GET was set, nothing is done). A negative error code for failure:
+ *
+- * -ENOMEM FOLL_GET or FOLL_PIN was set, but the page could not
++ * -ENOMEM FOLL_GET or FOLL_PIN was set, but the folio could not
+ * be grabbed.
++ *
++ * It is called when we have a stable reference for the folio, typically in
++ * GUP slow path.
+ */
+-int __must_check try_grab_page(struct page *page, unsigned int flags)
++int __must_check try_grab_folio(struct folio *folio, int refs,
++ unsigned int flags)
+ {
+- struct folio *folio = page_folio(page);
+-
+ if (WARN_ON_ONCE(folio_ref_count(folio) <= 0))
+ return -ENOMEM;
+
+- if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))
++ if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(&folio->page)))
+ return -EREMOTEIO;
+
+ if (flags & FOLL_GET)
+- folio_ref_inc(folio);
++ folio_ref_add(folio, refs);
+ else if (flags & FOLL_PIN) {
+ /*
+ * Don't take a pin on the zero page - it's not going anywhere
+ * and it is used in a *lot* of places.
+ */
+- if (is_zero_page(page))
++ if (is_zero_folio(folio))
+ return 0;
+
+ /*
+- * Similar to try_grab_folio(): be sure to *also*
+- * increment the normal page refcount field at least once,
++ * Increment the normal page refcount field at least once,
+ * so that the page really is pinned.
+ */
+ if (folio_test_large(folio)) {
+- folio_ref_add(folio, 1);
+- atomic_add(1, &folio->_pincount);
++ folio_ref_add(folio, refs);
++ atomic_add(refs, &folio->_pincount);
+ } else {
+- folio_ref_add(folio, GUP_PIN_COUNTING_BIAS);
++ folio_ref_add(folio, refs * GUP_PIN_COUNTING_BIAS);
+ }
+
+- node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, 1);
++ node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
+ }
+
+ return 0;
+@@ -647,8 +559,8 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
+ VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
+ !PageAnonExclusive(page), page);
+
+- /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
+- ret = try_grab_page(page, flags);
++ /* try_grab_folio() does nothing unless FOLL_GET or FOLL_PIN is set. */
++ ret = try_grab_folio(page_folio(page), 1, flags);
+ if (unlikely(ret)) {
+ page = ERR_PTR(ret);
+ goto out;
+@@ -899,7 +811,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
+ goto unmap;
+ *page = pte_page(entry);
+ }
+- ret = try_grab_page(*page, gup_flags);
++ ret = try_grab_folio(page_folio(*page), 1, gup_flags);
+ if (unlikely(ret))
+ goto unmap;
+ out:
+@@ -1302,20 +1214,19 @@ static long __get_user_pages(struct mm_struct *mm,
+ * pages.
+ */
+ if (page_increm > 1) {
+- struct folio *folio;
++ struct folio *folio = page_folio(page);
+
+ /*
+ * Since we already hold refcount on the
+ * large folio, this should never fail.
+ */
+- folio = try_grab_folio(page, page_increm - 1,
+- foll_flags);
+- if (WARN_ON_ONCE(!folio)) {
++ if (try_grab_folio(folio, page_increm - 1,
++ foll_flags)) {
+ /*
+ * Release the 1st page ref if the
+ * folio is problematic, fail hard.
+ */
+- gup_put_folio(page_folio(page), 1,
++ gup_put_folio(folio, 1,
+ foll_flags);
+ ret = -EFAULT;
+ goto out;
+@@ -2541,6 +2452,102 @@ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
+ }
+ }
+
++/**
++ * try_grab_folio_fast() - Attempt to get or pin a folio in fast path.
++ * @page: pointer to page to be grabbed
++ * @refs: the value to (effectively) add to the folio's refcount
++ * @flags: gup flags: these are the FOLL_* flag values.
++ *
++ * "grab" names in this file mean, "look at flags to decide whether to use
++ * FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount.
++ *
++ * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
++ * same time. (That's true throughout the get_user_pages*() and
++ * pin_user_pages*() APIs.) Cases:
++ *
++ * FOLL_GET: folio's refcount will be incremented by @refs.
++ *
++ * FOLL_PIN on large folios: folio's refcount will be incremented by
++ * @refs, and its pincount will be incremented by @refs.
++ *
++ * FOLL_PIN on single-page folios: folio's refcount will be incremented by
++ * @refs * GUP_PIN_COUNTING_BIAS.
++ *
++ * Return: The folio containing @page (with refcount appropriately
++ * incremented) for success, or NULL upon failure. If neither FOLL_GET
++ * nor FOLL_PIN was set, that's considered failure, and furthermore,
++ * a likely bug in the caller, so a warning is also emitted.
++ *
++ * It uses add ref unless zero to elevate the folio refcount and must be called
++ * in fast path only.
++ */
++static struct folio *try_grab_folio_fast(struct page *page, int refs,
++ unsigned int flags)
++{
++ struct folio *folio;
++
++ /* Raise warn if it is not called in fast GUP */
++ VM_WARN_ON_ONCE(!irqs_disabled());
++
++ if (WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == 0))
++ return NULL;
++
++ if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)))
++ return NULL;
++
++ if (flags & FOLL_GET)
++ return try_get_folio(page, refs);
++
++ /* FOLL_PIN is set */
++
++ /*
++ * Don't take a pin on the zero page - it's not going anywhere
++ * and it is used in a *lot* of places.
++ */
++ if (is_zero_page(page))
++ return page_folio(page);
++
++ folio = try_get_folio(page, refs);
++ if (!folio)
++ return NULL;
++
++ /*
++ * Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a
++ * right zone, so fail and let the caller fall back to the slow
++ * path.
++ */
++ if (unlikely((flags & FOLL_LONGTERM) &&
++ !folio_is_longterm_pinnable(folio))) {
++ if (!put_devmap_managed_page_refs(&folio->page, refs))
++ folio_put_refs(folio, refs);
++ return NULL;
++ }
++
++ /*
++ * When pinning a large folio, use an exact count to track it.
++ *
++ * However, be sure to *also* increment the normal folio
++ * refcount field at least once, so that the folio really
++ * is pinned. That's why the refcount from the earlier
++ * try_get_folio() is left intact.
++ */
++ if (folio_test_large(folio))
++ atomic_add(refs, &folio->_pincount);
++ else
++ folio_ref_add(folio,
++ refs * (GUP_PIN_COUNTING_BIAS - 1));
++ /*
++ * Adjust the pincount before re-checking the PTE for changes.
++ * This is essentially a smp_mb() and is paired with a memory
++ * barrier in folio_try_share_anon_rmap_*().
++ */
++ smp_mb__after_atomic();
++
++ node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);
++
++ return folio;
++}
++
+ #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
+ /*
+ * Fast-gup relies on pte change detection to avoid concurrent pgtable
+@@ -2605,7 +2612,7 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
+ VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+ page = pte_page(pte);
+
+- folio = try_grab_folio(page, 1, flags);
++ folio = try_grab_folio_fast(page, 1, flags);
+ if (!folio)
+ goto pte_unmap;
+
+@@ -2699,7 +2706,7 @@ static int __gup_device_huge(unsigned long pfn, unsigned long addr,
+
+ SetPageReferenced(page);
+ pages[*nr] = page;
+- if (unlikely(try_grab_page(page, flags))) {
++ if (unlikely(try_grab_folio(page_folio(page), 1, flags))) {
+ undo_dev_pagemap(nr, nr_start, flags, pages);
+ break;
+ }
+@@ -2808,7 +2815,7 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
+ page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT);
+ refs = record_subpages(page, addr, end, pages + *nr);
+
+- folio = try_grab_folio(page, refs, flags);
++ folio = try_grab_folio_fast(page, refs, flags);
+ if (!folio)
+ return 0;
+
+@@ -2879,7 +2886,7 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
+ page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT);
+ refs = record_subpages(page, addr, end, pages + *nr);
+
+- folio = try_grab_folio(page, refs, flags);
++ folio = try_grab_folio_fast(page, refs, flags);
+ if (!folio)
+ return 0;
+
+@@ -2923,7 +2930,7 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
+ page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT);
+ refs = record_subpages(page, addr, end, pages + *nr);
+
+- folio = try_grab_folio(page, refs, flags);
++ folio = try_grab_folio_fast(page, refs, flags);
+ if (!folio)
+ return 0;
+
+@@ -2963,7 +2970,7 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
+ page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT);
+ refs = record_subpages(page, addr, end, pages + *nr);
+
+- folio = try_grab_folio(page, refs, flags);
++ folio = try_grab_folio_fast(page, refs, flags);
+ if (!folio)
+ return 0;
+
+diff --git a/mm/huge_memory.c b/mm/huge_memory.c
+index 79fbd6ddec49f..fc773b0c4438c 100644
+--- a/mm/huge_memory.c
++++ b/mm/huge_memory.c
+@@ -1052,7 +1052,7 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
+ if (!*pgmap)
+ return ERR_PTR(-EFAULT);
+ page = pfn_to_page(pfn);
+- ret = try_grab_page(page, flags);
++ ret = try_grab_folio(page_folio(page), 1, flags);
+ if (ret)
+ page = ERR_PTR(ret);
+
+@@ -1471,7 +1471,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
+ VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
+ !PageAnonExclusive(page), page);
+
+- ret = try_grab_page(page, flags);
++ ret = try_grab_folio(page_folio(page), 1, flags);
+ if (ret)
+ return ERR_PTR(ret);
+
+diff --git a/mm/hugetlb.c b/mm/hugetlb.c
+index a480affd475bf..ab040f8d19876 100644
+--- a/mm/hugetlb.c
++++ b/mm/hugetlb.c
+@@ -6532,7 +6532,7 @@ struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma,
+ * try_grab_page() should always be able to get the page here,
+ * because we hold the ptl lock and have verified pte_present().
+ */
+- ret = try_grab_page(page, flags);
++ ret = try_grab_folio(page_folio(page), 1, flags);
+
+ if (WARN_ON_ONCE(ret)) {
+ page = ERR_PTR(ret);
+diff --git a/mm/internal.h b/mm/internal.h
+index abed947f784b7..ef8d787a510c5 100644
+--- a/mm/internal.h
++++ b/mm/internal.h
+@@ -938,8 +938,8 @@ int migrate_device_coherent_page(struct page *page);
+ /*
+ * mm/gup.c
+ */
+-struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags);
+-int __must_check try_grab_page(struct page *page, unsigned int flags);
++int __must_check try_grab_folio(struct folio *folio, int refs,
++ unsigned int flags);
+
+ /*
+ * mm/huge_memory.c
+--
+2.43.0
+
projects / thirdparty / kernel / stable-queue.git / commitdiff
