+++ /dev/null
-From 94efde1d15399f5c88e576923db9bcd422d217f2 Mon Sep 17 00:00:00 2001
-From: John Hubbard <jhubbard@nvidia.com>
-Date: Mon, 4 Nov 2024 19:29:44 -0800
-Subject: mm/gup: avoid an unnecessary allocation call for FOLL_LONGTERM cases
-
-From: John Hubbard <jhubbard@nvidia.com>
-
-commit 94efde1d15399f5c88e576923db9bcd422d217f2 upstream.
-
-commit 53ba78de064b ("mm/gup: introduce
-check_and_migrate_movable_folios()") created a new constraint on the
-pin_user_pages*() API family: a potentially large internal allocation must
-now occur, for FOLL_LONGTERM cases.
-
-A user-visible consequence has now appeared: user space can no longer pin
-more than 2GB of memory anymore on x86_64. That's because, on a 4KB
-PAGE_SIZE system, when user space tries to (indirectly, via a device
-driver that calls pin_user_pages()) pin 2GB, this requires an allocation
-of a folio pointers array of MAX_PAGE_ORDER size, which is the limit for
-kmalloc().
-
-In addition to the directly visible effect described above, there is also
-the problem of adding an unnecessary allocation. The **pages array
-argument has already been allocated, and there is no need for a redundant
-**folios array allocation in this case.
-
-Fix this by avoiding the new allocation entirely. This is done by
-referring to either the original page[i] within **pages, or to the
-associated folio. Thanks to David Hildenbrand for suggesting this
-approach and for providing the initial implementation (which I've tested
-and adjusted slightly) as well.
-
-[jhubbard@nvidia.com: whitespace tweak, per David]
- Link: https://lkml.kernel.org/r/131cf9c8-ebc0-4cbb-b722-22fa8527bf3c@nvidia.com
-[jhubbard@nvidia.com: bypass pofs_get_folio(), per Oscar]
- Link: https://lkml.kernel.org/r/c1587c7f-9155-45be-bd62-1e36c0dd6923@nvidia.com
-Link: https://lkml.kernel.org/r/20241105032944.141488-2-jhubbard@nvidia.com
-Fixes: 53ba78de064b ("mm/gup: introduce check_and_migrate_movable_folios()")
-Signed-off-by: John Hubbard <jhubbard@nvidia.com>
-Suggested-by: David Hildenbrand <david@redhat.com>
-Acked-by: David Hildenbrand <david@redhat.com>
-Reviewed-by: Oscar Salvador <osalvador@suse.de>
-Cc: Vivek Kasireddy <vivek.kasireddy@intel.com>
-Cc: Dave Airlie <airlied@redhat.com>
-Cc: Gerd Hoffmann <kraxel@redhat.com>
-Cc: Matthew Wilcox <willy@infradead.org>
-Cc: Christoph Hellwig <hch@infradead.org>
-Cc: Jason Gunthorpe <jgg@nvidia.com>
-Cc: Peter Xu <peterx@redhat.com>
-Cc: Arnd Bergmann <arnd@arndb.de>
-Cc: Daniel Vetter <daniel.vetter@ffwll.ch>
-Cc: Dongwon Kim <dongwon.kim@intel.com>
-Cc: Hugh Dickins <hughd@google.com>
-Cc: Junxiao Chang <junxiao.chang@intel.com>
-Cc: <stable@vger.kernel.org>
-Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
-Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
----
- mm/gup.c | 114 ++++++++++++++++++++++++++++++++++++++++++---------------------
- 1 file changed, 77 insertions(+), 37 deletions(-)
-
---- a/mm/gup.c
-+++ b/mm/gup.c
-@@ -2282,20 +2282,57 @@ struct page *get_dump_page(unsigned long
- #endif /* CONFIG_ELF_CORE */
-
- #ifdef CONFIG_MIGRATION
-+
-+/*
-+ * An array of either pages or folios ("pofs"). Although it may seem tempting to
-+ * avoid this complication, by simply interpreting a list of folios as a list of
-+ * pages, that approach won't work in the longer term, because eventually the
-+ * layouts of struct page and struct folio will become completely different.
-+ * Furthermore, this pof approach avoids excessive page_folio() calls.
-+ */
-+struct pages_or_folios {
-+ union {
-+ struct page **pages;
-+ struct folio **folios;
-+ void **entries;
-+ };
-+ bool has_folios;
-+ long nr_entries;
-+};
-+
-+static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i)
-+{
-+ if (pofs->has_folios)
-+ return pofs->folios[i];
-+ return page_folio(pofs->pages[i]);
-+}
-+
-+static void pofs_clear_entry(struct pages_or_folios *pofs, long i)
-+{
-+ pofs->entries[i] = NULL;
-+}
-+
-+static void pofs_unpin(struct pages_or_folios *pofs)
-+{
-+ if (pofs->has_folios)
-+ unpin_folios(pofs->folios, pofs->nr_entries);
-+ else
-+ unpin_user_pages(pofs->pages, pofs->nr_entries);
-+}
-+
- /*
- * Returns the number of collected folios. Return value is always >= 0.
- */
- static unsigned long collect_longterm_unpinnable_folios(
-- struct list_head *movable_folio_list,
-- unsigned long nr_folios,
-- struct folio **folios)
-+ struct list_head *movable_folio_list,
-+ struct pages_or_folios *pofs)
- {
- unsigned long i, collected = 0;
- struct folio *prev_folio = NULL;
- bool drain_allow = true;
-
-- for (i = 0; i < nr_folios; i++) {
-- struct folio *folio = folios[i];
-+ for (i = 0; i < pofs->nr_entries; i++) {
-+ struct folio *folio = pofs_get_folio(pofs, i);
-
- if (folio == prev_folio)
- continue;
-@@ -2336,16 +2373,15 @@ static unsigned long collect_longterm_un
- * Returns -EAGAIN if all folios were successfully migrated or -errno for
- * failure (or partial success).
- */
--static int migrate_longterm_unpinnable_folios(
-- struct list_head *movable_folio_list,
-- unsigned long nr_folios,
-- struct folio **folios)
-+static int
-+migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list,
-+ struct pages_or_folios *pofs)
- {
- int ret;
- unsigned long i;
-
-- for (i = 0; i < nr_folios; i++) {
-- struct folio *folio = folios[i];
-+ for (i = 0; i < pofs->nr_entries; i++) {
-+ struct folio *folio = pofs_get_folio(pofs, i);
-
- if (folio_is_device_coherent(folio)) {
- /*
-@@ -2353,7 +2389,7 @@ static int migrate_longterm_unpinnable_f
- * convert the pin on the source folio to a normal
- * reference.
- */
-- folios[i] = NULL;
-+ pofs_clear_entry(pofs, i);
- folio_get(folio);
- gup_put_folio(folio, 1, FOLL_PIN);
-
-@@ -2372,8 +2408,8 @@ static int migrate_longterm_unpinnable_f
- * calling folio_isolate_lru() which takes a reference so the
- * folio won't be freed if it's migrating.
- */
-- unpin_folio(folios[i]);
-- folios[i] = NULL;
-+ unpin_folio(folio);
-+ pofs_clear_entry(pofs, i);
- }
-
- if (!list_empty(movable_folio_list)) {
-@@ -2396,12 +2432,26 @@ static int migrate_longterm_unpinnable_f
- return -EAGAIN;
-
- err:
-- unpin_folios(folios, nr_folios);
-+ pofs_unpin(pofs);
- putback_movable_pages(movable_folio_list);
-
- return ret;
- }
-
-+static long
-+check_and_migrate_movable_pages_or_folios(struct pages_or_folios *pofs)
-+{
-+ LIST_HEAD(movable_folio_list);
-+ unsigned long collected;
-+
-+ collected = collect_longterm_unpinnable_folios(&movable_folio_list,
-+ pofs);
-+ if (!collected)
-+ return 0;
-+
-+ return migrate_longterm_unpinnable_folios(&movable_folio_list, pofs);
-+}
-+
- /*
- * Check whether all folios are *allowed* to be pinned indefinitely (longterm).
- * Rather confusingly, all folios in the range are required to be pinned via
-@@ -2421,16 +2471,13 @@ err:
- static long check_and_migrate_movable_folios(unsigned long nr_folios,
- struct folio **folios)
- {
-- unsigned long collected;
-- LIST_HEAD(movable_folio_list);
-+ struct pages_or_folios pofs = {
-+ .folios = folios,
-+ .has_folios = true,
-+ .nr_entries = nr_folios,
-+ };
-
-- collected = collect_longterm_unpinnable_folios(&movable_folio_list,
-- nr_folios, folios);
-- if (!collected)
-- return 0;
--
-- return migrate_longterm_unpinnable_folios(&movable_folio_list,
-- nr_folios, folios);
-+ return check_and_migrate_movable_pages_or_folios(&pofs);
- }
-
- /*
-@@ -2442,20 +2489,13 @@ static long check_and_migrate_movable_fo
- static long check_and_migrate_movable_pages(unsigned long nr_pages,
- struct page **pages)
- {
-- struct folio **folios;
-- long i, ret;
-+ struct pages_or_folios pofs = {
-+ .pages = pages,
-+ .has_folios = false,
-+ .nr_entries = nr_pages,
-+ };
-
-- folios = kmalloc_array(nr_pages, sizeof(*folios), GFP_KERNEL);
-- if (!folios)
-- return -ENOMEM;
--
-- for (i = 0; i < nr_pages; i++)
-- folios[i] = page_folio(pages[i]);
--
-- ret = check_and_migrate_movable_folios(nr_pages, folios);
--
-- kfree(folios);
-- return ret;
-+ return check_and_migrate_movable_pages_or_folios(&pofs);
- }
- #else
- static long check_and_migrate_movable_pages(unsigned long nr_pages,
projects / thirdparty / kernel / stable-queue.git / commitdiff
