1 // SPDX-License-Identifier: GPL-2.0
3 * Device Memory Migration functionality.
5 * Originally written by Jérôme Glisse.
7 #include <linux/export.h>
8 #include <linux/memremap.h>
9 #include <linux/migrate.h>
11 #include <linux/mm_inline.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/oom.h>
14 #include <linux/pagewalk.h>
15 #include <linux/rmap.h>
16 #include <linux/swapops.h>
17 #include <asm/tlbflush.h>
20 static int migrate_vma_collect_skip(unsigned long start
,
24 struct migrate_vma
*migrate
= walk
->private;
27 for (addr
= start
; addr
< end
; addr
+= PAGE_SIZE
) {
28 migrate
->dst
[migrate
->npages
] = 0;
29 migrate
->src
[migrate
->npages
++] = 0;
35 static int migrate_vma_collect_hole(unsigned long start
,
37 __always_unused
int depth
,
40 struct migrate_vma
*migrate
= walk
->private;
43 /* Only allow populating anonymous memory. */
44 if (!vma_is_anonymous(walk
->vma
))
45 return migrate_vma_collect_skip(start
, end
, walk
);
47 for (addr
= start
; addr
< end
; addr
+= PAGE_SIZE
) {
48 migrate
->src
[migrate
->npages
] = MIGRATE_PFN_MIGRATE
;
49 migrate
->dst
[migrate
->npages
] = 0;
57 static int migrate_vma_collect_pmd(pmd_t
*pmdp
,
62 struct migrate_vma
*migrate
= walk
->private;
63 struct vm_area_struct
*vma
= walk
->vma
;
64 struct mm_struct
*mm
= vma
->vm_mm
;
65 unsigned long addr
= start
, unmapped
= 0;
71 return migrate_vma_collect_hole(start
, end
, -1, walk
);
73 if (pmd_trans_huge(*pmdp
)) {
76 ptl
= pmd_lock(mm
, pmdp
);
77 if (unlikely(!pmd_trans_huge(*pmdp
))) {
82 page
= pmd_page(*pmdp
);
83 if (is_huge_zero_page(page
)) {
85 split_huge_pmd(vma
, pmdp
, addr
);
86 if (pmd_trans_unstable(pmdp
))
87 return migrate_vma_collect_skip(start
, end
,
94 if (unlikely(!trylock_page(page
)))
95 return migrate_vma_collect_skip(start
, end
,
97 ret
= split_huge_page(page
);
101 return migrate_vma_collect_skip(start
, end
,
104 return migrate_vma_collect_hole(start
, end
, -1,
109 if (unlikely(pmd_bad(*pmdp
)))
110 return migrate_vma_collect_skip(start
, end
, walk
);
112 ptep
= pte_offset_map_lock(mm
, pmdp
, addr
, &ptl
);
113 arch_enter_lazy_mmu_mode();
115 for (; addr
< end
; addr
+= PAGE_SIZE
, ptep
++) {
116 unsigned long mpfn
= 0, pfn
;
124 if (vma_is_anonymous(vma
)) {
125 mpfn
= MIGRATE_PFN_MIGRATE
;
131 if (!pte_present(pte
)) {
133 * Only care about unaddressable device page special
134 * page table entry. Other special swap entries are not
135 * migratable, and we ignore regular swapped page.
137 entry
= pte_to_swp_entry(pte
);
138 if (!is_device_private_entry(entry
))
141 page
= pfn_swap_entry_to_page(entry
);
142 if (!(migrate
->flags
&
143 MIGRATE_VMA_SELECT_DEVICE_PRIVATE
) ||
144 page
->pgmap
->owner
!= migrate
->pgmap_owner
)
147 mpfn
= migrate_pfn(page_to_pfn(page
)) |
149 if (is_writable_device_private_entry(entry
))
150 mpfn
|= MIGRATE_PFN_WRITE
;
153 if (is_zero_pfn(pfn
) &&
154 (migrate
->flags
& MIGRATE_VMA_SELECT_SYSTEM
)) {
155 mpfn
= MIGRATE_PFN_MIGRATE
;
159 page
= vm_normal_page(migrate
->vma
, addr
, pte
);
160 if (page
&& !is_zone_device_page(page
) &&
161 !(migrate
->flags
& MIGRATE_VMA_SELECT_SYSTEM
))
163 else if (page
&& is_device_coherent_page(page
) &&
164 (!(migrate
->flags
& MIGRATE_VMA_SELECT_DEVICE_COHERENT
) ||
165 page
->pgmap
->owner
!= migrate
->pgmap_owner
))
167 mpfn
= migrate_pfn(pfn
) | MIGRATE_PFN_MIGRATE
;
168 mpfn
|= pte_write(pte
) ? MIGRATE_PFN_WRITE
: 0;
171 /* FIXME support THP */
172 if (!page
|| !page
->mapping
|| PageTransCompound(page
)) {
178 * By getting a reference on the page we pin it and that blocks
179 * any kind of migration. Side effect is that it "freezes" the
182 * We drop this reference after isolating the page from the lru
183 * for non device page (device page are not on the lru and thus
184 * can't be dropped from it).
189 * We rely on trylock_page() to avoid deadlock between
190 * concurrent migrations where each is waiting on the others
191 * page lock. If we can't immediately lock the page we fail this
192 * migration as it is only best effort anyway.
194 * If we can lock the page it's safe to set up a migration entry
195 * now. In the common case where the page is mapped once in a
196 * single process setting up the migration entry now is an
197 * optimisation to avoid walking the rmap later with
200 if (trylock_page(page
)) {
204 flush_cache_page(vma
, addr
, pte_pfn(*ptep
));
205 anon_exclusive
= PageAnon(page
) && PageAnonExclusive(page
);
206 if (anon_exclusive
) {
207 pte
= ptep_clear_flush(vma
, addr
, ptep
);
209 if (page_try_share_anon_rmap(page
)) {
210 set_pte_at(mm
, addr
, ptep
, pte
);
217 pte
= ptep_get_and_clear(mm
, addr
, ptep
);
222 /* Set the dirty flag on the folio now the pte is gone. */
224 folio_mark_dirty(page_folio(page
));
226 /* Setup special migration page table entry */
227 if (mpfn
& MIGRATE_PFN_WRITE
)
228 entry
= make_writable_migration_entry(
230 else if (anon_exclusive
)
231 entry
= make_readable_exclusive_migration_entry(
234 entry
= make_readable_migration_entry(
236 if (pte_present(pte
)) {
238 entry
= make_migration_entry_young(entry
);
240 entry
= make_migration_entry_dirty(entry
);
242 swp_pte
= swp_entry_to_pte(entry
);
243 if (pte_present(pte
)) {
244 if (pte_soft_dirty(pte
))
245 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
246 if (pte_uffd_wp(pte
))
247 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
249 if (pte_swp_soft_dirty(pte
))
250 swp_pte
= pte_swp_mksoft_dirty(swp_pte
);
251 if (pte_swp_uffd_wp(pte
))
252 swp_pte
= pte_swp_mkuffd_wp(swp_pte
);
254 set_pte_at(mm
, addr
, ptep
, swp_pte
);
257 * This is like regular unmap: we remove the rmap and
258 * drop page refcount. Page won't be freed, as we took
259 * a reference just above.
261 page_remove_rmap(page
, vma
, false);
264 if (pte_present(pte
))
272 migrate
->dst
[migrate
->npages
] = 0;
273 migrate
->src
[migrate
->npages
++] = mpfn
;
276 /* Only flush the TLB if we actually modified any entries */
278 flush_tlb_range(walk
->vma
, start
, end
);
280 arch_leave_lazy_mmu_mode();
281 pte_unmap_unlock(ptep
- 1, ptl
);
286 static const struct mm_walk_ops migrate_vma_walk_ops
= {
287 .pmd_entry
= migrate_vma_collect_pmd
,
288 .pte_hole
= migrate_vma_collect_hole
,
292 * migrate_vma_collect() - collect pages over a range of virtual addresses
293 * @migrate: migrate struct containing all migration information
295 * This will walk the CPU page table. For each virtual address backed by a
296 * valid page, it updates the src array and takes a reference on the page, in
297 * order to pin the page until we lock it and unmap it.
299 static void migrate_vma_collect(struct migrate_vma
*migrate
)
301 struct mmu_notifier_range range
;
304 * Note that the pgmap_owner is passed to the mmu notifier callback so
305 * that the registered device driver can skip invalidating device
306 * private page mappings that won't be migrated.
308 mmu_notifier_range_init_owner(&range
, MMU_NOTIFY_MIGRATE
, 0,
309 migrate
->vma
, migrate
->vma
->vm_mm
, migrate
->start
, migrate
->end
,
310 migrate
->pgmap_owner
);
311 mmu_notifier_invalidate_range_start(&range
);
313 walk_page_range(migrate
->vma
->vm_mm
, migrate
->start
, migrate
->end
,
314 &migrate_vma_walk_ops
, migrate
);
316 mmu_notifier_invalidate_range_end(&range
);
317 migrate
->end
= migrate
->start
+ (migrate
->npages
<< PAGE_SHIFT
);
321 * migrate_vma_check_page() - check if page is pinned or not
322 * @page: struct page to check
324 * Pinned pages cannot be migrated. This is the same test as in
325 * folio_migrate_mapping(), except that here we allow migration of a
328 static bool migrate_vma_check_page(struct page
*page
, struct page
*fault_page
)
331 * One extra ref because caller holds an extra reference, either from
332 * isolate_lru_page() for a regular page, or migrate_vma_collect() for
335 int extra
= 1 + (page
== fault_page
);
338 * FIXME support THP (transparent huge page), it is bit more complex to
339 * check them than regular pages, because they can be mapped with a pmd
340 * or with a pte (split pte mapping).
342 if (PageCompound(page
))
345 /* Page from ZONE_DEVICE have one extra reference */
346 if (is_zone_device_page(page
))
349 /* For file back page */
350 if (page_mapping(page
))
351 extra
+= 1 + page_has_private(page
);
353 if ((page_count(page
) - extra
) > page_mapcount(page
))
360 * Unmaps pages for migration. Returns number of unmapped pages.
362 static unsigned long migrate_device_unmap(unsigned long *src_pfns
,
363 unsigned long npages
,
364 struct page
*fault_page
)
366 unsigned long i
, restore
= 0;
367 bool allow_drain
= true;
368 unsigned long unmapped
= 0;
372 for (i
= 0; i
< npages
; i
++) {
373 struct page
*page
= migrate_pfn_to_page(src_pfns
[i
]);
379 /* ZONE_DEVICE pages are not on LRU */
380 if (!is_zone_device_page(page
)) {
381 if (!PageLRU(page
) && allow_drain
) {
382 /* Drain CPU's pagevec */
387 if (isolate_lru_page(page
)) {
388 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
393 /* Drop the reference we took in collect */
397 folio
= page_folio(page
);
398 if (folio_mapped(folio
))
399 try_to_migrate(folio
, 0);
401 if (page_mapped(page
) ||
402 !migrate_vma_check_page(page
, fault_page
)) {
403 if (!is_zone_device_page(page
)) {
405 putback_lru_page(page
);
408 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
416 for (i
= 0; i
< npages
&& restore
; i
++) {
417 struct page
*page
= migrate_pfn_to_page(src_pfns
[i
]);
420 if (!page
|| (src_pfns
[i
] & MIGRATE_PFN_MIGRATE
))
423 folio
= page_folio(page
);
424 remove_migration_ptes(folio
, folio
, false);
436 * migrate_vma_unmap() - replace page mapping with special migration pte entry
437 * @migrate: migrate struct containing all migration information
439 * Isolate pages from the LRU and replace mappings (CPU page table pte) with a
440 * special migration pte entry and check if it has been pinned. Pinned pages are
441 * restored because we cannot migrate them.
443 * This is the last step before we call the device driver callback to allocate
444 * destination memory and copy contents of original page over to new page.
446 static void migrate_vma_unmap(struct migrate_vma
*migrate
)
448 migrate
->cpages
= migrate_device_unmap(migrate
->src
, migrate
->npages
,
449 migrate
->fault_page
);
453 * migrate_vma_setup() - prepare to migrate a range of memory
454 * @args: contains the vma, start, and pfns arrays for the migration
456 * Returns: negative errno on failures, 0 when 0 or more pages were migrated
459 * Prepare to migrate a range of memory virtual address range by collecting all
460 * the pages backing each virtual address in the range, saving them inside the
461 * src array. Then lock those pages and unmap them. Once the pages are locked
462 * and unmapped, check whether each page is pinned or not. Pages that aren't
463 * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
464 * corresponding src array entry. Then restores any pages that are pinned, by
465 * remapping and unlocking those pages.
467 * The caller should then allocate destination memory and copy source memory to
468 * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
469 * flag set). Once these are allocated and copied, the caller must update each
470 * corresponding entry in the dst array with the pfn value of the destination
471 * page and with MIGRATE_PFN_VALID. Destination pages must be locked via
474 * Note that the caller does not have to migrate all the pages that are marked
475 * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
476 * device memory to system memory. If the caller cannot migrate a device page
477 * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
478 * consequences for the userspace process, so it must be avoided if at all
481 * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
482 * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
483 * allowing the caller to allocate device memory for those unbacked virtual
484 * addresses. For this the caller simply has to allocate device memory and
485 * properly set the destination entry like for regular migration. Note that
486 * this can still fail, and thus inside the device driver you must check if the
487 * migration was successful for those entries after calling migrate_vma_pages(),
488 * just like for regular migration.
490 * After that, the callers must call migrate_vma_pages() to go over each entry
491 * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
492 * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
493 * then migrate_vma_pages() to migrate struct page information from the source
494 * struct page to the destination struct page. If it fails to migrate the
495 * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
498 * At this point all successfully migrated pages have an entry in the src
499 * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
500 * array entry with MIGRATE_PFN_VALID flag set.
502 * Once migrate_vma_pages() returns the caller may inspect which pages were
503 * successfully migrated, and which were not. Successfully migrated pages will
504 * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
506 * It is safe to update device page table after migrate_vma_pages() because
507 * both destination and source page are still locked, and the mmap_lock is held
508 * in read mode (hence no one can unmap the range being migrated).
510 * Once the caller is done cleaning up things and updating its page table (if it
511 * chose to do so, this is not an obligation) it finally calls
512 * migrate_vma_finalize() to update the CPU page table to point to new pages
513 * for successfully migrated pages or otherwise restore the CPU page table to
514 * point to the original source pages.
516 int migrate_vma_setup(struct migrate_vma
*args
)
518 long nr_pages
= (args
->end
- args
->start
) >> PAGE_SHIFT
;
520 args
->start
&= PAGE_MASK
;
521 args
->end
&= PAGE_MASK
;
522 if (!args
->vma
|| is_vm_hugetlb_page(args
->vma
) ||
523 (args
->vma
->vm_flags
& VM_SPECIAL
) || vma_is_dax(args
->vma
))
527 if (args
->start
< args
->vma
->vm_start
||
528 args
->start
>= args
->vma
->vm_end
)
530 if (args
->end
<= args
->vma
->vm_start
|| args
->end
> args
->vma
->vm_end
)
532 if (!args
->src
|| !args
->dst
)
534 if (args
->fault_page
&& !is_device_private_page(args
->fault_page
))
537 memset(args
->src
, 0, sizeof(*args
->src
) * nr_pages
);
541 migrate_vma_collect(args
);
544 migrate_vma_unmap(args
);
547 * At this point pages are locked and unmapped, and thus they have
548 * stable content and can safely be copied to destination memory that
549 * is allocated by the drivers.
554 EXPORT_SYMBOL(migrate_vma_setup
);
557 * This code closely matches the code in:
558 * __handle_mm_fault()
560 * do_anonymous_page()
561 * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
562 * private or coherent page.
564 static void migrate_vma_insert_page(struct migrate_vma
*migrate
,
569 struct vm_area_struct
*vma
= migrate
->vma
;
570 struct mm_struct
*mm
= vma
->vm_mm
;
580 /* Only allow populating anonymous memory */
581 if (!vma_is_anonymous(vma
))
584 pgdp
= pgd_offset(mm
, addr
);
585 p4dp
= p4d_alloc(mm
, pgdp
, addr
);
588 pudp
= pud_alloc(mm
, p4dp
, addr
);
591 pmdp
= pmd_alloc(mm
, pudp
, addr
);
595 if (pmd_trans_huge(*pmdp
) || pmd_devmap(*pmdp
))
599 * Use pte_alloc() instead of pte_alloc_map(). We can't run
600 * pte_offset_map() on pmds where a huge pmd might be created
601 * from a different thread.
603 * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
604 * parallel threads are excluded by other means.
606 * Here we only have mmap_read_lock(mm).
608 if (pte_alloc(mm
, pmdp
))
611 /* See the comment in pte_alloc_one_map() */
612 if (unlikely(pmd_trans_unstable(pmdp
)))
615 if (unlikely(anon_vma_prepare(vma
)))
617 if (mem_cgroup_charge(page_folio(page
), vma
->vm_mm
, GFP_KERNEL
))
621 * The memory barrier inside __SetPageUptodate makes sure that
622 * preceding stores to the page contents become visible before
623 * the set_pte_at() write.
625 __SetPageUptodate(page
);
627 if (is_device_private_page(page
)) {
628 swp_entry_t swp_entry
;
630 if (vma
->vm_flags
& VM_WRITE
)
631 swp_entry
= make_writable_device_private_entry(
634 swp_entry
= make_readable_device_private_entry(
636 entry
= swp_entry_to_pte(swp_entry
);
638 if (is_zone_device_page(page
) &&
639 !is_device_coherent_page(page
)) {
640 pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
643 entry
= mk_pte(page
, vma
->vm_page_prot
);
644 if (vma
->vm_flags
& VM_WRITE
)
645 entry
= pte_mkwrite(pte_mkdirty(entry
));
648 ptep
= pte_offset_map_lock(mm
, pmdp
, addr
, &ptl
);
650 if (check_stable_address_space(mm
))
653 if (pte_present(*ptep
)) {
654 unsigned long pfn
= pte_pfn(*ptep
);
656 if (!is_zero_pfn(pfn
))
659 } else if (!pte_none(*ptep
))
663 * Check for userfaultfd but do not deliver the fault. Instead,
666 if (userfaultfd_missing(vma
))
669 inc_mm_counter(mm
, MM_ANONPAGES
);
670 page_add_new_anon_rmap(page
, vma
, addr
);
671 if (!is_zone_device_page(page
))
672 lru_cache_add_inactive_or_unevictable(page
, vma
);
676 flush_cache_page(vma
, addr
, pte_pfn(*ptep
));
677 ptep_clear_flush_notify(vma
, addr
, ptep
);
678 set_pte_at_notify(mm
, addr
, ptep
, entry
);
679 update_mmu_cache(vma
, addr
, ptep
);
681 /* No need to invalidate - it was non-present before */
682 set_pte_at(mm
, addr
, ptep
, entry
);
683 update_mmu_cache(vma
, addr
, ptep
);
686 pte_unmap_unlock(ptep
, ptl
);
687 *src
= MIGRATE_PFN_MIGRATE
;
691 pte_unmap_unlock(ptep
, ptl
);
693 *src
&= ~MIGRATE_PFN_MIGRATE
;
696 static void __migrate_device_pages(unsigned long *src_pfns
,
697 unsigned long *dst_pfns
, unsigned long npages
,
698 struct migrate_vma
*migrate
)
700 struct mmu_notifier_range range
;
702 bool notified
= false;
704 for (i
= 0; i
< npages
; i
++) {
705 struct page
*newpage
= migrate_pfn_to_page(dst_pfns
[i
]);
706 struct page
*page
= migrate_pfn_to_page(src_pfns
[i
]);
707 struct address_space
*mapping
;
711 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
718 if (!(src_pfns
[i
] & MIGRATE_PFN_MIGRATE
))
722 * The only time there is no vma is when called from
723 * migrate_device_coherent_page(). However this isn't
724 * called if the page could not be unmapped.
727 addr
= migrate
->start
+ i
*PAGE_SIZE
;
731 mmu_notifier_range_init_owner(&range
,
732 MMU_NOTIFY_MIGRATE
, 0, migrate
->vma
,
733 migrate
->vma
->vm_mm
, addr
, migrate
->end
,
734 migrate
->pgmap_owner
);
735 mmu_notifier_invalidate_range_start(&range
);
737 migrate_vma_insert_page(migrate
, addr
, newpage
,
742 mapping
= page_mapping(page
);
744 if (is_device_private_page(newpage
) ||
745 is_device_coherent_page(newpage
)) {
747 * For now only support anonymous memory migrating to
748 * device private or coherent memory.
751 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
754 } else if (is_zone_device_page(newpage
)) {
756 * Other types of ZONE_DEVICE page are not supported.
758 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
762 if (migrate
&& migrate
->fault_page
== page
)
763 r
= migrate_folio_extra(mapping
, page_folio(newpage
),
765 MIGRATE_SYNC_NO_COPY
, 1);
767 r
= migrate_folio(mapping
, page_folio(newpage
),
768 page_folio(page
), MIGRATE_SYNC_NO_COPY
);
769 if (r
!= MIGRATEPAGE_SUCCESS
)
770 src_pfns
[i
] &= ~MIGRATE_PFN_MIGRATE
;
774 * No need to double call mmu_notifier->invalidate_range() callback as
775 * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
776 * did already call it.
779 mmu_notifier_invalidate_range_only_end(&range
);
783 * migrate_device_pages() - migrate meta-data from src page to dst page
784 * @src_pfns: src_pfns returned from migrate_device_range()
785 * @dst_pfns: array of pfns allocated by the driver to migrate memory to
786 * @npages: number of pages in the range
788 * Equivalent to migrate_vma_pages(). This is called to migrate struct page
789 * meta-data from source struct page to destination.
791 void migrate_device_pages(unsigned long *src_pfns
, unsigned long *dst_pfns
,
792 unsigned long npages
)
794 __migrate_device_pages(src_pfns
, dst_pfns
, npages
, NULL
);
796 EXPORT_SYMBOL(migrate_device_pages
);
799 * migrate_vma_pages() - migrate meta-data from src page to dst page
800 * @migrate: migrate struct containing all migration information
802 * This migrates struct page meta-data from source struct page to destination
803 * struct page. This effectively finishes the migration from source page to the
806 void migrate_vma_pages(struct migrate_vma
*migrate
)
808 __migrate_device_pages(migrate
->src
, migrate
->dst
, migrate
->npages
, migrate
);
810 EXPORT_SYMBOL(migrate_vma_pages
);
813 * migrate_device_finalize() - complete page migration
814 * @src_pfns: src_pfns returned from migrate_device_range()
815 * @dst_pfns: array of pfns allocated by the driver to migrate memory to
816 * @npages: number of pages in the range
818 * Completes migration of the page by removing special migration entries.
819 * Drivers must ensure copying of page data is complete and visible to the CPU
820 * before calling this.
822 void migrate_device_finalize(unsigned long *src_pfns
,
823 unsigned long *dst_pfns
, unsigned long npages
)
827 for (i
= 0; i
< npages
; i
++) {
828 struct folio
*dst
, *src
;
829 struct page
*newpage
= migrate_pfn_to_page(dst_pfns
[i
]);
830 struct page
*page
= migrate_pfn_to_page(src_pfns
[i
]);
834 unlock_page(newpage
);
840 if (!(src_pfns
[i
] & MIGRATE_PFN_MIGRATE
) || !newpage
) {
842 unlock_page(newpage
);
848 src
= page_folio(page
);
849 dst
= page_folio(newpage
);
850 remove_migration_ptes(src
, dst
, false);
853 if (is_zone_device_page(page
))
856 putback_lru_page(page
);
858 if (newpage
!= page
) {
859 unlock_page(newpage
);
860 if (is_zone_device_page(newpage
))
863 putback_lru_page(newpage
);
867 EXPORT_SYMBOL(migrate_device_finalize
);
870 * migrate_vma_finalize() - restore CPU page table entry
871 * @migrate: migrate struct containing all migration information
873 * This replaces the special migration pte entry with either a mapping to the
874 * new page if migration was successful for that page, or to the original page
877 * This also unlocks the pages and puts them back on the lru, or drops the extra
878 * refcount, for device pages.
880 void migrate_vma_finalize(struct migrate_vma
*migrate
)
882 migrate_device_finalize(migrate
->src
, migrate
->dst
, migrate
->npages
);
884 EXPORT_SYMBOL(migrate_vma_finalize
);
887 * migrate_device_range() - migrate device private pfns to normal memory.
888 * @src_pfns: array large enough to hold migrating source device private pfns.
889 * @start: starting pfn in the range to migrate.
890 * @npages: number of pages to migrate.
892 * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that
893 * instead of looking up pages based on virtual address mappings a range of
894 * device pfns that should be migrated to system memory is used instead.
896 * This is useful when a driver needs to free device memory but doesn't know the
897 * virtual mappings of every page that may be in device memory. For example this
898 * is often the case when a driver is being unloaded or unbound from a device.
900 * Like migrate_vma_setup() this function will take a reference and lock any
901 * migrating pages that aren't free before unmapping them. Drivers may then
902 * allocate destination pages and start copying data from the device to CPU
903 * memory before calling migrate_device_pages().
905 int migrate_device_range(unsigned long *src_pfns
, unsigned long start
,
906 unsigned long npages
)
908 unsigned long i
, pfn
;
910 for (pfn
= start
, i
= 0; i
< npages
; pfn
++, i
++) {
911 struct page
*page
= pfn_to_page(pfn
);
913 if (!get_page_unless_zero(page
)) {
918 if (!trylock_page(page
)) {
924 src_pfns
[i
] = migrate_pfn(pfn
) | MIGRATE_PFN_MIGRATE
;
927 migrate_device_unmap(src_pfns
, npages
, NULL
);
931 EXPORT_SYMBOL(migrate_device_range
);
934 * Migrate a device coherent page back to normal memory. The caller should have
935 * a reference on page which will be copied to the new page if migration is
936 * successful or dropped on failure.
938 int migrate_device_coherent_page(struct page
*page
)
940 unsigned long src_pfn
, dst_pfn
= 0;
943 WARN_ON_ONCE(PageCompound(page
));
946 src_pfn
= migrate_pfn(page_to_pfn(page
)) | MIGRATE_PFN_MIGRATE
;
949 * We don't have a VMA and don't need to walk the page tables to find
950 * the source page. So call migrate_vma_unmap() directly to unmap the
951 * page as migrate_vma_setup() will fail if args.vma == NULL.
953 migrate_device_unmap(&src_pfn
, 1, NULL
);
954 if (!(src_pfn
& MIGRATE_PFN_MIGRATE
))
957 dpage
= alloc_page(GFP_USER
| __GFP_NOWARN
);
960 dst_pfn
= migrate_pfn(page_to_pfn(dpage
));
963 migrate_device_pages(&src_pfn
, &dst_pfn
, 1);
964 if (src_pfn
& MIGRATE_PFN_MIGRATE
)
965 copy_highpage(dpage
, page
);
966 migrate_device_finalize(&src_pfn
, &dst_pfn
, 1);
968 if (src_pfn
& MIGRATE_PFN_MIGRATE
)