mm/device-public-memory: device memory cache coherent with CPU

[thirdparty/kernel/stable.git] / mm / migrate.c

diff --git a/mm/migrate.c b/mm/migrate.c
index e581253ef3301164b8e1d2ed2e9a14c28022b853..618aeb5e9cde070858a45176d3b9f38c826b4aec 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,6 +36,7 @@
 #include <linux/hugetlb.h>
 #include <linux/hugetlb_cgroup.h>
 #include <linux/gfp.h>
+#include <linux/pfn_t.h>
 #include <linux/memremap.h>
 #include <linux/userfaultfd_k.h>
 #include <linux/balloon_compaction.h>
@@ -239,10 +240,14 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
                if (is_write_migration_entry(entry))
                        pte = maybe_mkwrite(pte, vma);
 
-               if (unlikely(is_zone_device_page(new)) &&
-                   is_device_private_page(new)) {
-                       entry = make_device_private_entry(new, pte_write(pte));
-                       pte = swp_entry_to_pte(entry);
+               if (unlikely(is_zone_device_page(new))) {
+                       if (is_device_private_page(new)) {
+                               entry = make_device_private_entry(new, pte_write(pte));
+                               pte = swp_entry_to_pte(entry);
+                       } else if (is_device_public_page(new)) {
+                               pte = pte_mkdevmap(pte);
+                               flush_dcache_page(new);
+                       }
                } else
                        flush_dcache_page(new);
 
@@ -437,12 +442,11 @@ int migrate_page_move_mapping(struct address_space *mapping,
        void **pslot;
 
        /*
-        * ZONE_DEVICE pages have 1 refcount always held by their device
-        *
-        * Note that DAX memory will never reach that point as it does not have
-        * the MEMORY_DEVICE_ALLOW_MIGRATE flag set (see memory_hotplug.h).
+        * Device public or private pages have an extra refcount as they are
+        * ZONE_DEVICE pages.
         */
-       expected_count += is_zone_device_page(page);
+       expected_count += is_device_private_page(page);
+       expected_count += is_device_public_page(page);
 
        if (!mapping) {
                /* Anonymous page without mapping */
@@ -2123,7 +2127,6 @@ out_unlock:
 
 #endif /* CONFIG_NUMA */
 
-
 struct migrate_vma {
        struct vm_area_struct   *vma;
        unsigned long           *dst;
@@ -2263,7 +2266,7 @@ again:
                                pfn = 0;
                                goto next;
                        }
-                       page = vm_normal_page(migrate->vma, addr, pte);
+                       page = _vm_normal_page(migrate->vma, addr, pte, true);
                        mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
                        mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
                }
@@ -2406,10 +2409,19 @@ static bool migrate_vma_check_page(struct page *page)
                if (is_device_private_page(page))
                        return true;
 
-               /* Other ZONE_DEVICE memory type are not supported */
-               return false;
+               /*
+                * Only allow device public page to be migrated and account for
+                * the extra reference count imply by ZONE_DEVICE pages.
+                */
+               if (!is_device_public_page(page))
+                       return false;
+               extra++;
        }
 
+       /* For file back page */
+       if (page_mapping(page))
+               extra += 1 + page_has_private(page);
+
        if ((page_count(page) - extra) > page_mapcount(page))
                return false;
 
@@ -2647,11 +2659,18 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
         */
        __SetPageUptodate(page);
 
-       if (is_zone_device_page(page) && is_device_private_page(page)) {
-               swp_entry_t swp_entry;
-
-               swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
-               entry = swp_entry_to_pte(swp_entry);
+       if (is_zone_device_page(page)) {
+               if (is_device_private_page(page)) {
+                       swp_entry_t swp_entry;
+
+                       swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
+                       entry = swp_entry_to_pte(swp_entry);
+               } else if (is_device_public_page(page)) {
+                       entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
+                       if (vma->vm_flags & VM_WRITE)
+                               entry = pte_mkwrite(pte_mkdirty(entry));
+                       entry = pte_mkdevmap(entry);
+               }
        } else {
                entry = mk_pte(page, vma->vm_page_prot);
                if (vma->vm_flags & VM_WRITE)
@@ -2768,7 +2787,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
                                        migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
                                        continue;
                                }
-                       } else {
+                       } else if (!is_device_public_page(newpage)) {
                                /*
                                 * Other types of ZONE_DEVICE page are not
                                 * supported.