1 // SPDX-License-Identifier: GPL-2.0
3 * This is a module to test the HMM (Heterogeneous Memory Management)
4 * mirror and zone device private memory migration APIs of the kernel.
5 * Userspace programs can register with the driver to mirror their own address
6 * space and can use the device to read/write any valid virtual address.
8 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/kernel.h>
13 #include <linux/cdev.h>
14 #include <linux/device.h>
15 #include <linux/memremap.h>
16 #include <linux/mutex.h>
17 #include <linux/rwsem.h>
18 #include <linux/sched.h>
19 #include <linux/slab.h>
20 #include <linux/highmem.h>
21 #include <linux/delay.h>
22 #include <linux/pagemap.h>
23 #include <linux/hmm.h>
24 #include <linux/vmalloc.h>
25 #include <linux/swap.h>
26 #include <linux/swapops.h>
27 #include <linux/sched/mm.h>
28 #include <linux/platform_device.h>
29 #include <linux/rmap.h>
30 #include <linux/mmu_notifier.h>
31 #include <linux/migrate.h>
33 #include "test_hmm_uapi.h"
35 #define DMIRROR_NDEVICES 4
36 #define DMIRROR_RANGE_FAULT_TIMEOUT 1000
37 #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
38 #define DEVMEM_CHUNKS_RESERVE 16
41 * For device_private pages, dpage is just a dummy struct page
42 * representing a piece of device memory. dmirror_devmem_alloc_page
43 * allocates a real system memory page as backing storage to fake a
44 * real device. zone_device_data points to that backing page. But
45 * for device_coherent memory, the struct page represents real
46 * physical CPU-accessible memory that we can use directly.
48 #define BACKING_PAGE(page) (is_device_private_page((page)) ? \
49 (page)->zone_device_data : (page))
51 static unsigned long spm_addr_dev0
;
52 module_param(spm_addr_dev0
, long, 0644);
53 MODULE_PARM_DESC(spm_addr_dev0
,
54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
56 static unsigned long spm_addr_dev1
;
57 module_param(spm_addr_dev1
, long, 0644);
58 MODULE_PARM_DESC(spm_addr_dev1
,
59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
61 static const struct dev_pagemap_ops dmirror_devmem_ops
;
62 static const struct mmu_interval_notifier_ops dmirror_min_ops
;
63 static dev_t dmirror_dev
;
65 struct dmirror_device
;
67 struct dmirror_bounce
{
74 #define DPT_XA_TAG_ATOMIC 1UL
75 #define DPT_XA_TAG_WRITE 3UL
78 * Data structure to track address ranges and register for mmu interval
81 struct dmirror_interval
{
82 struct mmu_interval_notifier notifier
;
83 struct dmirror
*dmirror
;
87 * Data attached to the open device file.
88 * Note that it might be shared after a fork().
91 struct dmirror_device
*mdevice
;
93 struct mmu_interval_notifier notifier
;
98 * ZONE_DEVICE pages for migration and simulating device memory.
100 struct dmirror_chunk
{
101 struct dev_pagemap pagemap
;
102 struct dmirror_device
*mdevice
;
109 struct dmirror_device
{
111 unsigned int zone_device_type
;
112 struct device device
;
114 unsigned int devmem_capacity
;
115 unsigned int devmem_count
;
116 struct dmirror_chunk
**devmem_chunks
;
117 struct mutex devmem_lock
; /* protects the above */
119 unsigned long calloc
;
121 struct page
*free_pages
;
122 spinlock_t lock
; /* protects the above */
125 static struct dmirror_device dmirror_devices
[DMIRROR_NDEVICES
];
127 static int dmirror_bounce_init(struct dmirror_bounce
*bounce
,
134 bounce
->ptr
= vmalloc(size
);
140 static bool dmirror_is_private_zone(struct dmirror_device
*mdevice
)
142 return (mdevice
->zone_device_type
==
143 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE
) ? true : false;
146 static enum migrate_vma_direction
147 dmirror_select_device(struct dmirror
*dmirror
)
149 return (dmirror
->mdevice
->zone_device_type
==
150 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE
) ?
151 MIGRATE_VMA_SELECT_DEVICE_PRIVATE
:
152 MIGRATE_VMA_SELECT_DEVICE_COHERENT
;
155 static void dmirror_bounce_fini(struct dmirror_bounce
*bounce
)
160 static int dmirror_fops_open(struct inode
*inode
, struct file
*filp
)
162 struct cdev
*cdev
= inode
->i_cdev
;
163 struct dmirror
*dmirror
;
166 /* Mirror this process address space */
167 dmirror
= kzalloc(sizeof(*dmirror
), GFP_KERNEL
);
171 dmirror
->mdevice
= container_of(cdev
, struct dmirror_device
, cdevice
);
172 mutex_init(&dmirror
->mutex
);
173 xa_init(&dmirror
->pt
);
175 ret
= mmu_interval_notifier_insert(&dmirror
->notifier
, current
->mm
,
176 0, ULONG_MAX
& PAGE_MASK
, &dmirror_min_ops
);
182 filp
->private_data
= dmirror
;
186 static int dmirror_fops_release(struct inode
*inode
, struct file
*filp
)
188 struct dmirror
*dmirror
= filp
->private_data
;
190 mmu_interval_notifier_remove(&dmirror
->notifier
);
191 xa_destroy(&dmirror
->pt
);
196 static struct dmirror_chunk
*dmirror_page_to_chunk(struct page
*page
)
198 return container_of(page
->pgmap
, struct dmirror_chunk
, pagemap
);
201 static struct dmirror_device
*dmirror_page_to_device(struct page
*page
)
204 return dmirror_page_to_chunk(page
)->mdevice
;
207 static int dmirror_do_fault(struct dmirror
*dmirror
, struct hmm_range
*range
)
209 unsigned long *pfns
= range
->hmm_pfns
;
212 for (pfn
= (range
->start
>> PAGE_SHIFT
);
213 pfn
< (range
->end
>> PAGE_SHIFT
);
219 * Since we asked for hmm_range_fault() to populate pages,
220 * it shouldn't return an error entry on success.
222 WARN_ON(*pfns
& HMM_PFN_ERROR
);
223 WARN_ON(!(*pfns
& HMM_PFN_VALID
));
225 page
= hmm_pfn_to_page(*pfns
);
229 if (*pfns
& HMM_PFN_WRITE
)
230 entry
= xa_tag_pointer(entry
, DPT_XA_TAG_WRITE
);
231 else if (WARN_ON(range
->default_flags
& HMM_PFN_WRITE
))
233 entry
= xa_store(&dmirror
->pt
, pfn
, entry
, GFP_ATOMIC
);
234 if (xa_is_err(entry
))
235 return xa_err(entry
);
241 static void dmirror_do_update(struct dmirror
*dmirror
, unsigned long start
,
248 * The XArray doesn't hold references to pages since it relies on
249 * the mmu notifier to clear page pointers when they become stale.
250 * Therefore, it is OK to just clear the entry.
252 xa_for_each_range(&dmirror
->pt
, pfn
, entry
, start
>> PAGE_SHIFT
,
254 xa_erase(&dmirror
->pt
, pfn
);
257 static bool dmirror_interval_invalidate(struct mmu_interval_notifier
*mni
,
258 const struct mmu_notifier_range
*range
,
259 unsigned long cur_seq
)
261 struct dmirror
*dmirror
= container_of(mni
, struct dmirror
, notifier
);
264 * Ignore invalidation callbacks for device private pages since
265 * the invalidation is handled as part of the migration process.
267 if (range
->event
== MMU_NOTIFY_MIGRATE
&&
268 range
->owner
== dmirror
->mdevice
)
271 if (mmu_notifier_range_blockable(range
))
272 mutex_lock(&dmirror
->mutex
);
273 else if (!mutex_trylock(&dmirror
->mutex
))
276 mmu_interval_set_seq(mni
, cur_seq
);
277 dmirror_do_update(dmirror
, range
->start
, range
->end
);
279 mutex_unlock(&dmirror
->mutex
);
283 static const struct mmu_interval_notifier_ops dmirror_min_ops
= {
284 .invalidate
= dmirror_interval_invalidate
,
287 static int dmirror_range_fault(struct dmirror
*dmirror
,
288 struct hmm_range
*range
)
290 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
291 unsigned long timeout
=
292 jiffies
+ msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT
);
296 if (time_after(jiffies
, timeout
)) {
301 range
->notifier_seq
= mmu_interval_read_begin(range
->notifier
);
303 ret
= hmm_range_fault(range
);
304 mmap_read_unlock(mm
);
311 mutex_lock(&dmirror
->mutex
);
312 if (mmu_interval_read_retry(range
->notifier
,
313 range
->notifier_seq
)) {
314 mutex_unlock(&dmirror
->mutex
);
320 ret
= dmirror_do_fault(dmirror
, range
);
322 mutex_unlock(&dmirror
->mutex
);
327 static int dmirror_fault(struct dmirror
*dmirror
, unsigned long start
,
328 unsigned long end
, bool write
)
330 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
332 unsigned long pfns
[64];
333 struct hmm_range range
= {
334 .notifier
= &dmirror
->notifier
,
338 HMM_PFN_REQ_FAULT
| (write
? HMM_PFN_REQ_WRITE
: 0),
339 .dev_private_owner
= dmirror
->mdevice
,
343 /* Since the mm is for the mirrored process, get a reference first. */
344 if (!mmget_not_zero(mm
))
347 for (addr
= start
; addr
< end
; addr
= range
.end
) {
349 range
.end
= min(addr
+ (ARRAY_SIZE(pfns
) << PAGE_SHIFT
), end
);
351 ret
= dmirror_range_fault(dmirror
, &range
);
360 static int dmirror_do_read(struct dmirror
*dmirror
, unsigned long start
,
361 unsigned long end
, struct dmirror_bounce
*bounce
)
366 ptr
= bounce
->ptr
+ ((start
- bounce
->addr
) & PAGE_MASK
);
368 for (pfn
= start
>> PAGE_SHIFT
; pfn
< (end
>> PAGE_SHIFT
); pfn
++) {
372 entry
= xa_load(&dmirror
->pt
, pfn
);
373 page
= xa_untag_pointer(entry
);
377 memcpy_from_page(ptr
, page
, 0, PAGE_SIZE
);
386 static int dmirror_read(struct dmirror
*dmirror
, struct hmm_dmirror_cmd
*cmd
)
388 struct dmirror_bounce bounce
;
389 unsigned long start
, end
;
390 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
398 ret
= dmirror_bounce_init(&bounce
, start
, size
);
403 mutex_lock(&dmirror
->mutex
);
404 ret
= dmirror_do_read(dmirror
, start
, end
, &bounce
);
405 mutex_unlock(&dmirror
->mutex
);
409 start
= cmd
->addr
+ (bounce
.cpages
<< PAGE_SHIFT
);
410 ret
= dmirror_fault(dmirror
, start
, end
, false);
417 if (copy_to_user(u64_to_user_ptr(cmd
->ptr
), bounce
.ptr
,
421 cmd
->cpages
= bounce
.cpages
;
422 dmirror_bounce_fini(&bounce
);
426 static int dmirror_do_write(struct dmirror
*dmirror
, unsigned long start
,
427 unsigned long end
, struct dmirror_bounce
*bounce
)
432 ptr
= bounce
->ptr
+ ((start
- bounce
->addr
) & PAGE_MASK
);
434 for (pfn
= start
>> PAGE_SHIFT
; pfn
< (end
>> PAGE_SHIFT
); pfn
++) {
438 entry
= xa_load(&dmirror
->pt
, pfn
);
439 page
= xa_untag_pointer(entry
);
440 if (!page
|| xa_pointer_tag(entry
) != DPT_XA_TAG_WRITE
)
443 memcpy_to_page(page
, 0, ptr
, PAGE_SIZE
);
452 static int dmirror_write(struct dmirror
*dmirror
, struct hmm_dmirror_cmd
*cmd
)
454 struct dmirror_bounce bounce
;
455 unsigned long start
, end
;
456 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
464 ret
= dmirror_bounce_init(&bounce
, start
, size
);
467 if (copy_from_user(bounce
.ptr
, u64_to_user_ptr(cmd
->ptr
),
474 mutex_lock(&dmirror
->mutex
);
475 ret
= dmirror_do_write(dmirror
, start
, end
, &bounce
);
476 mutex_unlock(&dmirror
->mutex
);
480 start
= cmd
->addr
+ (bounce
.cpages
<< PAGE_SHIFT
);
481 ret
= dmirror_fault(dmirror
, start
, end
, true);
488 cmd
->cpages
= bounce
.cpages
;
489 dmirror_bounce_fini(&bounce
);
493 static int dmirror_allocate_chunk(struct dmirror_device
*mdevice
,
496 struct dmirror_chunk
*devmem
;
497 struct resource
*res
= NULL
;
499 unsigned long pfn_first
;
500 unsigned long pfn_last
;
504 devmem
= kzalloc(sizeof(*devmem
), GFP_KERNEL
);
508 switch (mdevice
->zone_device_type
) {
509 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE
:
510 res
= request_free_mem_region(&iomem_resource
, DEVMEM_CHUNK_SIZE
,
512 if (IS_ERR_OR_NULL(res
))
514 devmem
->pagemap
.range
.start
= res
->start
;
515 devmem
->pagemap
.range
.end
= res
->end
;
516 devmem
->pagemap
.type
= MEMORY_DEVICE_PRIVATE
;
518 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT
:
519 devmem
->pagemap
.range
.start
= (MINOR(mdevice
->cdevice
.dev
) - 2) ?
522 devmem
->pagemap
.range
.end
= devmem
->pagemap
.range
.start
+
523 DEVMEM_CHUNK_SIZE
- 1;
524 devmem
->pagemap
.type
= MEMORY_DEVICE_COHERENT
;
531 devmem
->pagemap
.nr_range
= 1;
532 devmem
->pagemap
.ops
= &dmirror_devmem_ops
;
533 devmem
->pagemap
.owner
= mdevice
;
535 mutex_lock(&mdevice
->devmem_lock
);
537 if (mdevice
->devmem_count
== mdevice
->devmem_capacity
) {
538 struct dmirror_chunk
**new_chunks
;
539 unsigned int new_capacity
;
541 new_capacity
= mdevice
->devmem_capacity
+
542 DEVMEM_CHUNKS_RESERVE
;
543 new_chunks
= krealloc(mdevice
->devmem_chunks
,
544 sizeof(new_chunks
[0]) * new_capacity
,
548 mdevice
->devmem_capacity
= new_capacity
;
549 mdevice
->devmem_chunks
= new_chunks
;
551 ptr
= memremap_pages(&devmem
->pagemap
, numa_node_id());
552 if (IS_ERR_OR_NULL(ptr
)) {
560 devmem
->mdevice
= mdevice
;
561 pfn_first
= devmem
->pagemap
.range
.start
>> PAGE_SHIFT
;
562 pfn_last
= pfn_first
+ (range_len(&devmem
->pagemap
.range
) >> PAGE_SHIFT
);
563 mdevice
->devmem_chunks
[mdevice
->devmem_count
++] = devmem
;
565 mutex_unlock(&mdevice
->devmem_lock
);
567 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
568 DEVMEM_CHUNK_SIZE
/ (1024 * 1024),
569 mdevice
->devmem_count
,
570 mdevice
->devmem_count
* (DEVMEM_CHUNK_SIZE
/ (1024 * 1024)),
571 pfn_first
, pfn_last
);
573 spin_lock(&mdevice
->lock
);
574 for (pfn
= pfn_first
; pfn
< pfn_last
; pfn
++) {
575 struct page
*page
= pfn_to_page(pfn
);
577 page
->zone_device_data
= mdevice
->free_pages
;
578 mdevice
->free_pages
= page
;
581 *ppage
= mdevice
->free_pages
;
582 mdevice
->free_pages
= (*ppage
)->zone_device_data
;
585 spin_unlock(&mdevice
->lock
);
590 mutex_unlock(&mdevice
->devmem_lock
);
591 if (res
&& devmem
->pagemap
.type
== MEMORY_DEVICE_PRIVATE
)
592 release_mem_region(devmem
->pagemap
.range
.start
,
593 range_len(&devmem
->pagemap
.range
));
600 static struct page
*dmirror_devmem_alloc_page(struct dmirror_device
*mdevice
)
602 struct page
*dpage
= NULL
;
603 struct page
*rpage
= NULL
;
606 * For ZONE_DEVICE private type, this is a fake device so we allocate
607 * real system memory to store our device memory.
608 * For ZONE_DEVICE coherent type we use the actual dpage to store the
609 * data and ignore rpage.
611 if (dmirror_is_private_zone(mdevice
)) {
612 rpage
= alloc_page(GFP_HIGHUSER
);
616 spin_lock(&mdevice
->lock
);
618 if (mdevice
->free_pages
) {
619 dpage
= mdevice
->free_pages
;
620 mdevice
->free_pages
= dpage
->zone_device_data
;
622 spin_unlock(&mdevice
->lock
);
624 spin_unlock(&mdevice
->lock
);
625 if (dmirror_allocate_chunk(mdevice
, &dpage
))
629 zone_device_page_init(dpage
);
630 dpage
->zone_device_data
= rpage
;
639 static void dmirror_migrate_alloc_and_copy(struct migrate_vma
*args
,
640 struct dmirror
*dmirror
)
642 struct dmirror_device
*mdevice
= dmirror
->mdevice
;
643 const unsigned long *src
= args
->src
;
644 unsigned long *dst
= args
->dst
;
647 for (addr
= args
->start
; addr
< args
->end
; addr
+= PAGE_SIZE
,
653 if (!(*src
& MIGRATE_PFN_MIGRATE
))
657 * Note that spage might be NULL which is OK since it is an
658 * unallocated pte_none() or read-only zero page.
660 spage
= migrate_pfn_to_page(*src
);
661 if (WARN(spage
&& is_zone_device_page(spage
),
662 "page already in device spage pfn: 0x%lx\n",
666 dpage
= dmirror_devmem_alloc_page(mdevice
);
670 rpage
= BACKING_PAGE(dpage
);
672 copy_highpage(rpage
, spage
);
674 clear_highpage(rpage
);
677 * Normally, a device would use the page->zone_device_data to
678 * point to the mirror but here we use it to hold the page for
679 * the simulated device memory and that page holds the pointer
682 rpage
->zone_device_data
= dmirror
;
684 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
685 page_to_pfn(spage
), page_to_pfn(dpage
));
686 *dst
= migrate_pfn(page_to_pfn(dpage
));
687 if ((*src
& MIGRATE_PFN_WRITE
) ||
688 (!spage
&& args
->vma
->vm_flags
& VM_WRITE
))
689 *dst
|= MIGRATE_PFN_WRITE
;
693 static int dmirror_check_atomic(struct dmirror
*dmirror
, unsigned long start
,
698 for (pfn
= start
>> PAGE_SHIFT
; pfn
< (end
>> PAGE_SHIFT
); pfn
++) {
701 entry
= xa_load(&dmirror
->pt
, pfn
);
702 if (xa_pointer_tag(entry
) == DPT_XA_TAG_ATOMIC
)
709 static int dmirror_atomic_map(unsigned long start
, unsigned long end
,
710 struct page
**pages
, struct dmirror
*dmirror
)
712 unsigned long pfn
, mapped
= 0;
715 /* Map the migrated pages into the device's page tables. */
716 mutex_lock(&dmirror
->mutex
);
718 for (i
= 0, pfn
= start
>> PAGE_SHIFT
; pfn
< (end
>> PAGE_SHIFT
); pfn
++, i
++) {
725 entry
= xa_tag_pointer(entry
, DPT_XA_TAG_ATOMIC
);
726 entry
= xa_store(&dmirror
->pt
, pfn
, entry
, GFP_ATOMIC
);
727 if (xa_is_err(entry
)) {
728 mutex_unlock(&dmirror
->mutex
);
729 return xa_err(entry
);
735 mutex_unlock(&dmirror
->mutex
);
739 static int dmirror_migrate_finalize_and_map(struct migrate_vma
*args
,
740 struct dmirror
*dmirror
)
742 unsigned long start
= args
->start
;
743 unsigned long end
= args
->end
;
744 const unsigned long *src
= args
->src
;
745 const unsigned long *dst
= args
->dst
;
748 /* Map the migrated pages into the device's page tables. */
749 mutex_lock(&dmirror
->mutex
);
751 for (pfn
= start
>> PAGE_SHIFT
; pfn
< (end
>> PAGE_SHIFT
); pfn
++,
756 if (!(*src
& MIGRATE_PFN_MIGRATE
))
759 dpage
= migrate_pfn_to_page(*dst
);
763 entry
= BACKING_PAGE(dpage
);
764 if (*dst
& MIGRATE_PFN_WRITE
)
765 entry
= xa_tag_pointer(entry
, DPT_XA_TAG_WRITE
);
766 entry
= xa_store(&dmirror
->pt
, pfn
, entry
, GFP_ATOMIC
);
767 if (xa_is_err(entry
)) {
768 mutex_unlock(&dmirror
->mutex
);
769 return xa_err(entry
);
773 mutex_unlock(&dmirror
->mutex
);
777 static int dmirror_exclusive(struct dmirror
*dmirror
,
778 struct hmm_dmirror_cmd
*cmd
)
780 unsigned long start
, end
, addr
;
781 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
782 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
783 struct page
*pages
[64];
784 struct dmirror_bounce bounce
;
793 /* Since the mm is for the mirrored process, get a reference first. */
794 if (!mmget_not_zero(mm
))
798 for (addr
= start
; addr
< end
; addr
= next
) {
799 unsigned long mapped
= 0;
802 if (end
< addr
+ (ARRAY_SIZE(pages
) << PAGE_SHIFT
))
805 next
= addr
+ (ARRAY_SIZE(pages
) << PAGE_SHIFT
);
807 ret
= make_device_exclusive_range(mm
, addr
, next
, pages
, NULL
);
809 * Do dmirror_atomic_map() iff all pages are marked for
810 * exclusive access to avoid accessing uninitialized
813 if (ret
== (next
- addr
) >> PAGE_SHIFT
)
814 mapped
= dmirror_atomic_map(addr
, next
, pages
, dmirror
);
815 for (i
= 0; i
< ret
; i
++) {
817 unlock_page(pages
[i
]);
822 if (addr
+ (mapped
<< PAGE_SHIFT
) < next
) {
823 mmap_read_unlock(mm
);
828 mmap_read_unlock(mm
);
831 /* Return the migrated data for verification. */
832 ret
= dmirror_bounce_init(&bounce
, start
, size
);
835 mutex_lock(&dmirror
->mutex
);
836 ret
= dmirror_do_read(dmirror
, start
, end
, &bounce
);
837 mutex_unlock(&dmirror
->mutex
);
839 if (copy_to_user(u64_to_user_ptr(cmd
->ptr
), bounce
.ptr
,
844 cmd
->cpages
= bounce
.cpages
;
845 dmirror_bounce_fini(&bounce
);
849 static vm_fault_t
dmirror_devmem_fault_alloc_and_copy(struct migrate_vma
*args
,
850 struct dmirror
*dmirror
)
852 const unsigned long *src
= args
->src
;
853 unsigned long *dst
= args
->dst
;
854 unsigned long start
= args
->start
;
855 unsigned long end
= args
->end
;
858 for (addr
= start
; addr
< end
; addr
+= PAGE_SIZE
,
860 struct page
*dpage
, *spage
;
862 spage
= migrate_pfn_to_page(*src
);
863 if (!spage
|| !(*src
& MIGRATE_PFN_MIGRATE
))
866 if (WARN_ON(!is_device_private_page(spage
) &&
867 !is_device_coherent_page(spage
)))
869 spage
= BACKING_PAGE(spage
);
870 dpage
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, args
->vma
, addr
);
873 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
874 page_to_pfn(spage
), page_to_pfn(dpage
));
877 xa_erase(&dmirror
->pt
, addr
>> PAGE_SHIFT
);
878 copy_highpage(dpage
, spage
);
879 *dst
= migrate_pfn(page_to_pfn(dpage
));
880 if (*src
& MIGRATE_PFN_WRITE
)
881 *dst
|= MIGRATE_PFN_WRITE
;
887 dmirror_successful_migrated_pages(struct migrate_vma
*migrate
)
889 unsigned long cpages
= 0;
892 for (i
= 0; i
< migrate
->npages
; i
++) {
893 if (migrate
->src
[i
] & MIGRATE_PFN_VALID
&&
894 migrate
->src
[i
] & MIGRATE_PFN_MIGRATE
)
900 static int dmirror_migrate_to_system(struct dmirror
*dmirror
,
901 struct hmm_dmirror_cmd
*cmd
)
903 unsigned long start
, end
, addr
;
904 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
905 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
906 struct vm_area_struct
*vma
;
907 unsigned long src_pfns
[64] = { 0 };
908 unsigned long dst_pfns
[64] = { 0 };
909 struct migrate_vma args
= { 0 };
918 /* Since the mm is for the mirrored process, get a reference first. */
919 if (!mmget_not_zero(mm
))
924 for (addr
= start
; addr
< end
; addr
= next
) {
925 vma
= vma_lookup(mm
, addr
);
926 if (!vma
|| !(vma
->vm_flags
& VM_READ
)) {
930 next
= min(end
, addr
+ (ARRAY_SIZE(src_pfns
) << PAGE_SHIFT
));
931 if (next
> vma
->vm_end
)
939 args
.pgmap_owner
= dmirror
->mdevice
;
940 args
.flags
= dmirror_select_device(dmirror
);
942 ret
= migrate_vma_setup(&args
);
946 pr_debug("Migrating from device mem to sys mem\n");
947 dmirror_devmem_fault_alloc_and_copy(&args
, dmirror
);
949 migrate_vma_pages(&args
);
950 cmd
->cpages
+= dmirror_successful_migrated_pages(&args
);
951 migrate_vma_finalize(&args
);
954 mmap_read_unlock(mm
);
960 static int dmirror_migrate_to_device(struct dmirror
*dmirror
,
961 struct hmm_dmirror_cmd
*cmd
)
963 unsigned long start
, end
, addr
;
964 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
965 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
966 struct vm_area_struct
*vma
;
967 unsigned long src_pfns
[64] = { 0 };
968 unsigned long dst_pfns
[64] = { 0 };
969 struct dmirror_bounce bounce
;
970 struct migrate_vma args
= { 0 };
979 /* Since the mm is for the mirrored process, get a reference first. */
980 if (!mmget_not_zero(mm
))
984 for (addr
= start
; addr
< end
; addr
= next
) {
985 vma
= vma_lookup(mm
, addr
);
986 if (!vma
|| !(vma
->vm_flags
& VM_READ
)) {
990 next
= min(end
, addr
+ (ARRAY_SIZE(src_pfns
) << PAGE_SHIFT
));
991 if (next
> vma
->vm_end
)
999 args
.pgmap_owner
= dmirror
->mdevice
;
1000 args
.flags
= MIGRATE_VMA_SELECT_SYSTEM
;
1001 ret
= migrate_vma_setup(&args
);
1005 pr_debug("Migrating from sys mem to device mem\n");
1006 dmirror_migrate_alloc_and_copy(&args
, dmirror
);
1007 migrate_vma_pages(&args
);
1008 dmirror_migrate_finalize_and_map(&args
, dmirror
);
1009 migrate_vma_finalize(&args
);
1011 mmap_read_unlock(mm
);
1015 * Return the migrated data for verification.
1016 * Only for pages in device zone
1018 ret
= dmirror_bounce_init(&bounce
, start
, size
);
1021 mutex_lock(&dmirror
->mutex
);
1022 ret
= dmirror_do_read(dmirror
, start
, end
, &bounce
);
1023 mutex_unlock(&dmirror
->mutex
);
1025 if (copy_to_user(u64_to_user_ptr(cmd
->ptr
), bounce
.ptr
,
1029 cmd
->cpages
= bounce
.cpages
;
1030 dmirror_bounce_fini(&bounce
);
1034 mmap_read_unlock(mm
);
1039 static void dmirror_mkentry(struct dmirror
*dmirror
, struct hmm_range
*range
,
1040 unsigned char *perm
, unsigned long entry
)
1044 if (entry
& HMM_PFN_ERROR
) {
1045 *perm
= HMM_DMIRROR_PROT_ERROR
;
1048 if (!(entry
& HMM_PFN_VALID
)) {
1049 *perm
= HMM_DMIRROR_PROT_NONE
;
1053 page
= hmm_pfn_to_page(entry
);
1054 if (is_device_private_page(page
)) {
1055 /* Is the page migrated to this device or some other? */
1056 if (dmirror
->mdevice
== dmirror_page_to_device(page
))
1057 *perm
= HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL
;
1059 *perm
= HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE
;
1060 } else if (is_device_coherent_page(page
)) {
1061 /* Is the page migrated to this device or some other? */
1062 if (dmirror
->mdevice
== dmirror_page_to_device(page
))
1063 *perm
= HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL
;
1065 *perm
= HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE
;
1066 } else if (is_zero_pfn(page_to_pfn(page
)))
1067 *perm
= HMM_DMIRROR_PROT_ZERO
;
1069 *perm
= HMM_DMIRROR_PROT_NONE
;
1070 if (entry
& HMM_PFN_WRITE
)
1071 *perm
|= HMM_DMIRROR_PROT_WRITE
;
1073 *perm
|= HMM_DMIRROR_PROT_READ
;
1074 if (hmm_pfn_to_map_order(entry
) + PAGE_SHIFT
== PMD_SHIFT
)
1075 *perm
|= HMM_DMIRROR_PROT_PMD
;
1076 else if (hmm_pfn_to_map_order(entry
) + PAGE_SHIFT
== PUD_SHIFT
)
1077 *perm
|= HMM_DMIRROR_PROT_PUD
;
1080 static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier
*mni
,
1081 const struct mmu_notifier_range
*range
,
1082 unsigned long cur_seq
)
1084 struct dmirror_interval
*dmi
=
1085 container_of(mni
, struct dmirror_interval
, notifier
);
1086 struct dmirror
*dmirror
= dmi
->dmirror
;
1088 if (mmu_notifier_range_blockable(range
))
1089 mutex_lock(&dmirror
->mutex
);
1090 else if (!mutex_trylock(&dmirror
->mutex
))
1094 * Snapshots only need to set the sequence number since any
1095 * invalidation in the interval invalidates the whole snapshot.
1097 mmu_interval_set_seq(mni
, cur_seq
);
1099 mutex_unlock(&dmirror
->mutex
);
1103 static const struct mmu_interval_notifier_ops dmirror_mrn_ops
= {
1104 .invalidate
= dmirror_snapshot_invalidate
,
1107 static int dmirror_range_snapshot(struct dmirror
*dmirror
,
1108 struct hmm_range
*range
,
1109 unsigned char *perm
)
1111 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
1112 struct dmirror_interval notifier
;
1113 unsigned long timeout
=
1114 jiffies
+ msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT
);
1119 notifier
.dmirror
= dmirror
;
1120 range
->notifier
= ¬ifier
.notifier
;
1122 ret
= mmu_interval_notifier_insert(range
->notifier
, mm
,
1123 range
->start
, range
->end
- range
->start
,
1129 if (time_after(jiffies
, timeout
)) {
1134 range
->notifier_seq
= mmu_interval_read_begin(range
->notifier
);
1137 ret
= hmm_range_fault(range
);
1138 mmap_read_unlock(mm
);
1145 mutex_lock(&dmirror
->mutex
);
1146 if (mmu_interval_read_retry(range
->notifier
,
1147 range
->notifier_seq
)) {
1148 mutex_unlock(&dmirror
->mutex
);
1154 n
= (range
->end
- range
->start
) >> PAGE_SHIFT
;
1155 for (i
= 0; i
< n
; i
++)
1156 dmirror_mkentry(dmirror
, range
, perm
+ i
, range
->hmm_pfns
[i
]);
1158 mutex_unlock(&dmirror
->mutex
);
1160 mmu_interval_notifier_remove(range
->notifier
);
1164 static int dmirror_snapshot(struct dmirror
*dmirror
,
1165 struct hmm_dmirror_cmd
*cmd
)
1167 struct mm_struct
*mm
= dmirror
->notifier
.mm
;
1168 unsigned long start
, end
;
1169 unsigned long size
= cmd
->npages
<< PAGE_SHIFT
;
1172 unsigned long pfns
[64];
1173 unsigned char perm
[64];
1175 struct hmm_range range
= {
1177 .dev_private_owner
= dmirror
->mdevice
,
1186 /* Since the mm is for the mirrored process, get a reference first. */
1187 if (!mmget_not_zero(mm
))
1191 * Register a temporary notifier to detect invalidations even if it
1192 * overlaps with other mmu_interval_notifiers.
1194 uptr
= u64_to_user_ptr(cmd
->ptr
);
1195 for (addr
= start
; addr
< end
; addr
= next
) {
1198 next
= min(addr
+ (ARRAY_SIZE(pfns
) << PAGE_SHIFT
), end
);
1202 ret
= dmirror_range_snapshot(dmirror
, &range
, perm
);
1206 n
= (range
.end
- range
.start
) >> PAGE_SHIFT
;
1207 if (copy_to_user(uptr
, perm
, n
)) {
1220 static void dmirror_device_evict_chunk(struct dmirror_chunk
*chunk
)
1222 unsigned long start_pfn
= chunk
->pagemap
.range
.start
>> PAGE_SHIFT
;
1223 unsigned long end_pfn
= chunk
->pagemap
.range
.end
>> PAGE_SHIFT
;
1224 unsigned long npages
= end_pfn
- start_pfn
+ 1;
1226 unsigned long *src_pfns
;
1227 unsigned long *dst_pfns
;
1229 src_pfns
= kcalloc(npages
, sizeof(*src_pfns
), GFP_KERNEL
);
1230 dst_pfns
= kcalloc(npages
, sizeof(*dst_pfns
), GFP_KERNEL
);
1232 migrate_device_range(src_pfns
, start_pfn
, npages
);
1233 for (i
= 0; i
< npages
; i
++) {
1234 struct page
*dpage
, *spage
;
1236 spage
= migrate_pfn_to_page(src_pfns
[i
]);
1237 if (!spage
|| !(src_pfns
[i
] & MIGRATE_PFN_MIGRATE
))
1240 if (WARN_ON(!is_device_private_page(spage
) &&
1241 !is_device_coherent_page(spage
)))
1243 spage
= BACKING_PAGE(spage
);
1244 dpage
= alloc_page(GFP_HIGHUSER_MOVABLE
| __GFP_NOFAIL
);
1246 copy_highpage(dpage
, spage
);
1247 dst_pfns
[i
] = migrate_pfn(page_to_pfn(dpage
));
1248 if (src_pfns
[i
] & MIGRATE_PFN_WRITE
)
1249 dst_pfns
[i
] |= MIGRATE_PFN_WRITE
;
1251 migrate_device_pages(src_pfns
, dst_pfns
, npages
);
1252 migrate_device_finalize(src_pfns
, dst_pfns
, npages
);
1257 /* Removes free pages from the free list so they can't be re-allocated */
1258 static void dmirror_remove_free_pages(struct dmirror_chunk
*devmem
)
1260 struct dmirror_device
*mdevice
= devmem
->mdevice
;
1263 for (page
= mdevice
->free_pages
; page
; page
= page
->zone_device_data
)
1264 if (dmirror_page_to_chunk(page
) == devmem
)
1265 mdevice
->free_pages
= page
->zone_device_data
;
1268 static void dmirror_device_remove_chunks(struct dmirror_device
*mdevice
)
1272 mutex_lock(&mdevice
->devmem_lock
);
1273 if (mdevice
->devmem_chunks
) {
1274 for (i
= 0; i
< mdevice
->devmem_count
; i
++) {
1275 struct dmirror_chunk
*devmem
=
1276 mdevice
->devmem_chunks
[i
];
1278 spin_lock(&mdevice
->lock
);
1279 devmem
->remove
= true;
1280 dmirror_remove_free_pages(devmem
);
1281 spin_unlock(&mdevice
->lock
);
1283 dmirror_device_evict_chunk(devmem
);
1284 memunmap_pages(&devmem
->pagemap
);
1285 if (devmem
->pagemap
.type
== MEMORY_DEVICE_PRIVATE
)
1286 release_mem_region(devmem
->pagemap
.range
.start
,
1287 range_len(&devmem
->pagemap
.range
));
1290 mdevice
->devmem_count
= 0;
1291 mdevice
->devmem_capacity
= 0;
1292 mdevice
->free_pages
= NULL
;
1293 kfree(mdevice
->devmem_chunks
);
1294 mdevice
->devmem_chunks
= NULL
;
1296 mutex_unlock(&mdevice
->devmem_lock
);
1299 static long dmirror_fops_unlocked_ioctl(struct file
*filp
,
1300 unsigned int command
,
1303 void __user
*uarg
= (void __user
*)arg
;
1304 struct hmm_dmirror_cmd cmd
;
1305 struct dmirror
*dmirror
;
1308 dmirror
= filp
->private_data
;
1312 if (copy_from_user(&cmd
, uarg
, sizeof(cmd
)))
1315 if (cmd
.addr
& ~PAGE_MASK
)
1317 if (cmd
.addr
>= (cmd
.addr
+ (cmd
.npages
<< PAGE_SHIFT
)))
1324 case HMM_DMIRROR_READ
:
1325 ret
= dmirror_read(dmirror
, &cmd
);
1328 case HMM_DMIRROR_WRITE
:
1329 ret
= dmirror_write(dmirror
, &cmd
);
1332 case HMM_DMIRROR_MIGRATE_TO_DEV
:
1333 ret
= dmirror_migrate_to_device(dmirror
, &cmd
);
1336 case HMM_DMIRROR_MIGRATE_TO_SYS
:
1337 ret
= dmirror_migrate_to_system(dmirror
, &cmd
);
1340 case HMM_DMIRROR_EXCLUSIVE
:
1341 ret
= dmirror_exclusive(dmirror
, &cmd
);
1344 case HMM_DMIRROR_CHECK_EXCLUSIVE
:
1345 ret
= dmirror_check_atomic(dmirror
, cmd
.addr
,
1346 cmd
.addr
+ (cmd
.npages
<< PAGE_SHIFT
));
1349 case HMM_DMIRROR_SNAPSHOT
:
1350 ret
= dmirror_snapshot(dmirror
, &cmd
);
1353 case HMM_DMIRROR_RELEASE
:
1354 dmirror_device_remove_chunks(dmirror
->mdevice
);
1364 if (copy_to_user(uarg
, &cmd
, sizeof(cmd
)))
1370 static int dmirror_fops_mmap(struct file
*file
, struct vm_area_struct
*vma
)
1374 for (addr
= vma
->vm_start
; addr
< vma
->vm_end
; addr
+= PAGE_SIZE
) {
1378 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
1382 ret
= vm_insert_page(vma
, addr
, page
);
1393 static const struct file_operations dmirror_fops
= {
1394 .open
= dmirror_fops_open
,
1395 .release
= dmirror_fops_release
,
1396 .mmap
= dmirror_fops_mmap
,
1397 .unlocked_ioctl
= dmirror_fops_unlocked_ioctl
,
1398 .llseek
= default_llseek
,
1399 .owner
= THIS_MODULE
,
1402 static void dmirror_devmem_free(struct page
*page
)
1404 struct page
*rpage
= BACKING_PAGE(page
);
1405 struct dmirror_device
*mdevice
;
1410 mdevice
= dmirror_page_to_device(page
);
1411 spin_lock(&mdevice
->lock
);
1413 /* Return page to our allocator if not freeing the chunk */
1414 if (!dmirror_page_to_chunk(page
)->remove
) {
1416 page
->zone_device_data
= mdevice
->free_pages
;
1417 mdevice
->free_pages
= page
;
1419 spin_unlock(&mdevice
->lock
);
1422 static vm_fault_t
dmirror_devmem_fault(struct vm_fault
*vmf
)
1424 struct migrate_vma args
= { 0 };
1425 unsigned long src_pfns
= 0;
1426 unsigned long dst_pfns
= 0;
1428 struct dmirror
*dmirror
;
1432 * Normally, a device would use the page->zone_device_data to point to
1433 * the mirror but here we use it to hold the page for the simulated
1434 * device memory and that page holds the pointer to the mirror.
1436 rpage
= vmf
->page
->zone_device_data
;
1437 dmirror
= rpage
->zone_device_data
;
1439 /* FIXME demonstrate how we can adjust migrate range */
1440 args
.vma
= vmf
->vma
;
1441 args
.start
= vmf
->address
;
1442 args
.end
= args
.start
+ PAGE_SIZE
;
1443 args
.src
= &src_pfns
;
1444 args
.dst
= &dst_pfns
;
1445 args
.pgmap_owner
= dmirror
->mdevice
;
1446 args
.flags
= dmirror_select_device(dmirror
);
1447 args
.fault_page
= vmf
->page
;
1449 if (migrate_vma_setup(&args
))
1450 return VM_FAULT_SIGBUS
;
1452 ret
= dmirror_devmem_fault_alloc_and_copy(&args
, dmirror
);
1455 migrate_vma_pages(&args
);
1457 * No device finalize step is needed since
1458 * dmirror_devmem_fault_alloc_and_copy() will have already
1459 * invalidated the device page table.
1461 migrate_vma_finalize(&args
);
1465 static const struct dev_pagemap_ops dmirror_devmem_ops
= {
1466 .page_free
= dmirror_devmem_free
,
1467 .migrate_to_ram
= dmirror_devmem_fault
,
1470 static int dmirror_device_init(struct dmirror_device
*mdevice
, int id
)
1475 dev
= MKDEV(MAJOR(dmirror_dev
), id
);
1476 mutex_init(&mdevice
->devmem_lock
);
1477 spin_lock_init(&mdevice
->lock
);
1479 cdev_init(&mdevice
->cdevice
, &dmirror_fops
);
1480 mdevice
->cdevice
.owner
= THIS_MODULE
;
1481 device_initialize(&mdevice
->device
);
1482 mdevice
->device
.devt
= dev
;
1484 ret
= dev_set_name(&mdevice
->device
, "hmm_dmirror%u", id
);
1488 ret
= cdev_device_add(&mdevice
->cdevice
, &mdevice
->device
);
1492 /* Build a list of free ZONE_DEVICE struct pages */
1493 return dmirror_allocate_chunk(mdevice
, NULL
);
1496 static void dmirror_device_remove(struct dmirror_device
*mdevice
)
1498 dmirror_device_remove_chunks(mdevice
);
1499 cdev_device_del(&mdevice
->cdevice
, &mdevice
->device
);
1502 static int __init
hmm_dmirror_init(void)
1508 ret
= alloc_chrdev_region(&dmirror_dev
, 0, DMIRROR_NDEVICES
,
1513 memset(dmirror_devices
, 0, DMIRROR_NDEVICES
* sizeof(dmirror_devices
[0]));
1514 dmirror_devices
[ndevices
++].zone_device_type
=
1515 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE
;
1516 dmirror_devices
[ndevices
++].zone_device_type
=
1517 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE
;
1518 if (spm_addr_dev0
&& spm_addr_dev1
) {
1519 dmirror_devices
[ndevices
++].zone_device_type
=
1520 HMM_DMIRROR_MEMORY_DEVICE_COHERENT
;
1521 dmirror_devices
[ndevices
++].zone_device_type
=
1522 HMM_DMIRROR_MEMORY_DEVICE_COHERENT
;
1524 for (id
= 0; id
< ndevices
; id
++) {
1525 ret
= dmirror_device_init(dmirror_devices
+ id
, id
);
1530 pr_info("HMM test module loaded. This is only for testing HMM.\n");
1535 dmirror_device_remove(dmirror_devices
+ id
);
1536 unregister_chrdev_region(dmirror_dev
, DMIRROR_NDEVICES
);
1541 static void __exit
hmm_dmirror_exit(void)
1545 for (id
= 0; id
< DMIRROR_NDEVICES
; id
++)
1546 if (dmirror_devices
[id
].zone_device_type
)
1547 dmirror_device_remove(dmirror_devices
+ id
);
1548 unregister_chrdev_region(dmirror_dev
, DMIRROR_NDEVICES
);
1551 module_init(hmm_dmirror_init
);
1552 module_exit(hmm_dmirror_exit
);
1553 MODULE_LICENSE("GPL");