]>
Commit | Line | Data |
---|---|---|
b20a3503 CL |
1 | #ifndef _LINUX_MIGRATE_H |
2 | #define _LINUX_MIGRATE_H | |
3 | ||
b20a3503 | 4 | #include <linux/mm.h> |
906e0be1 | 5 | #include <linux/mempolicy.h> |
6536e312 | 6 | #include <linux/migrate_mode.h> |
8b913238 | 7 | #include <linux/hugetlb.h> |
b20a3503 | 8 | |
68711a74 DR |
9 | typedef struct page *new_page_t(struct page *page, unsigned long private, |
10 | int **reason); | |
11 | typedef void free_page_t(struct page *page, unsigned long private); | |
95a402c3 | 12 | |
78bd5209 RA |
13 | /* |
14 | * Return values from addresss_space_operations.migratepage(): | |
15 | * - negative errno on page migration failure; | |
16 | * - zero on page migration success; | |
17 | */ | |
18 | #define MIGRATEPAGE_SUCCESS 0 | |
d6d86c0a | 19 | |
7b2a2d4a MG |
20 | enum migrate_reason { |
21 | MR_COMPACTION, | |
22 | MR_MEMORY_FAILURE, | |
23 | MR_MEMORY_HOTPLUG, | |
24 | MR_SYSCALL, /* also applies to cpusets */ | |
25 | MR_MEMPOLICY_MBIND, | |
7039e1db | 26 | MR_NUMA_MISPLACED, |
7cd12b4a VB |
27 | MR_CMA, |
28 | MR_TYPES | |
7b2a2d4a | 29 | }; |
78bd5209 | 30 | |
7cd12b4a VB |
31 | /* In mm/debug.c; also keep sync with include/trace/events/migrate.h */ |
32 | extern char *migrate_reason_names[MR_TYPES]; | |
33 | ||
8b913238 MH |
34 | static inline struct page *new_page_nodemask(struct page *page, |
35 | int preferred_nid, nodemask_t *nodemask) | |
36 | { | |
0f556856 | 37 | gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL; |
8135d892 NH |
38 | unsigned int order = 0; |
39 | struct page *new_page = NULL; | |
8b913238 MH |
40 | |
41 | if (PageHuge(page)) | |
42 | return alloc_huge_page_nodemask(page_hstate(compound_head(page)), | |
3e59fcb0 | 43 | preferred_nid, nodemask); |
8b913238 | 44 | |
8135d892 NH |
45 | if (thp_migration_supported() && PageTransHuge(page)) { |
46 | order = HPAGE_PMD_ORDER; | |
47 | gfp_mask |= GFP_TRANSHUGE; | |
48 | } | |
49 | ||
8b913238 MH |
50 | if (PageHighMem(page) || (zone_idx(page_zone(page)) == ZONE_MOVABLE)) |
51 | gfp_mask |= __GFP_HIGHMEM; | |
52 | ||
8135d892 NH |
53 | new_page = __alloc_pages_nodemask(gfp_mask, order, |
54 | preferred_nid, nodemask); | |
55 | ||
56 | if (new_page && PageTransHuge(page)) | |
57 | prep_transhuge_page(new_page); | |
58 | ||
59 | return new_page; | |
8b913238 MH |
60 | } |
61 | ||
906e0be1 | 62 | #ifdef CONFIG_MIGRATION |
64cdd548 | 63 | |
5733c7d1 | 64 | extern void putback_movable_pages(struct list_head *l); |
9927e388 PJ |
65 | extern int migrate_page(struct address_space *mapping, |
66 | struct page *newpage, struct page *page, | |
67 | enum migrate_mode mode); | |
68711a74 | 68 | extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, |
9c620e2b | 69 | unsigned long private, enum migrate_mode mode, int reason); |
9e5bcd61 | 70 | extern int isolate_movable_page(struct page *page, isolate_mode_t mode); |
bda807d4 | 71 | extern void putback_movable_page(struct page *page); |
95a402c3 | 72 | |
b20a3503 | 73 | extern int migrate_prep(void); |
748446bb | 74 | extern int migrate_prep_local(void); |
2916ecc0 | 75 | extern void migrate_page_states(struct page *newpage, struct page *page); |
290408d4 NH |
76 | extern void migrate_page_copy(struct page *newpage, struct page *page); |
77 | extern int migrate_huge_page_move_mapping(struct address_space *mapping, | |
78 | struct page *newpage, struct page *page); | |
36bc08cc GZ |
79 | extern int migrate_page_move_mapping(struct address_space *mapping, |
80 | struct page *newpage, struct page *page, | |
8e321fef BL |
81 | struct buffer_head *head, enum migrate_mode mode, |
82 | int extra_count); | |
b20a3503 | 83 | #else |
64cdd548 | 84 | |
5733c7d1 | 85 | static inline void putback_movable_pages(struct list_head *l) {} |
68711a74 DR |
86 | static inline int migrate_pages(struct list_head *l, new_page_t new, |
87 | free_page_t free, unsigned long private, enum migrate_mode mode, | |
88 | int reason) | |
9c620e2b | 89 | { return -ENOSYS; } |
cbae0170 YX |
90 | static inline int isolate_movable_page(struct page *page, isolate_mode_t mode) |
91 | { return -EBUSY; } | |
9bf9e89c | 92 | |
b20a3503 | 93 | static inline int migrate_prep(void) { return -ENOSYS; } |
748446bb | 94 | static inline int migrate_prep_local(void) { return -ENOSYS; } |
b20a3503 | 95 | |
2916ecc0 JG |
96 | static inline void migrate_page_states(struct page *newpage, struct page *page) |
97 | { | |
98 | } | |
99 | ||
290408d4 NH |
100 | static inline void migrate_page_copy(struct page *newpage, |
101 | struct page *page) {} | |
102 | ||
6f39ce05 | 103 | static inline int migrate_huge_page_move_mapping(struct address_space *mapping, |
290408d4 NH |
104 | struct page *newpage, struct page *page) |
105 | { | |
106 | return -ENOSYS; | |
107 | } | |
108 | ||
b20a3503 | 109 | #endif /* CONFIG_MIGRATION */ |
7039e1db | 110 | |
dd4123f3 MK |
111 | #ifdef CONFIG_COMPACTION |
112 | extern int PageMovable(struct page *page); | |
113 | extern void __SetPageMovable(struct page *page, struct address_space *mapping); | |
114 | extern void __ClearPageMovable(struct page *page); | |
115 | #else | |
116 | static inline int PageMovable(struct page *page) { return 0; }; | |
117 | static inline void __SetPageMovable(struct page *page, | |
118 | struct address_space *mapping) | |
119 | { | |
120 | } | |
121 | static inline void __ClearPageMovable(struct page *page) | |
122 | { | |
123 | } | |
124 | #endif | |
125 | ||
7039e1db | 126 | #ifdef CONFIG_NUMA_BALANCING |
de466bd6 | 127 | extern bool pmd_trans_migrating(pmd_t pmd); |
1bc115d8 MG |
128 | extern int migrate_misplaced_page(struct page *page, |
129 | struct vm_area_struct *vma, int node); | |
7039e1db | 130 | #else |
de466bd6 MG |
131 | static inline bool pmd_trans_migrating(pmd_t pmd) |
132 | { | |
133 | return false; | |
134 | } | |
1bc115d8 MG |
135 | static inline int migrate_misplaced_page(struct page *page, |
136 | struct vm_area_struct *vma, int node) | |
7039e1db PZ |
137 | { |
138 | return -EAGAIN; /* can't migrate now */ | |
139 | } | |
220018d3 | 140 | #endif /* CONFIG_NUMA_BALANCING */ |
b32967ff | 141 | |
220018d3 MG |
142 | #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE) |
143 | extern int migrate_misplaced_transhuge_page(struct mm_struct *mm, | |
144 | struct vm_area_struct *vma, | |
145 | pmd_t *pmd, pmd_t entry, | |
146 | unsigned long address, | |
147 | struct page *page, int node); | |
148 | #else | |
b32967ff MG |
149 | static inline int migrate_misplaced_transhuge_page(struct mm_struct *mm, |
150 | struct vm_area_struct *vma, | |
151 | pmd_t *pmd, pmd_t entry, | |
152 | unsigned long address, | |
153 | struct page *page, int node) | |
154 | { | |
155 | return -EAGAIN; | |
156 | } | |
220018d3 | 157 | #endif /* CONFIG_NUMA_BALANCING && CONFIG_TRANSPARENT_HUGEPAGE*/ |
7039e1db | 158 | |
8763cb45 JG |
159 | |
160 | #ifdef CONFIG_MIGRATION | |
161 | ||
a5430dda JG |
162 | /* |
163 | * Watch out for PAE architecture, which has an unsigned long, and might not | |
164 | * have enough bits to store all physical address and flags. So far we have | |
165 | * enough room for all our flags. | |
166 | */ | |
8763cb45 JG |
167 | #define MIGRATE_PFN_VALID (1UL << 0) |
168 | #define MIGRATE_PFN_MIGRATE (1UL << 1) | |
169 | #define MIGRATE_PFN_LOCKED (1UL << 2) | |
170 | #define MIGRATE_PFN_WRITE (1UL << 3) | |
a5430dda JG |
171 | #define MIGRATE_PFN_DEVICE (1UL << 4) |
172 | #define MIGRATE_PFN_ERROR (1UL << 5) | |
173 | #define MIGRATE_PFN_SHIFT 6 | |
8763cb45 JG |
174 | |
175 | static inline struct page *migrate_pfn_to_page(unsigned long mpfn) | |
176 | { | |
177 | if (!(mpfn & MIGRATE_PFN_VALID)) | |
178 | return NULL; | |
179 | return pfn_to_page(mpfn >> MIGRATE_PFN_SHIFT); | |
180 | } | |
181 | ||
182 | static inline unsigned long migrate_pfn(unsigned long pfn) | |
183 | { | |
184 | return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID; | |
185 | } | |
186 | ||
187 | /* | |
188 | * struct migrate_vma_ops - migrate operation callback | |
189 | * | |
190 | * @alloc_and_copy: alloc destination memory and copy source memory to it | |
191 | * @finalize_and_map: allow caller to map the successfully migrated pages | |
192 | * | |
193 | * | |
194 | * The alloc_and_copy() callback happens once all source pages have been locked, | |
195 | * unmapped and checked (checked whether pinned or not). All pages that can be | |
196 | * migrated will have an entry in the src array set with the pfn value of the | |
197 | * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set (other | |
198 | * flags might be set but should be ignored by the callback). | |
199 | * | |
200 | * The alloc_and_copy() callback can then allocate destination memory and copy | |
201 | * source memory to it for all those entries (ie with MIGRATE_PFN_VALID and | |
202 | * MIGRATE_PFN_MIGRATE flag set). Once these are allocated and copied, the | |
203 | * callback must update each corresponding entry in the dst array with the pfn | |
204 | * value of the destination page and with the MIGRATE_PFN_VALID and | |
205 | * MIGRATE_PFN_LOCKED flags set (destination pages must have their struct pages | |
206 | * locked, via lock_page()). | |
207 | * | |
208 | * At this point the alloc_and_copy() callback is done and returns. | |
209 | * | |
210 | * Note that the callback does not have to migrate all the pages that are | |
211 | * marked with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration | |
212 | * from device memory to system memory (ie the MIGRATE_PFN_DEVICE flag is also | |
213 | * set in the src array entry). If the device driver cannot migrate a device | |
214 | * page back to system memory, then it must set the corresponding dst array | |
215 | * entry to MIGRATE_PFN_ERROR. This will trigger a SIGBUS if CPU tries to | |
216 | * access any of the virtual addresses originally backed by this page. Because | |
217 | * a SIGBUS is such a severe result for the userspace process, the device | |
218 | * driver should avoid setting MIGRATE_PFN_ERROR unless it is really in an | |
219 | * unrecoverable state. | |
220 | * | |
221 | * THE alloc_and_copy() CALLBACK MUST NOT CHANGE ANY OF THE SRC ARRAY ENTRIES | |
222 | * OR BAD THINGS WILL HAPPEN ! | |
223 | * | |
224 | * | |
225 | * The finalize_and_map() callback happens after struct page migration from | |
226 | * source to destination (destination struct pages are the struct pages for the | |
227 | * memory allocated by the alloc_and_copy() callback). Migration can fail, and | |
228 | * thus the finalize_and_map() allows the driver to inspect which pages were | |
229 | * successfully migrated, and which were not. Successfully migrated pages will | |
230 | * have the MIGRATE_PFN_MIGRATE flag set for their src array entry. | |
231 | * | |
232 | * It is safe to update device page table from within the finalize_and_map() | |
233 | * callback because both destination and source page are still locked, and the | |
234 | * mmap_sem is held in read mode (hence no one can unmap the range being | |
235 | * migrated). | |
236 | * | |
237 | * Once callback is done cleaning up things and updating its page table (if it | |
238 | * chose to do so, this is not an obligation) then it returns. At this point, | |
239 | * the HMM core will finish up the final steps, and the migration is complete. | |
240 | * | |
241 | * THE finalize_and_map() CALLBACK MUST NOT CHANGE ANY OF THE SRC OR DST ARRAY | |
242 | * ENTRIES OR BAD THINGS WILL HAPPEN ! | |
243 | */ | |
244 | struct migrate_vma_ops { | |
245 | void (*alloc_and_copy)(struct vm_area_struct *vma, | |
246 | const unsigned long *src, | |
247 | unsigned long *dst, | |
248 | unsigned long start, | |
249 | unsigned long end, | |
250 | void *private); | |
251 | void (*finalize_and_map)(struct vm_area_struct *vma, | |
252 | const unsigned long *src, | |
253 | const unsigned long *dst, | |
254 | unsigned long start, | |
255 | unsigned long end, | |
256 | void *private); | |
257 | }; | |
258 | ||
259 | int migrate_vma(const struct migrate_vma_ops *ops, | |
260 | struct vm_area_struct *vma, | |
261 | unsigned long start, | |
262 | unsigned long end, | |
263 | unsigned long *src, | |
264 | unsigned long *dst, | |
265 | void *private); | |
266 | ||
267 | #endif /* CONFIG_MIGRATION */ | |
268 | ||
b20a3503 | 269 | #endif /* _LINUX_MIGRATE_H */ |