2 #include <linux/highmem.h>
3 #include <linux/kernel.h>
4 #include <linux/mmdebug.h>
5 #include <linux/mm_types.h>
6 #include <linux/mm_inline.h>
7 #include <linux/pagemap.h>
8 #include <linux/rcupdate.h>
10 #include <linux/swap.h>
11 #include <linux/rmap.h>
13 #include <asm/pgalloc.h>
16 #ifndef CONFIG_MMU_GATHER_NO_GATHER
18 static bool tlb_next_batch(struct mmu_gather
*tlb
)
20 struct mmu_gather_batch
*batch
;
22 /* Limit batching if we have delayed rmaps pending */
23 if (tlb
->delayed_rmap
&& tlb
->active
!= &tlb
->local
)
28 tlb
->active
= batch
->next
;
32 if (tlb
->batch_count
== MAX_GATHER_BATCH_COUNT
)
35 batch
= (void *)__get_free_page(GFP_NOWAIT
| __GFP_NOWARN
);
42 batch
->max
= MAX_GATHER_BATCH
;
44 tlb
->active
->next
= batch
;
51 static void tlb_flush_rmap_batch(struct mmu_gather_batch
*batch
, struct vm_area_struct
*vma
)
53 for (int i
= 0; i
< batch
->nr
; i
++) {
54 struct encoded_page
*enc
= batch
->encoded_pages
[i
];
56 if (encoded_page_flags(enc
)) {
57 struct page
*page
= encoded_page_ptr(enc
);
58 page_remove_rmap(page
, vma
, false);
64 * tlb_flush_rmaps - do pending rmap removals after we have flushed the TLB
65 * @tlb: the current mmu_gather
66 * @vma: The memory area from which the pages are being removed.
68 * Note that because of how tlb_next_batch() above works, we will
69 * never start multiple new batches with pending delayed rmaps, so
70 * we only need to walk through the current active batch and the
73 void tlb_flush_rmaps(struct mmu_gather
*tlb
, struct vm_area_struct
*vma
)
75 if (!tlb
->delayed_rmap
)
78 tlb_flush_rmap_batch(&tlb
->local
, vma
);
79 if (tlb
->active
!= &tlb
->local
)
80 tlb_flush_rmap_batch(tlb
->active
, vma
);
81 tlb
->delayed_rmap
= 0;
85 static void tlb_batch_pages_flush(struct mmu_gather
*tlb
)
87 struct mmu_gather_batch
*batch
;
89 for (batch
= &tlb
->local
; batch
&& batch
->nr
; batch
= batch
->next
) {
90 struct encoded_page
**pages
= batch
->encoded_pages
;
94 * limit free batch count when PAGE_SIZE > 4K
96 unsigned int nr
= min(512U, batch
->nr
);
98 free_pages_and_swap_cache(pages
, nr
);
105 tlb
->active
= &tlb
->local
;
108 static void tlb_batch_list_free(struct mmu_gather
*tlb
)
110 struct mmu_gather_batch
*batch
, *next
;
112 for (batch
= tlb
->local
.next
; batch
; batch
= next
) {
114 free_pages((unsigned long)batch
, 0);
116 tlb
->local
.next
= NULL
;
119 bool __tlb_remove_page_size(struct mmu_gather
*tlb
, struct encoded_page
*page
, int page_size
)
121 struct mmu_gather_batch
*batch
;
123 VM_BUG_ON(!tlb
->end
);
125 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
126 VM_WARN_ON(tlb
->page_size
!= page_size
);
131 * Add the page and check if we are full. If so
134 batch
->encoded_pages
[batch
->nr
++] = page
;
135 if (batch
->nr
== batch
->max
) {
136 if (!tlb_next_batch(tlb
))
140 VM_BUG_ON_PAGE(batch
->nr
> batch
->max
, encoded_page_ptr(page
));
145 #endif /* MMU_GATHER_NO_GATHER */
147 #ifdef CONFIG_MMU_GATHER_TABLE_FREE
149 static void __tlb_remove_table_free(struct mmu_table_batch
*batch
)
153 for (i
= 0; i
< batch
->nr
; i
++)
154 __tlb_remove_table(batch
->tables
[i
]);
156 free_page((unsigned long)batch
);
159 #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
162 * Semi RCU freeing of the page directories.
164 * This is needed by some architectures to implement software pagetable walkers.
166 * gup_fast() and other software pagetable walkers do a lockless page-table
167 * walk and therefore needs some synchronization with the freeing of the page
168 * directories. The chosen means to accomplish that is by disabling IRQs over
171 * Architectures that use IPIs to flush TLBs will then automagically DTRT,
172 * since we unlink the page, flush TLBs, free the page. Since the disabling of
173 * IRQs delays the completion of the TLB flush we can never observe an already
176 * Architectures that do not have this (PPC) need to delay the freeing by some
177 * other means, this is that means.
179 * What we do is batch the freed directory pages (tables) and RCU free them.
180 * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
181 * holds off grace periods.
183 * However, in order to batch these pages we need to allocate storage, this
184 * allocation is deep inside the MM code and can thus easily fail on memory
185 * pressure. To guarantee progress we fall back to single table freeing, see
186 * the implementation of tlb_remove_table_one().
190 static void tlb_remove_table_smp_sync(void *arg
)
192 /* Simply deliver the interrupt */
195 void tlb_remove_table_sync_one(void)
198 * This isn't an RCU grace period and hence the page-tables cannot be
199 * assumed to be actually RCU-freed.
201 * It is however sufficient for software page-table walkers that rely on
204 smp_call_function(tlb_remove_table_smp_sync
, NULL
, 1);
207 static void tlb_remove_table_rcu(struct rcu_head
*head
)
209 __tlb_remove_table_free(container_of(head
, struct mmu_table_batch
, rcu
));
212 static void tlb_remove_table_free(struct mmu_table_batch
*batch
)
214 call_rcu(&batch
->rcu
, tlb_remove_table_rcu
);
217 #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
219 static void tlb_remove_table_free(struct mmu_table_batch
*batch
)
221 __tlb_remove_table_free(batch
);
224 #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
227 * If we want tlb_remove_table() to imply TLB invalidates.
229 static inline void tlb_table_invalidate(struct mmu_gather
*tlb
)
231 if (tlb_needs_table_invalidate()) {
233 * Invalidate page-table caches used by hardware walkers. Then
234 * we still need to RCU-sched wait while freeing the pages
235 * because software walkers can still be in-flight.
237 tlb_flush_mmu_tlbonly(tlb
);
241 static void tlb_remove_table_one(void *table
)
243 tlb_remove_table_sync_one();
244 __tlb_remove_table(table
);
247 static void tlb_table_flush(struct mmu_gather
*tlb
)
249 struct mmu_table_batch
**batch
= &tlb
->batch
;
252 tlb_table_invalidate(tlb
);
253 tlb_remove_table_free(*batch
);
258 void tlb_remove_table(struct mmu_gather
*tlb
, void *table
)
260 struct mmu_table_batch
**batch
= &tlb
->batch
;
262 if (*batch
== NULL
) {
263 *batch
= (struct mmu_table_batch
*)__get_free_page(GFP_NOWAIT
| __GFP_NOWARN
);
264 if (*batch
== NULL
) {
265 tlb_table_invalidate(tlb
);
266 tlb_remove_table_one(table
);
272 (*batch
)->tables
[(*batch
)->nr
++] = table
;
273 if ((*batch
)->nr
== MAX_TABLE_BATCH
)
274 tlb_table_flush(tlb
);
277 static inline void tlb_table_init(struct mmu_gather
*tlb
)
282 #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
284 static inline void tlb_table_flush(struct mmu_gather
*tlb
) { }
285 static inline void tlb_table_init(struct mmu_gather
*tlb
) { }
287 #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
289 static void tlb_flush_mmu_free(struct mmu_gather
*tlb
)
291 tlb_table_flush(tlb
);
292 #ifndef CONFIG_MMU_GATHER_NO_GATHER
293 tlb_batch_pages_flush(tlb
);
297 void tlb_flush_mmu(struct mmu_gather
*tlb
)
299 tlb_flush_mmu_tlbonly(tlb
);
300 tlb_flush_mmu_free(tlb
);
303 static void __tlb_gather_mmu(struct mmu_gather
*tlb
, struct mm_struct
*mm
,
307 tlb
->fullmm
= fullmm
;
309 #ifndef CONFIG_MMU_GATHER_NO_GATHER
310 tlb
->need_flush_all
= 0;
311 tlb
->local
.next
= NULL
;
313 tlb
->local
.max
= ARRAY_SIZE(tlb
->__pages
);
314 tlb
->active
= &tlb
->local
;
315 tlb
->batch_count
= 0;
317 tlb
->delayed_rmap
= 0;
320 #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
324 __tlb_reset_range(tlb
);
325 inc_tlb_flush_pending(tlb
->mm
);
329 * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
330 * @tlb: the mmu_gather structure to initialize
331 * @mm: the mm_struct of the target address space
333 * Called to initialize an (on-stack) mmu_gather structure for page-table
334 * tear-down from @mm.
336 void tlb_gather_mmu(struct mmu_gather
*tlb
, struct mm_struct
*mm
)
338 __tlb_gather_mmu(tlb
, mm
, false);
342 * tlb_gather_mmu_fullmm - initialize an mmu_gather structure for page-table tear-down
343 * @tlb: the mmu_gather structure to initialize
344 * @mm: the mm_struct of the target address space
346 * In this case, @mm is without users and we're going to destroy the
347 * full address space (exit/execve).
349 * Called to initialize an (on-stack) mmu_gather structure for page-table
350 * tear-down from @mm.
352 void tlb_gather_mmu_fullmm(struct mmu_gather
*tlb
, struct mm_struct
*mm
)
354 __tlb_gather_mmu(tlb
, mm
, true);
358 * tlb_finish_mmu - finish an mmu_gather structure
359 * @tlb: the mmu_gather structure to finish
361 * Called at the end of the shootdown operation to free up any resources that
364 void tlb_finish_mmu(struct mmu_gather
*tlb
)
367 * If there are parallel threads are doing PTE changes on same range
368 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
369 * flush by batching, one thread may end up seeing inconsistent PTEs
370 * and result in having stale TLB entries. So flush TLB forcefully
371 * if we detect parallel PTE batching threads.
373 * However, some syscalls, e.g. munmap(), may free page tables, this
374 * needs force flush everything in the given range. Otherwise this
375 * may result in having stale TLB entries for some architectures,
376 * e.g. aarch64, that could specify flush what level TLB.
378 if (mm_tlb_flush_nested(tlb
->mm
)) {
380 * The aarch64 yields better performance with fullmm by
381 * avoiding multiple CPUs spamming TLBI messages at the
384 * On x86 non-fullmm doesn't yield significant difference
388 __tlb_reset_range(tlb
);
389 tlb
->freed_tables
= 1;
394 #ifndef CONFIG_MMU_GATHER_NO_GATHER
395 tlb_batch_list_free(tlb
);
397 dec_tlb_flush_pending(tlb
->mm
);