]>
git.ipfire.org Git - thirdparty/linux.git/blob - mm/page_io.c
1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
7 * Swap reorganised 29.12.95,
8 * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 * Removed race in async swapping. 14.4.1996. Bruno Haible
10 * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
15 #include <linux/kernel_stat.h>
16 #include <linux/gfp.h>
17 #include <linux/pagemap.h>
18 #include <linux/swap.h>
19 #include <linux/bio.h>
20 #include <linux/swapops.h>
21 #include <linux/writeback.h>
22 #include <linux/frontswap.h>
23 #include <linux/blkdev.h>
24 #include <linux/psi.h>
25 #include <linux/uio.h>
26 #include <linux/sched/task.h>
27 #include <linux/delayacct.h>
30 static void __end_swap_bio_write(struct bio
*bio
)
32 struct page
*page
= bio_first_page_all(bio
);
37 * We failed to write the page out to swap-space.
38 * Re-dirty the page in order to avoid it being reclaimed.
39 * Also print a dire warning that things will go BAD (tm)
42 * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
45 pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
46 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
47 (unsigned long long)bio
->bi_iter
.bi_sector
);
48 ClearPageReclaim(page
);
50 end_page_writeback(page
);
53 static void end_swap_bio_write(struct bio
*bio
)
55 __end_swap_bio_write(bio
);
59 static void __end_swap_bio_read(struct bio
*bio
)
61 struct page
*page
= bio_first_page_all(bio
);
65 ClearPageUptodate(page
);
66 pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
67 MAJOR(bio_dev(bio
)), MINOR(bio_dev(bio
)),
68 (unsigned long long)bio
->bi_iter
.bi_sector
);
70 SetPageUptodate(page
);
75 static void end_swap_bio_read(struct bio
*bio
)
77 __end_swap_bio_read(bio
);
81 int generic_swapfile_activate(struct swap_info_struct
*sis
,
82 struct file
*swap_file
,
85 struct address_space
*mapping
= swap_file
->f_mapping
;
86 struct inode
*inode
= mapping
->host
;
87 unsigned blocks_per_page
;
88 unsigned long page_no
;
92 sector_t lowest_block
= -1;
93 sector_t highest_block
= 0;
97 blkbits
= inode
->i_blkbits
;
98 blocks_per_page
= PAGE_SIZE
>> blkbits
;
101 * Map all the blocks into the extent tree. This code doesn't try
106 last_block
= i_size_read(inode
) >> blkbits
;
107 while ((probe_block
+ blocks_per_page
) <= last_block
&&
108 page_no
< sis
->max
) {
109 unsigned block_in_page
;
110 sector_t first_block
;
114 first_block
= probe_block
;
115 ret
= bmap(inode
, &first_block
);
116 if (ret
|| !first_block
)
120 * It must be PAGE_SIZE aligned on-disk
122 if (first_block
& (blocks_per_page
- 1)) {
127 for (block_in_page
= 1; block_in_page
< blocks_per_page
;
131 block
= probe_block
+ block_in_page
;
132 ret
= bmap(inode
, &block
);
136 if (block
!= first_block
+ block_in_page
) {
143 first_block
>>= (PAGE_SHIFT
- blkbits
);
144 if (page_no
) { /* exclude the header page */
145 if (first_block
< lowest_block
)
146 lowest_block
= first_block
;
147 if (first_block
> highest_block
)
148 highest_block
= first_block
;
152 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
154 ret
= add_swap_extent(sis
, page_no
, 1, first_block
);
159 probe_block
+= blocks_per_page
;
164 *span
= 1 + highest_block
- lowest_block
;
166 page_no
= 1; /* force Empty message */
168 sis
->pages
= page_no
- 1;
169 sis
->highest_bit
= page_no
- 1;
173 pr_err("swapon: swapfile has holes\n");
179 * We may have stale swap cache pages in memory: notice
180 * them here and get rid of the unnecessary final write.
182 int swap_writepage(struct page
*page
, struct writeback_control
*wbc
)
184 struct folio
*folio
= page_folio(page
);
187 if (folio_free_swap(folio
)) {
192 * Arch code may have to preserve more data than just the page
193 * contents, e.g. memory tags.
195 ret
= arch_prepare_to_swap(&folio
->page
);
197 folio_mark_dirty(folio
);
201 if (frontswap_store(&folio
->page
) == 0) {
202 folio_start_writeback(folio
);
204 folio_end_writeback(folio
);
207 __swap_writepage(&folio
->page
, wbc
);
211 static inline void count_swpout_vm_event(struct page
*page
)
213 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
214 if (unlikely(PageTransHuge(page
)))
215 count_vm_event(THP_SWPOUT
);
217 count_vm_events(PSWPOUT
, thp_nr_pages(page
));
220 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
221 static void bio_associate_blkg_from_page(struct bio
*bio
, struct page
*page
)
223 struct cgroup_subsys_state
*css
;
224 struct mem_cgroup
*memcg
;
226 memcg
= page_memcg(page
);
231 css
= cgroup_e_css(memcg
->css
.cgroup
, &io_cgrp_subsys
);
232 bio_associate_blkg_from_css(bio
, css
);
236 #define bio_associate_blkg_from_page(bio, page) do { } while (0)
237 #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
241 struct bio_vec bvec
[SWAP_CLUSTER_MAX
];
245 static mempool_t
*sio_pool
;
247 int sio_pool_init(void)
250 mempool_t
*pool
= mempool_create_kmalloc_pool(
251 SWAP_CLUSTER_MAX
, sizeof(struct swap_iocb
));
252 if (cmpxchg(&sio_pool
, NULL
, pool
))
253 mempool_destroy(pool
);
260 static void sio_write_complete(struct kiocb
*iocb
, long ret
)
262 struct swap_iocb
*sio
= container_of(iocb
, struct swap_iocb
, iocb
);
263 struct page
*page
= sio
->bvec
[0].bv_page
;
266 if (ret
!= sio
->len
) {
268 * In the case of swap-over-nfs, this can be a
269 * temporary failure if the system has limited
270 * memory for allocating transmit buffers.
271 * Mark the page dirty and avoid
272 * folio_rotate_reclaimable but rate-limit the
273 * messages but do not flag PageError like
274 * the normal direct-to-bio case as it could
277 pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
278 ret
, page_file_offset(page
));
279 for (p
= 0; p
< sio
->pages
; p
++) {
280 page
= sio
->bvec
[p
].bv_page
;
281 set_page_dirty(page
);
282 ClearPageReclaim(page
);
285 for (p
= 0; p
< sio
->pages
; p
++)
286 count_swpout_vm_event(sio
->bvec
[p
].bv_page
);
289 for (p
= 0; p
< sio
->pages
; p
++)
290 end_page_writeback(sio
->bvec
[p
].bv_page
);
292 mempool_free(sio
, sio_pool
);
295 static void swap_writepage_fs(struct page
*page
, struct writeback_control
*wbc
)
297 struct swap_iocb
*sio
= NULL
;
298 struct swap_info_struct
*sis
= page_swap_info(page
);
299 struct file
*swap_file
= sis
->swap_file
;
300 loff_t pos
= page_file_offset(page
);
302 set_page_writeback(page
);
305 sio
= *wbc
->swap_plug
;
307 if (sio
->iocb
.ki_filp
!= swap_file
||
308 sio
->iocb
.ki_pos
+ sio
->len
!= pos
) {
309 swap_write_unplug(sio
);
314 sio
= mempool_alloc(sio_pool
, GFP_NOIO
);
315 init_sync_kiocb(&sio
->iocb
, swap_file
);
316 sio
->iocb
.ki_complete
= sio_write_complete
;
317 sio
->iocb
.ki_pos
= pos
;
321 bvec_set_page(&sio
->bvec
[sio
->pages
], page
, thp_size(page
), 0);
322 sio
->len
+= thp_size(page
);
324 if (sio
->pages
== ARRAY_SIZE(sio
->bvec
) || !wbc
->swap_plug
) {
325 swap_write_unplug(sio
);
329 *wbc
->swap_plug
= sio
;
332 static void swap_writepage_bdev_sync(struct page
*page
,
333 struct writeback_control
*wbc
, struct swap_info_struct
*sis
)
338 bio_init(&bio
, sis
->bdev
, &bv
, 1,
339 REQ_OP_WRITE
| REQ_SWAP
| wbc_to_write_flags(wbc
));
340 bio
.bi_iter
.bi_sector
= swap_page_sector(page
);
341 bio_add_page(&bio
, page
, thp_size(page
), 0);
343 bio_associate_blkg_from_page(&bio
, page
);
344 count_swpout_vm_event(page
);
346 set_page_writeback(page
);
349 submit_bio_wait(&bio
);
350 __end_swap_bio_write(&bio
);
353 static void swap_writepage_bdev_async(struct page
*page
,
354 struct writeback_control
*wbc
, struct swap_info_struct
*sis
)
358 bio
= bio_alloc(sis
->bdev
, 1,
359 REQ_OP_WRITE
| REQ_SWAP
| wbc_to_write_flags(wbc
),
361 bio
->bi_iter
.bi_sector
= swap_page_sector(page
);
362 bio
->bi_end_io
= end_swap_bio_write
;
363 bio_add_page(bio
, page
, thp_size(page
), 0);
365 bio_associate_blkg_from_page(bio
, page
);
366 count_swpout_vm_event(page
);
367 set_page_writeback(page
);
372 void __swap_writepage(struct page
*page
, struct writeback_control
*wbc
)
374 struct swap_info_struct
*sis
= page_swap_info(page
);
376 VM_BUG_ON_PAGE(!PageSwapCache(page
), page
);
378 * ->flags can be updated non-atomicially (scan_swap_map_slots),
379 * but that will never affect SWP_FS_OPS, so the data_race
382 if (data_race(sis
->flags
& SWP_FS_OPS
))
383 swap_writepage_fs(page
, wbc
);
384 else if (sis
->flags
& SWP_SYNCHRONOUS_IO
)
385 swap_writepage_bdev_sync(page
, wbc
, sis
);
387 swap_writepage_bdev_async(page
, wbc
, sis
);
390 void swap_write_unplug(struct swap_iocb
*sio
)
392 struct iov_iter from
;
393 struct address_space
*mapping
= sio
->iocb
.ki_filp
->f_mapping
;
396 iov_iter_bvec(&from
, ITER_SOURCE
, sio
->bvec
, sio
->pages
, sio
->len
);
397 ret
= mapping
->a_ops
->swap_rw(&sio
->iocb
, &from
);
398 if (ret
!= -EIOCBQUEUED
)
399 sio_write_complete(&sio
->iocb
, ret
);
402 static void sio_read_complete(struct kiocb
*iocb
, long ret
)
404 struct swap_iocb
*sio
= container_of(iocb
, struct swap_iocb
, iocb
);
407 if (ret
== sio
->len
) {
408 for (p
= 0; p
< sio
->pages
; p
++) {
409 struct page
*page
= sio
->bvec
[p
].bv_page
;
411 SetPageUptodate(page
);
414 count_vm_events(PSWPIN
, sio
->pages
);
416 for (p
= 0; p
< sio
->pages
; p
++) {
417 struct page
*page
= sio
->bvec
[p
].bv_page
;
420 ClearPageUptodate(page
);
423 pr_alert_ratelimited("Read-error on swap-device\n");
425 mempool_free(sio
, sio_pool
);
428 static void swap_readpage_fs(struct page
*page
,
429 struct swap_iocb
**plug
)
431 struct swap_info_struct
*sis
= page_swap_info(page
);
432 struct swap_iocb
*sio
= NULL
;
433 loff_t pos
= page_file_offset(page
);
438 if (sio
->iocb
.ki_filp
!= sis
->swap_file
||
439 sio
->iocb
.ki_pos
+ sio
->len
!= pos
) {
440 swap_read_unplug(sio
);
445 sio
= mempool_alloc(sio_pool
, GFP_KERNEL
);
446 init_sync_kiocb(&sio
->iocb
, sis
->swap_file
);
447 sio
->iocb
.ki_pos
= pos
;
448 sio
->iocb
.ki_complete
= sio_read_complete
;
452 bvec_set_page(&sio
->bvec
[sio
->pages
], page
, thp_size(page
), 0);
453 sio
->len
+= thp_size(page
);
455 if (sio
->pages
== ARRAY_SIZE(sio
->bvec
) || !plug
) {
456 swap_read_unplug(sio
);
463 static void swap_readpage_bdev_sync(struct page
*page
,
464 struct swap_info_struct
*sis
)
469 bio_init(&bio
, sis
->bdev
, &bv
, 1, REQ_OP_READ
);
470 bio
.bi_iter
.bi_sector
= swap_page_sector(page
);
471 bio_add_page(&bio
, page
, thp_size(page
), 0);
473 * Keep this task valid during swap readpage because the oom killer may
474 * attempt to access it in the page fault retry time check.
476 get_task_struct(current
);
477 count_vm_event(PSWPIN
);
478 submit_bio_wait(&bio
);
479 __end_swap_bio_read(&bio
);
480 put_task_struct(current
);
483 static void swap_readpage_bdev_async(struct page
*page
,
484 struct swap_info_struct
*sis
)
488 bio
= bio_alloc(sis
->bdev
, 1, REQ_OP_READ
, GFP_KERNEL
);
489 bio
->bi_iter
.bi_sector
= swap_page_sector(page
);
490 bio
->bi_end_io
= end_swap_bio_read
;
491 bio_add_page(bio
, page
, thp_size(page
), 0);
492 count_vm_event(PSWPIN
);
496 void swap_readpage(struct page
*page
, bool synchronous
, struct swap_iocb
**plug
)
498 struct swap_info_struct
*sis
= page_swap_info(page
);
499 bool workingset
= PageWorkingset(page
);
500 unsigned long pflags
;
503 VM_BUG_ON_PAGE(!PageSwapCache(page
) && !synchronous
, page
);
504 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
505 VM_BUG_ON_PAGE(PageUptodate(page
), page
);
508 * Count submission time as memory stall and delay. When the device
509 * is congested, or the submitting cgroup IO-throttled, submission
510 * can be a significant part of overall IO time.
513 delayacct_thrashing_start(&in_thrashing
);
514 psi_memstall_enter(&pflags
);
516 delayacct_swapin_start();
518 if (frontswap_load(page
) == 0) {
519 SetPageUptodate(page
);
521 } else if (data_race(sis
->flags
& SWP_FS_OPS
)) {
522 swap_readpage_fs(page
, plug
);
523 } else if (synchronous
|| (sis
->flags
& SWP_SYNCHRONOUS_IO
)) {
524 swap_readpage_bdev_sync(page
, sis
);
526 swap_readpage_bdev_async(page
, sis
);
530 delayacct_thrashing_end(&in_thrashing
);
531 psi_memstall_leave(&pflags
);
533 delayacct_swapin_end();
536 void __swap_read_unplug(struct swap_iocb
*sio
)
538 struct iov_iter from
;
539 struct address_space
*mapping
= sio
->iocb
.ki_filp
->f_mapping
;
542 iov_iter_bvec(&from
, ITER_DEST
, sio
->bvec
, sio
->pages
, sio
->len
);
543 ret
= mapping
->a_ops
->swap_rw(&sio
->iocb
, &from
);
544 if (ret
!= -EIOCBQUEUED
)
545 sio_read_complete(&sio
->iocb
, ret
);