]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * linux/mm/page_io.c | |
4 | * | |
5 | * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds | |
6 | * | |
7 | * Swap reorganised 29.12.95, | |
8 | * Asynchronous swapping added 30.12.95. Stephen Tweedie | |
9 | * Removed race in async swapping. 14.4.1996. Bruno Haible | |
10 | * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie | |
11 | * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman | |
12 | */ | |
13 | ||
14 | #include <linux/mm.h> | |
15 | #include <linux/kernel_stat.h> | |
5a0e3ad6 | 16 | #include <linux/gfp.h> |
1da177e4 LT |
17 | #include <linux/pagemap.h> |
18 | #include <linux/swap.h> | |
19 | #include <linux/bio.h> | |
20 | #include <linux/swapops.h> | |
21 | #include <linux/writeback.h> | |
b430e9d1 | 22 | #include <linux/blkdev.h> |
93779069 | 23 | #include <linux/psi.h> |
e2e40f2c | 24 | #include <linux/uio.h> |
b0ba2d0f | 25 | #include <linux/sched/task.h> |
a3d5dc90 | 26 | #include <linux/delayacct.h> |
42c06a0e | 27 | #include <linux/zswap.h> |
014bb1de | 28 | #include "swap.h" |
1da177e4 | 29 | |
3222d8c2 | 30 | static void __end_swap_bio_write(struct bio *bio) |
1da177e4 | 31 | { |
a3ed1e9b | 32 | struct folio *folio = bio_first_folio_all(bio); |
1da177e4 | 33 | |
4e4cbee9 | 34 | if (bio->bi_status) { |
6ddab3b9 PZ |
35 | /* |
36 | * We failed to write the page out to swap-space. | |
37 | * Re-dirty the page in order to avoid it being reclaimed. | |
38 | * Also print a dire warning that things will go BAD (tm) | |
39 | * very quickly. | |
40 | * | |
575ced1c | 41 | * Also clear PG_reclaim to avoid folio_rotate_reclaimable() |
6ddab3b9 | 42 | */ |
a3ed1e9b | 43 | folio_mark_dirty(folio); |
25eaab43 GD |
44 | pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", |
45 | MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), | |
46 | (unsigned long long)bio->bi_iter.bi_sector); | |
a3ed1e9b | 47 | folio_clear_reclaim(folio); |
6ddab3b9 | 48 | } |
a3ed1e9b | 49 | folio_end_writeback(folio); |
3222d8c2 CH |
50 | } |
51 | ||
52 | static void end_swap_bio_write(struct bio *bio) | |
53 | { | |
54 | __end_swap_bio_write(bio); | |
1da177e4 | 55 | bio_put(bio); |
1da177e4 LT |
56 | } |
57 | ||
9b4e30bd | 58 | static void __end_swap_bio_read(struct bio *bio) |
1da177e4 | 59 | { |
bc74b53f | 60 | struct folio *folio = bio_first_folio_all(bio); |
1da177e4 | 61 | |
4e4cbee9 | 62 | if (bio->bi_status) { |
25eaab43 GD |
63 | pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", |
64 | MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), | |
65 | (unsigned long long)bio->bi_iter.bi_sector); | |
9b4e30bd | 66 | } else { |
bc74b53f | 67 | folio_mark_uptodate(folio); |
1da177e4 | 68 | } |
bc74b53f | 69 | folio_unlock(folio); |
9b4e30bd CH |
70 | } |
71 | ||
72 | static void end_swap_bio_read(struct bio *bio) | |
73 | { | |
74 | __end_swap_bio_read(bio); | |
1da177e4 | 75 | bio_put(bio); |
1da177e4 LT |
76 | } |
77 | ||
a509bc1a MG |
78 | int generic_swapfile_activate(struct swap_info_struct *sis, |
79 | struct file *swap_file, | |
80 | sector_t *span) | |
81 | { | |
82 | struct address_space *mapping = swap_file->f_mapping; | |
83 | struct inode *inode = mapping->host; | |
84 | unsigned blocks_per_page; | |
85 | unsigned long page_no; | |
86 | unsigned blkbits; | |
87 | sector_t probe_block; | |
88 | sector_t last_block; | |
89 | sector_t lowest_block = -1; | |
90 | sector_t highest_block = 0; | |
91 | int nr_extents = 0; | |
92 | int ret; | |
93 | ||
94 | blkbits = inode->i_blkbits; | |
95 | blocks_per_page = PAGE_SIZE >> blkbits; | |
96 | ||
97 | /* | |
4efaceb1 | 98 | * Map all the blocks into the extent tree. This code doesn't try |
a509bc1a MG |
99 | * to be very smart. |
100 | */ | |
101 | probe_block = 0; | |
102 | page_no = 0; | |
103 | last_block = i_size_read(inode) >> blkbits; | |
104 | while ((probe_block + blocks_per_page) <= last_block && | |
105 | page_no < sis->max) { | |
106 | unsigned block_in_page; | |
107 | sector_t first_block; | |
108 | ||
7e4411bf MP |
109 | cond_resched(); |
110 | ||
30460e1e CM |
111 | first_block = probe_block; |
112 | ret = bmap(inode, &first_block); | |
113 | if (ret || !first_block) | |
a509bc1a MG |
114 | goto bad_bmap; |
115 | ||
116 | /* | |
117 | * It must be PAGE_SIZE aligned on-disk | |
118 | */ | |
119 | if (first_block & (blocks_per_page - 1)) { | |
120 | probe_block++; | |
121 | goto reprobe; | |
122 | } | |
123 | ||
124 | for (block_in_page = 1; block_in_page < blocks_per_page; | |
125 | block_in_page++) { | |
126 | sector_t block; | |
127 | ||
30460e1e CM |
128 | block = probe_block + block_in_page; |
129 | ret = bmap(inode, &block); | |
130 | if (ret || !block) | |
a509bc1a | 131 | goto bad_bmap; |
30460e1e | 132 | |
a509bc1a MG |
133 | if (block != first_block + block_in_page) { |
134 | /* Discontiguity */ | |
135 | probe_block++; | |
136 | goto reprobe; | |
137 | } | |
138 | } | |
139 | ||
140 | first_block >>= (PAGE_SHIFT - blkbits); | |
141 | if (page_no) { /* exclude the header page */ | |
142 | if (first_block < lowest_block) | |
143 | lowest_block = first_block; | |
144 | if (first_block > highest_block) | |
145 | highest_block = first_block; | |
146 | } | |
147 | ||
148 | /* | |
149 | * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks | |
150 | */ | |
151 | ret = add_swap_extent(sis, page_no, 1, first_block); | |
152 | if (ret < 0) | |
153 | goto out; | |
154 | nr_extents += ret; | |
155 | page_no++; | |
156 | probe_block += blocks_per_page; | |
157 | reprobe: | |
158 | continue; | |
159 | } | |
160 | ret = nr_extents; | |
161 | *span = 1 + highest_block - lowest_block; | |
162 | if (page_no == 0) | |
163 | page_no = 1; /* force Empty message */ | |
164 | sis->max = page_no; | |
165 | sis->pages = page_no - 1; | |
166 | sis->highest_bit = page_no - 1; | |
167 | out: | |
168 | return ret; | |
169 | bad_bmap: | |
1170532b | 170 | pr_err("swapon: swapfile has holes\n"); |
a509bc1a MG |
171 | ret = -EINVAL; |
172 | goto out; | |
173 | } | |
174 | ||
1da177e4 LT |
175 | /* |
176 | * We may have stale swap cache pages in memory: notice | |
177 | * them here and get rid of the unnecessary final write. | |
178 | */ | |
179 | int swap_writepage(struct page *page, struct writeback_control *wbc) | |
180 | { | |
71fa1a53 | 181 | struct folio *folio = page_folio(page); |
e3e2762b | 182 | int ret; |
1da177e4 | 183 | |
71fa1a53 MWO |
184 | if (folio_free_swap(folio)) { |
185 | folio_unlock(folio); | |
e3e2762b | 186 | return 0; |
1da177e4 | 187 | } |
8a84802e SP |
188 | /* |
189 | * Arch code may have to preserve more data than just the page | |
190 | * contents, e.g. memory tags. | |
191 | */ | |
f238b8c3 | 192 | ret = arch_prepare_to_swap(folio); |
8a84802e | 193 | if (ret) { |
71fa1a53 MWO |
194 | folio_mark_dirty(folio); |
195 | folio_unlock(folio); | |
e3e2762b | 196 | return ret; |
8a84802e | 197 | } |
34f4c198 | 198 | if (zswap_store(folio)) { |
71fa1a53 MWO |
199 | folio_start_writeback(folio); |
200 | folio_unlock(folio); | |
201 | folio_end_writeback(folio); | |
e3e2762b | 202 | return 0; |
38b5faf4 | 203 | } |
501a06fe NP |
204 | if (!mem_cgroup_zswap_writeback_enabled(folio_memcg(folio))) { |
205 | folio_mark_dirty(folio); | |
206 | return AOP_WRITEPAGE_ACTIVATE; | |
207 | } | |
208 | ||
b99b4e0d | 209 | __swap_writepage(folio, wbc); |
e3e2762b | 210 | return 0; |
2f772e6c SJ |
211 | } |
212 | ||
9b72b134 | 213 | static inline void count_swpout_vm_event(struct folio *folio) |
225311a4 HY |
214 | { |
215 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
811244a5 XH |
216 | if (unlikely(folio_test_pmd_mappable(folio))) { |
217 | count_memcg_folio_events(folio, THP_SWPOUT, 1); | |
225311a4 | 218 | count_vm_event(THP_SWPOUT); |
811244a5 | 219 | } |
d0f048ac | 220 | count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_SWPOUT); |
225311a4 | 221 | #endif |
9b72b134 | 222 | count_vm_events(PSWPOUT, folio_nr_pages(folio)); |
225311a4 HY |
223 | } |
224 | ||
a18b9b15 | 225 | #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) |
98630cfd | 226 | static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) |
a18b9b15 CH |
227 | { |
228 | struct cgroup_subsys_state *css; | |
bcfe06bf | 229 | struct mem_cgroup *memcg; |
a18b9b15 | 230 | |
98630cfd | 231 | memcg = folio_memcg(folio); |
bcfe06bf | 232 | if (!memcg) |
a18b9b15 CH |
233 | return; |
234 | ||
235 | rcu_read_lock(); | |
bcfe06bf | 236 | css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); |
a18b9b15 CH |
237 | bio_associate_blkg_from_css(bio, css); |
238 | rcu_read_unlock(); | |
239 | } | |
240 | #else | |
98630cfd | 241 | #define bio_associate_blkg_from_page(bio, folio) do { } while (0) |
a18b9b15 CH |
242 | #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ |
243 | ||
e1209d3a N |
244 | struct swap_iocb { |
245 | struct kiocb iocb; | |
5169b844 N |
246 | struct bio_vec bvec[SWAP_CLUSTER_MAX]; |
247 | int pages; | |
a1a0dfd5 | 248 | int len; |
e1209d3a N |
249 | }; |
250 | static mempool_t *sio_pool; | |
251 | ||
252 | int sio_pool_init(void) | |
2f772e6c | 253 | { |
e1209d3a N |
254 | if (!sio_pool) { |
255 | mempool_t *pool = mempool_create_kmalloc_pool( | |
256 | SWAP_CLUSTER_MAX, sizeof(struct swap_iocb)); | |
257 | if (cmpxchg(&sio_pool, NULL, pool)) | |
258 | mempool_destroy(pool); | |
259 | } | |
260 | if (!sio_pool) | |
261 | return -ENOMEM; | |
262 | return 0; | |
263 | } | |
62c230bc | 264 | |
7eadabc0 N |
265 | static void sio_write_complete(struct kiocb *iocb, long ret) |
266 | { | |
267 | struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); | |
5169b844 | 268 | struct page *page = sio->bvec[0].bv_page; |
2282679f | 269 | int p; |
62c230bc | 270 | |
a1a0dfd5 | 271 | if (ret != sio->len) { |
7eadabc0 N |
272 | /* |
273 | * In the case of swap-over-nfs, this can be a | |
274 | * temporary failure if the system has limited | |
275 | * memory for allocating transmit buffers. | |
276 | * Mark the page dirty and avoid | |
277 | * folio_rotate_reclaimable but rate-limit the | |
278 | * messages but do not flag PageError like | |
279 | * the normal direct-to-bio case as it could | |
280 | * be temporary. | |
281 | */ | |
7eadabc0 N |
282 | pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n", |
283 | ret, page_file_offset(page)); | |
2282679f N |
284 | for (p = 0; p < sio->pages; p++) { |
285 | page = sio->bvec[p].bv_page; | |
2d30d31e | 286 | set_page_dirty(page); |
0cdc444a | 287 | ClearPageReclaim(page); |
62c230bc | 288 | } |
62c230bc MG |
289 | } |
290 | ||
2282679f N |
291 | for (p = 0; p < sio->pages; p++) |
292 | end_page_writeback(sio->bvec[p].bv_page); | |
293 | ||
7eadabc0 N |
294 | mempool_free(sio, sio_pool); |
295 | } | |
296 | ||
bfcd44d5 | 297 | static void swap_writepage_fs(struct folio *folio, struct writeback_control *wbc) |
7eadabc0 | 298 | { |
2282679f | 299 | struct swap_iocb *sio = NULL; |
bfcd44d5 | 300 | struct swap_info_struct *sis = swp_swap_info(folio->swap); |
7eadabc0 | 301 | struct file *swap_file = sis->swap_file; |
bfcd44d5 | 302 | loff_t pos = folio_file_pos(folio); |
7eadabc0 | 303 | |
bfcd44d5 MWO |
304 | count_swpout_vm_event(folio); |
305 | folio_start_writeback(folio); | |
306 | folio_unlock(folio); | |
2282679f N |
307 | if (wbc->swap_plug) |
308 | sio = *wbc->swap_plug; | |
309 | if (sio) { | |
310 | if (sio->iocb.ki_filp != swap_file || | |
a1a0dfd5 | 311 | sio->iocb.ki_pos + sio->len != pos) { |
2282679f N |
312 | swap_write_unplug(sio); |
313 | sio = NULL; | |
314 | } | |
315 | } | |
316 | if (!sio) { | |
317 | sio = mempool_alloc(sio_pool, GFP_NOIO); | |
318 | init_sync_kiocb(&sio->iocb, swap_file); | |
319 | sio->iocb.ki_complete = sio_write_complete; | |
320 | sio->iocb.ki_pos = pos; | |
321 | sio->pages = 0; | |
a1a0dfd5 | 322 | sio->len = 0; |
2282679f | 323 | } |
bfcd44d5 MWO |
324 | bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0); |
325 | sio->len += folio_size(folio); | |
2282679f N |
326 | sio->pages += 1; |
327 | if (sio->pages == ARRAY_SIZE(sio->bvec) || !wbc->swap_plug) { | |
328 | swap_write_unplug(sio); | |
329 | sio = NULL; | |
330 | } | |
331 | if (wbc->swap_plug) | |
332 | *wbc->swap_plug = sio; | |
7eadabc0 N |
333 | } |
334 | ||
6de62c7b | 335 | static void swap_writepage_bdev_sync(struct folio *folio, |
05cda97e | 336 | struct writeback_control *wbc, struct swap_info_struct *sis) |
2f772e6c | 337 | { |
3222d8c2 CH |
338 | struct bio_vec bv; |
339 | struct bio bio; | |
62c230bc | 340 | |
3222d8c2 CH |
341 | bio_init(&bio, sis->bdev, &bv, 1, |
342 | REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc)); | |
3a61e6f6 | 343 | bio.bi_iter.bi_sector = swap_folio_sector(folio); |
6de62c7b | 344 | bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); |
62c230bc | 345 | |
98630cfd | 346 | bio_associate_blkg_from_page(&bio, folio); |
9b72b134 | 347 | count_swpout_vm_event(folio); |
3222d8c2 | 348 | |
f54fcaab Z |
349 | folio_start_writeback(folio); |
350 | folio_unlock(folio); | |
3222d8c2 CH |
351 | |
352 | submit_bio_wait(&bio); | |
353 | __end_swap_bio_write(&bio); | |
354 | } | |
355 | ||
ee1b1d9b | 356 | static void swap_writepage_bdev_async(struct folio *folio, |
3222d8c2 CH |
357 | struct writeback_control *wbc, struct swap_info_struct *sis) |
358 | { | |
359 | struct bio *bio; | |
dd6bd0d9 | 360 | |
07888c66 CH |
361 | bio = bio_alloc(sis->bdev, 1, |
362 | REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc), | |
363 | GFP_NOIO); | |
3a61e6f6 | 364 | bio->bi_iter.bi_sector = swap_folio_sector(folio); |
cf1e3fe4 | 365 | bio->bi_end_io = end_swap_bio_write; |
ee1b1d9b | 366 | bio_add_folio_nofail(bio, folio, folio_size(folio), 0); |
48d15436 | 367 | |
98630cfd | 368 | bio_associate_blkg_from_page(bio, folio); |
9b72b134 | 369 | count_swpout_vm_event(folio); |
2675251d Z |
370 | folio_start_writeback(folio); |
371 | folio_unlock(folio); | |
4e49ea4a | 372 | submit_bio(bio); |
1da177e4 | 373 | } |
548d9782 | 374 | |
b99b4e0d | 375 | void __swap_writepage(struct folio *folio, struct writeback_control *wbc) |
05cda97e | 376 | { |
b99b4e0d | 377 | struct swap_info_struct *sis = swp_swap_info(folio->swap); |
05cda97e | 378 | |
b99b4e0d | 379 | VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); |
05cda97e CH |
380 | /* |
381 | * ->flags can be updated non-atomicially (scan_swap_map_slots), | |
382 | * but that will never affect SWP_FS_OPS, so the data_race | |
383 | * is safe. | |
384 | */ | |
385 | if (data_race(sis->flags & SWP_FS_OPS)) | |
bfcd44d5 | 386 | swap_writepage_fs(folio, wbc); |
3222d8c2 | 387 | else if (sis->flags & SWP_SYNCHRONOUS_IO) |
6de62c7b | 388 | swap_writepage_bdev_sync(folio, wbc, sis); |
05cda97e | 389 | else |
ee1b1d9b | 390 | swap_writepage_bdev_async(folio, wbc, sis); |
1da177e4 LT |
391 | } |
392 | ||
2282679f N |
393 | void swap_write_unplug(struct swap_iocb *sio) |
394 | { | |
395 | struct iov_iter from; | |
396 | struct address_space *mapping = sio->iocb.ki_filp->f_mapping; | |
397 | int ret; | |
398 | ||
de4eda9d | 399 | iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len); |
2282679f N |
400 | ret = mapping->a_ops->swap_rw(&sio->iocb, &from); |
401 | if (ret != -EIOCBQUEUED) | |
402 | sio_write_complete(&sio->iocb, ret); | |
403 | } | |
404 | ||
e1209d3a N |
405 | static void sio_read_complete(struct kiocb *iocb, long ret) |
406 | { | |
407 | struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb); | |
5169b844 | 408 | int p; |
e1209d3a | 409 | |
a1a0dfd5 | 410 | if (ret == sio->len) { |
5169b844 | 411 | for (p = 0; p < sio->pages; p++) { |
6a8c0687 | 412 | struct folio *folio = page_folio(sio->bvec[p].bv_page); |
5169b844 | 413 | |
6a8c0687 Z |
414 | folio_mark_uptodate(folio); |
415 | folio_unlock(folio); | |
5169b844 N |
416 | } |
417 | count_vm_events(PSWPIN, sio->pages); | |
e1209d3a | 418 | } else { |
5169b844 | 419 | for (p = 0; p < sio->pages; p++) { |
6a8c0687 | 420 | struct folio *folio = page_folio(sio->bvec[p].bv_page); |
5169b844 | 421 | |
6a8c0687 | 422 | folio_unlock(folio); |
5169b844 N |
423 | } |
424 | pr_alert_ratelimited("Read-error on swap-device\n"); | |
e1209d3a | 425 | } |
e1209d3a N |
426 | mempool_free(sio, sio_pool); |
427 | } | |
428 | ||
c9bdf768 | 429 | static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug) |
e1209d3a | 430 | { |
64a24e55 | 431 | struct swap_info_struct *sis = swp_swap_info(folio->swap); |
5169b844 | 432 | struct swap_iocb *sio = NULL; |
64a24e55 | 433 | loff_t pos = folio_file_pos(folio); |
e1209d3a | 434 | |
5169b844 N |
435 | if (plug) |
436 | sio = *plug; | |
437 | if (sio) { | |
438 | if (sio->iocb.ki_filp != sis->swap_file || | |
a1a0dfd5 | 439 | sio->iocb.ki_pos + sio->len != pos) { |
5169b844 N |
440 | swap_read_unplug(sio); |
441 | sio = NULL; | |
442 | } | |
443 | } | |
444 | if (!sio) { | |
445 | sio = mempool_alloc(sio_pool, GFP_KERNEL); | |
446 | init_sync_kiocb(&sio->iocb, sis->swap_file); | |
447 | sio->iocb.ki_pos = pos; | |
448 | sio->iocb.ki_complete = sio_read_complete; | |
449 | sio->pages = 0; | |
a1a0dfd5 | 450 | sio->len = 0; |
5169b844 | 451 | } |
64a24e55 MWO |
452 | bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0); |
453 | sio->len += folio_size(folio); | |
5169b844 N |
454 | sio->pages += 1; |
455 | if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) { | |
456 | swap_read_unplug(sio); | |
457 | sio = NULL; | |
458 | } | |
459 | if (plug) | |
460 | *plug = sio; | |
e1209d3a N |
461 | } |
462 | ||
c9bdf768 | 463 | static void swap_read_folio_bdev_sync(struct folio *folio, |
14bd75f5 | 464 | struct swap_info_struct *sis) |
1da177e4 | 465 | { |
9b4e30bd CH |
466 | struct bio_vec bv; |
467 | struct bio bio; | |
62c230bc | 468 | |
9b4e30bd | 469 | bio_init(&bio, sis->bdev, &bv, 1, REQ_OP_READ); |
3a61e6f6 | 470 | bio.bi_iter.bi_sector = swap_folio_sector(folio); |
2c184d82 | 471 | bio_add_folio_nofail(&bio, folio, folio_size(folio), 0); |
b0ba2d0f TH |
472 | /* |
473 | * Keep this task valid during swap readpage because the oom killer may | |
474 | * attempt to access it in the page fault retry time check. | |
475 | */ | |
9b4e30bd | 476 | get_task_struct(current); |
f8891e5e | 477 | count_vm_event(PSWPIN); |
9b4e30bd CH |
478 | submit_bio_wait(&bio); |
479 | __end_swap_bio_read(&bio); | |
480 | put_task_struct(current); | |
481 | } | |
482 | ||
c9bdf768 | 483 | static void swap_read_folio_bdev_async(struct folio *folio, |
9b4e30bd | 484 | struct swap_info_struct *sis) |
1da177e4 LT |
485 | { |
486 | struct bio *bio; | |
23955622 | 487 | |
9b4e30bd | 488 | bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); |
3a61e6f6 | 489 | bio->bi_iter.bi_sector = swap_folio_sector(folio); |
9b4e30bd | 490 | bio->bi_end_io = end_swap_bio_read; |
3c3ebd82 | 491 | bio_add_folio_nofail(bio, folio, folio_size(folio), 0); |
9b4e30bd CH |
492 | count_vm_event(PSWPIN); |
493 | submit_bio(bio); | |
14bd75f5 CH |
494 | } |
495 | ||
c9bdf768 MWO |
496 | void swap_read_folio(struct folio *folio, bool synchronous, |
497 | struct swap_iocb **plug) | |
14bd75f5 | 498 | { |
c9bdf768 | 499 | struct swap_info_struct *sis = swp_swap_info(folio->swap); |
fbcec6a3 | 500 | bool workingset = folio_test_workingset(folio); |
93779069 | 501 | unsigned long pflags; |
3a9bb7b1 | 502 | bool in_thrashing; |
1da177e4 | 503 | |
fbcec6a3 MWO |
504 | VM_BUG_ON_FOLIO(!folio_test_swapcache(folio) && !synchronous, folio); |
505 | VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); | |
506 | VM_BUG_ON_FOLIO(folio_test_uptodate(folio), folio); | |
93779069 MK |
507 | |
508 | /* | |
3a9bb7b1 YY |
509 | * Count submission time as memory stall and delay. When the device |
510 | * is congested, or the submitting cgroup IO-throttled, submission | |
511 | * can be a significant part of overall IO time. | |
93779069 | 512 | */ |
3a9bb7b1 YY |
513 | if (workingset) { |
514 | delayacct_thrashing_start(&in_thrashing); | |
d8c47cc7 | 515 | psi_memstall_enter(&pflags); |
3a9bb7b1 | 516 | } |
a3d5dc90 | 517 | delayacct_swapin_start(); |
93779069 | 518 | |
ca54f6d8 | 519 | if (zswap_load(folio)) { |
fbcec6a3 MWO |
520 | folio_mark_uptodate(folio); |
521 | folio_unlock(folio); | |
14bd75f5 | 522 | } else if (data_race(sis->flags & SWP_FS_OPS)) { |
c9bdf768 | 523 | swap_read_folio_fs(folio, plug); |
3222d8c2 | 524 | } else if (synchronous || (sis->flags & SWP_SYNCHRONOUS_IO)) { |
c9bdf768 | 525 | swap_read_folio_bdev_sync(folio, sis); |
14bd75f5 | 526 | } else { |
c9bdf768 | 527 | swap_read_folio_bdev_async(folio, sis); |
23955622 | 528 | } |
23955622 | 529 | |
3a9bb7b1 YY |
530 | if (workingset) { |
531 | delayacct_thrashing_end(&in_thrashing); | |
d8c47cc7 | 532 | psi_memstall_leave(&pflags); |
3a9bb7b1 | 533 | } |
a3d5dc90 | 534 | delayacct_swapin_end(); |
1da177e4 | 535 | } |
62c230bc | 536 | |
5169b844 | 537 | void __swap_read_unplug(struct swap_iocb *sio) |
62c230bc | 538 | { |
5169b844 N |
539 | struct iov_iter from; |
540 | struct address_space *mapping = sio->iocb.ki_filp->f_mapping; | |
541 | int ret; | |
cc30c5d6 | 542 | |
de4eda9d | 543 | iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len); |
5169b844 N |
544 | ret = mapping->a_ops->swap_rw(&sio->iocb, &from); |
545 | if (ret != -EIOCBQUEUED) | |
546 | sio_read_complete(&sio->iocb, ret); | |
62c230bc | 547 | } |