]> git.ipfire.org Git - people/ms/linux.git/blame - fs/f2fs/data.c
projects / people / ms / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs: clean up f2fs_abort_atomic_write()
[people/ms/linux.git] / fs / f2fs / data.c
CommitLineData
7c1a000d 1// SPDX-License-Identifier: GPL-2.0
0a8165d7 2/*
eb47b800
JK		 3 * fs/f2fs/data.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
eb47b800
JK		 7 */
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/buffer_head.h>
4034247a 11#include <linux/sched/mm.h>
eb47b800
JK		 12#include <linux/mpage.h>
13#include <linux/writeback.h>
8f46dcae 14#include <linux/pagevec.h>
eb47b800
JK		 15#include <linux/blkdev.h>
16#include <linux/bio.h>
27aacd28 17#include <linux/blk-crypto.h>
4969c06a 18#include <linux/swap.h>
690e4a3e 19#include <linux/prefetch.h>
e2e40f2c 20#include <linux/uio.h>
174cd4b1 21#include <linux/sched/signal.h>
10c5db28 22#include <linux/fiemap.h>
1517c1a7 23#include <linux/iomap.h>
eb47b800
JK		 24
25#include "f2fs.h"
26#include "node.h"
27#include "segment.h"
52118743 28#include "iostat.h"
848753aa 29#include <trace/events/f2fs.h>
eb47b800 30
6dbb1796
EB		 31#define NUM_PREALLOC_POST_READ_CTXS 128
32
33static struct kmem_cache *bio_post_read_ctx_cache;
0b20fcec 34static struct kmem_cache *bio_entry_slab;
6dbb1796 35static mempool_t *bio_post_read_ctx_pool;
f543805f
CY		 36static struct bio_set f2fs_bioset;
37
38#define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE
39
40int __init f2fs_init_bioset(void)
41{
42 if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE,
43 0, BIOSET_NEED_BVECS))
44 return -ENOMEM;
45 return 0;
46}
47
48void f2fs_destroy_bioset(void)
49{
50 bioset_exit(&f2fs_bioset);
51}
52
36951b38
CY		 53static bool __is_cp_guaranteed(struct page *page)
54{
55 struct address_space *mapping = page->mapping;
56 struct inode *inode;
57 struct f2fs_sb_info *sbi;
58
59 if (!mapping)
60 return false;
61
62 inode = mapping->host;
63 sbi = F2FS_I_SB(inode);
64
65 if (inode->i_ino == F2FS_META_INO(sbi) ||
a87aff1d 66 inode->i_ino == F2FS_NODE_INO(sbi) ||
b763f3be
CY		 67 S_ISDIR(inode->i_mode))
68 return true;
69
70 if (f2fs_is_compressed_page(page))
71 return false;
3db1de0e 72 if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) ||
b763f3be 73 page_private_gcing(page))
36951b38
CY		 74 return true;
75 return false;
76}
77
5f9abab4
JK		 78static enum count_type __read_io_type(struct page *page)
79{
4969c06a 80 struct address_space *mapping = page_file_mapping(page);
5f9abab4
JK		 81
82 if (mapping) {
83 struct inode *inode = mapping->host;
84 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
85
86 if (inode->i_ino == F2FS_META_INO(sbi))
87 return F2FS_RD_META;
88
89 if (inode->i_ino == F2FS_NODE_INO(sbi))
90 return F2FS_RD_NODE;
91 }
92 return F2FS_RD_DATA;
93}
94
6dbb1796
EB		 95/* postprocessing steps for read bios */
96enum bio_post_read_step {
7f59b277
EB		 97#ifdef CONFIG_FS_ENCRYPTION
98 STEP_DECRYPT = 1 << 0,
99#else
100 STEP_DECRYPT = 0, /* compile out the decryption-related code */
101#endif
102#ifdef CONFIG_F2FS_FS_COMPRESSION
103 STEP_DECOMPRESS = 1 << 1,
104#else
105 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */
106#endif
107#ifdef CONFIG_FS_VERITY
108 STEP_VERITY = 1 << 2,
109#else
110 STEP_VERITY = 0, /* compile out the verity-related code */
111#endif
6dbb1796
EB		 112};
113
114struct bio_post_read_ctx {
115 struct bio *bio;
4c8ff709 116 struct f2fs_sb_info *sbi;
6dbb1796 117 struct work_struct work;
6dbb1796 118 unsigned int enabled_steps;
4931e0c9 119 block_t fs_blkaddr;
6dbb1796
EB		 120};
121
bff139b4 122static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
93dfe2ac 123{
6dbb1796 124 struct bio_vec *bv;
6dc4f100 125 struct bvec_iter_all iter_all;
93dfe2ac 126
7f59b277
EB		 127 /*
128 * Update and unlock the bio's pagecache pages, and put the
129 * decompression context for any compressed pages.
130 */
2b070cfe 131 bio_for_each_segment_all(bv, bio, iter_all) {
7f59b277 132 struct page *page = bv->bv_page;
6dbb1796 133
7f59b277
EB		 134 if (f2fs_is_compressed_page(page)) {
135 if (bio->bi_status)
bff139b4
DJ		 136 f2fs_end_read_compressed_page(page, true, 0,
137 in_task);
138 f2fs_put_page_dic(page, in_task);
4c8ff709
CY		 139 continue;
140 }
4c8ff709 141
7f59b277 142 /* PG_error was set if decryption or verity failed. */
6dbb1796
EB		 143 if (bio->bi_status || PageError(page)) {
144 ClearPageUptodate(page);
fb7d70db
JK		 145 /* will re-read again later */
146 ClearPageError(page);
6dbb1796
EB		 147 } else {
148 SetPageUptodate(page);
149 }
5f9abab4 150 dec_page_count(F2FS_P_SB(page), __read_io_type(page));
6dbb1796
EB		 151 unlock_page(page);
152 }
79bbefb1 153
7f59b277
EB		 154 if (bio->bi_private)
155 mempool_free(bio->bi_private, bio_post_read_ctx_pool);
156 bio_put(bio);
4c8ff709 157}
4c8ff709 158
7f59b277 159static void f2fs_verify_bio(struct work_struct *work)
6dbb1796
EB		 160{
161 struct bio_post_read_ctx *ctx =
162 container_of(work, struct bio_post_read_ctx, work);
644c8c92 163 struct bio *bio = ctx->bio;
7f59b277 164 bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS);
644c8c92
EB		 165
166 /*
704528d8 167 * fsverity_verify_bio() may call readahead() again, and while verity
7f59b277
EB		 168 * will be disabled for this, decryption and/or decompression may still
169 * be needed, resulting in another bio_post_read_ctx being allocated.
170 * So to prevent deadlocks we need to release the current ctx to the
171 * mempool first. This assumes that verity is the last post-read step.
644c8c92
EB		 172 */
173 mempool_free(ctx, bio_post_read_ctx_pool);
174 bio->bi_private = NULL;
6dbb1796 175
7f59b277
EB		 176 /*
177 * Verify the bio's pages with fs-verity. Exclude compressed pages,
178 * as those were handled separately by f2fs_end_read_compressed_page().
179 */
180 if (may_have_compressed_pages) {
181 struct bio_vec *bv;
182 struct bvec_iter_all iter_all;
183
184 bio_for_each_segment_all(bv, bio, iter_all) {
185 struct page *page = bv->bv_page;
186
187 if (!f2fs_is_compressed_page(page) &&
188 !PageError(page) && !fsverity_verify_page(page))
189 SetPageError(page);
190 }
191 } else {
192 fsverity_verify_bio(bio);
4c8ff709 193 }
6dbb1796 194
bff139b4 195 f2fs_finish_read_bio(bio, true);
6dbb1796
EB		 196}
197
7f59b277
EB		 198/*
199 * If the bio's data needs to be verified with fs-verity, then enqueue the
200 * verity work for the bio. Otherwise finish the bio now.
201 *
202 * Note that to avoid deadlocks, the verity work can't be done on the
203 * decryption/decompression workqueue. This is because verifying the data pages
204 * can involve reading verity metadata pages from the file, and these verity
205 * metadata pages may be encrypted and/or compressed.
206 */
bff139b4 207static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task)
95ae251f 208{
7f59b277 209 struct bio_post_read_ctx *ctx = bio->bi_private;
4c8ff709 210
7f59b277
EB		 211 if (ctx && (ctx->enabled_steps & STEP_VERITY)) {
212 INIT_WORK(&ctx->work, f2fs_verify_bio);
4c8ff709 213 fsverity_enqueue_verify_work(&ctx->work);
7f59b277 214 } else {
bff139b4 215 f2fs_finish_read_bio(bio, in_task);
4c8ff709 216 }
4c8ff709 217}
95ae251f 218
7f59b277
EB		 219/*
220 * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last
221 * remaining page was read by @ctx->bio.
222 *
223 * Note that a bio may span clusters (even a mix of compressed and uncompressed
224 * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates
225 * that the bio includes at least one compressed page. The actual decompression
226 * is done on a per-cluster basis, not a per-bio basis.
227 */
bff139b4
DJ		 228static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx,
229 bool in_task)
6dbb1796 230{
7f59b277
EB		 231 struct bio_vec *bv;
232 struct bvec_iter_all iter_all;
233 bool all_compressed = true;
4931e0c9 234 block_t blkaddr = ctx->fs_blkaddr;
4c8ff709 235
7f59b277
EB		 236 bio_for_each_segment_all(bv, ctx->bio, iter_all) {
237 struct page *page = bv->bv_page;
4c8ff709 238
7f59b277
EB		 239 /* PG_error was set if decryption failed. */
240 if (f2fs_is_compressed_page(page))
6ce19aff 241 f2fs_end_read_compressed_page(page, PageError(page),
bff139b4 242 blkaddr, in_task);
7f59b277
EB		 243 else
244 all_compressed = false;
6ce19aff
CY		 245
246 blkaddr++;
6dbb1796 247 }
4c8ff709 248
7f59b277
EB		 249 /*
250 * Optimization: if all the bio's pages are compressed, then scheduling
251 * the per-bio verity work is unnecessary, as verity will be fully
252 * handled at the compression cluster level.
253 */
254 if (all_compressed)
255 ctx->enabled_steps &= ~STEP_VERITY;
6dbb1796
EB		 256}
257
7f59b277 258static void f2fs_post_read_work(struct work_struct *work)
6dbb1796 259{
7f59b277
EB		 260 struct bio_post_read_ctx *ctx =
261 container_of(work, struct bio_post_read_ctx, work);
262
263 if (ctx->enabled_steps & STEP_DECRYPT)
264 fscrypt_decrypt_bio(ctx->bio);
265
266 if (ctx->enabled_steps & STEP_DECOMPRESS)
bff139b4 267 f2fs_handle_step_decompress(ctx, true);
7f59b277 268
bff139b4 269 f2fs_verify_and_finish_bio(ctx->bio, true);
6dbb1796
EB		 270}
271
272static void f2fs_read_end_io(struct bio *bio)
273{
c45d6002 274 struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio));
a4b68176 275 struct bio_post_read_ctx *ctx;
bff139b4 276 bool intask = in_task();
a4b68176
DJ		 277
278 iostat_update_and_unbind_ctx(bio, 0);
279 ctx = bio->bi_private;
c45d6002
CY		 280
281 if (time_to_inject(sbi, FAULT_READ_IO)) {
282 f2fs_show_injection_info(sbi, FAULT_READ_IO);
4e4cbee9 283 bio->bi_status = BLK_STS_IOERR;
55523519 284 }
8b038c70 285
7f59b277 286 if (bio->bi_status) {
bff139b4 287 f2fs_finish_read_bio(bio, intask);
6dbb1796 288 return;
f1e88660 289 }
6dbb1796 290
bff139b4
DJ		 291 if (ctx) {
292 unsigned int enabled_steps = ctx->enabled_steps &
293 (STEP_DECRYPT | STEP_DECOMPRESS);
294
295 /*
296 * If we have only decompression step between decompression and
297 * decrypt, we don't need post processing for this.
298 */
299 if (enabled_steps == STEP_DECOMPRESS &&
300 !f2fs_low_mem_mode(sbi)) {
301 f2fs_handle_step_decompress(ctx, intask);
302 } else if (enabled_steps) {
303 INIT_WORK(&ctx->work, f2fs_post_read_work);
304 queue_work(ctx->sbi->post_read_wq, &ctx->work);
305 return;
306 }
7f59b277 307 }
bff139b4
DJ		 308
309 f2fs_verify_and_finish_bio(bio, intask);
f1e88660
JK		 310}
311
4246a0b6 312static void f2fs_write_end_io(struct bio *bio)
93dfe2ac 313{
a4b68176 314 struct f2fs_sb_info *sbi;
f568849e 315 struct bio_vec *bvec;
6dc4f100 316 struct bvec_iter_all iter_all;
93dfe2ac 317
a4b68176
DJ		 318 iostat_update_and_unbind_ctx(bio, 1);
319 sbi = bio->bi_private;
320
6f5c2ed0 321 if (time_to_inject(sbi, FAULT_WRITE_IO)) {
c45d6002 322 f2fs_show_injection_info(sbi, FAULT_WRITE_IO);
6f5c2ed0
CY		 323 bio->bi_status = BLK_STS_IOERR;
324 }
325
2b070cfe 326 bio_for_each_segment_all(bvec, bio, iter_all) {
93dfe2ac 327 struct page *page = bvec->bv_page;
36951b38 328 enum count_type type = WB_DATA_TYPE(page);
93dfe2ac 329
b763f3be
CY		 330 if (page_private_dummy(page)) {
331 clear_page_private_dummy(page);
0a595eba
JK		 332 unlock_page(page);
333 mempool_free(page, sbi->write_io_dummy);
334
4e4cbee9 335 if (unlikely(bio->bi_status))
0a595eba
JK		 336 f2fs_stop_checkpoint(sbi, true);
337 continue;
338 }
339
d2d0727b 340 fscrypt_finalize_bounce_page(&page);
4375a336 341
4c8ff709
CY		 342#ifdef CONFIG_F2FS_FS_COMPRESSION
343 if (f2fs_is_compressed_page(page)) {
344 f2fs_compress_write_end_io(bio, page);
345 continue;
346 }
347#endif
348
4e4cbee9 349 if (unlikely(bio->bi_status)) {
5114a97a 350 mapping_set_error(page->mapping, -EIO);
b1ca321d
JK		 351 if (type == F2FS_WB_CP_DATA)
352 f2fs_stop_checkpoint(sbi, true);
93dfe2ac 353 }
7dff55d2
YH		 354
355 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
356 page->index != nid_of_node(page));
357
36951b38 358 dec_page_count(sbi, type);
50fa53ec
CY		 359 if (f2fs_in_warm_node_list(sbi, page))
360 f2fs_del_fsync_node_entry(sbi, page);
b763f3be 361 clear_page_private_gcing(page);
93dfe2ac 362 end_page_writeback(page);
f568849e 363 }
36951b38 364 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
f5730184 365 wq_has_sleeper(&sbi->cp_wait))
93dfe2ac
JK		 366 wake_up(&sbi->cp_wait);
367
368 bio_put(bio);
369}
370
3c62be17 371struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
5189810a 372 block_t blk_addr, sector_t *sector)
3c62be17
JK		 373{
374 struct block_device *bdev = sbi->sb->s_bdev;
375 int i;
376
0916878d
DLM		 377 if (f2fs_is_multi_device(sbi)) {
378 for (i = 0; i < sbi->s_ndevs; i++) {
379 if (FDEV(i).start_blk <= blk_addr &&
380 FDEV(i).end_blk >= blk_addr) {
381 blk_addr -= FDEV(i).start_blk;
382 bdev = FDEV(i).bdev;
383 break;
384 }
3c62be17
JK		 385 }
386 }
5189810a
CH		 387
388 if (sector)
389 *sector = SECTOR_FROM_BLOCK(blk_addr);
3c62be17
JK		 390 return bdev;
391}
392
393int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
394{
395 int i;
396
0916878d
DLM		 397 if (!f2fs_is_multi_device(sbi))
398 return 0;
399
3c62be17
JK		 400 for (i = 0; i < sbi->s_ndevs; i++)
401 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
402 return i;
403 return 0;
404}
405
0adc2ab0 406static unsigned int f2fs_io_flags(struct f2fs_io_info *fio)
64bf0eef
CH		 407{
408 unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1;
0adc2ab0
JK		 409 unsigned int fua_flag, meta_flag, io_flag;
410 unsigned int op_flags = 0;
411
412 if (fio->op != REQ_OP_WRITE)
413 return 0;
414 if (fio->type == DATA)
415 io_flag = fio->sbi->data_io_flag;
416 else if (fio->type == NODE)
417 io_flag = fio->sbi->node_io_flag;
418 else
419 return 0;
420
421 fua_flag = io_flag & temp_mask;
422 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask;
64bf0eef
CH		 423
424 /*
425 * data/node io flag bits per temp:
426 * REQ_META | REQ_FUA |
427 * 5 | 4 | 3 | 2 | 1 | 0 |
428 * Cold | Warm | Hot | Cold | Warm | Hot |
429 */
430 if ((1 << fio->temp) & meta_flag)
0adc2ab0 431 op_flags |= REQ_META;
64bf0eef 432 if ((1 << fio->temp) & fua_flag)
0adc2ab0
JK		 433 op_flags |= REQ_FUA;
434 return op_flags;
64bf0eef
CH		 435}
436
b757f6ed 437static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
940a6d34 438{
b757f6ed 439 struct f2fs_sb_info *sbi = fio->sbi;
5189810a
CH		 440 struct block_device *bdev;
441 sector_t sector;
940a6d34
GZ		 442 struct bio *bio;
443
5189810a 444 bdev = f2fs_target_device(sbi, fio->new_blkaddr, &sector);
0adc2ab0
JK		 445 bio = bio_alloc_bioset(bdev, npages,
446 fio->op | fio->op_flags | f2fs_io_flags(fio),
447 GFP_NOIO, &f2fs_bioset);
5189810a 448 bio->bi_iter.bi_sector = sector;
b757f6ed 449 if (is_read_io(fio->op)) {
0cdd3195
HL		 450 bio->bi_end_io = f2fs_read_end_io;
451 bio->bi_private = NULL;
452 } else {
453 bio->bi_end_io = f2fs_write_end_io;
454 bio->bi_private = sbi;
0cdd3195 455 }
a4b68176
DJ		 456 iostat_alloc_and_bind_ctx(sbi, bio, NULL);
457
b757f6ed
CY		 458 if (fio->io_wbc)
459 wbc_init_bio(fio->io_wbc, bio);
940a6d34
GZ		 460
461 return bio;
462}
463
27aacd28
ST		 464static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
465 pgoff_t first_idx,
466 const struct f2fs_io_info *fio,
467 gfp_t gfp_mask)
468{
469 /*
470 * The f2fs garbage collector sets ->encrypted_page when it wants to
471 * read/write raw data without encryption.
472 */
473 if (!fio || !fio->encrypted_page)
474 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask);
475}
476
477static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode,
478 pgoff_t next_idx,
479 const struct f2fs_io_info *fio)
480{
481 /*
482 * The f2fs garbage collector sets ->encrypted_page when it wants to
483 * read/write raw data without encryption.
484 */
485 if (fio && fio->encrypted_page)
486 return !bio_has_crypt_ctx(bio);
487
488 return fscrypt_mergeable_bio(bio, inode, next_idx);
489}
490
4fc29c1a
LT		 491static inline void __submit_bio(struct f2fs_sb_info *sbi,
492 struct bio *bio, enum page_type type)
f5730184 493{
4fc29c1a 494 if (!is_read_io(bio_op(bio))) {
0a595eba
JK		 495 unsigned int start;
496
0a595eba
JK		 497 if (type != DATA && type != NODE)
498 goto submit_io;
499
b0332a0f 500 if (f2fs_lfs_mode(sbi) && current->plug)
3bb09a0e
TY		 501 blk_finish_plug(current->plug);
502
39f71b7e 503 if (!F2FS_IO_ALIGNED(sbi))
8223ecc4
CY		 504 goto submit_io;
505
0a595eba
JK		 506 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
507 start %= F2FS_IO_SIZE(sbi);
508
509 if (start == 0)
510 goto submit_io;
511
512 /* fill dummy pages */
513 for (; start < F2FS_IO_SIZE(sbi); start++) {
514 struct page *page =
515 mempool_alloc(sbi->write_io_dummy,
bc73a4b2 516 GFP_NOIO | __GFP_NOFAIL);
0a595eba
JK		 517 f2fs_bug_on(sbi, !page);
518
0a595eba 519 lock_page(page);
b763f3be
CY		 520
521 zero_user_segment(page, 0, PAGE_SIZE);
522 set_page_private_dummy(page);
523
0a595eba
JK		 524 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
525 f2fs_bug_on(sbi, 1);
526 }
527 /*
528 * In the NODE case, we lose next block address chain. So, we
529 * need to do checkpoint in f2fs_sync_file.
530 */
531 if (type == NODE)
532 set_sbi_flag(sbi, SBI_NEED_CP);
19a5f5e2 533 }
0a595eba 534submit_io:
554b5125
JK		 535 if (is_read_io(bio_op(bio)))
536 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
537 else
538 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
a4b68176
DJ		 539
540 iostat_update_submit_ctx(bio, type);
4e49ea4a 541 submit_bio(bio);
f5730184
JK		 542}
543
4c8ff709
CY		 544void f2fs_submit_bio(struct f2fs_sb_info *sbi,
545 struct bio *bio, enum page_type type)
546{
547 __submit_bio(sbi, bio, type);
548}
549
458e6197 550static void __submit_merged_bio(struct f2fs_bio_info *io)
93dfe2ac 551{
458e6197 552 struct f2fs_io_info *fio = &io->fio;
93dfe2ac
JK		 553
554 if (!io->bio)
555 return;
556
04d328de 557 if (is_read_io(fio->op))
554b5125 558 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
6a8f8ca5 559 else
554b5125 560 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
04d328de 561
4fc29c1a 562 __submit_bio(io->sbi, io->bio, fio->type);
93dfe2ac
JK		 563 io->bio = NULL;
564}
565
8648de2c 566static bool __has_merged_page(struct bio *bio, struct inode *inode,
bab475c5 567 struct page *page, nid_t ino)
0fd785eb 568{
0fd785eb 569 struct bio_vec *bvec;
6dc4f100 570 struct bvec_iter_all iter_all;
0fd785eb 571
8648de2c 572 if (!bio)
0fd785eb 573 return false;
0c3a5797 574
bab475c5 575 if (!inode && !page && !ino)
0c3a5797 576 return true;
0fd785eb 577
8648de2c 578 bio_for_each_segment_all(bvec, bio, iter_all) {
4c8ff709 579 struct page *target = bvec->bv_page;
0fd785eb 580
4c8ff709 581 if (fscrypt_is_bounce_page(target)) {
d2d0727b 582 target = fscrypt_pagecache_page(target);
4c8ff709
CY		 583 if (IS_ERR(target))
584 continue;
585 }
586 if (f2fs_is_compressed_page(target)) {
587 target = f2fs_compress_control_page(target);
588 if (IS_ERR(target))
589 continue;
590 }
0fd785eb 591
0c3a5797
CY		 592 if (inode && inode == target->mapping->host)
593 return true;
bab475c5
CY		 594 if (page && page == target)
595 return true;
0c3a5797 596 if (ino && ino == ino_of_node(target))
0fd785eb 597 return true;
0fd785eb
CY		 598 }
599
0fd785eb
CY		 600 return false;
601}
602
908ea654
YY		 603int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi)
604{
605 int i;
606
607 for (i = 0; i < NR_PAGE_TYPE; i++) {
608 int n = (i == META) ? 1 : NR_TEMP_TYPE;
609 int j;
610
611 sbi->write_io[i] = f2fs_kmalloc(sbi,
612 array_size(n, sizeof(struct f2fs_bio_info)),
613 GFP_KERNEL);
614 if (!sbi->write_io[i])
615 return -ENOMEM;
616
617 for (j = HOT; j < n; j++) {
618 init_f2fs_rwsem(&sbi->write_io[i][j].io_rwsem);
619 sbi->write_io[i][j].sbi = sbi;
620 sbi->write_io[i][j].bio = NULL;
621 spin_lock_init(&sbi->write_io[i][j].io_lock);
622 INIT_LIST_HEAD(&sbi->write_io[i][j].io_list);
623 INIT_LIST_HEAD(&sbi->write_io[i][j].bio_list);
624 init_f2fs_rwsem(&sbi->write_io[i][j].bio_list_lock);
625 }
626 }
627
628 return 0;
629}
630
b9109b0e 631static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
a912b54d 632 enum page_type type, enum temp_type temp)
93dfe2ac
JK		 633{
634 enum page_type btype = PAGE_TYPE_OF_BIO(type);
a912b54d 635 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
93dfe2ac 636
e4544b63 637 f2fs_down_write(&io->io_rwsem);
458e6197
JK		 638
639 /* change META to META_FLUSH in the checkpoint procedure */
640 if (type >= META_FLUSH) {
641 io->fio.type = META_FLUSH;
64bf0eef 642 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC;
70fd7614 643 if (!test_opt(sbi, NOBARRIER))
64bf0eef 644 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA;
458e6197
JK		 645 }
646 __submit_merged_bio(io);
e4544b63 647 f2fs_up_write(&io->io_rwsem);
93dfe2ac
JK		 648}
649
a912b54d 650static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
bab475c5
CY		 651 struct inode *inode, struct page *page,
652 nid_t ino, enum page_type type, bool force)
0c3a5797 653{
a912b54d 654 enum temp_type temp;
1e771e83 655 bool ret = true;
a912b54d
JK		 656
657 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
1e771e83
YS		 658 if (!force) {
659 enum page_type btype = PAGE_TYPE_OF_BIO(type);
660 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
a912b54d 661
e4544b63 662 f2fs_down_read(&io->io_rwsem);
8648de2c 663 ret = __has_merged_page(io->bio, inode, page, ino);
e4544b63 664 f2fs_up_read(&io->io_rwsem);
1e771e83
YS		 665 }
666 if (ret)
667 __f2fs_submit_merged_write(sbi, type, temp);
a912b54d
JK		 668
669 /* TODO: use HOT temp only for meta pages now. */
670 if (type >= META)
671 break;
672 }
0c3a5797
CY		 673}
674
b9109b0e 675void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
0c3a5797 676{
adcc00f7 677 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true);
0c3a5797
CY		 678}
679
b9109b0e 680void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
bab475c5
CY		 681 struct inode *inode, struct page *page,
682 nid_t ino, enum page_type type)
0c3a5797 683{
bab475c5 684 __submit_merged_write_cond(sbi, inode, page, ino, type, false);
0c3a5797
CY		 685}
686
b9109b0e 687void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
406657dd 688{
b9109b0e
JK		 689 f2fs_submit_merged_write(sbi, DATA);
690 f2fs_submit_merged_write(sbi, NODE);
691 f2fs_submit_merged_write(sbi, META);
406657dd
CY		 692}
693
93dfe2ac
JK		 694/*
695 * Fill the locked page with data located in the block address.
771a9a71 696 * A caller needs to unlock the page on failure.
93dfe2ac 697 */
05ca3632 698int f2fs_submit_page_bio(struct f2fs_io_info *fio)
93dfe2ac 699{
93dfe2ac 700 struct bio *bio;
0b81d077
JK		 701 struct page *page = fio->encrypted_page ?
702 fio->encrypted_page : fio->page;
93dfe2ac 703
c9b60788 704 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
93770ab7
CY		 705 fio->is_por ? META_POR : (__is_meta_io(fio) ?
706 META_GENERIC : DATA_GENERIC_ENHANCE)))
10f966bb 707 return -EFSCORRUPTED;
c9b60788 708
2ace38e0 709 trace_f2fs_submit_page_bio(page, fio);
93dfe2ac
JK		 710
711 /* Allocate a new bio */
b757f6ed 712 bio = __bio_alloc(fio, 1);
93dfe2ac 713
27aacd28
ST		 714 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
715 fio->page->index, fio, GFP_NOIO);
716
09cbfeaf 717 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
93dfe2ac 718 bio_put(bio);
93dfe2ac
JK		 719 return -EFAULT;
720 }
78efac53
CY		 721
722 if (fio->io_wbc && !is_read_io(fio->op))
34e51a5e 723 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
78efac53 724
5f9abab4
JK		 725 inc_page_count(fio->sbi, is_read_io(fio->op) ?
726 __read_io_type(page): WB_DATA_TYPE(fio->page));
4c58ed07
CY		 727
728 __submit_bio(fio->sbi, bio, fio->type);
93dfe2ac
JK		 729 return 0;
730}
731
8896cbdf
CY		 732static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
733 block_t last_blkaddr, block_t cur_blkaddr)
734{
10208567
JK		 735 if (unlikely(sbi->max_io_bytes &&
736 bio->bi_iter.bi_size >= sbi->max_io_bytes))
737 return false;
8896cbdf
CY		 738 if (last_blkaddr + 1 != cur_blkaddr)
739 return false;
309dca30 740 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL);
8896cbdf
CY		 741}
742
743static bool io_type_is_mergeable(struct f2fs_bio_info *io,
744 struct f2fs_io_info *fio)
745{
746 if (io->fio.op != fio->op)
747 return false;
748 return io->fio.op_flags == fio->op_flags;
749}
750
751static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio,
752 struct f2fs_bio_info *io,
753 struct f2fs_io_info *fio,
754 block_t last_blkaddr,
755 block_t cur_blkaddr)
756{
c72db71e
CY		 757 if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) {
758 unsigned int filled_blocks =
759 F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size);
760 unsigned int io_size = F2FS_IO_SIZE(sbi);
761 unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt;
762
763 /* IOs in bio is aligned and left space of vectors is not enough */
764 if (!(filled_blocks % io_size) && left_vecs < io_size)
765 return false;
766 }
8896cbdf
CY		 767 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr))
768 return false;
769 return io_type_is_mergeable(io, fio);
770}
771
0b20fcec
CY		 772static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio,
773 struct page *page, enum temp_type temp)
774{
775 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
776 struct bio_entry *be;
777
32410577 778 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL);
0b20fcec
CY		 779 be->bio = bio;
780 bio_get(bio);
781
782 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE)
783 f2fs_bug_on(sbi, 1);
784
e4544b63 785 f2fs_down_write(&io->bio_list_lock);
0b20fcec 786 list_add_tail(&be->list, &io->bio_list);
e4544b63 787 f2fs_up_write(&io->bio_list_lock);
0b20fcec
CY		 788}
789
790static void del_bio_entry(struct bio_entry *be)
791{
792 list_del(&be->list);
793 kmem_cache_free(bio_entry_slab, be);
794}
795
27aacd28 796static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio,
0b20fcec
CY		 797 struct page *page)
798{
27aacd28 799 struct f2fs_sb_info *sbi = fio->sbi;
0b20fcec
CY		 800 enum temp_type temp;
801 bool found = false;
802 int ret = -EAGAIN;
803
804 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
805 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
806 struct list_head *head = &io->bio_list;
807 struct bio_entry *be;
808
e4544b63 809 f2fs_down_write(&io->bio_list_lock);
0b20fcec
CY		 810 list_for_each_entry(be, head, list) {
811 if (be->bio != *bio)
812 continue;
813
814 found = true;
815
27aacd28
ST		 816 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio,
817 *fio->last_block,
818 fio->new_blkaddr));
819 if (f2fs_crypt_mergeable_bio(*bio,
820 fio->page->mapping->host,
821 fio->page->index, fio) &&
822 bio_add_page(*bio, page, PAGE_SIZE, 0) ==
823 PAGE_SIZE) {
0b20fcec
CY		 824 ret = 0;
825 break;
826 }
827
27aacd28 828 /* page can't be merged into bio; submit the bio */
0b20fcec
CY		 829 del_bio_entry(be);
830 __submit_bio(sbi, *bio, DATA);
831 break;
832 }
e4544b63 833 f2fs_up_write(&io->bio_list_lock);
0b20fcec
CY		 834 }
835
836 if (ret) {
837 bio_put(*bio);
838 *bio = NULL;
839 }
840
841 return ret;
842}
843
844void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
845 struct bio **bio, struct page *page)
846{
847 enum temp_type temp;
848 bool found = false;
849 struct bio *target = bio ? *bio : NULL;
850
851 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) {
852 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp;
853 struct list_head *head = &io->bio_list;
854 struct bio_entry *be;
855
856 if (list_empty(head))
857 continue;
858
e4544b63 859 f2fs_down_read(&io->bio_list_lock);
0b20fcec
CY		 860 list_for_each_entry(be, head, list) {
861 if (target)
862 found = (target == be->bio);
863 else
864 found = __has_merged_page(be->bio, NULL,
865 page, 0);
866 if (found)
867 break;
868 }
e4544b63 869 f2fs_up_read(&io->bio_list_lock);
0b20fcec
CY		 870
871 if (!found)
872 continue;
873
874 found = false;
875
e4544b63 876 f2fs_down_write(&io->bio_list_lock);
0b20fcec
CY		 877 list_for_each_entry(be, head, list) {
878 if (target)
879 found = (target == be->bio);
880 else
881 found = __has_merged_page(be->bio, NULL,
882 page, 0);
883 if (found) {
884 target = be->bio;
885 del_bio_entry(be);
886 break;
887 }
888 }
e4544b63 889 f2fs_up_write(&io->bio_list_lock);
0b20fcec
CY		 890 }
891
892 if (found)
893 __submit_bio(sbi, target, DATA);
894 if (bio && *bio) {
895 bio_put(*bio);
896 *bio = NULL;
897 }
898}
899
8648de2c
CY		 900int f2fs_merge_page_bio(struct f2fs_io_info *fio)
901{
902 struct bio *bio = *fio->bio;
903 struct page *page = fio->encrypted_page ?
904 fio->encrypted_page : fio->page;
905
906 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr,
907 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC))
10f966bb 908 return -EFSCORRUPTED;
8648de2c
CY		 909
910 trace_f2fs_submit_page_bio(page, fio);
8648de2c 911
8896cbdf 912 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block,
0b20fcec
CY		 913 fio->new_blkaddr))
914 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL);
8648de2c
CY		 915alloc_new:
916 if (!bio) {
a8affc03 917 bio = __bio_alloc(fio, BIO_MAX_VECS);
27aacd28
ST		 918 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host,
919 fio->page->index, fio, GFP_NOIO);
8648de2c 920
0b20fcec
CY		 921 add_bio_entry(fio->sbi, bio, page, fio->temp);
922 } else {
27aacd28 923 if (add_ipu_page(fio, &bio, page))
0b20fcec 924 goto alloc_new;
8648de2c
CY		 925 }
926
927 if (fio->io_wbc)
9637d517 928 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE);
8648de2c
CY		 929
930 inc_page_count(fio->sbi, WB_DATA_TYPE(page));
931
932 *fio->last_block = fio->new_blkaddr;
933 *fio->bio = bio;
934
935 return 0;
936}
937
fe16efe6 938void f2fs_submit_page_write(struct f2fs_io_info *fio)
93dfe2ac 939{
05ca3632 940 struct f2fs_sb_info *sbi = fio->sbi;
458e6197 941 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
a912b54d 942 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
4375a336 943 struct page *bio_page;
93dfe2ac 944
b9109b0e 945 f2fs_bug_on(sbi, is_read_io(fio->op));
93dfe2ac 946
e4544b63 947 f2fs_down_write(&io->io_rwsem);
fb830fc5
CY		 948next:
949 if (fio->in_list) {
950 spin_lock(&io->io_lock);
951 if (list_empty(&io->io_list)) {
952 spin_unlock(&io->io_lock);
fe16efe6 953 goto out;
fb830fc5
CY		 954 }
955 fio = list_first_entry(&io->io_list,
956 struct f2fs_io_info, list);
957 list_del(&fio->list);
958 spin_unlock(&io->io_lock);
959 }
93dfe2ac 960
93770ab7 961 verify_fio_blkaddr(fio);
93dfe2ac 962
4c8ff709
CY		 963 if (fio->encrypted_page)
964 bio_page = fio->encrypted_page;
965 else if (fio->compressed_page)
966 bio_page = fio->compressed_page;
967 else
968 bio_page = fio->page;
36951b38 969
ebf7c522
TM		 970 /* set submitted = true as a return value */
971 fio->submitted = true;
d68f735b 972
b9109b0e 973 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
93dfe2ac 974
27aacd28
ST		 975 if (io->bio &&
976 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio,
977 fio->new_blkaddr) ||
978 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host,
979 bio_page->index, fio)))
458e6197 980 __submit_merged_bio(io);
93dfe2ac
JK		 981alloc_new:
982 if (io->bio == NULL) {
8223ecc4
CY		 983 if (F2FS_IO_ALIGNED(sbi) &&
984 (fio->type == DATA || fio->type == NODE) &&
0a595eba 985 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
b9109b0e 986 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
fe16efe6
CY		 987 fio->retry = true;
988 goto skip;
0a595eba 989 }
a8affc03 990 io->bio = __bio_alloc(fio, BIO_MAX_VECS);
27aacd28
ST		 991 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host,
992 bio_page->index, fio, GFP_NOIO);
458e6197 993 io->fio = *fio;
93dfe2ac
JK		 994 }
995
a912b54d 996 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
458e6197 997 __submit_merged_bio(io);
93dfe2ac
JK		 998 goto alloc_new;
999 }
1000
578c6478 1001 if (fio->io_wbc)
34e51a5e 1002 wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE);
578c6478 1003
7a9d7548 1004 io->last_block_in_bio = fio->new_blkaddr;
fb830fc5
CY		 1005
1006 trace_f2fs_submit_page_write(fio->page, fio);
fe16efe6 1007skip:
fb830fc5
CY		 1008 if (fio->in_list)
1009 goto next;
fe16efe6 1010out:
4354994f 1011 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
00e09c0b 1012 !f2fs_is_checkpoint_ready(sbi))
5ce80586 1013 __submit_merged_bio(io);
e4544b63 1014 f2fs_up_write(&io->io_rwsem);
93dfe2ac
JK		 1015}
1016
13ba41e3 1017static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
95ae251f 1018 unsigned nr_pages, unsigned op_flag,
7f59b277 1019 pgoff_t first_idx, bool for_write)
13ba41e3
JK		 1020{
1021 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
13ba41e3 1022 struct bio *bio;
a4b68176 1023 struct bio_post_read_ctx *ctx = NULL;
6dbb1796 1024 unsigned int post_read_steps = 0;
5189810a
CH		 1025 sector_t sector;
1026 struct block_device *bdev = f2fs_target_device(sbi, blkaddr, &sector);
13ba41e3 1027
64bf0eef
CH		 1028 bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages),
1029 REQ_OP_READ | op_flag,
609be106 1030 for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset);
6dbb1796 1031 if (!bio)
13ba41e3 1032 return ERR_PTR(-ENOMEM);
5189810a 1033 bio->bi_iter.bi_sector = sector;
27aacd28 1034 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS);
13ba41e3 1035 bio->bi_end_io = f2fs_read_end_io;
13ba41e3 1036
27aacd28 1037 if (fscrypt_inode_uses_fs_layer_crypto(inode))
7f59b277 1038 post_read_steps |= STEP_DECRYPT;
95ae251f 1039
7f59b277
EB		 1040 if (f2fs_need_verity(inode, first_idx))
1041 post_read_steps |= STEP_VERITY;
1042
1043 /*
1044 * STEP_DECOMPRESS is handled specially, since a compressed file might
1045 * contain both compressed and uncompressed clusters. We'll allocate a
1046 * bio_post_read_ctx if the file is compressed, but the caller is
1047 * responsible for enabling STEP_DECOMPRESS if it's actually needed.
1048 */
1049
1050 if (post_read_steps || f2fs_compressed_file(inode)) {
e8ce5749 1051 /* Due to the mempool, this never fails. */
6dbb1796 1052 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS);
6dbb1796 1053 ctx->bio = bio;
4c8ff709 1054 ctx->sbi = sbi;
6dbb1796 1055 ctx->enabled_steps = post_read_steps;
4931e0c9 1056 ctx->fs_blkaddr = blkaddr;
6dbb1796 1057 bio->bi_private = ctx;
6dbb1796 1058 }
a4b68176 1059 iostat_alloc_and_bind_ctx(sbi, bio, ctx);
6dbb1796 1060
13ba41e3
JK		 1061 return bio;
1062}
1063
1064/* This can handle encryption stuffs */
1065static int f2fs_submit_page_read(struct inode *inode, struct page *page,
b7973091 1066 block_t blkaddr, int op_flags, bool for_write)
13ba41e3 1067{
93770ab7
CY		 1068 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1069 struct bio *bio;
13ba41e3 1070
b7973091 1071 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags,
7f59b277 1072 page->index, for_write);
13ba41e3
JK		 1073 if (IS_ERR(bio))
1074 return PTR_ERR(bio);
1075
0ded69f6
JK		 1076 /* wait for GCed page writeback via META_MAPPING */
1077 f2fs_wait_on_block_writeback(inode, blkaddr);
1078
13ba41e3
JK		 1079 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
1080 bio_put(bio);
1081 return -EFAULT;
1082 }
fb7d70db 1083 ClearPageError(page);
93770ab7 1084 inc_page_count(sbi, F2FS_RD_DATA);
8b83ac81 1085 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
93770ab7 1086 __submit_bio(sbi, bio, DATA);
13ba41e3
JK		 1087 return 0;
1088}
1089
46008c6d
CY		 1090static void __set_data_blkaddr(struct dnode_of_data *dn)
1091{
1092 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
1093 __le32 *addr_array;
7a2af766
CY		 1094 int base = 0;
1095
1096 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
1097 base = get_extra_isize(dn->inode);
46008c6d
CY		 1098
1099 /* Get physical address of data block */
1100 addr_array = blkaddr_in_node(rn);
7a2af766 1101 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
46008c6d
CY		 1102}
1103
0a8165d7 1104/*
eb47b800
JK		 1105 * Lock ordering for the change of data block address:
1106 * ->data_page
1107 * ->node_page
1108 * update block addresses in the node page
1109 */
4d57b86d 1110void f2fs_set_data_blkaddr(struct dnode_of_data *dn)
eb47b800 1111{
bae0ee7a 1112 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
46008c6d
CY		 1113 __set_data_blkaddr(dn);
1114 if (set_page_dirty(dn->node_page))
12719ae1 1115 dn->node_changed = true;
eb47b800
JK		 1116}
1117
f28b3434
CY		 1118void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
1119{
1120 dn->data_blkaddr = blkaddr;
4d57b86d 1121 f2fs_set_data_blkaddr(dn);
f28b3434
CY		 1122 f2fs_update_extent_cache(dn);
1123}
1124
46008c6d 1125/* dn->ofs_in_node will be returned with up-to-date last block pointer */
4d57b86d 1126int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
eb47b800 1127{
4081363f 1128 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
0abd675e 1129 int err;
eb47b800 1130
46008c6d
CY		 1131 if (!count)
1132 return 0;
1133
91942321 1134 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
eb47b800 1135 return -EPERM;
0abd675e
CY		 1136 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
1137 return err;
eb47b800 1138
46008c6d
CY		 1139 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
1140 dn->ofs_in_node, count);
1141
bae0ee7a 1142 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true);
46008c6d
CY		 1143
1144 for (; count > 0; dn->ofs_in_node++) {
a2ced1ce 1145 block_t blkaddr = f2fs_data_blkaddr(dn);
5f029c04 1146
46008c6d
CY		 1147 if (blkaddr == NULL_ADDR) {
1148 dn->data_blkaddr = NEW_ADDR;
1149 __set_data_blkaddr(dn);
1150 count--;
1151 }
1152 }
1153
1154 if (set_page_dirty(dn->node_page))
1155 dn->node_changed = true;
eb47b800
JK		 1156 return 0;
1157}
1158
46008c6d 1159/* Should keep dn->ofs_in_node unchanged */
4d57b86d 1160int f2fs_reserve_new_block(struct dnode_of_data *dn)
46008c6d
CY		 1161{
1162 unsigned int ofs_in_node = dn->ofs_in_node;
1163 int ret;
1164
4d57b86d 1165 ret = f2fs_reserve_new_blocks(dn, 1);
46008c6d
CY		 1166 dn->ofs_in_node = ofs_in_node;
1167 return ret;
1168}
1169
b600965c
HL		 1170int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
1171{
1172 bool need_put = dn->inode_page ? false : true;
1173 int err;
1174
4d57b86d 1175 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE);
b600965c
HL		 1176 if (err)
1177 return err;
a8865372 1178
b600965c 1179 if (dn->data_blkaddr == NULL_ADDR)
4d57b86d 1180 err = f2fs_reserve_new_block(dn);
a8865372 1181 if (err || need_put)
b600965c
HL		 1182 f2fs_put_dnode(dn);
1183 return err;
1184}
1185
759af1c9 1186int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
eb47b800 1187{
94afd6d6 1188 struct extent_info ei = {0, };
759af1c9 1189 struct inode *inode = dn->inode;
028a41e8 1190
759af1c9
FL		 1191 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1192 dn->data_blkaddr = ei.blk + index - ei.fofs;
1193 return 0;
429511cd 1194 }
028a41e8 1195
759af1c9 1196 return f2fs_reserve_block(dn, index);
eb47b800
JK		 1197}
1198
4d57b86d 1199struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index,
04d328de 1200 int op_flags, bool for_write)
eb47b800 1201{
eb47b800
JK		 1202 struct address_space *mapping = inode->i_mapping;
1203 struct dnode_of_data dn;
1204 struct page *page;
94afd6d6 1205 struct extent_info ei = {0, };
eb47b800 1206 int err;
4375a336 1207
a56c7c6f 1208 page = f2fs_grab_cache_page(mapping, index, for_write);
650495de
JK		 1209 if (!page)
1210 return ERR_PTR(-ENOMEM);
1211
cb3bc9ee
CY		 1212 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1213 dn.data_blkaddr = ei.blk + index - ei.fofs;
93770ab7
CY		 1214 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr,
1215 DATA_GENERIC_ENHANCE_READ)) {
10f966bb 1216 err = -EFSCORRUPTED;
93770ab7
CY		 1217 goto put_err;
1218 }
cb3bc9ee
CY		 1219 goto got_it;
1220 }
1221
eb47b800 1222 set_new_dnode(&dn, inode, NULL, NULL, 0);
4d57b86d 1223 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
86531d6b
JK		 1224 if (err)
1225 goto put_err;
eb47b800
JK		 1226 f2fs_put_dnode(&dn);
1227
6bacf52f 1228 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
86531d6b
JK		 1229 err = -ENOENT;
1230 goto put_err;
650495de 1231 }
93770ab7
CY		 1232 if (dn.data_blkaddr != NEW_ADDR &&
1233 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode),
1234 dn.data_blkaddr,
1235 DATA_GENERIC_ENHANCE)) {
10f966bb 1236 err = -EFSCORRUPTED;
93770ab7
CY		 1237 goto put_err;
1238 }
cb3bc9ee 1239got_it:
43f3eae1
JK		 1240 if (PageUptodate(page)) {
1241 unlock_page(page);
eb47b800 1242 return page;
43f3eae1 1243 }
eb47b800 1244
d59ff4df
JK		 1245 /*
1246 * A new dentry page is allocated but not able to be written, since its
1247 * new inode page couldn't be allocated due to -ENOSPC.
1248 * In such the case, its blkaddr can be remained as NEW_ADDR.
4d57b86d
CY		 1249 * see, f2fs_add_link -> f2fs_get_new_data_page ->
1250 * f2fs_init_inode_metadata.
d59ff4df
JK		 1251 */
1252 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 1253 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 1254 if (!PageUptodate(page))
1255 SetPageUptodate(page);
43f3eae1 1256 unlock_page(page);
d59ff4df
JK		 1257 return page;
1258 }
eb47b800 1259
b7973091
JY		 1260 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr,
1261 op_flags, for_write);
393ff91f 1262 if (err)
86531d6b 1263 goto put_err;
43f3eae1 1264 return page;
86531d6b
JK		 1265
1266put_err:
1267 f2fs_put_page(page, 1);
1268 return ERR_PTR(err);
43f3eae1
JK		 1269}
1270
4d57b86d 1271struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index)
43f3eae1
JK		 1272{
1273 struct address_space *mapping = inode->i_mapping;
1274 struct page *page;
1275
1276 page = find_get_page(mapping, index);
1277 if (page && PageUptodate(page))
1278 return page;
1279 f2fs_put_page(page, 0);
1280
4d57b86d 1281 page = f2fs_get_read_data_page(inode, index, 0, false);
43f3eae1
JK		 1282 if (IS_ERR(page))
1283 return page;
1284
1285 if (PageUptodate(page))
1286 return page;
1287
1288 wait_on_page_locked(page);
1289 if (unlikely(!PageUptodate(page))) {
1290 f2fs_put_page(page, 0);
1291 return ERR_PTR(-EIO);
1292 }
1293 return page;
1294}
1295
1296/*
1297 * If it tries to access a hole, return an error.
1298 * Because, the callers, functions in dir.c and GC, should be able to know
1299 * whether this page exists or not.
1300 */
4d57b86d 1301struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index,
a56c7c6f 1302 bool for_write)
43f3eae1
JK		 1303{
1304 struct address_space *mapping = inode->i_mapping;
1305 struct page *page;
1306repeat:
4d57b86d 1307 page = f2fs_get_read_data_page(inode, index, 0, for_write);
43f3eae1
JK		 1308 if (IS_ERR(page))
1309 return page;
393ff91f 1310
43f3eae1 1311 /* wait for read completion */
393ff91f 1312 lock_page(page);
6bacf52f 1313 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 1314 f2fs_put_page(page, 1);
1315 goto repeat;
eb47b800 1316 }
1563ac75
CY		 1317 if (unlikely(!PageUptodate(page))) {
1318 f2fs_put_page(page, 1);
1319 return ERR_PTR(-EIO);
1320 }
eb47b800
JK		 1321 return page;
1322}
1323
0a8165d7 1324/*
eb47b800
JK		 1325 * Caller ensures that this data page is never allocated.
1326 * A new zero-filled data page is allocated in the page cache.
39936837 1327 *
4f4124d0
CY		 1328 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
1329 * f2fs_unlock_op().
470f00e9
CY		 1330 * Note that, ipage is set only by make_empty_dir, and if any error occur,
1331 * ipage should be released by this function.
eb47b800 1332 */
4d57b86d 1333struct page *f2fs_get_new_data_page(struct inode *inode,
a8865372 1334 struct page *ipage, pgoff_t index, bool new_i_size)
eb47b800 1335{
eb47b800
JK		 1336 struct address_space *mapping = inode->i_mapping;
1337 struct page *page;
1338 struct dnode_of_data dn;
1339 int err;
7612118a 1340
a56c7c6f 1341 page = f2fs_grab_cache_page(mapping, index, true);
470f00e9
CY		 1342 if (!page) {
1343 /*
1344 * before exiting, we should make sure ipage will be released
1345 * if any error occur.
1346 */
1347 f2fs_put_page(ipage, 1);
01f28610 1348 return ERR_PTR(-ENOMEM);
470f00e9 1349 }
eb47b800 1350
a8865372 1351 set_new_dnode(&dn, inode, ipage, NULL, 0);
b600965c 1352 err = f2fs_reserve_block(&dn, index);
01f28610
JK		 1353 if (err) {
1354 f2fs_put_page(page, 1);
eb47b800 1355 return ERR_PTR(err);
a8865372 1356 }
01f28610
JK		 1357 if (!ipage)
1358 f2fs_put_dnode(&dn);
eb47b800
JK		 1359
1360 if (PageUptodate(page))
01f28610 1361 goto got_it;
eb47b800
JK		 1362
1363 if (dn.data_blkaddr == NEW_ADDR) {
09cbfeaf 1364 zero_user_segment(page, 0, PAGE_SIZE);
237c0790
JK		 1365 if (!PageUptodate(page))
1366 SetPageUptodate(page);
eb47b800 1367 } else {
4375a336 1368 f2fs_put_page(page, 1);
a8865372 1369
7612118a
JK		 1370 /* if ipage exists, blkaddr should be NEW_ADDR */
1371 f2fs_bug_on(F2FS_I_SB(inode), ipage);
4d57b86d 1372 page = f2fs_get_lock_data_page(inode, index, true);
4375a336 1373 if (IS_ERR(page))
7612118a 1374 return page;
eb47b800 1375 }
01f28610 1376got_it:
9edcdabf 1377 if (new_i_size && i_size_read(inode) <
ee6d182f 1378 ((loff_t)(index + 1) << PAGE_SHIFT))
fc9581c8 1379 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
eb47b800
JK		 1380 return page;
1381}
1382
d5097be5 1383static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
bfad7c2d 1384{
4081363f 1385 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
bfad7c2d 1386 struct f2fs_summary sum;
bfad7c2d 1387 struct node_info ni;
6aa58d8a 1388 block_t old_blkaddr;
46008c6d 1389 blkcnt_t count = 1;
0abd675e 1390 int err;
bfad7c2d 1391
91942321 1392 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
bfad7c2d 1393 return -EPERM;
df6136ef 1394
a9419b63 1395 err = f2fs_get_node_info(sbi, dn->nid, &ni, false);
7735730d
CY		 1396 if (err)
1397 return err;
1398
a2ced1ce 1399 dn->data_blkaddr = f2fs_data_blkaddr(dn);
f847c699 1400 if (dn->data_blkaddr != NULL_ADDR)
df6136ef
CY		 1401 goto alloc;
1402
0abd675e
CY		 1403 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
1404 return err;
bfad7c2d 1405
df6136ef 1406alloc:
bfad7c2d 1407 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
6aa58d8a
CY		 1408 old_blkaddr = dn->data_blkaddr;
1409 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr,
093749e2 1410 &sum, seg_type, NULL);
6ce19aff 1411 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
6aa58d8a
CY		 1412 invalidate_mapping_pages(META_MAPPING(sbi),
1413 old_blkaddr, old_blkaddr);
6ce19aff
CY		 1414 f2fs_invalidate_compress_page(sbi, old_blkaddr);
1415 }
86f35dc3 1416 f2fs_update_data_blkaddr(dn, dn->data_blkaddr);
bfad7c2d
JK		 1417 return 0;
1418}
1419
0ef81833 1420void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
59c9081b
YH		 1421{
1422 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
1423 if (lock)
e4544b63 1424 f2fs_down_read(&sbi->node_change);
59c9081b 1425 else
e4544b63 1426 f2fs_up_read(&sbi->node_change);
59c9081b
YH		 1427 } else {
1428 if (lock)
1429 f2fs_lock_op(sbi);
1430 else
1431 f2fs_unlock_op(sbi);
1432 }
1433}
1434
0a8165d7 1435/*
7a88ddb5
CY		 1436 * f2fs_map_blocks() tries to find or build mapping relationship which
1437 * maps continuous logical blocks to physical blocks, and return such
1438 * info via f2fs_map_blocks structure.
eb47b800 1439 */
d323d005 1440int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
e2b4e2bc 1441 int create, int flag)
eb47b800 1442{
003a3e1d 1443 unsigned int maxblocks = map->m_len;
eb47b800 1444 struct dnode_of_data dn;
f9811703 1445 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
f9d6d059 1446 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE;
46008c6d 1447 pgoff_t pgofs, end_offset, end;
bfad7c2d 1448 int err = 0, ofs = 1;
46008c6d
CY		 1449 unsigned int ofs_in_node, last_ofs_in_node;
1450 blkcnt_t prealloc;
94afd6d6 1451 struct extent_info ei = {0, };
7df3a431 1452 block_t blkaddr;
c4020b2d 1453 unsigned int start_pgofs;
71f2c820 1454 int bidx = 0;
eb47b800 1455
dfd02e4d
CY		 1456 if (!maxblocks)
1457 return 0;
1458
71f2c820
CY		 1459 map->m_bdev = inode->i_sb->s_bdev;
1460 map->m_multidev_dio =
1461 f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag);
1462
003a3e1d
JK		 1463 map->m_len = 0;
1464 map->m_flags = 0;
1465
1466 /* it only supports block size == page size */
1467 pgofs = (pgoff_t)map->m_lblk;
46008c6d 1468 end = pgofs + maxblocks;
eb47b800 1469
24b84912 1470 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
b0332a0f 1471 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
f4f0b677
JZ		 1472 map->m_may_create)
1473 goto next_dnode;
1474
003a3e1d
JK		 1475 map->m_pblk = ei.blk + pgofs - ei.fofs;
1476 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
1477 map->m_flags = F2FS_MAP_MAPPED;
c4020b2d
CY		 1478 if (map->m_next_extent)
1479 *map->m_next_extent = pgofs + map->m_len;
1e78e8bd
ST		 1480
1481 /* for hardware encryption, but to avoid potential issue in future */
0d5b9d81 1482 if (flag == F2FS_GET_BLOCK_DIO)
1e78e8bd
ST		 1483 f2fs_wait_on_block_writeback_range(inode,
1484 map->m_pblk, map->m_len);
71f2c820
CY		 1485
1486 if (map->m_multidev_dio) {
1487 block_t blk_addr = map->m_pblk;
1488
1489 bidx = f2fs_target_device_index(sbi, map->m_pblk);
1490
1491 map->m_bdev = FDEV(bidx).bdev;
1492 map->m_pblk -= FDEV(bidx).start_blk;
1493 map->m_len = min(map->m_len,
1494 FDEV(bidx).end_blk + 1 - map->m_pblk);
1495
1496 if (map->m_may_create)
1497 f2fs_update_device_state(sbi, inode->i_ino,
1498 blk_addr, map->m_len);
1499 }
bfad7c2d 1500 goto out;
a2e7d1bf 1501 }
bfad7c2d 1502
4fe71e88 1503next_dnode:
f9d6d059 1504 if (map->m_may_create)
0ef81833 1505 f2fs_do_map_lock(sbi, flag, true);
eb47b800
JK		 1506
1507 /* When reading holes, we need its node page */
1508 set_new_dnode(&dn, inode, NULL, NULL, 0);
4d57b86d 1509 err = f2fs_get_dnode_of_data(&dn, pgofs, mode);
1ec79083 1510 if (err) {
43473f96
CY		 1511 if (flag == F2FS_GET_BLOCK_BMAP)
1512 map->m_pblk = 0;
adf9ea89 1513
da85985c 1514 if (err == -ENOENT) {
adf9ea89
CY		 1515 /*
1516 * There is one exceptional case that read_node_page()
1517 * may return -ENOENT due to filesystem has been
1518 * shutdown or cp_error, so force to convert error
1519 * number to EIO for such case.
1520 */
1521 if (map->m_may_create &&
1522 (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) ||
1523 f2fs_cp_error(sbi))) {
1524 err = -EIO;
1525 goto unlock_out;
1526 }
1527
bfad7c2d 1528 err = 0;
da85985c
CY		 1529 if (map->m_next_pgofs)
1530 *map->m_next_pgofs =
4d57b86d 1531 f2fs_get_next_page_offset(&dn, pgofs);
c4020b2d
CY		 1532 if (map->m_next_extent)
1533 *map->m_next_extent =
4d57b86d 1534 f2fs_get_next_page_offset(&dn, pgofs);
da85985c 1535 }
bfad7c2d 1536 goto unlock_out;
848753aa 1537 }
973163fc 1538
c4020b2d 1539 start_pgofs = pgofs;
46008c6d 1540 prealloc = 0;
230436b3 1541 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
81ca7350 1542 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
4fe71e88
CY		 1543
1544next_block:
a2ced1ce 1545 blkaddr = f2fs_data_blkaddr(&dn);
4fe71e88 1546
c9b60788 1547 if (__is_valid_data_blkaddr(blkaddr) &&
93770ab7 1548 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) {
10f966bb 1549 err = -EFSCORRUPTED;
c9b60788
CY		 1550 goto sync_out;
1551 }
1552
93770ab7 1553 if (__is_valid_data_blkaddr(blkaddr)) {
f847c699 1554 /* use out-place-update for driect IO under LFS mode */
b0332a0f 1555 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO &&
f9d6d059 1556 map->m_may_create) {
f847c699 1557 err = __allocate_data_block(&dn, map->m_seg_type);
05e36006
CY		 1558 if (err)
1559 goto sync_out;
1560 blkaddr = dn.data_blkaddr;
1561 set_inode_flag(inode, FI_APPEND_WRITE);
f847c699
CY		 1562 }
1563 } else {
973163fc 1564 if (create) {
f9811703
CY		 1565 if (unlikely(f2fs_cp_error(sbi))) {
1566 err = -EIO;
4fe71e88 1567 goto sync_out;
f9811703 1568 }
24b84912 1569 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
46008c6d
CY		 1570 if (blkaddr == NULL_ADDR) {
1571 prealloc++;
1572 last_ofs_in_node = dn.ofs_in_node;
1573 }
24b84912 1574 } else {
0a4daae5
JK		 1575 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO &&
1576 flag != F2FS_GET_BLOCK_DIO);
d5097be5
HL		 1577 err = __allocate_data_block(&dn,
1578 map->m_seg_type);
d4dd19ec
JK		 1579 if (!err) {
1580 if (flag == F2FS_GET_BLOCK_PRE_DIO)
1581 file_need_truncate(inode);
91942321 1582 set_inode_flag(inode, FI_APPEND_WRITE);
d4dd19ec 1583 }
24b84912 1584 }
973163fc 1585 if (err)
4fe71e88 1586 goto sync_out;
3f2be043 1587 map->m_flags |= F2FS_MAP_NEW;
4fe71e88 1588 blkaddr = dn.data_blkaddr;
973163fc 1589 } else {
bbe1da7e
CY		 1590 if (f2fs_compressed_file(inode) &&
1591 f2fs_sanity_check_cluster(&dn) &&
1592 (flag != F2FS_GET_BLOCK_FIEMAP ||
1593 IS_ENABLED(CONFIG_F2FS_CHECK_FS))) {
1594 err = -EFSCORRUPTED;
1595 goto sync_out;
1596 }
43473f96
CY		 1597 if (flag == F2FS_GET_BLOCK_BMAP) {
1598 map->m_pblk = 0;
1599 goto sync_out;
1600 }
c4020b2d
CY		 1601 if (flag == F2FS_GET_BLOCK_PRECACHE)
1602 goto sync_out;
da85985c
CY		 1603 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1604 blkaddr == NULL_ADDR) {
1605 if (map->m_next_pgofs)
1606 *map->m_next_pgofs = pgofs + 1;
4c2ac6a8 1607 goto sync_out;
da85985c 1608 }
f3d98e74
CY		 1609 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1610 /* for defragment case */
1611 if (map->m_next_pgofs)
1612 *map->m_next_pgofs = pgofs + 1;
4fe71e88 1613 goto sync_out;
f3d98e74 1614 }
e2b4e2bc 1615 }
e2b4e2bc 1616 }
eb47b800 1617
46008c6d
CY		 1618 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1619 goto skip;
1620
71f2c820
CY		 1621 if (map->m_multidev_dio)
1622 bidx = f2fs_target_device_index(sbi, blkaddr);
1623
4fe71e88
CY		 1624 if (map->m_len == 0) {
1625 /* preallocated unwritten block should be mapped for fiemap. */
1626 if (blkaddr == NEW_ADDR)
1627 map->m_flags |= F2FS_MAP_UNWRITTEN;
1628 map->m_flags |= F2FS_MAP_MAPPED;
1629
1630 map->m_pblk = blkaddr;
1631 map->m_len = 1;
71f2c820
CY		 1632
1633 if (map->m_multidev_dio)
1634 map->m_bdev = FDEV(bidx).bdev;
4fe71e88
CY		 1635 } else if ((map->m_pblk != NEW_ADDR &&
1636 blkaddr == (map->m_pblk + ofs)) ||
b439b103 1637 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
46008c6d 1638 flag == F2FS_GET_BLOCK_PRE_DIO) {
71f2c820
CY		 1639 if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev)
1640 goto sync_out;
4fe71e88
CY		 1641 ofs++;
1642 map->m_len++;
1643 } else {
1644 goto sync_out;
1645 }
bfad7c2d 1646
46008c6d 1647skip:
bfad7c2d
JK		 1648 dn.ofs_in_node++;
1649 pgofs++;
1650
46008c6d
CY		 1651 /* preallocate blocks in batch for one dnode page */
1652 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1653 (pgofs == end || dn.ofs_in_node == end_offset)) {
7df3a431 1654
46008c6d 1655 dn.ofs_in_node = ofs_in_node;
4d57b86d 1656 err = f2fs_reserve_new_blocks(&dn, prealloc);
46008c6d
CY		 1657 if (err)
1658 goto sync_out;
bfad7c2d 1659
46008c6d
CY		 1660 map->m_len += dn.ofs_in_node - ofs_in_node;
1661 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1662 err = -ENOSPC;
1663 goto sync_out;
3104af35 1664 }
46008c6d
CY		 1665 dn.ofs_in_node = end_offset;
1666 }
1667
1668 if (pgofs >= end)
1669 goto sync_out;
1670 else if (dn.ofs_in_node < end_offset)
1671 goto next_block;
1672
c4020b2d
CY		 1673 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1674 if (map->m_flags & F2FS_MAP_MAPPED) {
1675 unsigned int ofs = start_pgofs - map->m_lblk;
1676
1677 f2fs_update_extent_cache_range(&dn,
1678 start_pgofs, map->m_pblk + ofs,
1679 map->m_len - ofs);
1680 }
1681 }
1682
46008c6d
CY		 1683 f2fs_put_dnode(&dn);
1684
f9d6d059 1685 if (map->m_may_create) {
0ef81833 1686 f2fs_do_map_lock(sbi, flag, false);
6f2d8ed6 1687 f2fs_balance_fs(sbi, dn.node_changed);
eb47b800 1688 }
46008c6d 1689 goto next_dnode;
7df3a431 1690
bfad7c2d 1691sync_out:
1e78e8bd 1692
71f2c820
CY		 1693 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) {
1694 /*
1695 * for hardware encryption, but to avoid potential issue
1696 * in future
1697 */
1e78e8bd
ST		 1698 f2fs_wait_on_block_writeback_range(inode,
1699 map->m_pblk, map->m_len);
1700
71f2c820
CY		 1701 if (map->m_multidev_dio) {
1702 block_t blk_addr = map->m_pblk;
1703
1704 bidx = f2fs_target_device_index(sbi, map->m_pblk);
1705
1706 map->m_bdev = FDEV(bidx).bdev;
1707 map->m_pblk -= FDEV(bidx).start_blk;
1708
1709 if (map->m_may_create)
1710 f2fs_update_device_state(sbi, inode->i_ino,
1711 blk_addr, map->m_len);
1712
1713 f2fs_bug_on(sbi, blk_addr + map->m_len >
1714 FDEV(bidx).end_blk + 1);
1715 }
1716 }
1717
c4020b2d
CY		 1718 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1719 if (map->m_flags & F2FS_MAP_MAPPED) {
1720 unsigned int ofs = start_pgofs - map->m_lblk;
1721
1722 f2fs_update_extent_cache_range(&dn,
1723 start_pgofs, map->m_pblk + ofs,
1724 map->m_len - ofs);
1725 }
1726 if (map->m_next_extent)
1727 *map->m_next_extent = pgofs + 1;
1728 }
eb47b800 1729 f2fs_put_dnode(&dn);
bfad7c2d 1730unlock_out:
f9d6d059 1731 if (map->m_may_create) {
0ef81833 1732 f2fs_do_map_lock(sbi, flag, false);
6f2d8ed6 1733 f2fs_balance_fs(sbi, dn.node_changed);
2a340760 1734 }
bfad7c2d 1735out:
71f2c820 1736 trace_f2fs_map_blocks(inode, map, create, flag, err);
bfad7c2d 1737 return err;
eb47b800
JK		 1738}
1739
b91050a8
HL		 1740bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1741{
1742 struct f2fs_map_blocks map;
1743 block_t last_lblk;
1744 int err;
1745
1746 if (pos + len > i_size_read(inode))
1747 return false;
1748
1749 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1750 map.m_next_pgofs = NULL;
1751 map.m_next_extent = NULL;
1752 map.m_seg_type = NO_CHECK_TYPE;
f4f0b677 1753 map.m_may_create = false;
b91050a8
HL		 1754 last_lblk = F2FS_BLK_ALIGN(pos + len);
1755
1756 while (map.m_lblk < last_lblk) {
1757 map.m_len = last_lblk - map.m_lblk;
1758 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1759 if (err || map.m_len == 0)
1760 return false;
1761 map.m_lblk += map.m_len;
1762 }
1763 return true;
1764}
1765
43b9d4b4
JK		 1766static inline u64 bytes_to_blks(struct inode *inode, u64 bytes)
1767{
1768 return (bytes >> inode->i_blkbits);
1769}
1770
1771static inline u64 blks_to_bytes(struct inode *inode, u64 blks)
1772{
1773 return (blks << inode->i_blkbits);
1774}
1775
442a9dbd
CY		 1776static int f2fs_xattr_fiemap(struct inode *inode,
1777 struct fiemap_extent_info *fieinfo)
1778{
1779 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1780 struct page *page;
1781 struct node_info ni;
1782 __u64 phys = 0, len;
1783 __u32 flags;
1784 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1785 int err = 0;
1786
1787 if (f2fs_has_inline_xattr(inode)) {
1788 int offset;
1789
1790 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1791 inode->i_ino, false);
1792 if (!page)
1793 return -ENOMEM;
1794
a9419b63 1795 err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false);
7735730d
CY		 1796 if (err) {
1797 f2fs_put_page(page, 1);
1798 return err;
1799 }
442a9dbd 1800
6cbfcab5 1801 phys = blks_to_bytes(inode, ni.blk_addr);
442a9dbd
CY		 1802 offset = offsetof(struct f2fs_inode, i_addr) +
1803 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
b323fd28 1804 get_inline_xattr_addrs(inode));
442a9dbd
CY		 1805
1806 phys += offset;
1807 len = inline_xattr_size(inode);
1808
1809 f2fs_put_page(page, 1);
1810
1811 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1812
1813 if (!xnid)
1814 flags |= FIEMAP_EXTENT_LAST;
1815
1816 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
dd5a09bd 1817 trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
442a9dbd
CY		 1818 if (err || err == 1)
1819 return err;
1820 }
1821
1822 if (xnid) {
1823 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1824 if (!page)
1825 return -ENOMEM;
1826
a9419b63 1827 err = f2fs_get_node_info(sbi, xnid, &ni, false);
7735730d
CY		 1828 if (err) {
1829 f2fs_put_page(page, 1);
1830 return err;
1831 }
442a9dbd 1832
6cbfcab5 1833 phys = blks_to_bytes(inode, ni.blk_addr);
442a9dbd
CY		 1834 len = inode->i_sb->s_blocksize;
1835
1836 f2fs_put_page(page, 1);
1837
1838 flags = FIEMAP_EXTENT_LAST;
1839 }
1840
dd5a09bd 1841 if (phys) {
442a9dbd 1842 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
dd5a09bd
CY		 1843 trace_f2fs_fiemap(inode, 0, phys, len, flags, err);
1844 }
442a9dbd
CY		 1845
1846 return (err < 0 ? err : 0);
1847}
1848
bf38fbad
CY		 1849static loff_t max_inode_blocks(struct inode *inode)
1850{
1851 loff_t result = ADDRS_PER_INODE(inode);
1852 loff_t leaf_count = ADDRS_PER_BLOCK(inode);
1853
1854 /* two direct node blocks */
1855 result += (leaf_count * 2);
1856
1857 /* two indirect node blocks */
1858 leaf_count *= NIDS_PER_BLOCK;
1859 result += (leaf_count * 2);
1860
1861 /* one double indirect node block */
1862 leaf_count *= NIDS_PER_BLOCK;
1863 result += leaf_count;
1864
1865 return result;
1866}
1867
9ab70134
JK		 1868int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1869 u64 start, u64 len)
1870{
b876f4c9 1871 struct f2fs_map_blocks map;
7f63eb77 1872 sector_t start_blk, last_blk;
da85985c 1873 pgoff_t next_pgofs;
7f63eb77
JK		 1874 u64 logical = 0, phys = 0, size = 0;
1875 u32 flags = 0;
7f63eb77 1876 int ret = 0;
093f0bac 1877 bool compr_cluster = false, compr_appended;
bf38fbad 1878 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size;
093f0bac 1879 unsigned int count_in_cluster = 0;
0bb2045c 1880 loff_t maxbytes;
7f63eb77 1881
c4020b2d
CY		 1882 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1883 ret = f2fs_precache_extents(inode);
1884 if (ret)
1885 return ret;
1886 }
1887
45dd052e 1888 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR);
7f63eb77
JK		 1889 if (ret)
1890 return ret;
1891
f1b43d4c
CY		 1892 inode_lock(inode);
1893
0bb2045c
CX		 1894 maxbytes = max_file_blocks(inode) << F2FS_BLKSIZE_BITS;
1895 if (start > maxbytes) {
1896 ret = -EFBIG;
1897 goto out;
1898 }
1899
1900 if (len > maxbytes || (maxbytes - len) < start)
1901 len = maxbytes - start;
1902
442a9dbd
CY		 1903 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1904 ret = f2fs_xattr_fiemap(inode, fieinfo);
1905 goto out;
1906 }
1907
7975f349 1908 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
67f8cf3c
JK		 1909 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1910 if (ret != -EAGAIN)
f1b43d4c 1911 goto out;
67f8cf3c
JK		 1912 }
1913
6cbfcab5
JK		 1914 if (bytes_to_blks(inode, len) == 0)
1915 len = blks_to_bytes(inode, 1);
7f63eb77 1916
6cbfcab5
JK		 1917 start_blk = bytes_to_blks(inode, start);
1918 last_blk = bytes_to_blks(inode, start + len - 1);
9a950d52 1919
7f63eb77 1920next:
b876f4c9
JK		 1921 memset(&map, 0, sizeof(map));
1922 map.m_lblk = start_blk;
1923 map.m_len = bytes_to_blks(inode, len);
1924 map.m_next_pgofs = &next_pgofs;
1925 map.m_seg_type = NO_CHECK_TYPE;
7f63eb77 1926
093f0bac
DJ		 1927 if (compr_cluster) {
1928 map.m_lblk += 1;
1929 map.m_len = cluster_size - count_in_cluster;
1930 }
bf38fbad 1931
b876f4c9 1932 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
7f63eb77
JK		 1933 if (ret)
1934 goto out;
1935
1936 /* HOLE */
093f0bac 1937 if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) {
da85985c 1938 start_blk = next_pgofs;
58736fa6 1939
6cbfcab5 1940 if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode,
bf38fbad 1941 max_inode_blocks(inode)))
9a950d52 1942 goto prep_next;
58736fa6 1943
9a950d52
FL		 1944 flags |= FIEMAP_EXTENT_LAST;
1945 }
7f63eb77 1946
093f0bac
DJ		 1947 compr_appended = false;
1948 /* In a case of compressed cluster, append this to the last extent */
1949 if (compr_cluster && ((map.m_flags & F2FS_MAP_UNWRITTEN) ||
1950 !(map.m_flags & F2FS_MAP_FLAGS))) {
1951 compr_appended = true;
1952 goto skip_fill;
1953 }
1954
da5af127 1955 if (size) {
0953fe86 1956 flags |= FIEMAP_EXTENT_MERGED;
62230e0d 1957 if (IS_ENCRYPTED(inode))
da5af127
CY		 1958 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1959
9a950d52
FL		 1960 ret = fiemap_fill_next_extent(fieinfo, logical,
1961 phys, size, flags);
dd5a09bd 1962 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret);
bf38fbad
CY		 1963 if (ret)
1964 goto out;
1965 size = 0;
da5af127 1966 }
7f63eb77 1967
bf38fbad 1968 if (start_blk > last_blk)
9a950d52 1969 goto out;
7f63eb77 1970
093f0bac 1971skip_fill:
b876f4c9 1972 if (map.m_pblk == COMPRESS_ADDR) {
bf38fbad 1973 compr_cluster = true;
093f0bac
DJ		 1974 count_in_cluster = 1;
1975 } else if (compr_appended) {
1976 unsigned int appended_blks = cluster_size -
1977 count_in_cluster + 1;
1978 size += blks_to_bytes(inode, appended_blks);
1979 start_blk += appended_blks;
1980 compr_cluster = false;
1981 } else {
1982 logical = blks_to_bytes(inode, start_blk);
1983 phys = __is_valid_data_blkaddr(map.m_pblk) ?
1984 blks_to_bytes(inode, map.m_pblk) : 0;
1985 size = blks_to_bytes(inode, map.m_len);
1986 flags = 0;
1987
1988 if (compr_cluster) {
1989 flags = FIEMAP_EXTENT_ENCODED;
1990 count_in_cluster += map.m_len;
1991 if (count_in_cluster == cluster_size) {
1992 compr_cluster = false;
1993 size += blks_to_bytes(inode, 1);
1994 }
1995 } else if (map.m_flags & F2FS_MAP_UNWRITTEN) {
1996 flags = FIEMAP_EXTENT_UNWRITTEN;
1997 }
7f63eb77 1998
093f0bac
DJ		 1999 start_blk += bytes_to_blks(inode, size);
2000 }
7f63eb77 2001
9a950d52 2002prep_next:
7f63eb77
JK		 2003 cond_resched();
2004 if (fatal_signal_pending(current))
2005 ret = -EINTR;
2006 else
2007 goto next;
2008out:
2009 if (ret == 1)
2010 ret = 0;
2011
5955102c 2012 inode_unlock(inode);
7f63eb77 2013 return ret;
9ab70134
JK		 2014}
2015
95ae251f
EB		 2016static inline loff_t f2fs_readpage_limit(struct inode *inode)
2017{
2018 if (IS_ENABLED(CONFIG_FS_VERITY) &&
2019 (IS_VERITY(inode) || f2fs_verity_in_progress(inode)))
2020 return inode->i_sb->s_maxbytes;
2021
2022 return i_size_read(inode);
2023}
2024
2df0ab04
CY		 2025static int f2fs_read_single_page(struct inode *inode, struct page *page,
2026 unsigned nr_pages,
2027 struct f2fs_map_blocks *map,
2028 struct bio **bio_ret,
2029 sector_t *last_block_in_bio,
2030 bool is_readahead)
2031{
2032 struct bio *bio = *bio_ret;
43b9d4b4 2033 const unsigned blocksize = blks_to_bytes(inode, 1);
2df0ab04
CY		 2034 sector_t block_in_file;
2035 sector_t last_block;
2036 sector_t last_block_in_file;
2037 sector_t block_nr;
2038 int ret = 0;
2039
4969c06a 2040 block_in_file = (sector_t)page_index(page);
2df0ab04 2041 last_block = block_in_file + nr_pages;
43b9d4b4
JK		 2042 last_block_in_file = bytes_to_blks(inode,
2043 f2fs_readpage_limit(inode) + blocksize - 1);
2df0ab04
CY		 2044 if (last_block > last_block_in_file)
2045 last_block = last_block_in_file;
2046
2047 /* just zeroing out page which is beyond EOF */
2048 if (block_in_file >= last_block)
2049 goto zero_out;
2050 /*
2051 * Map blocks using the previous result first.
2052 */
2053 if ((map->m_flags & F2FS_MAP_MAPPED) &&
2054 block_in_file > map->m_lblk &&
2055 block_in_file < (map->m_lblk + map->m_len))
2056 goto got_it;
2057
2058 /*
2059 * Then do more f2fs_map_blocks() calls until we are
2060 * done with this page.
2061 */
2062 map->m_lblk = block_in_file;
2063 map->m_len = last_block - block_in_file;
2064
2065 ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT);
2066 if (ret)
2067 goto out;
2068got_it:
2069 if ((map->m_flags & F2FS_MAP_MAPPED)) {
2070 block_nr = map->m_pblk + block_in_file - map->m_lblk;
2071 SetPageMappedToDisk(page);
2072
2df0ab04 2073 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr,
93770ab7 2074 DATA_GENERIC_ENHANCE_READ)) {
10f966bb 2075 ret = -EFSCORRUPTED;
2df0ab04
CY		 2076 goto out;
2077 }
2078 } else {
2079zero_out:
2080 zero_user_segment(page, 0, PAGE_SIZE);
95ae251f
EB		 2081 if (f2fs_need_verity(inode, page->index) &&
2082 !fsverity_verify_page(page)) {
2083 ret = -EIO;
2084 goto out;
2085 }
2df0ab04
CY		 2086 if (!PageUptodate(page))
2087 SetPageUptodate(page);
2088 unlock_page(page);
2089 goto out;
2090 }
2091
2092 /*
2093 * This page will go to BIO. Do we need to send this
2094 * BIO off first?
2095 */
27aacd28
ST		 2096 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio,
2097 *last_block_in_bio, block_nr) ||
2098 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
2df0ab04
CY		 2099submit_and_realloc:
2100 __submit_bio(F2FS_I_SB(inode), bio, DATA);
2101 bio = NULL;
2102 }
2103 if (bio == NULL) {
2104 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages,
0683728a 2105 is_readahead ? REQ_RAHEAD : 0, page->index,
7f59b277 2106 false);
2df0ab04
CY		 2107 if (IS_ERR(bio)) {
2108 ret = PTR_ERR(bio);
2109 bio = NULL;
2110 goto out;
2111 }
2112 }
2113
2114 /*
2115 * If the page is under writeback, we need to wait for
2116 * its completion to see the correct decrypted data.
2117 */
2118 f2fs_wait_on_block_writeback(inode, block_nr);
2119
2120 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
2121 goto submit_and_realloc;
2122
2123 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA);
8b83ac81 2124 f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE);
2df0ab04
CY		 2125 ClearPageError(page);
2126 *last_block_in_bio = block_nr;
2127 goto out;
2df0ab04
CY		 2128out:
2129 *bio_ret = bio;
2130 return ret;
2131}
2132
4c8ff709
CY		 2133#ifdef CONFIG_F2FS_FS_COMPRESSION
2134int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret,
2135 unsigned nr_pages, sector_t *last_block_in_bio,
0683728a 2136 bool is_readahead, bool for_write)
4c8ff709
CY		 2137{
2138 struct dnode_of_data dn;
2139 struct inode *inode = cc->inode;
2140 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2141 struct bio *bio = *bio_ret;
2142 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size;
2143 sector_t last_block_in_file;
43b9d4b4 2144 const unsigned blocksize = blks_to_bytes(inode, 1);
4c8ff709 2145 struct decompress_io_ctx *dic = NULL;
94afd6d6
CY		 2146 struct extent_info ei = {0, };
2147 bool from_dnode = true;
4c8ff709
CY		 2148 int i;
2149 int ret = 0;
2150
2151 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc));
2152
43b9d4b4
JK		 2153 last_block_in_file = bytes_to_blks(inode,
2154 f2fs_readpage_limit(inode) + blocksize - 1);
4c8ff709
CY		 2155
2156 /* get rid of pages beyond EOF */
2157 for (i = 0; i < cc->cluster_size; i++) {
2158 struct page *page = cc->rpages[i];
2159
2160 if (!page)
2161 continue;
2162 if ((sector_t)page->index >= last_block_in_file) {
2163 zero_user_segment(page, 0, PAGE_SIZE);
2164 if (!PageUptodate(page))
2165 SetPageUptodate(page);
2166 } else if (!PageUptodate(page)) {
2167 continue;
2168 }
2169 unlock_page(page);
9605f75c
JK		 2170 if (for_write)
2171 put_page(page);
4c8ff709
CY		 2172 cc->rpages[i] = NULL;
2173 cc->nr_rpages--;
2174 }
2175
2176 /* we are done since all pages are beyond EOF */
2177 if (f2fs_cluster_is_empty(cc))
2178 goto out;
2179
94afd6d6
CY		 2180 if (f2fs_lookup_extent_cache(inode, start_idx, &ei))
2181 from_dnode = false;
2182
2183 if (!from_dnode)
2184 goto skip_reading_dnode;
2185
4c8ff709
CY		 2186 set_new_dnode(&dn, inode, NULL, NULL, 0);
2187 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
2188 if (ret)
2189 goto out;
2190
a86d27dd 2191 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR);
4c8ff709 2192
94afd6d6 2193skip_reading_dnode:
4c8ff709
CY		 2194 for (i = 1; i < cc->cluster_size; i++) {
2195 block_t blkaddr;
2196
94afd6d6
CY		 2197 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
2198 dn.ofs_in_node + i) :
2199 ei.blk + i - 1;
4c8ff709
CY		 2200
2201 if (!__is_valid_data_blkaddr(blkaddr))
2202 break;
2203
2204 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) {
2205 ret = -EFAULT;
2206 goto out_put_dnode;
2207 }
2208 cc->nr_cpages++;
94afd6d6
CY		 2209
2210 if (!from_dnode && i >= ei.c_len)
2211 break;
4c8ff709
CY		 2212 }
2213
2214 /* nothing to decompress */
2215 if (cc->nr_cpages == 0) {
2216 ret = 0;
2217 goto out_put_dnode;
2218 }
2219
2220 dic = f2fs_alloc_dic(cc);
2221 if (IS_ERR(dic)) {
2222 ret = PTR_ERR(dic);
2223 goto out_put_dnode;
2224 }
2225
6ce19aff 2226 for (i = 0; i < cc->nr_cpages; i++) {
4c8ff709
CY		 2227 struct page *page = dic->cpages[i];
2228 block_t blkaddr;
7f59b277 2229 struct bio_post_read_ctx *ctx;
4c8ff709 2230
94afd6d6
CY		 2231 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page,
2232 dn.ofs_in_node + i + 1) :
2233 ei.blk + i;
4c8ff709 2234
6ce19aff
CY		 2235 f2fs_wait_on_block_writeback(inode, blkaddr);
2236
2237 if (f2fs_load_compressed_page(sbi, page, blkaddr)) {
2238 if (atomic_dec_and_test(&dic->remaining_pages))
bff139b4 2239 f2fs_decompress_cluster(dic, true);
6ce19aff
CY		 2240 continue;
2241 }
2242
27aacd28
ST		 2243 if (bio && (!page_is_mergeable(sbi, bio,
2244 *last_block_in_bio, blkaddr) ||
2245 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) {
4c8ff709
CY		 2246submit_and_realloc:
2247 __submit_bio(sbi, bio, DATA);
2248 bio = NULL;
2249 }
2250
2251 if (!bio) {
2252 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages,
2253 is_readahead ? REQ_RAHEAD : 0,
7f59b277 2254 page->index, for_write);
4c8ff709
CY		 2255 if (IS_ERR(bio)) {
2256 ret = PTR_ERR(bio);
bff139b4 2257 f2fs_decompress_end_io(dic, ret, true);
4c8ff709 2258 f2fs_put_dnode(&dn);
f3494345 2259 *bio_ret = NULL;
4c8ff709
CY		 2260 return ret;
2261 }
2262 }
2263
4c8ff709
CY		 2264 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
2265 goto submit_and_realloc;
2266
a4b68176 2267 ctx = get_post_read_ctx(bio);
7f59b277
EB		 2268 ctx->enabled_steps |= STEP_DECOMPRESS;
2269 refcount_inc(&dic->refcnt);
03382f1a 2270
4c8ff709 2271 inc_page_count(sbi, F2FS_RD_DATA);
8b83ac81 2272 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
9c122384 2273 f2fs_update_iostat(sbi, FS_CDATA_READ_IO, F2FS_BLKSIZE);
4c8ff709
CY		 2274 ClearPageError(page);
2275 *last_block_in_bio = blkaddr;
2276 }
2277
94afd6d6
CY		 2278 if (from_dnode)
2279 f2fs_put_dnode(&dn);
4c8ff709
CY		 2280
2281 *bio_ret = bio;
2282 return 0;
2283
2284out_put_dnode:
94afd6d6
CY		 2285 if (from_dnode)
2286 f2fs_put_dnode(&dn);
4c8ff709 2287out:
7f59b277
EB		 2288 for (i = 0; i < cc->cluster_size; i++) {
2289 if (cc->rpages[i]) {
2290 ClearPageUptodate(cc->rpages[i]);
2291 ClearPageError(cc->rpages[i]);
2292 unlock_page(cc->rpages[i]);
2293 }
2294 }
4c8ff709
CY		 2295 *bio_ret = bio;
2296 return ret;
2297}
2298#endif
2299
f1e88660
JK		 2300/*
2301 * This function was originally taken from fs/mpage.c, and customized for f2fs.
2302 * Major change was from block_size == page_size in f2fs by default.
2303 */
e20a7693 2304static int f2fs_mpage_readpages(struct inode *inode,
23323196 2305 struct readahead_control *rac, struct page *page)
f1e88660
JK		 2306{
2307 struct bio *bio = NULL;
f1e88660 2308 sector_t last_block_in_bio = 0;
f1e88660 2309 struct f2fs_map_blocks map;
4c8ff709
CY		 2310#ifdef CONFIG_F2FS_FS_COMPRESSION
2311 struct compress_ctx cc = {
2312 .inode = inode,
2313 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2314 .cluster_size = F2FS_I(inode)->i_cluster_size,
2315 .cluster_idx = NULL_CLUSTER,
2316 .rpages = NULL,
2317 .cpages = NULL,
2318 .nr_rpages = 0,
2319 .nr_cpages = 0,
2320 };
a2649315 2321 pgoff_t nc_cluster_idx = NULL_CLUSTER;
4c8ff709 2322#endif
23323196 2323 unsigned nr_pages = rac ? readahead_count(rac) : 1;
4c8ff709 2324 unsigned max_nr_pages = nr_pages;
2df0ab04 2325 int ret = 0;
f1e88660
JK		 2326
2327 map.m_pblk = 0;
2328 map.m_lblk = 0;
2329 map.m_len = 0;
2330 map.m_flags = 0;
da85985c 2331 map.m_next_pgofs = NULL;
c4020b2d 2332 map.m_next_extent = NULL;
d5097be5 2333 map.m_seg_type = NO_CHECK_TYPE;
f9d6d059 2334 map.m_may_create = false;
f1e88660 2335
736c0a74 2336 for (; nr_pages; nr_pages--) {
23323196
MWO		 2337 if (rac) {
2338 page = readahead_page(rac);
a83d50bc 2339 prefetchw(&page->flags);
f1e88660
JK		 2340 }
2341
4c8ff709
CY		 2342#ifdef CONFIG_F2FS_FS_COMPRESSION
2343 if (f2fs_compressed_file(inode)) {
2344 /* there are remained comressed pages, submit them */
2345 if (!f2fs_cluster_can_merge_page(&cc, page->index)) {
2346 ret = f2fs_read_multi_pages(&cc, &bio,
2347 max_nr_pages,
2348 &last_block_in_bio,
23323196 2349 rac != NULL, false);
8bfbfb0d 2350 f2fs_destroy_compress_ctx(&cc, false);
4c8ff709
CY		 2351 if (ret)
2352 goto set_error_page;
2353 }
a2649315
FC		 2354 if (cc.cluster_idx == NULL_CLUSTER) {
2355 if (nc_cluster_idx ==
2356 page->index >> cc.log_cluster_size) {
2357 goto read_single_page;
2358 }
4c8ff709 2359
a2649315
FC		 2360 ret = f2fs_is_compressed_cluster(inode, page->index);
2361 if (ret < 0)
2362 goto set_error_page;
2363 else if (!ret) {
2364 nc_cluster_idx =
2365 page->index >> cc.log_cluster_size;
2366 goto read_single_page;
2367 }
4c8ff709 2368
a2649315
FC		 2369 nc_cluster_idx = NULL_CLUSTER;
2370 }
4c8ff709
CY		 2371 ret = f2fs_init_compress_ctx(&cc);
2372 if (ret)
2373 goto set_error_page;
2374
2375 f2fs_compress_ctx_add_page(&cc, page);
2376
2377 goto next_page;
2378 }
2379read_single_page:
2380#endif
2381
2382 ret = f2fs_read_single_page(inode, page, max_nr_pages, &map,
23323196 2383 &bio, &last_block_in_bio, rac);
2df0ab04 2384 if (ret) {
4c8ff709
CY		 2385#ifdef CONFIG_F2FS_FS_COMPRESSION
2386set_error_page:
2387#endif
2df0ab04 2388 SetPageError(page);
09cbfeaf 2389 zero_user_segment(page, 0, PAGE_SIZE);
f1e88660 2390 unlock_page(page);
f1e88660 2391 }
23323196 2392#ifdef CONFIG_F2FS_FS_COMPRESSION
f1e88660 2393next_page:
23323196
MWO		 2394#endif
2395 if (rac)
09cbfeaf 2396 put_page(page);
4c8ff709
CY		 2397
2398#ifdef CONFIG_F2FS_FS_COMPRESSION
2399 if (f2fs_compressed_file(inode)) {
2400 /* last page */
2401 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) {
2402 ret = f2fs_read_multi_pages(&cc, &bio,
2403 max_nr_pages,
2404 &last_block_in_bio,
23323196 2405 rac != NULL, false);
8bfbfb0d 2406 f2fs_destroy_compress_ctx(&cc, false);
4c8ff709
CY		 2407 }
2408 }
2409#endif
f1e88660 2410 }
f1e88660 2411 if (bio)
4fc29c1a 2412 __submit_bio(F2FS_I_SB(inode), bio, DATA);
23323196 2413 return ret;
f1e88660
JK		 2414}
2415
be05584f 2416static int f2fs_read_data_folio(struct file *file, struct folio *folio)
eb47b800 2417{
be05584f 2418 struct page *page = &folio->page;
4969c06a 2419 struct inode *inode = page_file_mapping(page)->host;
b3d208f9 2420 int ret = -EAGAIN;
9ffe0fb5 2421
c20e89cd
CY		 2422 trace_f2fs_readpage(page, DATA);
2423
4c8ff709
CY		 2424 if (!f2fs_is_compress_backend_ready(inode)) {
2425 unlock_page(page);
2426 return -EOPNOTSUPP;
2427 }
2428
e1c42045 2429 /* If the file has inline data, try to read it directly */
9ffe0fb5
HL		 2430 if (f2fs_has_inline_data(inode))
2431 ret = f2fs_read_inline_data(inode, page);
b3d208f9 2432 if (ret == -EAGAIN)
e20a7693 2433 ret = f2fs_mpage_readpages(inode, NULL, page);
9ffe0fb5 2434 return ret;
eb47b800
JK		 2435}
2436
23323196 2437static void f2fs_readahead(struct readahead_control *rac)
eb47b800 2438{
23323196 2439 struct inode *inode = rac->mapping->host;
b8c29400 2440
23323196 2441 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac));
9ffe0fb5 2442
4c8ff709 2443 if (!f2fs_is_compress_backend_ready(inode))
23323196 2444 return;
4c8ff709 2445
704528d8 2446 /* If the file has inline data, skip readahead */
9ffe0fb5 2447 if (f2fs_has_inline_data(inode))
23323196 2448 return;
9ffe0fb5 2449
e20a7693 2450 f2fs_mpage_readpages(inode, rac, NULL);
eb47b800
JK		 2451}
2452
4c8ff709 2453int f2fs_encrypt_one_page(struct f2fs_io_info *fio)
7eab0c0d
HP		 2454{
2455 struct inode *inode = fio->page->mapping->host;
4c8ff709 2456 struct page *mpage, *page;
7eab0c0d
HP		 2457 gfp_t gfp_flags = GFP_NOFS;
2458
1958593e 2459 if (!f2fs_encrypted_file(inode))
7eab0c0d
HP		 2460 return 0;
2461
4c8ff709
CY		 2462 page = fio->compressed_page ? fio->compressed_page : fio->page;
2463
6dbb1796 2464 /* wait for GCed page writeback via META_MAPPING */
0ded69f6 2465 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr);
7eab0c0d 2466
27aacd28
ST		 2467 if (fscrypt_inode_uses_inline_crypto(inode))
2468 return 0;
2469
7eab0c0d 2470retry_encrypt:
4c8ff709
CY		 2471 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page,
2472 PAGE_SIZE, 0, gfp_flags);
6aa58d8a
CY		 2473 if (IS_ERR(fio->encrypted_page)) {
2474 /* flush pending IOs and wait for a while in the ENOMEM case */
2475 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
2476 f2fs_flush_merged_writes(fio->sbi);
4034247a 2477 memalloc_retry_wait(GFP_NOFS);
6aa58d8a
CY		 2478 gfp_flags |= __GFP_NOFAIL;
2479 goto retry_encrypt;
2480 }
2481 return PTR_ERR(fio->encrypted_page);
2482 }
7eab0c0d 2483
6aa58d8a
CY		 2484 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr);
2485 if (mpage) {
2486 if (PageUptodate(mpage))
2487 memcpy(page_address(mpage),
2488 page_address(fio->encrypted_page), PAGE_SIZE);
2489 f2fs_put_page(mpage, 1);
7eab0c0d 2490 }
6aa58d8a 2491 return 0;
7eab0c0d
HP		 2492}
2493
bb9e3bb8
CY		 2494static inline bool check_inplace_update_policy(struct inode *inode,
2495 struct f2fs_io_info *fio)
7eab0c0d 2496{
bb9e3bb8
CY		 2497 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2498 unsigned int policy = SM_I(sbi)->ipu_policy;
7eab0c0d 2499
1018a546
CY		 2500 if (policy & (0x1 << F2FS_IPU_HONOR_OPU_WRITE) &&
2501 is_inode_flag_set(inode, FI_OPU_WRITE))
2502 return false;
bb9e3bb8
CY		 2503 if (policy & (0x1 << F2FS_IPU_FORCE))
2504 return true;
4d57b86d 2505 if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi))
bb9e3bb8
CY		 2506 return true;
2507 if (policy & (0x1 << F2FS_IPU_UTIL) &&
2508 utilization(sbi) > SM_I(sbi)->min_ipu_util)
2509 return true;
4d57b86d 2510 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) &&
bb9e3bb8
CY		 2511 utilization(sbi) > SM_I(sbi)->min_ipu_util)
2512 return true;
2513
2514 /*
2515 * IPU for rewrite async pages
2516 */
2517 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
2518 fio && fio->op == REQ_OP_WRITE &&
2519 !(fio->op_flags & REQ_SYNC) &&
62230e0d 2520 !IS_ENCRYPTED(inode))
bb9e3bb8
CY		 2521 return true;
2522
2523 /* this is only set during fdatasync */
2524 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
2525 is_inode_flag_set(inode, FI_NEED_IPU))
2526 return true;
2527
4354994f
DR		 2528 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2529 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2530 return true;
2531
bb9e3bb8
CY		 2532 return false;
2533}
2534
4d57b86d 2535bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
bb9e3bb8 2536{
859fca6b
CY		 2537 /* swap file is migrating in aligned write mode */
2538 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
2539 return false;
2540
1ad71a27
JK		 2541 if (f2fs_is_pinned_file(inode))
2542 return true;
bb9e3bb8
CY		 2543
2544 /* if this is cold file, we should overwrite to avoid fragmentation */
2545 if (file_is_cold(inode))
2546 return true;
2547
2548 return check_inplace_update_policy(inode, fio);
2549}
2550
4d57b86d 2551bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
bb9e3bb8
CY		 2552{
2553 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2554
19bdba52
JK		 2555 /* The below cases were checked when setting it. */
2556 if (f2fs_is_pinned_file(inode))
2557 return false;
2558 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK))
2559 return true;
b0332a0f 2560 if (f2fs_lfs_mode(sbi))
bb9e3bb8
CY		 2561 return true;
2562 if (S_ISDIR(inode->i_mode))
2563 return true;
af033b2a
CY		 2564 if (IS_NOQUOTA(inode))
2565 return true;
bb9e3bb8
CY		 2566 if (f2fs_is_atomic_file(inode))
2567 return true;
859fca6b
CY		 2568
2569 /* swap file is migrating in aligned write mode */
2570 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
2571 return true;
2572
1018a546
CY		 2573 if (is_inode_flag_set(inode, FI_OPU_WRITE))
2574 return true;
2575
bb9e3bb8 2576 if (fio) {
b763f3be 2577 if (page_private_gcing(fio->page))
bb9e3bb8 2578 return true;
b763f3be 2579 if (page_private_dummy(fio->page))
bb9e3bb8 2580 return true;
4354994f
DR		 2581 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2582 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr)))
2583 return true;
bb9e3bb8
CY		 2584 }
2585 return false;
2586}
2587
7eab0c0d
HP		 2588static inline bool need_inplace_update(struct f2fs_io_info *fio)
2589{
2590 struct inode *inode = fio->page->mapping->host;
2591
4d57b86d 2592 if (f2fs_should_update_outplace(inode, fio))
7eab0c0d
HP		 2593 return false;
2594
4d57b86d 2595 return f2fs_should_update_inplace(inode, fio);
7eab0c0d
HP		 2596}
2597
4d57b86d 2598int f2fs_do_write_data_page(struct f2fs_io_info *fio)
eb47b800 2599{
05ca3632 2600 struct page *page = fio->page;
eb47b800 2601 struct inode *inode = page->mapping->host;
eb47b800 2602 struct dnode_of_data dn;
94afd6d6 2603 struct extent_info ei = {0, };
7735730d 2604 struct node_info ni;
e959c8f5 2605 bool ipu_force = false;
eb47b800
JK		 2606 int err = 0;
2607
3db1de0e
DJ		 2608 /* Use COW inode to make dnode_of_data for atomic write */
2609 if (f2fs_is_atomic_file(inode))
2610 set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0);
2611 else
2612 set_new_dnode(&dn, inode, NULL, NULL, 0);
2613
e959c8f5
HP		 2614 if (need_inplace_update(fio) &&
2615 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
2616 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
a817737e 2617
c9b60788 2618 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
93770ab7 2619 DATA_GENERIC_ENHANCE))
10f966bb 2620 return -EFSCORRUPTED;
c9b60788
CY		 2621
2622 ipu_force = true;
2623 fio->need_lock = LOCK_DONE;
2624 goto got_it;
e959c8f5 2625 }
279d6df2 2626
d29460e5
JK		 2627 /* Deadlock due to between page->lock and f2fs_lock_op */
2628 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
2629 return -EAGAIN;
279d6df2 2630
4d57b86d 2631 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
eb47b800 2632 if (err)
279d6df2 2633 goto out;
eb47b800 2634
28bc106b 2635 fio->old_blkaddr = dn.data_blkaddr;
eb47b800
JK		 2636
2637 /* This page is already truncated */
7a9d7548 2638 if (fio->old_blkaddr == NULL_ADDR) {
2bca1e23 2639 ClearPageUptodate(page);
b763f3be 2640 clear_page_private_gcing(page);
eb47b800 2641 goto out_writepage;
2bca1e23 2642 }
e959c8f5 2643got_it:
c9b60788
CY		 2644 if (__is_valid_data_blkaddr(fio->old_blkaddr) &&
2645 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr,
93770ab7 2646 DATA_GENERIC_ENHANCE)) {
10f966bb 2647 err = -EFSCORRUPTED;
c9b60788
CY		 2648 goto out_writepage;
2649 }
3db1de0e 2650
eb47b800
JK		 2651 /*
2652 * If current allocation needs SSR,
2653 * it had better in-place writes for updated data.
2654 */
93770ab7
CY		 2655 if (ipu_force ||
2656 (__is_valid_data_blkaddr(fio->old_blkaddr) &&
7b525dd0 2657 need_inplace_update(fio))) {
4c8ff709 2658 err = f2fs_encrypt_one_page(fio);
cc15620b
JK		 2659 if (err)
2660 goto out_writepage;
2661
2662 set_page_writeback(page);
17c50035 2663 ClearPageError(page);
279d6df2 2664 f2fs_put_dnode(&dn);
cc15620b 2665 if (fio->need_lock == LOCK_REQ)
279d6df2 2666 f2fs_unlock_op(fio->sbi);
4d57b86d 2667 err = f2fs_inplace_write_data(fio);
6492a335 2668 if (err) {
27aacd28 2669 if (fscrypt_inode_uses_fs_layer_crypto(inode))
d2d0727b 2670 fscrypt_finalize_bounce_page(&fio->encrypted_page);
6492a335
CY		 2671 if (PageWriteback(page))
2672 end_page_writeback(page);
cd23ffa9
CY		 2673 } else {
2674 set_inode_flag(inode, FI_UPDATE_WRITE);
6492a335 2675 }
7eab0c0d 2676 trace_f2fs_do_write_data_page(fio->page, IPU);
279d6df2 2677 return err;
eb47b800 2678 }
279d6df2 2679
cc15620b
JK		 2680 if (fio->need_lock == LOCK_RETRY) {
2681 if (!f2fs_trylock_op(fio->sbi)) {
2682 err = -EAGAIN;
2683 goto out_writepage;
2684 }
2685 fio->need_lock = LOCK_REQ;
2686 }
2687
a9419b63 2688 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false);
7735730d
CY		 2689 if (err)
2690 goto out_writepage;
2691
2692 fio->version = ni.version;
2693
4c8ff709 2694 err = f2fs_encrypt_one_page(fio);
cc15620b
JK		 2695 if (err)
2696 goto out_writepage;
2697
2698 set_page_writeback(page);
17c50035 2699 ClearPageError(page);
cc15620b 2700
4c8ff709
CY		 2701 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR)
2702 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false);
2703
279d6df2 2704 /* LFS mode write path */
4d57b86d 2705 f2fs_outplace_write_data(&dn, fio);
279d6df2
HP		 2706 trace_f2fs_do_write_data_page(page, OPU);
2707 set_inode_flag(inode, FI_APPEND_WRITE);
2708 if (page->index == 0)
2709 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
eb47b800
JK		 2710out_writepage:
2711 f2fs_put_dnode(&dn);
279d6df2 2712out:
cc15620b 2713 if (fio->need_lock == LOCK_REQ)
279d6df2 2714 f2fs_unlock_op(fio->sbi);
eb47b800
JK		 2715 return err;
2716}
2717
4c8ff709 2718int f2fs_write_single_data_page(struct page *page, int *submitted,
8648de2c
CY		 2719 struct bio **bio,
2720 sector_t *last_block,
b0af6d49 2721 struct writeback_control *wbc,
4c8ff709 2722 enum iostat_type io_type,
3afae09f
CY		 2723 int compr_blocks,
2724 bool allow_balance)
eb47b800
JK		 2725{
2726 struct inode *inode = page->mapping->host;
4081363f 2727 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
eb47b800 2728 loff_t i_size = i_size_read(inode);
4c8ff709 2729 const pgoff_t end_index = ((unsigned long long)i_size)
09cbfeaf 2730 >> PAGE_SHIFT;
1f0d5c91 2731 loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT;
9ffe0fb5 2732 unsigned offset = 0;
39936837 2733 bool need_balance_fs = false;
eb47b800 2734 int err = 0;
458e6197 2735 struct f2fs_io_info fio = {
05ca3632 2736 .sbi = sbi,
39d787be 2737 .ino = inode->i_ino,
458e6197 2738 .type = DATA,
04d328de 2739 .op = REQ_OP_WRITE,
7637241e 2740 .op_flags = wbc_to_write_flags(wbc),
e959c8f5 2741 .old_blkaddr = NULL_ADDR,
05ca3632 2742 .page = page,
4375a336 2743 .encrypted_page = NULL,
d68f735b 2744 .submitted = false,
4c8ff709 2745 .compr_blocks = compr_blocks,
cc15620b 2746 .need_lock = LOCK_RETRY,
0d5b9d81 2747 .post_read = f2fs_post_read_required(inode),
b0af6d49 2748 .io_type = io_type,
578c6478 2749 .io_wbc = wbc,
8648de2c
CY		 2750 .bio = bio,
2751 .last_block = last_block,
458e6197 2752 };
eb47b800 2753
ecda0de3
CY		 2754 trace_f2fs_writepage(page, DATA);
2755
db198ae0
CY		 2756 /* we should bypass data pages to proceed the kworkder jobs */
2757 if (unlikely(f2fs_cp_error(sbi))) {
2758 mapping_set_error(page->mapping, -EIO);
1174abfd
CY		 2759 /*
2760 * don't drop any dirty dentry pages for keeping lastest
2761 * directory structure.
2762 */
2763 if (S_ISDIR(inode->i_mode))
2764 goto redirty_out;
db198ae0
CY		 2765 goto out;
2766 }
2767
0771fcc7
CY		 2768 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2769 goto redirty_out;
2770
4c8ff709
CY		 2771 if (page->index < end_index ||
2772 f2fs_verity_in_progress(inode) ||
2773 compr_blocks)
39936837 2774 goto write;
eb47b800
JK		 2775
2776 /*
2777 * If the offset is out-of-range of file size,
2778 * this page does not have to be written to disk.
2779 */
09cbfeaf 2780 offset = i_size & (PAGE_SIZE - 1);
76f60268 2781 if ((page->index >= end_index + 1) || !offset)
39936837 2782 goto out;
eb47b800 2783
09cbfeaf 2784 zero_user_segment(page, offset, PAGE_SIZE);
39936837 2785write:
1e84371f
JK		 2786 if (f2fs_is_drop_cache(inode))
2787 goto out;
eb47b800 2788
435cbab9
JK		 2789 /* Dentry/quota blocks are controlled by checkpoint */
2790 if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) {
79963d96
CY		 2791 /*
2792 * We need to wait for node_write to avoid block allocation during
2793 * checkpoint. This can only happen to quota writes which can cause
2794 * the below discard race condition.
2795 */
2796 if (IS_NOQUOTA(inode))
e4544b63 2797 f2fs_down_read(&sbi->node_write);
79963d96 2798
cc15620b 2799 fio.need_lock = LOCK_DONE;
4d57b86d 2800 err = f2fs_do_write_data_page(&fio);
79963d96
CY		 2801
2802 if (IS_NOQUOTA(inode))
e4544b63 2803 f2fs_up_read(&sbi->node_write);
79963d96 2804
8618b881
JK		 2805 goto done;
2806 }
9ffe0fb5 2807
8618b881 2808 if (!wbc->for_reclaim)
39936837 2809 need_balance_fs = true;
7f3037a5 2810 else if (has_not_enough_free_secs(sbi, 0, 0))
39936837 2811 goto redirty_out;
ef095d19
JK		 2812 else
2813 set_inode_flag(inode, FI_HOT_DATA);
eb47b800 2814
b3d208f9 2815 err = -EAGAIN;
dd7b2333 2816 if (f2fs_has_inline_data(inode)) {
b3d208f9 2817 err = f2fs_write_inline_data(inode, page);
dd7b2333
YH		 2818 if (!err)
2819 goto out;
2820 }
279d6df2 2821
cc15620b 2822 if (err == -EAGAIN) {
4d57b86d 2823 err = f2fs_do_write_data_page(&fio);
cc15620b
JK		 2824 if (err == -EAGAIN) {
2825 fio.need_lock = LOCK_REQ;
4d57b86d 2826 err = f2fs_do_write_data_page(&fio);
cc15620b
JK		 2827 }
2828 }
a0d00fad 2829
eb449797
CY		 2830 if (err) {
2831 file_set_keep_isize(inode);
2832 } else {
c10c9820 2833 spin_lock(&F2FS_I(inode)->i_size_lock);
eb449797
CY		 2834 if (F2FS_I(inode)->last_disk_size < psize)
2835 F2FS_I(inode)->last_disk_size = psize;
c10c9820 2836 spin_unlock(&F2FS_I(inode)->i_size_lock);
eb449797 2837 }
279d6df2 2838
8618b881
JK		 2839done:
2840 if (err && err != -ENOENT)
2841 goto redirty_out;
eb47b800 2842
39936837 2843out:
a7ffdbe2 2844 inode_dec_dirty_pages(inode);
2baf0781 2845 if (err) {
2bca1e23 2846 ClearPageUptodate(page);
b763f3be 2847 clear_page_private_gcing(page);
2baf0781 2848 }
0c3a5797
CY		 2849
2850 if (wbc->for_reclaim) {
bab475c5 2851 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA);
ef095d19 2852 clear_inode_flag(inode, FI_HOT_DATA);
4d57b86d 2853 f2fs_remove_dirty_inode(inode);
d68f735b 2854 submitted = NULL;
0c3a5797 2855 }
eb47b800 2856 unlock_page(page);
186857c5 2857 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) &&
3afae09f 2858 !F2FS_I(inode)->cp_task && allow_balance)
a7881893 2859 f2fs_balance_fs(sbi, need_balance_fs);
0c3a5797 2860
d68f735b 2861 if (unlikely(f2fs_cp_error(sbi))) {
b9109b0e 2862 f2fs_submit_merged_write(sbi, DATA);
0b20fcec 2863 f2fs_submit_merged_ipu_write(sbi, bio, NULL);
d68f735b
JK		 2864 submitted = NULL;
2865 }
2866
2867 if (submitted)
4c8ff709 2868 *submitted = fio.submitted ? 1 : 0;
0c3a5797 2869
eb47b800
JK		 2870 return 0;
2871
eb47b800 2872redirty_out:
76f60268 2873 redirty_page_for_writepage(wbc, page);
5b19d284
JK		 2874 /*
2875 * pageout() in MM traslates EAGAIN, so calls handle_write_error()
2876 * -> mapping_set_error() -> set_bit(AS_EIO, ...).
2877 * file_write_and_wait_range() will see EIO error, which is critical
2878 * to return value of fsync() followed by atomic_write failure to user.
2879 */
2880 if (!err || wbc->for_reclaim)
0002b61b 2881 return AOP_WRITEPAGE_ACTIVATE;
b230e6ca
JK		 2882 unlock_page(page);
2883 return err;
fa9150a8
NJ		 2884}
2885
f566bae8
JK		 2886static int f2fs_write_data_page(struct page *page,
2887 struct writeback_control *wbc)
2888{
4c8ff709
CY		 2889#ifdef CONFIG_F2FS_FS_COMPRESSION
2890 struct inode *inode = page->mapping->host;
2891
2892 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode))))
2893 goto out;
2894
2895 if (f2fs_compressed_file(inode)) {
2896 if (f2fs_is_compressed_cluster(inode, page->index)) {
2897 redirty_page_for_writepage(wbc, page);
2898 return AOP_WRITEPAGE_ACTIVATE;
2899 }
2900 }
2901out:
2902#endif
2903
2904 return f2fs_write_single_data_page(page, NULL, NULL, NULL,
3afae09f 2905 wbc, FS_DATA_IO, 0, true);
f566bae8
JK		 2906}
2907
8f46dcae
CY		 2908/*
2909 * This function was copied from write_cche_pages from mm/page-writeback.c.
2910 * The major change is making write step of cold data page separately from
2911 * warm/hot data page.
2912 */
2913static int f2fs_write_cache_pages(struct address_space *mapping,
b0af6d49
CY		 2914 struct writeback_control *wbc,
2915 enum iostat_type io_type)
8f46dcae
CY		 2916{
2917 int ret = 0;
4c8ff709 2918 int done = 0, retry = 0;
8f46dcae 2919 struct pagevec pvec;
c29fd0c0 2920 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
8648de2c
CY		 2921 struct bio *bio = NULL;
2922 sector_t last_block;
4c8ff709
CY		 2923#ifdef CONFIG_F2FS_FS_COMPRESSION
2924 struct inode *inode = mapping->host;
2925 struct compress_ctx cc = {
2926 .inode = inode,
2927 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
2928 .cluster_size = F2FS_I(inode)->i_cluster_size,
2929 .cluster_idx = NULL_CLUSTER,
2930 .rpages = NULL,
2931 .nr_rpages = 0,
2932 .cpages = NULL,
3271d7eb 2933 .valid_nr_cpages = 0,
4c8ff709
CY		 2934 .rbuf = NULL,
2935 .cbuf = NULL,
2936 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size,
2937 .private = NULL,
2938 };
2939#endif
8f46dcae 2940 int nr_pages;
8f46dcae
CY		 2941 pgoff_t index;
2942 pgoff_t end; /* Inclusive */
2943 pgoff_t done_index;
8f46dcae 2944 int range_whole = 0;
10bbd235 2945 xa_mark_t tag;
bab475c5 2946 int nwritten = 0;
4c8ff709
CY		 2947 int submitted = 0;
2948 int i;
8f46dcae 2949
86679820 2950 pagevec_init(&pvec);
46ae957f 2951
ef095d19
JK		 2952 if (get_dirty_pages(mapping->host) <=
2953 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
2954 set_inode_flag(mapping->host, FI_HOT_DATA);
2955 else
2956 clear_inode_flag(mapping->host, FI_HOT_DATA);
2957
8f46dcae 2958 if (wbc->range_cyclic) {
4df7a75f 2959 index = mapping->writeback_index; /* prev offset */
8f46dcae
CY		 2960 end = -1;
2961 } else {
09cbfeaf
KS		 2962 index = wbc->range_start >> PAGE_SHIFT;
2963 end = wbc->range_end >> PAGE_SHIFT;
8f46dcae
CY		 2964 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2965 range_whole = 1;
8f46dcae
CY		 2966 }
2967 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2968 tag = PAGECACHE_TAG_TOWRITE;
2969 else
2970 tag = PAGECACHE_TAG_DIRTY;
2971retry:
4c8ff709 2972 retry = 0;
8f46dcae
CY		 2973 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2974 tag_pages_for_writeback(mapping, index, end);
2975 done_index = index;
4c8ff709 2976 while (!done && !retry && (index <= end)) {
69c4f35d 2977 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
67fd707f 2978 tag);
8f46dcae
CY		 2979 if (nr_pages == 0)
2980 break;
2981
2982 for (i = 0; i < nr_pages; i++) {
2983 struct page *page = pvec.pages[i];
4c8ff709
CY		 2984 bool need_readd;
2985readd:
2986 need_readd = false;
2987#ifdef CONFIG_F2FS_FS_COMPRESSION
2988 if (f2fs_compressed_file(inode)) {
b368cc5e
FC		 2989 void *fsdata = NULL;
2990 struct page *pagep;
2991 int ret2;
2992
4c8ff709
CY		 2993 ret = f2fs_init_compress_ctx(&cc);
2994 if (ret) {
2995 done = 1;
2996 break;
2997 }
2998
2999 if (!f2fs_cluster_can_merge_page(&cc,
3000 page->index)) {
3001 ret = f2fs_write_multi_pages(&cc,
3002 &submitted, wbc, io_type);
3003 if (!ret)
3004 need_readd = true;
3005 goto result;
3006 }
8f46dcae 3007
4c8ff709
CY		 3008 if (unlikely(f2fs_cp_error(sbi)))
3009 goto lock_page;
3010
b368cc5e
FC		 3011 if (!f2fs_cluster_is_empty(&cc))
3012 goto lock_page;
4c8ff709 3013
b368cc5e 3014 ret2 = f2fs_prepare_compress_overwrite(
4c8ff709
CY		 3015 inode, &pagep,
3016 page->index, &fsdata);
b368cc5e
FC		 3017 if (ret2 < 0) {
3018 ret = ret2;
3019 done = 1;
3020 break;
3021 } else if (ret2 &&
3022 (!f2fs_compress_write_end(inode,
3023 fsdata, page->index, 1) ||
3024 !f2fs_all_cluster_page_loaded(&cc,
3025 &pvec, i, nr_pages))) {
3026 retry = 1;
3027 break;
4c8ff709
CY		 3028 }
3029 }
3030#endif
f8de4331 3031 /* give a priority to WB_SYNC threads */
c29fd0c0 3032 if (atomic_read(&sbi->wb_sync_req[DATA]) &&
f8de4331
CY		 3033 wbc->sync_mode == WB_SYNC_NONE) {
3034 done = 1;
3035 break;
3036 }
4c8ff709
CY		 3037#ifdef CONFIG_F2FS_FS_COMPRESSION
3038lock_page:
3039#endif
8f46dcae 3040 done_index = page->index;
d29460e5 3041retry_write:
8f46dcae
CY		 3042 lock_page(page);
3043
3044 if (unlikely(page->mapping != mapping)) {
3045continue_unlock:
3046 unlock_page(page);
3047 continue;
3048 }
3049
3050 if (!PageDirty(page)) {
3051 /* someone wrote it for us */
3052 goto continue_unlock;
3053 }
3054
8f46dcae 3055 if (PageWriteback(page)) {
0b20fcec 3056 if (wbc->sync_mode != WB_SYNC_NONE)
fec1d657 3057 f2fs_wait_on_page_writeback(page,
bae0ee7a 3058 DATA, true, true);
0b20fcec 3059 else
8f46dcae
CY		 3060 goto continue_unlock;
3061 }
3062
8f46dcae
CY		 3063 if (!clear_page_dirty_for_io(page))
3064 goto continue_unlock;
3065
4c8ff709
CY		 3066#ifdef CONFIG_F2FS_FS_COMPRESSION
3067 if (f2fs_compressed_file(inode)) {
3068 get_page(page);
3069 f2fs_compress_ctx_add_page(&cc, page);
3070 continue;
3071 }
3072#endif
3073 ret = f2fs_write_single_data_page(page, &submitted,
3afae09f
CY		 3074 &bio, &last_block, wbc, io_type,
3075 0, true);
4c8ff709
CY		 3076 if (ret == AOP_WRITEPAGE_ACTIVATE)
3077 unlock_page(page);
3078#ifdef CONFIG_F2FS_FS_COMPRESSION
3079result:
3080#endif
3081 nwritten += submitted;
3082 wbc->nr_to_write -= submitted;
3083
8f46dcae 3084 if (unlikely(ret)) {
0002b61b
CY		 3085 /*
3086 * keep nr_to_write, since vfs uses this to
3087 * get # of written pages.
3088 */
3089 if (ret == AOP_WRITEPAGE_ACTIVATE) {
0002b61b 3090 ret = 0;
4c8ff709 3091 goto next;
d29460e5
JK		 3092 } else if (ret == -EAGAIN) {
3093 ret = 0;
3094 if (wbc->sync_mode == WB_SYNC_ALL) {
a64239d0 3095 f2fs_io_schedule_timeout(
5df7731f 3096 DEFAULT_IO_TIMEOUT);
d29460e5
JK		 3097 goto retry_write;
3098 }
4c8ff709 3099 goto next;
0002b61b 3100 }
b230e6ca
JK		 3101 done_index = page->index + 1;
3102 done = 1;
3103 break;
8f46dcae
CY		 3104 }
3105
4c8ff709 3106 if (wbc->nr_to_write <= 0 &&
687de7f1 3107 wbc->sync_mode == WB_SYNC_NONE) {
8f46dcae
CY		 3108 done = 1;
3109 break;
3110 }
4c8ff709
CY		 3111next:
3112 if (need_readd)
3113 goto readd;
8f46dcae
CY		 3114 }
3115 pagevec_release(&pvec);
3116 cond_resched();
3117 }
4c8ff709
CY		 3118#ifdef CONFIG_F2FS_FS_COMPRESSION
3119 /* flush remained pages in compress cluster */
3120 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) {
3121 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type);
3122 nwritten += submitted;
3123 wbc->nr_to_write -= submitted;
3124 if (ret) {
3125 done = 1;
3126 retry = 0;
3127 }
3128 }
adfc6943 3129 if (f2fs_compressed_file(inode))
8bfbfb0d 3130 f2fs_destroy_compress_ctx(&cc, false);
4c8ff709 3131#endif
e78790f8 3132 if (retry) {
8f46dcae 3133 index = 0;
e78790f8 3134 end = -1;
8f46dcae
CY		 3135 goto retry;
3136 }
e78790f8
ST		 3137 if (wbc->range_cyclic && !done)
3138 done_index = 0;
8f46dcae
CY		 3139 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
3140 mapping->writeback_index = done_index;
3141
bab475c5 3142 if (nwritten)
b9109b0e 3143 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
bab475c5 3144 NULL, 0, DATA);
8648de2c
CY		 3145 /* submit cached bio of IPU write */
3146 if (bio)
0b20fcec 3147 f2fs_submit_merged_ipu_write(sbi, &bio, NULL);
6ca56ca4 3148
8f46dcae
CY		 3149 return ret;
3150}
3151
853137ce
JK		 3152static inline bool __should_serialize_io(struct inode *inode,
3153 struct writeback_control *wbc)
3154{
b13f67ff
CY		 3155 /* to avoid deadlock in path of data flush */
3156 if (F2FS_I(inode)->cp_task)
3157 return false;
3158
853137ce
JK		 3159 if (!S_ISREG(inode->i_mode))
3160 return false;
af033b2a
CY		 3161 if (IS_NOQUOTA(inode))
3162 return false;
b13f67ff 3163
602a16d5 3164 if (f2fs_need_compress_data(inode))
b13f67ff 3165 return true;
853137ce
JK		 3166 if (wbc->sync_mode != WB_SYNC_ALL)
3167 return true;
3168 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks)
3169 return true;
3170 return false;
3171}
3172
fc99fe27 3173static int __f2fs_write_data_pages(struct address_space *mapping,
b0af6d49
CY		 3174 struct writeback_control *wbc,
3175 enum iostat_type io_type)
eb47b800
JK		 3176{
3177 struct inode *inode = mapping->host;
4081363f 3178 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
9dfa1baf 3179 struct blk_plug plug;
eb47b800 3180 int ret;
853137ce 3181 bool locked = false;
eb47b800 3182
cfb185a1 3183 /* deal with chardevs and other special file */
3184 if (!mapping->a_ops->writepage)
3185 return 0;
3186
6a290544
CY		 3187 /* skip writing if there is no dirty page in this inode */
3188 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
3189 return 0;
3190
0771fcc7
CY		 3191 /* during POR, we don't need to trigger writepage at all. */
3192 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
3193 goto skip_write;
3194
af033b2a
CY		 3195 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) &&
3196 wbc->sync_mode == WB_SYNC_NONE &&
a1257023 3197 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
4d57b86d 3198 f2fs_available_free_memory(sbi, DIRTY_DENTS))
a1257023
JK		 3199 goto skip_write;
3200
1018a546
CY		 3201 /* skip writing in file defragment preparing stage */
3202 if (is_inode_flag_set(inode, FI_SKIP_WRITES))
d323d005
CY		 3203 goto skip_write;
3204
d31c7c3f
YH		 3205 trace_f2fs_writepages(mapping->host, wbc, DATA);
3206
687de7f1
JK		 3207 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
3208 if (wbc->sync_mode == WB_SYNC_ALL)
c29fd0c0 3209 atomic_inc(&sbi->wb_sync_req[DATA]);
34415099
CY		 3210 else if (atomic_read(&sbi->wb_sync_req[DATA])) {
3211 /* to avoid potential deadlock */
3212 if (current->plug)
3213 blk_finish_plug(current->plug);
687de7f1 3214 goto skip_write;
34415099 3215 }
687de7f1 3216
853137ce
JK		 3217 if (__should_serialize_io(inode, wbc)) {
3218 mutex_lock(&sbi->writepages);
3219 locked = true;
3220 }
3221
9dfa1baf 3222 blk_start_plug(&plug);
b0af6d49 3223 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
9dfa1baf 3224 blk_finish_plug(&plug);
687de7f1 3225
853137ce
JK		 3226 if (locked)
3227 mutex_unlock(&sbi->writepages);
3228
687de7f1 3229 if (wbc->sync_mode == WB_SYNC_ALL)
c29fd0c0 3230 atomic_dec(&sbi->wb_sync_req[DATA]);
28ea6162
JK		 3231 /*
3232 * if some pages were truncated, we cannot guarantee its mapping->host
3233 * to detect pending bios.
3234 */
458e6197 3235
4d57b86d 3236 f2fs_remove_dirty_inode(inode);
eb47b800 3237 return ret;
d3baf95d
JK		 3238
3239skip_write:
a7ffdbe2 3240 wbc->pages_skipped += get_dirty_pages(inode);
d31c7c3f 3241 trace_f2fs_writepages(mapping->host, wbc, DATA);
d3baf95d 3242 return 0;
eb47b800
JK		 3243}
3244
b0af6d49
CY		 3245static int f2fs_write_data_pages(struct address_space *mapping,
3246 struct writeback_control *wbc)
3247{
3248 struct inode *inode = mapping->host;
3249
3250 return __f2fs_write_data_pages(mapping, wbc,
3251 F2FS_I(inode)->cp_task == current ?
3252 FS_CP_DATA_IO : FS_DATA_IO);
3253}
3254
a1e09b03 3255void f2fs_write_failed(struct inode *inode, loff_t to)
3aab8f82 3256{
819d9153 3257 loff_t i_size = i_size_read(inode);
3aab8f82 3258
3f188c23
JK		 3259 if (IS_NOQUOTA(inode))
3260 return;
3261
95ae251f
EB		 3262 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */
3263 if (to > i_size && !f2fs_verity_in_progress(inode)) {
e4544b63 3264 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
6abaa83c 3265 filemap_invalidate_lock(inode->i_mapping);
a33c1502 3266
819d9153 3267 truncate_pagecache(inode, i_size);
3f188c23 3268 f2fs_truncate_blocks(inode, i_size, true);
a33c1502 3269
6abaa83c 3270 filemap_invalidate_unlock(inode->i_mapping);
e4544b63 3271 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
3aab8f82
CY		 3272 }
3273}
3274
2aadac08
JK		 3275static int prepare_write_begin(struct f2fs_sb_info *sbi,
3276 struct page *page, loff_t pos, unsigned len,
3277 block_t *blk_addr, bool *node_changed)
3278{
3279 struct inode *inode = page->mapping->host;
3280 pgoff_t index = page->index;
3281 struct dnode_of_data dn;
3282 struct page *ipage;
b4d07a3e 3283 bool locked = false;
94afd6d6 3284 struct extent_info ei = {0, };
2aadac08 3285 int err = 0;
2866fb16 3286 int flag;
2aadac08 3287
24b84912 3288 /*
3d697a4a
EB		 3289 * If a whole page is being written and we already preallocated all the
3290 * blocks, then there is no need to get a block address now.
24b84912 3291 */
3d697a4a 3292 if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL))
24b84912
JK		 3293 return 0;
3294
2866fb16
SY		 3295 /* f2fs_lock_op avoids race between write CP and convert_inline_page */
3296 if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode))
3297 flag = F2FS_GET_BLOCK_DEFAULT;
3298 else
3299 flag = F2FS_GET_BLOCK_PRE_AIO;
3300
b4d07a3e 3301 if (f2fs_has_inline_data(inode) ||
09cbfeaf 3302 (pos & PAGE_MASK) >= i_size_read(inode)) {
0ef81833 3303 f2fs_do_map_lock(sbi, flag, true);
b4d07a3e
JK		 3304 locked = true;
3305 }
4c8ff709 3306
b4d07a3e 3307restart:
2aadac08 3308 /* check inline_data */
4d57b86d 3309 ipage = f2fs_get_node_page(sbi, inode->i_ino);
2aadac08
JK		 3310 if (IS_ERR(ipage)) {
3311 err = PTR_ERR(ipage);
3312 goto unlock_out;
3313 }
3314
3315 set_new_dnode(&dn, inode, ipage, ipage, 0);
3316
3317 if (f2fs_has_inline_data(inode)) {
f2470371 3318 if (pos + len <= MAX_INLINE_DATA(inode)) {
4d57b86d 3319 f2fs_do_read_inline_data(page, ipage);
91942321 3320 set_inode_flag(inode, FI_DATA_EXIST);
ab47036d 3321 if (inode->i_nlink)
b763f3be 3322 set_page_private_inline(ipage);
2aadac08
JK		 3323 } else {
3324 err = f2fs_convert_inline_page(&dn, page);
3325 if (err)
b4d07a3e
JK		 3326 goto out;
3327 if (dn.data_blkaddr == NULL_ADDR)
3328 err = f2fs_get_block(&dn, index);
3329 }
3330 } else if (locked) {
3331 err = f2fs_get_block(&dn, index);
3332 } else {
3333 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
3334 dn.data_blkaddr = ei.blk + index - ei.fofs;
3335 } else {
b4d07a3e 3336 /* hole case */
4d57b86d 3337 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
4da7bf5a 3338 if (err || dn.data_blkaddr == NULL_ADDR) {
b4d07a3e 3339 f2fs_put_dnode(&dn);
0ef81833 3340 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
59c9081b 3341 true);
2866fb16 3342 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO);
b4d07a3e
JK		 3343 locked = true;
3344 goto restart;
3345 }
2aadac08
JK		 3346 }
3347 }
b4d07a3e 3348
2aadac08
JK		 3349 /* convert_inline_page can make node_changed */
3350 *blk_addr = dn.data_blkaddr;
3351 *node_changed = dn.node_changed;
b4d07a3e 3352out:
2aadac08
JK		 3353 f2fs_put_dnode(&dn);
3354unlock_out:
b4d07a3e 3355 if (locked)
0ef81833 3356 f2fs_do_map_lock(sbi, flag, false);
2aadac08
JK		 3357 return err;
3358}
3359
3db1de0e
DJ		 3360static int __find_data_block(struct inode *inode, pgoff_t index,
3361 block_t *blk_addr)
3362{
3363 struct dnode_of_data dn;
3364 struct page *ipage;
3365 struct extent_info ei = {0, };
3366 int err = 0;
3367
3368 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino);
3369 if (IS_ERR(ipage))
3370 return PTR_ERR(ipage);
3371
3372 set_new_dnode(&dn, inode, ipage, ipage, 0);
3373
3374 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
3375 dn.data_blkaddr = ei.blk + index - ei.fofs;
3376 } else {
3377 /* hole case */
3378 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
3379 if (err) {
3380 dn.data_blkaddr = NULL_ADDR;
3381 err = 0;
3382 }
3383 }
3384 *blk_addr = dn.data_blkaddr;
3385 f2fs_put_dnode(&dn);
3386 return err;
3387}
3388
3389static int __reserve_data_block(struct inode *inode, pgoff_t index,
3390 block_t *blk_addr, bool *node_changed)
3391{
3392 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3393 struct dnode_of_data dn;
3394 struct page *ipage;
3395 int err = 0;
3396
3397 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
3398
3399 ipage = f2fs_get_node_page(sbi, inode->i_ino);
3400 if (IS_ERR(ipage)) {
3401 err = PTR_ERR(ipage);
3402 goto unlock_out;
3403 }
3404 set_new_dnode(&dn, inode, ipage, ipage, 0);
3405
3406 err = f2fs_get_block(&dn, index);
3407
3408 *blk_addr = dn.data_blkaddr;
3409 *node_changed = dn.node_changed;
3410 f2fs_put_dnode(&dn);
3411
3412unlock_out:
3413 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
3414 return err;
3415}
3416
3417static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi,
3418 struct page *page, loff_t pos, unsigned int len,
3419 block_t *blk_addr, bool *node_changed)
3420{
3421 struct inode *inode = page->mapping->host;
3422 struct inode *cow_inode = F2FS_I(inode)->cow_inode;
3423 pgoff_t index = page->index;
3424 int err = 0;
f8e2f32b 3425 block_t ori_blk_addr = NULL_ADDR;
3db1de0e
DJ		 3426
3427 /* If pos is beyond the end of file, reserve a new block in COW inode */
3428 if ((pos & PAGE_MASK) >= i_size_read(inode))
f8e2f32b 3429 goto reserve_block;
3db1de0e
DJ		 3430
3431 /* Look for the block in COW inode first */
3432 err = __find_data_block(cow_inode, index, blk_addr);
3433 if (err)
3434 return err;
3435 else if (*blk_addr != NULL_ADDR)
3436 return 0;
3437
3438 /* Look for the block in the original inode */
3439 err = __find_data_block(inode, index, &ori_blk_addr);
3440 if (err)
3441 return err;
3442
f8e2f32b 3443reserve_block:
3db1de0e
DJ		 3444 /* Finally, we should reserve a new block in COW inode for the update */
3445 err = __reserve_data_block(cow_inode, index, blk_addr, node_changed);
3446 if (err)
3447 return err;
f8e2f32b 3448 inc_atomic_write_cnt(inode);
3db1de0e
DJ		 3449
3450 if (ori_blk_addr != NULL_ADDR)
3451 *blk_addr = ori_blk_addr;
3452 return 0;
3453}
3454
eb47b800 3455static int f2fs_write_begin(struct file *file, struct address_space *mapping,
9d6b0cd7 3456 loff_t pos, unsigned len, struct page **pagep, void **fsdata)
eb47b800
JK		 3457{
3458 struct inode *inode = mapping->host;
4081363f 3459 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
86531d6b 3460 struct page *page = NULL;
09cbfeaf 3461 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
3db1de0e 3462 bool need_balance = false;
2aadac08 3463 block_t blkaddr = NULL_ADDR;
eb47b800
JK		 3464 int err = 0;
3465
9d6b0cd7 3466 trace_f2fs_write_begin(inode, pos, len);
62aed044 3467
00e09c0b
CY		 3468 if (!f2fs_is_checkpoint_ready(sbi)) {
3469 err = -ENOSPC;
4354994f 3470 goto fail;
00e09c0b 3471 }
4354994f 3472
5f727395
JK		 3473 /*
3474 * We should check this at this moment to avoid deadlock on inode page
3475 * and #0 page. The locking rule for inline_data conversion should be:
3476 * lock_page(page #0) -> lock_page(inode_page)
3477 */
3478 if (index != 0) {
3479 err = f2fs_convert_inline_inode(inode);
3480 if (err)
3481 goto fail;
3482 }
4c8ff709
CY		 3483
3484#ifdef CONFIG_F2FS_FS_COMPRESSION
3485 if (f2fs_compressed_file(inode)) {
3486 int ret;
3487
3488 *fsdata = NULL;
3489
9b56adcf 3490 if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode)))
7eab7a69
FC		 3491 goto repeat;
3492
4c8ff709
CY		 3493 ret = f2fs_prepare_compress_overwrite(inode, pagep,
3494 index, fsdata);
3495 if (ret < 0) {
3496 err = ret;
3497 goto fail;
3498 } else if (ret) {
3499 return 0;
3500 }
3501 }
3502#endif
3503
afcb7ca0 3504repeat:
86d54795
JK		 3505 /*
3506 * Do not use grab_cache_page_write_begin() to avoid deadlock due to
3507 * wait_for_stable_page. Will wait that below with our IO control.
3508 */
01eccef7 3509 page = f2fs_pagecache_get_page(mapping, index,
86d54795 3510 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
3aab8f82
CY		 3511 if (!page) {
3512 err = -ENOMEM;
3513 goto fail;
3514 }
d5f66990 3515
4c8ff709
CY		 3516 /* TODO: cluster can be compressed due to race with .writepage */
3517
eb47b800
JK		 3518 *pagep = page;
3519
3db1de0e
DJ		 3520 if (f2fs_is_atomic_file(inode))
3521 err = prepare_atomic_write_begin(sbi, page, pos, len,
3522 &blkaddr, &need_balance);
3523 else
3524 err = prepare_write_begin(sbi, page, pos, len,
2aadac08 3525 &blkaddr, &need_balance);
9ba69cf9 3526 if (err)
2aadac08 3527 goto fail;
9ba69cf9 3528
af033b2a
CY		 3529 if (need_balance && !IS_NOQUOTA(inode) &&
3530 has_not_enough_free_secs(sbi, 0, 0)) {
2a340760 3531 unlock_page(page);
2c4db1a6 3532 f2fs_balance_fs(sbi, true);
2a340760
JK		 3533 lock_page(page);
3534 if (page->mapping != mapping) {
3535 /* The page got truncated from under us */
3536 f2fs_put_page(page, 1);
3537 goto repeat;
3538 }
3539 }
3540
bae0ee7a 3541 f2fs_wait_on_page_writeback(page, DATA, false, true);
b3d208f9 3542
649d7df2
JK		 3543 if (len == PAGE_SIZE || PageUptodate(page))
3544 return 0;
eb47b800 3545
95ae251f
EB		 3546 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) &&
3547 !f2fs_verity_in_progress(inode)) {
746e2403
YH		 3548 zero_user_segment(page, len, PAGE_SIZE);
3549 return 0;
3550 }
3551
2aadac08 3552 if (blkaddr == NEW_ADDR) {
09cbfeaf 3553 zero_user_segment(page, 0, PAGE_SIZE);
649d7df2 3554 SetPageUptodate(page);
eb47b800 3555 } else {
93770ab7
CY		 3556 if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
3557 DATA_GENERIC_ENHANCE_READ)) {
10f966bb 3558 err = -EFSCORRUPTED;
93770ab7
CY		 3559 goto fail;
3560 }
b7973091 3561 err = f2fs_submit_page_read(inode, page, blkaddr, 0, true);
13ba41e3 3562 if (err)
78682f79 3563 goto fail;
d54c795b 3564
393ff91f 3565 lock_page(page);
6bacf52f 3566 if (unlikely(page->mapping != mapping)) {
afcb7ca0
JK		 3567 f2fs_put_page(page, 1);
3568 goto repeat;
eb47b800 3569 }
1563ac75
CY		 3570 if (unlikely(!PageUptodate(page))) {
3571 err = -EIO;
3572 goto fail;
4375a336 3573 }
eb47b800 3574 }
eb47b800 3575 return 0;
9ba69cf9 3576
3aab8f82 3577fail:
86531d6b 3578 f2fs_put_page(page, 1);
3e679dc7 3579 f2fs_write_failed(inode, pos + len);
3aab8f82 3580 return err;
eb47b800
JK		 3581}
3582
a1dd3c13
JK		 3583static int f2fs_write_end(struct file *file,
3584 struct address_space *mapping,
3585 loff_t pos, unsigned len, unsigned copied,
3586 struct page *page, void *fsdata)
3587{
3588 struct inode *inode = page->mapping->host;
3589
dfb2bf38
CY		 3590 trace_f2fs_write_end(inode, pos, len, copied);
3591
649d7df2
JK		 3592 /*
3593 * This should be come from len == PAGE_SIZE, and we expect copied
3594 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
3595 * let generic_perform_write() try to copy data again through copied=0.
3596 */
3597 if (!PageUptodate(page)) {
746e2403 3598 if (unlikely(copied != len))
649d7df2
JK		 3599 copied = 0;
3600 else
3601 SetPageUptodate(page);
3602 }
4c8ff709
CY		 3603
3604#ifdef CONFIG_F2FS_FS_COMPRESSION
3605 /* overwrite compressed file */
3606 if (f2fs_compressed_file(inode) && fsdata) {
3607 f2fs_compress_write_end(inode, fsdata, page->index, copied);
3608 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
944dd22e
CY		 3609
3610 if (pos + copied > i_size_read(inode) &&
3611 !f2fs_verity_in_progress(inode))
3612 f2fs_i_size_write(inode, pos + copied);
4c8ff709
CY		 3613 return copied;
3614 }
3615#endif
3616
649d7df2
JK		 3617 if (!copied)
3618 goto unlock_out;
3619
34ba94ba 3620 set_page_dirty(page);
a1dd3c13 3621
95ae251f 3622 if (pos + copied > i_size_read(inode) &&
3db1de0e 3623 !f2fs_verity_in_progress(inode)) {
fc9581c8 3624 f2fs_i_size_write(inode, pos + copied);
3db1de0e
DJ		 3625 if (f2fs_is_atomic_file(inode))
3626 f2fs_i_size_write(F2FS_I(inode)->cow_inode,
3627 pos + copied);
3628 }
649d7df2 3629unlock_out:
3024c9a1 3630 f2fs_put_page(page, 1);
d0239e1b 3631 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
a1dd3c13
JK		 3632 return copied;
3633}
3634
91503996 3635void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
eb47b800 3636{
91503996 3637 struct inode *inode = folio->mapping->host;
487261f3 3638 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
a7ffdbe2 3639
487261f3 3640 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
91503996 3641 (offset || length != folio_size(folio)))
a7ffdbe2
JK		 3642 return;
3643
91503996 3644 if (folio_test_dirty(folio)) {
933439c8 3645 if (inode->i_ino == F2FS_META_INO(sbi)) {
487261f3 3646 dec_page_count(sbi, F2FS_DIRTY_META);
933439c8 3647 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
487261f3 3648 dec_page_count(sbi, F2FS_DIRTY_NODES);
933439c8 3649 } else {
487261f3 3650 inode_dec_dirty_pages(inode);
4d57b86d 3651 f2fs_remove_dirty_inode(inode);
933439c8 3652 }
487261f3 3653 }
decd36b6 3654
91503996 3655 clear_page_private_gcing(&folio->page);
2baf0781 3656
2a64e303
CY		 3657 if (test_opt(sbi, COMPRESS_CACHE) &&
3658 inode->i_ino == F2FS_COMPRESS_INO(sbi))
91503996 3659 clear_page_private_data(&folio->page);
6ce19aff 3660
91503996 3661 folio_detach_private(folio);
eb47b800
JK		 3662}
3663
c26cd045 3664bool f2fs_release_folio(struct folio *folio, gfp_t wait)
eb47b800 3665{
c26cd045
MWO		 3666 struct f2fs_sb_info *sbi;
3667
3668 /* If this is dirty folio, keep private data */
3669 if (folio_test_dirty(folio))
3670 return false;
f68daeeb 3671
c26cd045
MWO		 3672 sbi = F2FS_M_SB(folio->mapping);
3673 if (test_opt(sbi, COMPRESS_CACHE)) {
3674 struct inode *inode = folio->mapping->host;
6ce19aff 3675
c26cd045
MWO		 3676 if (inode->i_ino == F2FS_COMPRESS_INO(sbi))
3677 clear_page_private_data(&folio->page);
6ce19aff
CY		 3678 }
3679
c26cd045 3680 clear_page_private_gcing(&folio->page);
b763f3be 3681
c26cd045
MWO		 3682 folio_detach_private(folio);
3683 return true;
eb47b800
JK		 3684}
3685
4f5e34f7
MWO		 3686static bool f2fs_dirty_data_folio(struct address_space *mapping,
3687 struct folio *folio)
eb47b800 3688{
4f5e34f7 3689 struct inode *inode = mapping->host;
eb47b800 3690
4f5e34f7 3691 trace_f2fs_set_page_dirty(&folio->page, DATA);
26c6b887 3692
4f5e34f7
MWO		 3693 if (!folio_test_uptodate(folio))
3694 folio_mark_uptodate(folio);
3695 BUG_ON(folio_test_swapcache(folio));
34ba94ba 3696
4f5e34f7
MWO		 3697 if (!folio_test_dirty(folio)) {
3698 filemap_dirty_folio(mapping, folio);
3699 f2fs_update_dirty_folio(inode, folio);
3700 return true;
eb47b800 3701 }
0fb5b2eb 3702 return false;
eb47b800
JK		 3703}
3704
c1c63387
CY		 3705
3706static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block)
3707{
3708#ifdef CONFIG_F2FS_FS_COMPRESSION
3709 struct dnode_of_data dn;
3710 sector_t start_idx, blknr = 0;
3711 int ret;
3712
3713 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size);
3714
3715 set_new_dnode(&dn, inode, NULL, NULL, 0);
3716 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
3717 if (ret)
3718 return 0;
3719
3720 if (dn.data_blkaddr != COMPRESS_ADDR) {
3721 dn.ofs_in_node += block - start_idx;
3722 blknr = f2fs_data_blkaddr(&dn);
3723 if (!__is_valid_data_blkaddr(blknr))
3724 blknr = 0;
3725 }
3726
3727 f2fs_put_dnode(&dn);
c1c63387
CY		 3728 return blknr;
3729#else
250e84d7 3730 return 0;
c1c63387
CY		 3731#endif
3732}
3733
3734
c01e54b7
JK		 3735static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
3736{
454ae7e5 3737 struct inode *inode = mapping->host;
b79b0a31 3738 sector_t blknr = 0;
454ae7e5 3739
1d373a0e 3740 if (f2fs_has_inline_data(inode))
b79b0a31 3741 goto out;
1d373a0e
JK		 3742
3743 /* make sure allocating whole blocks */
3744 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
3745 filemap_write_and_wait(mapping);
3746
4eda1682 3747 /* Block number less than F2FS MAX BLOCKS */
6d1451bf 3748 if (unlikely(block >= max_file_blocks(inode)))
4eda1682 3749 goto out;
c1c63387 3750
4eda1682
DJ		 3751 if (f2fs_compressed_file(inode)) {
3752 blknr = f2fs_bmap_compress(inode, block);
3753 } else {
b876f4c9
JK		 3754 struct f2fs_map_blocks map;
3755
3756 memset(&map, 0, sizeof(map));
3757 map.m_lblk = block;
3758 map.m_len = 1;
3759 map.m_next_pgofs = NULL;
3760 map.m_seg_type = NO_CHECK_TYPE;
3761
3762 if (!f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_BMAP))
3763 blknr = map.m_pblk;
4eda1682 3764 }
b79b0a31
CY		 3765out:
3766 trace_f2fs_bmap(inode, block, blknr);
3767 return blknr;
429511cd
CY		 3768}
3769
5b7a487c
WG		 3770#ifdef CONFIG_MIGRATION
3771#include <linux/migrate.h>
3772
3773int f2fs_migrate_page(struct address_space *mapping,
3774 struct page *newpage, struct page *page, enum migrate_mode mode)
3775{
3db1de0e 3776 int rc, extra_count = 0;
5b7a487c
WG		 3777
3778 BUG_ON(PageWriteback(page));
3779
5b7a487c 3780 rc = migrate_page_move_mapping(mapping, newpage,
37109694 3781 page, extra_count);
3db1de0e 3782 if (rc != MIGRATEPAGE_SUCCESS)
5b7a487c 3783 return rc;
5b7a487c 3784
c9ebd3df
JK		 3785 /* guarantee to start from no stale private field */
3786 set_page_private(newpage, 0);
240a5915 3787 if (PagePrivate(page)) {
b763f3be
CY		 3788 set_page_private(newpage, page_private(page));
3789 SetPagePrivate(newpage);
3790 get_page(newpage);
3791
3792 set_page_private(page, 0);
3793 ClearPagePrivate(page);
3794 put_page(page);
240a5915 3795 }
5b7a487c 3796
2916ecc0
JG		 3797 if (mode != MIGRATE_SYNC_NO_COPY)
3798 migrate_page_copy(newpage, page);
3799 else
3800 migrate_page_states(newpage, page);
5b7a487c
WG		 3801
3802 return MIGRATEPAGE_SUCCESS;
3803}
3804#endif
3805
4969c06a 3806#ifdef CONFIG_SWAP
859fca6b
CY		 3807static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk,
3808 unsigned int blkcnt)
3809{
3810 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3811 unsigned int blkofs;
3812 unsigned int blk_per_sec = BLKS_PER_SEC(sbi);
3813 unsigned int secidx = start_blk / blk_per_sec;
3814 unsigned int end_sec = secidx + blkcnt / blk_per_sec;
3815 int ret = 0;
3816
e4544b63 3817 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
edc6d01b 3818 filemap_invalidate_lock(inode->i_mapping);
859fca6b
CY		 3819
3820 set_inode_flag(inode, FI_ALIGNED_WRITE);
1018a546 3821 set_inode_flag(inode, FI_OPU_WRITE);
859fca6b
CY		 3822
3823 for (; secidx < end_sec; secidx++) {
e4544b63 3824 f2fs_down_write(&sbi->pin_sem);
859fca6b
CY		 3825
3826 f2fs_lock_op(sbi);
3827 f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3828 f2fs_unlock_op(sbi);
3829
1018a546 3830 set_inode_flag(inode, FI_SKIP_WRITES);
859fca6b
CY		 3831
3832 for (blkofs = 0; blkofs < blk_per_sec; blkofs++) {
3833 struct page *page;
3834 unsigned int blkidx = secidx * blk_per_sec + blkofs;
3835
3836 page = f2fs_get_lock_data_page(inode, blkidx, true);
3837 if (IS_ERR(page)) {
e4544b63 3838 f2fs_up_write(&sbi->pin_sem);
859fca6b
CY		 3839 ret = PTR_ERR(page);
3840 goto done;
3841 }
3842
3843 set_page_dirty(page);
3844 f2fs_put_page(page, 1);
3845 }
3846
1018a546 3847 clear_inode_flag(inode, FI_SKIP_WRITES);
859fca6b
CY		 3848
3849 ret = filemap_fdatawrite(inode->i_mapping);
3850
e4544b63 3851 f2fs_up_write(&sbi->pin_sem);
859fca6b
CY		 3852
3853 if (ret)
3854 break;
3855 }
3856
3857done:
1018a546
CY		 3858 clear_inode_flag(inode, FI_SKIP_WRITES);
3859 clear_inode_flag(inode, FI_OPU_WRITE);
859fca6b
CY		 3860 clear_inode_flag(inode, FI_ALIGNED_WRITE);
3861
edc6d01b 3862 filemap_invalidate_unlock(inode->i_mapping);
e4544b63 3863 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
859fca6b
CY		 3864
3865 return ret;
3866}
3867
0b8fc006 3868static int check_swap_activate(struct swap_info_struct *sis,
af4b6b8e
CY		 3869 struct file *swap_file, sector_t *span)
3870{
3871 struct address_space *mapping = swap_file->f_mapping;
3872 struct inode *inode = mapping->host;
36e4d958 3873 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
af4b6b8e
CY		 3874 sector_t cur_lblock;
3875 sector_t last_lblock;
3876 sector_t pblock;
3877 sector_t lowest_pblock = -1;
3878 sector_t highest_pblock = 0;
3879 int nr_extents = 0;
3880 unsigned long nr_pblocks;
859fca6b
CY		 3881 unsigned int blks_per_sec = BLKS_PER_SEC(sbi);
3882 unsigned int sec_blks_mask = BLKS_PER_SEC(sbi) - 1;
ca298241 3883 unsigned int not_aligned = 0;
36e4d958 3884 int ret = 0;
af4b6b8e
CY		 3885
3886 /*
3887 * Map all the blocks into the extent list. This code doesn't try
3888 * to be very smart.
3889 */
3890 cur_lblock = 0;
6cbfcab5 3891 last_lblock = bytes_to_blks(inode, i_size_read(inode));
af4b6b8e 3892
1da66103 3893 while (cur_lblock < last_lblock && cur_lblock < sis->max) {
b876f4c9 3894 struct f2fs_map_blocks map;
859fca6b 3895retry:
af4b6b8e
CY		 3896 cond_resched();
3897
b876f4c9
JK		 3898 memset(&map, 0, sizeof(map));
3899 map.m_lblk = cur_lblock;
36e4d958 3900 map.m_len = last_lblock - cur_lblock;
3901 map.m_next_pgofs = NULL;
3902 map.m_next_extent = NULL;
b876f4c9 3903 map.m_seg_type = NO_CHECK_TYPE;
36e4d958 3904 map.m_may_create = false;
af4b6b8e 3905
b876f4c9 3906 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP);
af4b6b8e 3907 if (ret)
36e4d958 3908 goto out;
af4b6b8e
CY		 3909
3910 /* hole */
36e4d958 3911 if (!(map.m_flags & F2FS_MAP_FLAGS)) {
833dcd35 3912 f2fs_err(sbi, "Swapfile has holes");
f395183f 3913 ret = -EINVAL;
36e4d958 3914 goto out;
3915 }
af4b6b8e 3916
b876f4c9
JK		 3917 pblock = map.m_pblk;
3918 nr_pblocks = map.m_len;
af4b6b8e 3919
859fca6b
CY		 3920 if ((pblock - SM_I(sbi)->main_blkaddr) & sec_blks_mask ||
3921 nr_pblocks & sec_blks_mask) {
ca298241 3922 not_aligned++;
36e4d958 3923
859fca6b
CY		 3924 nr_pblocks = roundup(nr_pblocks, blks_per_sec);
3925 if (cur_lblock + nr_pblocks > sis->max)
3926 nr_pblocks -= blks_per_sec;
3927
3928 if (!nr_pblocks) {
3929 /* this extent is last one */
3930 nr_pblocks = map.m_len;
3931 f2fs_warn(sbi, "Swapfile: last extent is not aligned to section");
3932 goto next;
3933 }
3934
3935 ret = f2fs_migrate_blocks(inode, cur_lblock,
3936 nr_pblocks);
3937 if (ret)
3938 goto out;
3939 goto retry;
3940 }
3941next:
af4b6b8e
CY		 3942 if (cur_lblock + nr_pblocks >= sis->max)
3943 nr_pblocks = sis->max - cur_lblock;
3944
3945 if (cur_lblock) { /* exclude the header page */
3946 if (pblock < lowest_pblock)
3947 lowest_pblock = pblock;
3948 if (pblock + nr_pblocks - 1 > highest_pblock)
3949 highest_pblock = pblock + nr_pblocks - 1;
3950 }
3951
3952 /*
3953 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
3954 */
3955 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock);
3956 if (ret < 0)
3957 goto out;
3958 nr_extents += ret;
3959 cur_lblock += nr_pblocks;
3960 }
3961 ret = nr_extents;
3962 *span = 1 + highest_pblock - lowest_pblock;
3963 if (cur_lblock == 0)
3964 cur_lblock = 1; /* force Empty message */
3965 sis->max = cur_lblock;
3966 sis->pages = cur_lblock - 1;
3967 sis->highest_bit = cur_lblock - 1;
3968out:
859fca6b
CY		 3969 if (not_aligned)
3970 f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%u * N)",
3971 not_aligned, blks_per_sec * F2FS_BLKSIZE);
af4b6b8e 3972 return ret;
af4b6b8e
CY		 3973}
3974
4969c06a
JK		 3975static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
3976 sector_t *span)
3977{
3978 struct inode *inode = file_inode(file);
3979 int ret;
3980
3981 if (!S_ISREG(inode->i_mode))
3982 return -EINVAL;
3983
3984 if (f2fs_readonly(F2FS_I_SB(inode)->sb))
3985 return -EROFS;
3986
d927ccfc
SK		 3987 if (f2fs_lfs_mode(F2FS_I_SB(inode))) {
3988 f2fs_err(F2FS_I_SB(inode),
3989 "Swapfile not supported in LFS mode");
3990 return -EINVAL;
3991 }
3992
4969c06a
JK		 3993 ret = f2fs_convert_inline_inode(inode);
3994 if (ret)
3995 return ret;
3996
78134d03 3997 if (!f2fs_disable_compressed_file(inode))
4c8ff709
CY		 3998 return -EINVAL;
3999
0b979f1b
CY		 4000 f2fs_precache_extents(inode);
4001
3e5e479a
CY		 4002 ret = check_swap_activate(sis, file, span);
4003 if (ret < 0)
4969c06a
JK		 4004 return ret;
4005
4006 set_inode_flag(inode, FI_PIN_FILE);
4969c06a 4007 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
3e5e479a 4008 return ret;
4969c06a
JK		 4009}
4010
4011static void f2fs_swap_deactivate(struct file *file)
4012{
4013 struct inode *inode = file_inode(file);
4014
4015 clear_inode_flag(inode, FI_PIN_FILE);
4016}
4017#else
4018static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file,
4019 sector_t *span)
4020{
4021 return -EOPNOTSUPP;
4022}
4023
4024static void f2fs_swap_deactivate(struct file *file)
4025{
4026}
4027#endif
4028
eb47b800 4029const struct address_space_operations f2fs_dblock_aops = {
be05584f 4030 .read_folio = f2fs_read_data_folio,
23323196 4031 .readahead = f2fs_readahead,
eb47b800
JK		 4032 .writepage = f2fs_write_data_page,
4033 .writepages = f2fs_write_data_pages,
4034 .write_begin = f2fs_write_begin,
a1dd3c13 4035 .write_end = f2fs_write_end,
4f5e34f7 4036 .dirty_folio = f2fs_dirty_data_folio,
91503996 4037 .invalidate_folio = f2fs_invalidate_folio,
c26cd045 4038 .release_folio = f2fs_release_folio,
a1e09b03 4039 .direct_IO = noop_direct_IO,
c01e54b7 4040 .bmap = f2fs_bmap,
4969c06a
JK		 4041 .swap_activate = f2fs_swap_activate,
4042 .swap_deactivate = f2fs_swap_deactivate,
5b7a487c
WG		 4043#ifdef CONFIG_MIGRATION
4044 .migratepage = f2fs_migrate_page,
4045#endif
eb47b800 4046};
6dbb1796 4047
5ec2d99d 4048void f2fs_clear_page_cache_dirty_tag(struct page *page)
aec2f729
CY		 4049{
4050 struct address_space *mapping = page_mapping(page);
4051 unsigned long flags;
4052
4053 xa_lock_irqsave(&mapping->i_pages, flags);
5ec2d99d 4054 __xa_clear_mark(&mapping->i_pages, page_index(page),
aec2f729
CY		 4055 PAGECACHE_TAG_DIRTY);
4056 xa_unlock_irqrestore(&mapping->i_pages, flags);
4057}
4058
6dbb1796
EB		 4059int __init f2fs_init_post_read_processing(void)
4060{
95ae251f
EB		 4061 bio_post_read_ctx_cache =
4062 kmem_cache_create("f2fs_bio_post_read_ctx",
4063 sizeof(struct bio_post_read_ctx), 0, 0, NULL);
6dbb1796
EB		 4064 if (!bio_post_read_ctx_cache)
4065 goto fail;
4066 bio_post_read_ctx_pool =
4067 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS,
4068 bio_post_read_ctx_cache);
4069 if (!bio_post_read_ctx_pool)
4070 goto fail_free_cache;
4071 return 0;
4072
4073fail_free_cache:
4074 kmem_cache_destroy(bio_post_read_ctx_cache);
4075fail:
4076 return -ENOMEM;
4077}
4078
0b20fcec 4079void f2fs_destroy_post_read_processing(void)
6dbb1796
EB		 4080{
4081 mempool_destroy(bio_post_read_ctx_pool);
4082 kmem_cache_destroy(bio_post_read_ctx_cache);
4083}
0b20fcec 4084
4c8ff709
CY		 4085int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi)
4086{
4087 if (!f2fs_sb_has_encrypt(sbi) &&
4088 !f2fs_sb_has_verity(sbi) &&
4089 !f2fs_sb_has_compression(sbi))
4090 return 0;
4091
4092 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq",
4093 WQ_UNBOUND | WQ_HIGHPRI,
4094 num_online_cpus());
4095 if (!sbi->post_read_wq)
4096 return -ENOMEM;
4097 return 0;
4098}
4099
4100void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi)
4101{
4102 if (sbi->post_read_wq)
4103 destroy_workqueue(sbi->post_read_wq);
4104}
4105
0b20fcec
CY		 4106int __init f2fs_init_bio_entry_cache(void)
4107{
98510003 4108 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab",
0b20fcec
CY		 4109 sizeof(struct bio_entry));
4110 if (!bio_entry_slab)
4111 return -ENOMEM;
4112 return 0;
4113}
4114
f543805f 4115void f2fs_destroy_bio_entry_cache(void)
0b20fcec
CY		 4116{
4117 kmem_cache_destroy(bio_entry_slab);
4118}
1517c1a7
EB		 4119
4120static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
4121 unsigned int flags, struct iomap *iomap,
4122 struct iomap *srcmap)
4123{
4124 struct f2fs_map_blocks map = {};
4125 pgoff_t next_pgofs = 0;
4126 int err;
4127
4128 map.m_lblk = bytes_to_blks(inode, offset);
4129 map.m_len = bytes_to_blks(inode, offset + length - 1) - map.m_lblk + 1;
4130 map.m_next_pgofs = &next_pgofs;
4131 map.m_seg_type = f2fs_rw_hint_to_seg_type(inode->i_write_hint);
4132 if (flags & IOMAP_WRITE)
4133 map.m_may_create = true;
4134
4135 err = f2fs_map_blocks(inode, &map, flags & IOMAP_WRITE,
4136 F2FS_GET_BLOCK_DIO);
4137 if (err)
4138 return err;
4139
4140 iomap->offset = blks_to_bytes(inode, map.m_lblk);
4141
8a2c77bc
EB		 4142 /*
4143 * When inline encryption is enabled, sometimes I/O to an encrypted file
4144 * has to be broken up to guarantee DUN contiguity. Handle this by
4145 * limiting the length of the mapping returned.
4146 */
4147 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
4148
1517c1a7
EB		 4149 if (map.m_flags & (F2FS_MAP_MAPPED | F2FS_MAP_UNWRITTEN)) {
4150 iomap->length = blks_to_bytes(inode, map.m_len);
4151 if (map.m_flags & F2FS_MAP_MAPPED) {
4152 iomap->type = IOMAP_MAPPED;
4153 iomap->flags |= IOMAP_F_MERGED;
4154 } else {
4155 iomap->type = IOMAP_UNWRITTEN;
4156 }
4157 if (WARN_ON_ONCE(!__is_valid_data_blkaddr(map.m_pblk)))
4158 return -EINVAL;
4159
4160 iomap->bdev = map.m_bdev;
4161 iomap->addr = blks_to_bytes(inode, map.m_pblk);
4162 } else {
4163 iomap->length = blks_to_bytes(inode, next_pgofs) -
4164 iomap->offset;
4165 iomap->type = IOMAP_HOLE;
4166 iomap->addr = IOMAP_NULL_ADDR;
4167 }
4168
4169 if (map.m_flags & F2FS_MAP_NEW)
4170 iomap->flags |= IOMAP_F_NEW;
4171 if ((inode->i_state & I_DIRTY_DATASYNC) ||
4172 offset + length > i_size_read(inode))
4173 iomap->flags |= IOMAP_F_DIRTY;
4174
4175 return 0;
4176}
4177
4178const struct iomap_ops f2fs_iomap_ops = {
4179 .iomap_begin = f2fs_iomap_begin,
4180};
Michael's Linux tree.