1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2010 Red Hat, Inc.
4 * Copyright (C) 2016-2019 Christoph Hellwig.
6 #include <linux/module.h>
7 #include <linux/compiler.h>
9 #include <linux/iomap.h>
10 #include <linux/pagemap.h>
11 #include <linux/uio.h>
12 #include <linux/buffer_head.h>
13 #include <linux/dax.h>
14 #include <linux/writeback.h>
15 #include <linux/list_sort.h>
16 #include <linux/swap.h>
17 #include <linux/bio.h>
18 #include <linux/sched/signal.h>
19 #include <linux/migrate.h>
22 #include "../internal.h"
24 #define IOEND_BATCH_SIZE 4096
26 typedef int (*iomap_punch_t
)(struct inode
*inode
, loff_t offset
, loff_t length
);
28 * Structure allocated for each folio to track per-block uptodate, dirty state
29 * and I/O completions.
31 struct iomap_folio_state
{
32 atomic_t read_bytes_pending
;
33 atomic_t write_bytes_pending
;
34 spinlock_t state_lock
;
37 * Each block has two bits in this bitmap:
38 * Bits [0..blocks_per_folio) has the uptodate status.
39 * Bits [b_p_f...(2*b_p_f)) has the dirty status.
41 unsigned long state
[];
44 static struct bio_set iomap_ioend_bioset
;
46 static inline bool ifs_is_fully_uptodate(struct folio
*folio
,
47 struct iomap_folio_state
*ifs
)
49 struct inode
*inode
= folio
->mapping
->host
;
51 return bitmap_full(ifs
->state
, i_blocks_per_folio(inode
, folio
));
54 static inline bool ifs_block_is_uptodate(struct iomap_folio_state
*ifs
,
57 return test_bit(block
, ifs
->state
);
60 static void ifs_set_range_uptodate(struct folio
*folio
,
61 struct iomap_folio_state
*ifs
, size_t off
, size_t len
)
63 struct inode
*inode
= folio
->mapping
->host
;
64 unsigned int first_blk
= off
>> inode
->i_blkbits
;
65 unsigned int last_blk
= (off
+ len
- 1) >> inode
->i_blkbits
;
66 unsigned int nr_blks
= last_blk
- first_blk
+ 1;
69 spin_lock_irqsave(&ifs
->state_lock
, flags
);
70 bitmap_set(ifs
->state
, first_blk
, nr_blks
);
71 if (ifs_is_fully_uptodate(folio
, ifs
))
72 folio_mark_uptodate(folio
);
73 spin_unlock_irqrestore(&ifs
->state_lock
, flags
);
76 static void iomap_set_range_uptodate(struct folio
*folio
, size_t off
,
79 struct iomap_folio_state
*ifs
= folio
->private;
82 ifs_set_range_uptodate(folio
, ifs
, off
, len
);
84 folio_mark_uptodate(folio
);
87 static inline bool ifs_block_is_dirty(struct folio
*folio
,
88 struct iomap_folio_state
*ifs
, int block
)
90 struct inode
*inode
= folio
->mapping
->host
;
91 unsigned int blks_per_folio
= i_blocks_per_folio(inode
, folio
);
93 return test_bit(block
+ blks_per_folio
, ifs
->state
);
96 static void ifs_clear_range_dirty(struct folio
*folio
,
97 struct iomap_folio_state
*ifs
, size_t off
, size_t len
)
99 struct inode
*inode
= folio
->mapping
->host
;
100 unsigned int blks_per_folio
= i_blocks_per_folio(inode
, folio
);
101 unsigned int first_blk
= (off
>> inode
->i_blkbits
);
102 unsigned int last_blk
= (off
+ len
- 1) >> inode
->i_blkbits
;
103 unsigned int nr_blks
= last_blk
- first_blk
+ 1;
106 spin_lock_irqsave(&ifs
->state_lock
, flags
);
107 bitmap_clear(ifs
->state
, first_blk
+ blks_per_folio
, nr_blks
);
108 spin_unlock_irqrestore(&ifs
->state_lock
, flags
);
111 static void iomap_clear_range_dirty(struct folio
*folio
, size_t off
, size_t len
)
113 struct iomap_folio_state
*ifs
= folio
->private;
116 ifs_clear_range_dirty(folio
, ifs
, off
, len
);
119 static void ifs_set_range_dirty(struct folio
*folio
,
120 struct iomap_folio_state
*ifs
, size_t off
, size_t len
)
122 struct inode
*inode
= folio
->mapping
->host
;
123 unsigned int blks_per_folio
= i_blocks_per_folio(inode
, folio
);
124 unsigned int first_blk
= (off
>> inode
->i_blkbits
);
125 unsigned int last_blk
= (off
+ len
- 1) >> inode
->i_blkbits
;
126 unsigned int nr_blks
= last_blk
- first_blk
+ 1;
129 spin_lock_irqsave(&ifs
->state_lock
, flags
);
130 bitmap_set(ifs
->state
, first_blk
+ blks_per_folio
, nr_blks
);
131 spin_unlock_irqrestore(&ifs
->state_lock
, flags
);
134 static void iomap_set_range_dirty(struct folio
*folio
, size_t off
, size_t len
)
136 struct iomap_folio_state
*ifs
= folio
->private;
139 ifs_set_range_dirty(folio
, ifs
, off
, len
);
142 static struct iomap_folio_state
*ifs_alloc(struct inode
*inode
,
143 struct folio
*folio
, unsigned int flags
)
145 struct iomap_folio_state
*ifs
= folio
->private;
146 unsigned int nr_blocks
= i_blocks_per_folio(inode
, folio
);
149 if (ifs
|| nr_blocks
<= 1)
152 if (flags
& IOMAP_NOWAIT
)
155 gfp
= GFP_NOFS
| __GFP_NOFAIL
;
158 * ifs->state tracks two sets of state flags when the
159 * filesystem block size is smaller than the folio size.
160 * The first state tracks per-block uptodate and the
161 * second tracks per-block dirty state.
163 ifs
= kzalloc(struct_size(ifs
, state
,
164 BITS_TO_LONGS(2 * nr_blocks
)), gfp
);
168 spin_lock_init(&ifs
->state_lock
);
169 if (folio_test_uptodate(folio
))
170 bitmap_set(ifs
->state
, 0, nr_blocks
);
171 if (folio_test_dirty(folio
))
172 bitmap_set(ifs
->state
, nr_blocks
, nr_blocks
);
173 folio_attach_private(folio
, ifs
);
178 static void ifs_free(struct folio
*folio
)
180 struct iomap_folio_state
*ifs
= folio_detach_private(folio
);
184 WARN_ON_ONCE(atomic_read(&ifs
->read_bytes_pending
));
185 WARN_ON_ONCE(atomic_read(&ifs
->write_bytes_pending
));
186 WARN_ON_ONCE(ifs_is_fully_uptodate(folio
, ifs
) !=
187 folio_test_uptodate(folio
));
192 * Calculate the range inside the folio that we actually need to read.
194 static void iomap_adjust_read_range(struct inode
*inode
, struct folio
*folio
,
195 loff_t
*pos
, loff_t length
, size_t *offp
, size_t *lenp
)
197 struct iomap_folio_state
*ifs
= folio
->private;
198 loff_t orig_pos
= *pos
;
199 loff_t isize
= i_size_read(inode
);
200 unsigned block_bits
= inode
->i_blkbits
;
201 unsigned block_size
= (1 << block_bits
);
202 size_t poff
= offset_in_folio(folio
, *pos
);
203 size_t plen
= min_t(loff_t
, folio_size(folio
) - poff
, length
);
204 unsigned first
= poff
>> block_bits
;
205 unsigned last
= (poff
+ plen
- 1) >> block_bits
;
208 * If the block size is smaller than the page size, we need to check the
209 * per-block uptodate status and adjust the offset and length if needed
210 * to avoid reading in already uptodate ranges.
215 /* move forward for each leading block marked uptodate */
216 for (i
= first
; i
<= last
; i
++) {
217 if (!ifs_block_is_uptodate(ifs
, i
))
225 /* truncate len if we find any trailing uptodate block(s) */
226 for ( ; i
<= last
; i
++) {
227 if (ifs_block_is_uptodate(ifs
, i
)) {
228 plen
-= (last
- i
+ 1) * block_size
;
236 * If the extent spans the block that contains the i_size, we need to
237 * handle both halves separately so that we properly zero data in the
238 * page cache for blocks that are entirely outside of i_size.
240 if (orig_pos
<= isize
&& orig_pos
+ length
> isize
) {
241 unsigned end
= offset_in_folio(folio
, isize
- 1) >> block_bits
;
243 if (first
<= end
&& last
> end
)
244 plen
-= (last
- end
) * block_size
;
251 static void iomap_finish_folio_read(struct folio
*folio
, size_t offset
,
252 size_t len
, int error
)
254 struct iomap_folio_state
*ifs
= folio
->private;
256 if (unlikely(error
)) {
257 folio_clear_uptodate(folio
);
258 folio_set_error(folio
);
260 iomap_set_range_uptodate(folio
, offset
, len
);
263 if (!ifs
|| atomic_sub_and_test(len
, &ifs
->read_bytes_pending
))
267 static void iomap_read_end_io(struct bio
*bio
)
269 int error
= blk_status_to_errno(bio
->bi_status
);
270 struct folio_iter fi
;
272 bio_for_each_folio_all(fi
, bio
)
273 iomap_finish_folio_read(fi
.folio
, fi
.offset
, fi
.length
, error
);
277 struct iomap_readpage_ctx
{
278 struct folio
*cur_folio
;
279 bool cur_folio_in_bio
;
281 struct readahead_control
*rac
;
285 * iomap_read_inline_data - copy inline data into the page cache
286 * @iter: iteration structure
287 * @folio: folio to copy to
289 * Copy the inline data in @iter into @folio and zero out the rest of the folio.
290 * Only a single IOMAP_INLINE extent is allowed at the end of each file.
291 * Returns zero for success to complete the read, or the usual negative errno.
293 static int iomap_read_inline_data(const struct iomap_iter
*iter
,
296 const struct iomap
*iomap
= iomap_iter_srcmap(iter
);
297 size_t size
= i_size_read(iter
->inode
) - iomap
->offset
;
298 size_t poff
= offset_in_page(iomap
->offset
);
299 size_t offset
= offset_in_folio(folio
, iomap
->offset
);
302 if (folio_test_uptodate(folio
))
305 if (WARN_ON_ONCE(size
> PAGE_SIZE
- poff
))
307 if (WARN_ON_ONCE(size
> PAGE_SIZE
-
308 offset_in_page(iomap
->inline_data
)))
310 if (WARN_ON_ONCE(size
> iomap
->length
))
313 ifs_alloc(iter
->inode
, folio
, iter
->flags
);
315 addr
= kmap_local_folio(folio
, offset
);
316 memcpy(addr
, iomap
->inline_data
, size
);
317 memset(addr
+ size
, 0, PAGE_SIZE
- poff
- size
);
319 iomap_set_range_uptodate(folio
, offset
, PAGE_SIZE
- poff
);
323 static inline bool iomap_block_needs_zeroing(const struct iomap_iter
*iter
,
326 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
328 return srcmap
->type
!= IOMAP_MAPPED
||
329 (srcmap
->flags
& IOMAP_F_NEW
) ||
330 pos
>= i_size_read(iter
->inode
);
333 static loff_t
iomap_readpage_iter(const struct iomap_iter
*iter
,
334 struct iomap_readpage_ctx
*ctx
, loff_t offset
)
336 const struct iomap
*iomap
= &iter
->iomap
;
337 loff_t pos
= iter
->pos
+ offset
;
338 loff_t length
= iomap_length(iter
) - offset
;
339 struct folio
*folio
= ctx
->cur_folio
;
340 struct iomap_folio_state
*ifs
;
341 loff_t orig_pos
= pos
;
345 if (iomap
->type
== IOMAP_INLINE
)
346 return iomap_read_inline_data(iter
, folio
);
348 /* zero post-eof blocks as the page may be mapped */
349 ifs
= ifs_alloc(iter
->inode
, folio
, iter
->flags
);
350 iomap_adjust_read_range(iter
->inode
, folio
, &pos
, length
, &poff
, &plen
);
354 if (iomap_block_needs_zeroing(iter
, pos
)) {
355 folio_zero_range(folio
, poff
, plen
);
356 iomap_set_range_uptodate(folio
, poff
, plen
);
360 ctx
->cur_folio_in_bio
= true;
362 atomic_add(plen
, &ifs
->read_bytes_pending
);
364 sector
= iomap_sector(iomap
, pos
);
366 bio_end_sector(ctx
->bio
) != sector
||
367 !bio_add_folio(ctx
->bio
, folio
, plen
, poff
)) {
368 gfp_t gfp
= mapping_gfp_constraint(folio
->mapping
, GFP_KERNEL
);
369 gfp_t orig_gfp
= gfp
;
370 unsigned int nr_vecs
= DIV_ROUND_UP(length
, PAGE_SIZE
);
373 submit_bio(ctx
->bio
);
375 if (ctx
->rac
) /* same as readahead_gfp_mask */
376 gfp
|= __GFP_NORETRY
| __GFP_NOWARN
;
377 ctx
->bio
= bio_alloc(iomap
->bdev
, bio_max_segs(nr_vecs
),
380 * If the bio_alloc fails, try it again for a single page to
381 * avoid having to deal with partial page reads. This emulates
382 * what do_mpage_read_folio does.
385 ctx
->bio
= bio_alloc(iomap
->bdev
, 1, REQ_OP_READ
,
389 ctx
->bio
->bi_opf
|= REQ_RAHEAD
;
390 ctx
->bio
->bi_iter
.bi_sector
= sector
;
391 ctx
->bio
->bi_end_io
= iomap_read_end_io
;
392 bio_add_folio_nofail(ctx
->bio
, folio
, plen
, poff
);
397 * Move the caller beyond our range so that it keeps making progress.
398 * For that, we have to include any leading non-uptodate ranges, but
399 * we can skip trailing ones as they will be handled in the next
402 return pos
- orig_pos
+ plen
;
405 int iomap_read_folio(struct folio
*folio
, const struct iomap_ops
*ops
)
407 struct iomap_iter iter
= {
408 .inode
= folio
->mapping
->host
,
409 .pos
= folio_pos(folio
),
410 .len
= folio_size(folio
),
412 struct iomap_readpage_ctx ctx
= {
417 trace_iomap_readpage(iter
.inode
, 1);
419 while ((ret
= iomap_iter(&iter
, ops
)) > 0)
420 iter
.processed
= iomap_readpage_iter(&iter
, &ctx
, 0);
423 folio_set_error(folio
);
427 WARN_ON_ONCE(!ctx
.cur_folio_in_bio
);
429 WARN_ON_ONCE(ctx
.cur_folio_in_bio
);
434 * Just like mpage_readahead and block_read_full_folio, we always
435 * return 0 and just set the folio error flag on errors. This
436 * should be cleaned up throughout the stack eventually.
440 EXPORT_SYMBOL_GPL(iomap_read_folio
);
442 static loff_t
iomap_readahead_iter(const struct iomap_iter
*iter
,
443 struct iomap_readpage_ctx
*ctx
)
445 loff_t length
= iomap_length(iter
);
448 for (done
= 0; done
< length
; done
+= ret
) {
449 if (ctx
->cur_folio
&&
450 offset_in_folio(ctx
->cur_folio
, iter
->pos
+ done
) == 0) {
451 if (!ctx
->cur_folio_in_bio
)
452 folio_unlock(ctx
->cur_folio
);
453 ctx
->cur_folio
= NULL
;
455 if (!ctx
->cur_folio
) {
456 ctx
->cur_folio
= readahead_folio(ctx
->rac
);
457 ctx
->cur_folio_in_bio
= false;
459 ret
= iomap_readpage_iter(iter
, ctx
, done
);
468 * iomap_readahead - Attempt to read pages from a file.
469 * @rac: Describes the pages to be read.
470 * @ops: The operations vector for the filesystem.
472 * This function is for filesystems to call to implement their readahead
473 * address_space operation.
475 * Context: The @ops callbacks may submit I/O (eg to read the addresses of
476 * blocks from disc), and may wait for it. The caller may be trying to
477 * access a different page, and so sleeping excessively should be avoided.
478 * It may allocate memory, but should avoid costly allocations. This
479 * function is called with memalloc_nofs set, so allocations will not cause
480 * the filesystem to be reentered.
482 void iomap_readahead(struct readahead_control
*rac
, const struct iomap_ops
*ops
)
484 struct iomap_iter iter
= {
485 .inode
= rac
->mapping
->host
,
486 .pos
= readahead_pos(rac
),
487 .len
= readahead_length(rac
),
489 struct iomap_readpage_ctx ctx
= {
493 trace_iomap_readahead(rac
->mapping
->host
, readahead_count(rac
));
495 while (iomap_iter(&iter
, ops
) > 0)
496 iter
.processed
= iomap_readahead_iter(&iter
, &ctx
);
501 if (!ctx
.cur_folio_in_bio
)
502 folio_unlock(ctx
.cur_folio
);
505 EXPORT_SYMBOL_GPL(iomap_readahead
);
508 * iomap_is_partially_uptodate checks whether blocks within a folio are
511 * Returns true if all blocks which correspond to the specified part
512 * of the folio are uptodate.
514 bool iomap_is_partially_uptodate(struct folio
*folio
, size_t from
, size_t count
)
516 struct iomap_folio_state
*ifs
= folio
->private;
517 struct inode
*inode
= folio
->mapping
->host
;
518 unsigned first
, last
, i
;
523 /* Caller's range may extend past the end of this folio */
524 count
= min(folio_size(folio
) - from
, count
);
526 /* First and last blocks in range within folio */
527 first
= from
>> inode
->i_blkbits
;
528 last
= (from
+ count
- 1) >> inode
->i_blkbits
;
530 for (i
= first
; i
<= last
; i
++)
531 if (!ifs_block_is_uptodate(ifs
, i
))
535 EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate
);
538 * iomap_get_folio - get a folio reference for writing
539 * @iter: iteration structure
540 * @pos: start offset of write
541 * @len: Suggested size of folio to create.
543 * Returns a locked reference to the folio at @pos, or an error pointer if the
544 * folio could not be obtained.
546 struct folio
*iomap_get_folio(struct iomap_iter
*iter
, loff_t pos
, size_t len
)
548 fgf_t fgp
= FGP_WRITEBEGIN
| FGP_NOFS
;
550 if (iter
->flags
& IOMAP_NOWAIT
)
552 fgp
|= fgf_set_order(len
);
554 return __filemap_get_folio(iter
->inode
->i_mapping
, pos
>> PAGE_SHIFT
,
555 fgp
, mapping_gfp_mask(iter
->inode
->i_mapping
));
557 EXPORT_SYMBOL_GPL(iomap_get_folio
);
559 bool iomap_release_folio(struct folio
*folio
, gfp_t gfp_flags
)
561 trace_iomap_release_folio(folio
->mapping
->host
, folio_pos(folio
),
565 * If the folio is dirty, we refuse to release our metadata because
566 * it may be partially dirty. Once we track per-block dirty state,
567 * we can release the metadata if every block is dirty.
569 if (folio_test_dirty(folio
))
574 EXPORT_SYMBOL_GPL(iomap_release_folio
);
576 void iomap_invalidate_folio(struct folio
*folio
, size_t offset
, size_t len
)
578 trace_iomap_invalidate_folio(folio
->mapping
->host
,
579 folio_pos(folio
) + offset
, len
);
582 * If we're invalidating the entire folio, clear the dirty state
583 * from it and release it to avoid unnecessary buildup of the LRU.
585 if (offset
== 0 && len
== folio_size(folio
)) {
586 WARN_ON_ONCE(folio_test_writeback(folio
));
587 folio_cancel_dirty(folio
);
591 EXPORT_SYMBOL_GPL(iomap_invalidate_folio
);
593 bool iomap_dirty_folio(struct address_space
*mapping
, struct folio
*folio
)
595 struct inode
*inode
= mapping
->host
;
596 size_t len
= folio_size(folio
);
598 ifs_alloc(inode
, folio
, 0);
599 iomap_set_range_dirty(folio
, 0, len
);
600 return filemap_dirty_folio(mapping
, folio
);
602 EXPORT_SYMBOL_GPL(iomap_dirty_folio
);
605 iomap_write_failed(struct inode
*inode
, loff_t pos
, unsigned len
)
607 loff_t i_size
= i_size_read(inode
);
610 * Only truncate newly allocated pages beyoned EOF, even if the
611 * write started inside the existing inode size.
613 if (pos
+ len
> i_size
)
614 truncate_pagecache_range(inode
, max(pos
, i_size
),
618 static int iomap_read_folio_sync(loff_t block_start
, struct folio
*folio
,
619 size_t poff
, size_t plen
, const struct iomap
*iomap
)
624 bio_init(&bio
, iomap
->bdev
, &bvec
, 1, REQ_OP_READ
);
625 bio
.bi_iter
.bi_sector
= iomap_sector(iomap
, block_start
);
626 bio_add_folio_nofail(&bio
, folio
, plen
, poff
);
627 return submit_bio_wait(&bio
);
630 static int __iomap_write_begin(const struct iomap_iter
*iter
, loff_t pos
,
631 size_t len
, struct folio
*folio
)
633 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
634 struct iomap_folio_state
*ifs
;
635 loff_t block_size
= i_blocksize(iter
->inode
);
636 loff_t block_start
= round_down(pos
, block_size
);
637 loff_t block_end
= round_up(pos
+ len
, block_size
);
638 unsigned int nr_blocks
= i_blocks_per_folio(iter
->inode
, folio
);
639 size_t from
= offset_in_folio(folio
, pos
), to
= from
+ len
;
643 * If the write completely overlaps the current folio, then
644 * entire folio will be dirtied so there is no need for
645 * per-block state tracking structures to be attached to this folio.
647 if (pos
<= folio_pos(folio
) &&
648 pos
+ len
>= folio_pos(folio
) + folio_size(folio
))
651 ifs
= ifs_alloc(iter
->inode
, folio
, iter
->flags
);
652 if ((iter
->flags
& IOMAP_NOWAIT
) && !ifs
&& nr_blocks
> 1)
655 if (folio_test_uptodate(folio
))
657 folio_clear_error(folio
);
660 iomap_adjust_read_range(iter
->inode
, folio
, &block_start
,
661 block_end
- block_start
, &poff
, &plen
);
665 if (!(iter
->flags
& IOMAP_UNSHARE
) &&
666 (from
<= poff
|| from
>= poff
+ plen
) &&
667 (to
<= poff
|| to
>= poff
+ plen
))
670 if (iomap_block_needs_zeroing(iter
, block_start
)) {
671 if (WARN_ON_ONCE(iter
->flags
& IOMAP_UNSHARE
))
673 folio_zero_segments(folio
, poff
, from
, to
, poff
+ plen
);
677 if (iter
->flags
& IOMAP_NOWAIT
)
680 status
= iomap_read_folio_sync(block_start
, folio
,
685 iomap_set_range_uptodate(folio
, poff
, plen
);
686 } while ((block_start
+= plen
) < block_end
);
691 static struct folio
*__iomap_get_folio(struct iomap_iter
*iter
, loff_t pos
,
694 const struct iomap_folio_ops
*folio_ops
= iter
->iomap
.folio_ops
;
696 if (folio_ops
&& folio_ops
->get_folio
)
697 return folio_ops
->get_folio(iter
, pos
, len
);
699 return iomap_get_folio(iter
, pos
, len
);
702 static void __iomap_put_folio(struct iomap_iter
*iter
, loff_t pos
, size_t ret
,
705 const struct iomap_folio_ops
*folio_ops
= iter
->iomap
.folio_ops
;
707 if (folio_ops
&& folio_ops
->put_folio
) {
708 folio_ops
->put_folio(iter
->inode
, pos
, ret
, folio
);
715 static int iomap_write_begin_inline(const struct iomap_iter
*iter
,
718 /* needs more work for the tailpacking case; disable for now */
719 if (WARN_ON_ONCE(iomap_iter_srcmap(iter
)->offset
!= 0))
721 return iomap_read_inline_data(iter
, folio
);
724 static int iomap_write_begin(struct iomap_iter
*iter
, loff_t pos
,
725 size_t len
, struct folio
**foliop
)
727 const struct iomap_folio_ops
*folio_ops
= iter
->iomap
.folio_ops
;
728 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
732 BUG_ON(pos
+ len
> iter
->iomap
.offset
+ iter
->iomap
.length
);
733 if (srcmap
!= &iter
->iomap
)
734 BUG_ON(pos
+ len
> srcmap
->offset
+ srcmap
->length
);
736 if (fatal_signal_pending(current
))
739 if (!mapping_large_folio_support(iter
->inode
->i_mapping
))
740 len
= min_t(size_t, len
, PAGE_SIZE
- offset_in_page(pos
));
742 folio
= __iomap_get_folio(iter
, pos
, len
);
744 return PTR_ERR(folio
);
747 * Now we have a locked folio, before we do anything with it we need to
748 * check that the iomap we have cached is not stale. The inode extent
749 * mapping can change due to concurrent IO in flight (e.g.
750 * IOMAP_UNWRITTEN state can change and memory reclaim could have
751 * reclaimed a previously partially written page at this index after IO
752 * completion before this write reaches this file offset) and hence we
753 * could do the wrong thing here (zero a page range incorrectly or fail
754 * to zero) and corrupt data.
756 if (folio_ops
&& folio_ops
->iomap_valid
) {
757 bool iomap_valid
= folio_ops
->iomap_valid(iter
->inode
,
760 iter
->iomap
.flags
|= IOMAP_F_STALE
;
766 if (pos
+ len
> folio_pos(folio
) + folio_size(folio
))
767 len
= folio_pos(folio
) + folio_size(folio
) - pos
;
769 if (srcmap
->type
== IOMAP_INLINE
)
770 status
= iomap_write_begin_inline(iter
, folio
);
771 else if (srcmap
->flags
& IOMAP_F_BUFFER_HEAD
)
772 status
= __block_write_begin_int(folio
, pos
, len
, NULL
, srcmap
);
774 status
= __iomap_write_begin(iter
, pos
, len
, folio
);
776 if (unlikely(status
))
783 __iomap_put_folio(iter
, pos
, 0, folio
);
784 iomap_write_failed(iter
->inode
, pos
, len
);
789 static size_t __iomap_write_end(struct inode
*inode
, loff_t pos
, size_t len
,
790 size_t copied
, struct folio
*folio
)
792 flush_dcache_folio(folio
);
795 * The blocks that were entirely written will now be uptodate, so we
796 * don't have to worry about a read_folio reading them and overwriting a
797 * partial write. However, if we've encountered a short write and only
798 * partially written into a block, it will not be marked uptodate, so a
799 * read_folio might come in and destroy our partial write.
801 * Do the simplest thing and just treat any short write to a
802 * non-uptodate page as a zero-length write, and force the caller to
803 * redo the whole thing.
805 if (unlikely(copied
< len
&& !folio_test_uptodate(folio
)))
807 iomap_set_range_uptodate(folio
, offset_in_folio(folio
, pos
), len
);
808 iomap_set_range_dirty(folio
, offset_in_folio(folio
, pos
), copied
);
809 filemap_dirty_folio(inode
->i_mapping
, folio
);
813 static size_t iomap_write_end_inline(const struct iomap_iter
*iter
,
814 struct folio
*folio
, loff_t pos
, size_t copied
)
816 const struct iomap
*iomap
= &iter
->iomap
;
819 WARN_ON_ONCE(!folio_test_uptodate(folio
));
820 BUG_ON(!iomap_inline_data_valid(iomap
));
822 flush_dcache_folio(folio
);
823 addr
= kmap_local_folio(folio
, pos
);
824 memcpy(iomap_inline_data(iomap
, pos
), addr
, copied
);
827 mark_inode_dirty(iter
->inode
);
831 /* Returns the number of bytes copied. May be 0. Cannot be an errno. */
832 static size_t iomap_write_end(struct iomap_iter
*iter
, loff_t pos
, size_t len
,
833 size_t copied
, struct folio
*folio
)
835 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
836 loff_t old_size
= iter
->inode
->i_size
;
839 if (srcmap
->type
== IOMAP_INLINE
) {
840 ret
= iomap_write_end_inline(iter
, folio
, pos
, copied
);
841 } else if (srcmap
->flags
& IOMAP_F_BUFFER_HEAD
) {
842 ret
= block_write_end(NULL
, iter
->inode
->i_mapping
, pos
, len
,
843 copied
, &folio
->page
, NULL
);
845 ret
= __iomap_write_end(iter
->inode
, pos
, len
, copied
, folio
);
849 * Update the in-memory inode size after copying the data into the page
850 * cache. It's up to the file system to write the updated size to disk,
851 * preferably after I/O completion so that no stale data is exposed.
853 if (pos
+ ret
> old_size
) {
854 i_size_write(iter
->inode
, pos
+ ret
);
855 iter
->iomap
.flags
|= IOMAP_F_SIZE_CHANGED
;
857 __iomap_put_folio(iter
, pos
, ret
, folio
);
860 pagecache_isize_extended(iter
->inode
, old_size
, pos
);
862 iomap_write_failed(iter
->inode
, pos
+ ret
, len
- ret
);
866 static loff_t
iomap_write_iter(struct iomap_iter
*iter
, struct iov_iter
*i
)
868 loff_t length
= iomap_length(iter
);
869 size_t chunk
= PAGE_SIZE
<< MAX_PAGECACHE_ORDER
;
870 loff_t pos
= iter
->pos
;
873 struct address_space
*mapping
= iter
->inode
->i_mapping
;
874 unsigned int bdp_flags
= (iter
->flags
& IOMAP_NOWAIT
) ? BDP_ASYNC
: 0;
878 size_t offset
; /* Offset into folio */
879 size_t bytes
; /* Bytes to write to folio */
880 size_t copied
; /* Bytes copied from user */
882 offset
= pos
& (chunk
- 1);
883 bytes
= min(chunk
- offset
, iov_iter_count(i
));
884 status
= balance_dirty_pages_ratelimited_flags(mapping
,
886 if (unlikely(status
))
893 * Bring in the user page that we'll copy from _first_.
894 * Otherwise there's a nasty deadlock on copying from the
895 * same page as we're writing to, without it being marked
898 * For async buffered writes the assumption is that the user
899 * page has already been faulted in. This can be optimized by
900 * faulting the user page.
902 if (unlikely(fault_in_iov_iter_readable(i
, bytes
) == bytes
)) {
907 status
= iomap_write_begin(iter
, pos
, bytes
, &folio
);
908 if (unlikely(status
))
910 if (iter
->iomap
.flags
& IOMAP_F_STALE
)
913 offset
= offset_in_folio(folio
, pos
);
914 if (bytes
> folio_size(folio
) - offset
)
915 bytes
= folio_size(folio
) - offset
;
917 if (mapping_writably_mapped(mapping
))
918 flush_dcache_folio(folio
);
920 copied
= copy_folio_from_iter_atomic(folio
, offset
, bytes
, i
);
921 status
= iomap_write_end(iter
, pos
, bytes
, copied
, folio
);
923 if (unlikely(copied
!= status
))
924 iov_iter_revert(i
, copied
- status
);
927 if (unlikely(status
== 0)) {
929 * A short copy made iomap_write_end() reject the
930 * thing entirely. Might be memory poisoning
931 * halfway through, might be a race with munmap,
932 * might be severe memory pressure.
936 if (chunk
> PAGE_SIZE
)
943 } while (iov_iter_count(i
) && length
);
945 if (status
== -EAGAIN
) {
946 iov_iter_revert(i
, written
);
949 return written
? written
: status
;
953 iomap_file_buffered_write(struct kiocb
*iocb
, struct iov_iter
*i
,
954 const struct iomap_ops
*ops
)
956 struct iomap_iter iter
= {
957 .inode
= iocb
->ki_filp
->f_mapping
->host
,
959 .len
= iov_iter_count(i
),
960 .flags
= IOMAP_WRITE
,
964 if (iocb
->ki_flags
& IOCB_NOWAIT
)
965 iter
.flags
|= IOMAP_NOWAIT
;
967 while ((ret
= iomap_iter(&iter
, ops
)) > 0)
968 iter
.processed
= iomap_write_iter(&iter
, i
);
970 if (unlikely(iter
.pos
== iocb
->ki_pos
))
972 ret
= iter
.pos
- iocb
->ki_pos
;
973 iocb
->ki_pos
= iter
.pos
;
976 EXPORT_SYMBOL_GPL(iomap_file_buffered_write
);
978 static int iomap_write_delalloc_ifs_punch(struct inode
*inode
,
979 struct folio
*folio
, loff_t start_byte
, loff_t end_byte
,
982 unsigned int first_blk
, last_blk
, i
;
984 u8 blkbits
= inode
->i_blkbits
;
985 struct iomap_folio_state
*ifs
;
989 * When we have per-block dirty tracking, there can be
990 * blocks within a folio which are marked uptodate
991 * but not dirty. In that case it is necessary to punch
992 * out such blocks to avoid leaking any delalloc blocks.
994 ifs
= folio
->private;
998 last_byte
= min_t(loff_t
, end_byte
- 1,
999 folio_pos(folio
) + folio_size(folio
) - 1);
1000 first_blk
= offset_in_folio(folio
, start_byte
) >> blkbits
;
1001 last_blk
= offset_in_folio(folio
, last_byte
) >> blkbits
;
1002 for (i
= first_blk
; i
<= last_blk
; i
++) {
1003 if (!ifs_block_is_dirty(folio
, ifs
, i
)) {
1004 ret
= punch(inode
, folio_pos(folio
) + (i
<< blkbits
),
1015 static int iomap_write_delalloc_punch(struct inode
*inode
, struct folio
*folio
,
1016 loff_t
*punch_start_byte
, loff_t start_byte
, loff_t end_byte
,
1017 iomap_punch_t punch
)
1021 if (!folio_test_dirty(folio
))
1024 /* if dirty, punch up to offset */
1025 if (start_byte
> *punch_start_byte
) {
1026 ret
= punch(inode
, *punch_start_byte
,
1027 start_byte
- *punch_start_byte
);
1032 /* Punch non-dirty blocks within folio */
1033 ret
= iomap_write_delalloc_ifs_punch(inode
, folio
, start_byte
,
1039 * Make sure the next punch start is correctly bound to
1040 * the end of this data range, not the end of the folio.
1042 *punch_start_byte
= min_t(loff_t
, end_byte
,
1043 folio_pos(folio
) + folio_size(folio
));
1049 * Scan the data range passed to us for dirty page cache folios. If we find a
1050 * dirty folio, punch out the preceeding range and update the offset from which
1051 * the next punch will start from.
1053 * We can punch out storage reservations under clean pages because they either
1054 * contain data that has been written back - in which case the delalloc punch
1055 * over that range is a no-op - or they have been read faults in which case they
1056 * contain zeroes and we can remove the delalloc backing range and any new
1057 * writes to those pages will do the normal hole filling operation...
1059 * This makes the logic simple: we only need to keep the delalloc extents only
1060 * over the dirty ranges of the page cache.
1062 * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1063 * simplify range iterations.
1065 static int iomap_write_delalloc_scan(struct inode
*inode
,
1066 loff_t
*punch_start_byte
, loff_t start_byte
, loff_t end_byte
,
1067 iomap_punch_t punch
)
1069 while (start_byte
< end_byte
) {
1070 struct folio
*folio
;
1073 /* grab locked page */
1074 folio
= filemap_lock_folio(inode
->i_mapping
,
1075 start_byte
>> PAGE_SHIFT
);
1076 if (IS_ERR(folio
)) {
1077 start_byte
= ALIGN_DOWN(start_byte
, PAGE_SIZE
) +
1082 ret
= iomap_write_delalloc_punch(inode
, folio
, punch_start_byte
,
1083 start_byte
, end_byte
, punch
);
1085 folio_unlock(folio
);
1090 /* move offset to start of next folio in range */
1091 start_byte
= folio_next_index(folio
) << PAGE_SHIFT
;
1092 folio_unlock(folio
);
1099 * Punch out all the delalloc blocks in the range given except for those that
1100 * have dirty data still pending in the page cache - those are going to be
1101 * written and so must still retain the delalloc backing for writeback.
1103 * As we are scanning the page cache for data, we don't need to reimplement the
1104 * wheel - mapping_seek_hole_data() does exactly what we need to identify the
1105 * start and end of data ranges correctly even for sub-folio block sizes. This
1106 * byte range based iteration is especially convenient because it means we
1107 * don't have to care about variable size folios, nor where the start or end of
1108 * the data range lies within a folio, if they lie within the same folio or even
1109 * if there are multiple discontiguous data ranges within the folio.
1111 * It should be noted that mapping_seek_hole_data() is not aware of EOF, and so
1112 * can return data ranges that exist in the cache beyond EOF. e.g. a page fault
1113 * spanning EOF will initialise the post-EOF data to zeroes and mark it up to
1114 * date. A write page fault can then mark it dirty. If we then fail a write()
1115 * beyond EOF into that up to date cached range, we allocate a delalloc block
1116 * beyond EOF and then have to punch it out. Because the range is up to date,
1117 * mapping_seek_hole_data() will return it, and we will skip the punch because
1118 * the folio is dirty. THis is incorrect - we always need to punch out delalloc
1119 * beyond EOF in this case as writeback will never write back and covert that
1120 * delalloc block beyond EOF. Hence we limit the cached data scan range to EOF,
1121 * resulting in always punching out the range from the EOF to the end of the
1122 * range the iomap spans.
1124 * Intervals are of the form [start_byte, end_byte) (i.e. open ended) because it
1125 * matches the intervals returned by mapping_seek_hole_data(). i.e. SEEK_DATA
1126 * returns the start of a data range (start_byte), and SEEK_HOLE(start_byte)
1127 * returns the end of the data range (data_end). Using closed intervals would
1128 * require sprinkling this code with magic "+ 1" and "- 1" arithmetic and expose
1129 * the code to subtle off-by-one bugs....
1131 static int iomap_write_delalloc_release(struct inode
*inode
,
1132 loff_t start_byte
, loff_t end_byte
, iomap_punch_t punch
)
1134 loff_t punch_start_byte
= start_byte
;
1135 loff_t scan_end_byte
= min(i_size_read(inode
), end_byte
);
1139 * Lock the mapping to avoid races with page faults re-instantiating
1140 * folios and dirtying them via ->page_mkwrite whilst we walk the
1141 * cache and perform delalloc extent removal. Failing to do this can
1142 * leave dirty pages with no space reservation in the cache.
1144 filemap_invalidate_lock(inode
->i_mapping
);
1145 while (start_byte
< scan_end_byte
) {
1148 start_byte
= mapping_seek_hole_data(inode
->i_mapping
,
1149 start_byte
, scan_end_byte
, SEEK_DATA
);
1151 * If there is no more data to scan, all that is left is to
1152 * punch out the remaining range.
1154 if (start_byte
== -ENXIO
|| start_byte
== scan_end_byte
)
1156 if (start_byte
< 0) {
1160 WARN_ON_ONCE(start_byte
< punch_start_byte
);
1161 WARN_ON_ONCE(start_byte
> scan_end_byte
);
1164 * We find the end of this contiguous cached data range by
1165 * seeking from start_byte to the beginning of the next hole.
1167 data_end
= mapping_seek_hole_data(inode
->i_mapping
, start_byte
,
1168 scan_end_byte
, SEEK_HOLE
);
1173 WARN_ON_ONCE(data_end
<= start_byte
);
1174 WARN_ON_ONCE(data_end
> scan_end_byte
);
1176 error
= iomap_write_delalloc_scan(inode
, &punch_start_byte
,
1177 start_byte
, data_end
, punch
);
1181 /* The next data search starts at the end of this one. */
1182 start_byte
= data_end
;
1185 if (punch_start_byte
< end_byte
)
1186 error
= punch(inode
, punch_start_byte
,
1187 end_byte
- punch_start_byte
);
1189 filemap_invalidate_unlock(inode
->i_mapping
);
1194 * When a short write occurs, the filesystem may need to remove reserved space
1195 * that was allocated in ->iomap_begin from it's ->iomap_end method. For
1196 * filesystems that use delayed allocation, we need to punch out delalloc
1197 * extents from the range that are not dirty in the page cache. As the write can
1198 * race with page faults, there can be dirty pages over the delalloc extent
1199 * outside the range of a short write but still within the delalloc extent
1200 * allocated for this iomap.
1202 * This function uses [start_byte, end_byte) intervals (i.e. open ended) to
1203 * simplify range iterations.
1205 * The punch() callback *must* only punch delalloc extents in the range passed
1206 * to it. It must skip over all other types of extents in the range and leave
1207 * them completely unchanged. It must do this punch atomically with respect to
1208 * other extent modifications.
1210 * The punch() callback may be called with a folio locked to prevent writeback
1211 * extent allocation racing at the edge of the range we are currently punching.
1212 * The locked folio may or may not cover the range being punched, so it is not
1213 * safe for the punch() callback to lock folios itself.
1217 * inode->i_rwsem (shared or exclusive)
1218 * inode->i_mapping->invalidate_lock (exclusive)
1221 * internal filesystem allocation lock
1223 int iomap_file_buffered_write_punch_delalloc(struct inode
*inode
,
1224 struct iomap
*iomap
, loff_t pos
, loff_t length
,
1225 ssize_t written
, iomap_punch_t punch
)
1229 unsigned int blocksize
= i_blocksize(inode
);
1231 if (iomap
->type
!= IOMAP_DELALLOC
)
1234 /* If we didn't reserve the blocks, we're not allowed to punch them. */
1235 if (!(iomap
->flags
& IOMAP_F_NEW
))
1239 * start_byte refers to the first unused block after a short write. If
1240 * nothing was written, round offset down to point at the first block in
1243 if (unlikely(!written
))
1244 start_byte
= round_down(pos
, blocksize
);
1246 start_byte
= round_up(pos
+ written
, blocksize
);
1247 end_byte
= round_up(pos
+ length
, blocksize
);
1249 /* Nothing to do if we've written the entire delalloc extent */
1250 if (start_byte
>= end_byte
)
1253 return iomap_write_delalloc_release(inode
, start_byte
, end_byte
,
1256 EXPORT_SYMBOL_GPL(iomap_file_buffered_write_punch_delalloc
);
1258 static loff_t
iomap_unshare_iter(struct iomap_iter
*iter
)
1260 struct iomap
*iomap
= &iter
->iomap
;
1261 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
1262 loff_t pos
= iter
->pos
;
1263 loff_t length
= iomap_length(iter
);
1267 /* don't bother with blocks that are not shared to start with */
1268 if (!(iomap
->flags
& IOMAP_F_SHARED
))
1270 /* don't bother with holes or unwritten extents */
1271 if (srcmap
->type
== IOMAP_HOLE
|| srcmap
->type
== IOMAP_UNWRITTEN
)
1275 unsigned long offset
= offset_in_page(pos
);
1276 unsigned long bytes
= min_t(loff_t
, PAGE_SIZE
- offset
, length
);
1277 struct folio
*folio
;
1279 status
= iomap_write_begin(iter
, pos
, bytes
, &folio
);
1280 if (unlikely(status
))
1282 if (iter
->iomap
.flags
& IOMAP_F_STALE
)
1285 status
= iomap_write_end(iter
, pos
, bytes
, bytes
, folio
);
1286 if (WARN_ON_ONCE(status
== 0))
1295 balance_dirty_pages_ratelimited(iter
->inode
->i_mapping
);
1302 iomap_file_unshare(struct inode
*inode
, loff_t pos
, loff_t len
,
1303 const struct iomap_ops
*ops
)
1305 struct iomap_iter iter
= {
1309 .flags
= IOMAP_WRITE
| IOMAP_UNSHARE
,
1313 while ((ret
= iomap_iter(&iter
, ops
)) > 0)
1314 iter
.processed
= iomap_unshare_iter(&iter
);
1317 EXPORT_SYMBOL_GPL(iomap_file_unshare
);
1319 static loff_t
iomap_zero_iter(struct iomap_iter
*iter
, bool *did_zero
)
1321 const struct iomap
*srcmap
= iomap_iter_srcmap(iter
);
1322 loff_t pos
= iter
->pos
;
1323 loff_t length
= iomap_length(iter
);
1326 /* already zeroed? we're done. */
1327 if (srcmap
->type
== IOMAP_HOLE
|| srcmap
->type
== IOMAP_UNWRITTEN
)
1331 struct folio
*folio
;
1334 size_t bytes
= min_t(u64
, SIZE_MAX
, length
);
1336 status
= iomap_write_begin(iter
, pos
, bytes
, &folio
);
1339 if (iter
->iomap
.flags
& IOMAP_F_STALE
)
1342 offset
= offset_in_folio(folio
, pos
);
1343 if (bytes
> folio_size(folio
) - offset
)
1344 bytes
= folio_size(folio
) - offset
;
1346 folio_zero_range(folio
, offset
, bytes
);
1347 folio_mark_accessed(folio
);
1349 bytes
= iomap_write_end(iter
, pos
, bytes
, bytes
, folio
);
1350 if (WARN_ON_ONCE(bytes
== 0))
1356 } while (length
> 0);
1364 iomap_zero_range(struct inode
*inode
, loff_t pos
, loff_t len
, bool *did_zero
,
1365 const struct iomap_ops
*ops
)
1367 struct iomap_iter iter
= {
1371 .flags
= IOMAP_ZERO
,
1375 while ((ret
= iomap_iter(&iter
, ops
)) > 0)
1376 iter
.processed
= iomap_zero_iter(&iter
, did_zero
);
1379 EXPORT_SYMBOL_GPL(iomap_zero_range
);
1382 iomap_truncate_page(struct inode
*inode
, loff_t pos
, bool *did_zero
,
1383 const struct iomap_ops
*ops
)
1385 unsigned int blocksize
= i_blocksize(inode
);
1386 unsigned int off
= pos
& (blocksize
- 1);
1388 /* Block boundary? Nothing to do */
1391 return iomap_zero_range(inode
, pos
, blocksize
- off
, did_zero
, ops
);
1393 EXPORT_SYMBOL_GPL(iomap_truncate_page
);
1395 static loff_t
iomap_folio_mkwrite_iter(struct iomap_iter
*iter
,
1396 struct folio
*folio
)
1398 loff_t length
= iomap_length(iter
);
1401 if (iter
->iomap
.flags
& IOMAP_F_BUFFER_HEAD
) {
1402 ret
= __block_write_begin_int(folio
, iter
->pos
, length
, NULL
,
1406 block_commit_write(&folio
->page
, 0, length
);
1408 WARN_ON_ONCE(!folio_test_uptodate(folio
));
1409 folio_mark_dirty(folio
);
1415 vm_fault_t
iomap_page_mkwrite(struct vm_fault
*vmf
, const struct iomap_ops
*ops
)
1417 struct iomap_iter iter
= {
1418 .inode
= file_inode(vmf
->vma
->vm_file
),
1419 .flags
= IOMAP_WRITE
| IOMAP_FAULT
,
1421 struct folio
*folio
= page_folio(vmf
->page
);
1425 ret
= folio_mkwrite_check_truncate(folio
, iter
.inode
);
1428 iter
.pos
= folio_pos(folio
);
1430 while ((ret
= iomap_iter(&iter
, ops
)) > 0)
1431 iter
.processed
= iomap_folio_mkwrite_iter(&iter
, folio
);
1435 folio_wait_stable(folio
);
1436 return VM_FAULT_LOCKED
;
1438 folio_unlock(folio
);
1439 return vmf_fs_error(ret
);
1441 EXPORT_SYMBOL_GPL(iomap_page_mkwrite
);
1443 static void iomap_finish_folio_write(struct inode
*inode
, struct folio
*folio
,
1444 size_t len
, int error
)
1446 struct iomap_folio_state
*ifs
= folio
->private;
1449 folio_set_error(folio
);
1450 mapping_set_error(inode
->i_mapping
, error
);
1453 WARN_ON_ONCE(i_blocks_per_folio(inode
, folio
) > 1 && !ifs
);
1454 WARN_ON_ONCE(ifs
&& atomic_read(&ifs
->write_bytes_pending
) <= 0);
1456 if (!ifs
|| atomic_sub_and_test(len
, &ifs
->write_bytes_pending
))
1457 folio_end_writeback(folio
);
1461 * We're now finished for good with this ioend structure. Update the page
1462 * state, release holds on bios, and finally free up memory. Do not use the
1466 iomap_finish_ioend(struct iomap_ioend
*ioend
, int error
)
1468 struct inode
*inode
= ioend
->io_inode
;
1469 struct bio
*bio
= &ioend
->io_inline_bio
;
1470 struct bio
*last
= ioend
->io_bio
, *next
;
1471 u64 start
= bio
->bi_iter
.bi_sector
;
1472 loff_t offset
= ioend
->io_offset
;
1473 bool quiet
= bio_flagged(bio
, BIO_QUIET
);
1474 u32 folio_count
= 0;
1476 for (bio
= &ioend
->io_inline_bio
; bio
; bio
= next
) {
1477 struct folio_iter fi
;
1480 * For the last bio, bi_private points to the ioend, so we
1481 * need to explicitly end the iteration here.
1486 next
= bio
->bi_private
;
1488 /* walk all folios in bio, ending page IO on them */
1489 bio_for_each_folio_all(fi
, bio
) {
1490 iomap_finish_folio_write(inode
, fi
.folio
, fi
.length
,
1496 /* The ioend has been freed by bio_put() */
1498 if (unlikely(error
&& !quiet
)) {
1499 printk_ratelimited(KERN_ERR
1500 "%s: writeback error on inode %lu, offset %lld, sector %llu",
1501 inode
->i_sb
->s_id
, inode
->i_ino
, offset
, start
);
1507 * Ioend completion routine for merged bios. This can only be called from task
1508 * contexts as merged ioends can be of unbound length. Hence we have to break up
1509 * the writeback completions into manageable chunks to avoid long scheduler
1510 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
1511 * good batch processing throughput without creating adverse scheduler latency
1515 iomap_finish_ioends(struct iomap_ioend
*ioend
, int error
)
1517 struct list_head tmp
;
1522 list_replace_init(&ioend
->io_list
, &tmp
);
1523 completions
= iomap_finish_ioend(ioend
, error
);
1525 while (!list_empty(&tmp
)) {
1526 if (completions
> IOEND_BATCH_SIZE
* 8) {
1530 ioend
= list_first_entry(&tmp
, struct iomap_ioend
, io_list
);
1531 list_del_init(&ioend
->io_list
);
1532 completions
+= iomap_finish_ioend(ioend
, error
);
1535 EXPORT_SYMBOL_GPL(iomap_finish_ioends
);
1538 * We can merge two adjacent ioends if they have the same set of work to do.
1541 iomap_ioend_can_merge(struct iomap_ioend
*ioend
, struct iomap_ioend
*next
)
1543 if (ioend
->io_bio
->bi_status
!= next
->io_bio
->bi_status
)
1545 if ((ioend
->io_flags
& IOMAP_F_SHARED
) ^
1546 (next
->io_flags
& IOMAP_F_SHARED
))
1548 if ((ioend
->io_type
== IOMAP_UNWRITTEN
) ^
1549 (next
->io_type
== IOMAP_UNWRITTEN
))
1551 if (ioend
->io_offset
+ ioend
->io_size
!= next
->io_offset
)
1554 * Do not merge physically discontiguous ioends. The filesystem
1555 * completion functions will have to iterate the physical
1556 * discontiguities even if we merge the ioends at a logical level, so
1557 * we don't gain anything by merging physical discontiguities here.
1559 * We cannot use bio->bi_iter.bi_sector here as it is modified during
1560 * submission so does not point to the start sector of the bio at
1563 if (ioend
->io_sector
+ (ioend
->io_size
>> 9) != next
->io_sector
)
1569 iomap_ioend_try_merge(struct iomap_ioend
*ioend
, struct list_head
*more_ioends
)
1571 struct iomap_ioend
*next
;
1573 INIT_LIST_HEAD(&ioend
->io_list
);
1575 while ((next
= list_first_entry_or_null(more_ioends
, struct iomap_ioend
,
1577 if (!iomap_ioend_can_merge(ioend
, next
))
1579 list_move_tail(&next
->io_list
, &ioend
->io_list
);
1580 ioend
->io_size
+= next
->io_size
;
1583 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge
);
1586 iomap_ioend_compare(void *priv
, const struct list_head
*a
,
1587 const struct list_head
*b
)
1589 struct iomap_ioend
*ia
= container_of(a
, struct iomap_ioend
, io_list
);
1590 struct iomap_ioend
*ib
= container_of(b
, struct iomap_ioend
, io_list
);
1592 if (ia
->io_offset
< ib
->io_offset
)
1594 if (ia
->io_offset
> ib
->io_offset
)
1600 iomap_sort_ioends(struct list_head
*ioend_list
)
1602 list_sort(NULL
, ioend_list
, iomap_ioend_compare
);
1604 EXPORT_SYMBOL_GPL(iomap_sort_ioends
);
1606 static void iomap_writepage_end_bio(struct bio
*bio
)
1608 struct iomap_ioend
*ioend
= bio
->bi_private
;
1610 iomap_finish_ioend(ioend
, blk_status_to_errno(bio
->bi_status
));
1614 * Submit the final bio for an ioend.
1616 * If @error is non-zero, it means that we have a situation where some part of
1617 * the submission process has failed after we've marked pages for writeback
1618 * and unlocked them. In this situation, we need to fail the bio instead of
1619 * submitting it. This typically only happens on a filesystem shutdown.
1622 iomap_submit_ioend(struct iomap_writepage_ctx
*wpc
, struct iomap_ioend
*ioend
,
1625 ioend
->io_bio
->bi_private
= ioend
;
1626 ioend
->io_bio
->bi_end_io
= iomap_writepage_end_bio
;
1628 if (wpc
->ops
->prepare_ioend
)
1629 error
= wpc
->ops
->prepare_ioend(ioend
, error
);
1632 * If we're failing the IO now, just mark the ioend with an
1633 * error and finish it. This will run IO completion immediately
1634 * as there is only one reference to the ioend at this point in
1637 ioend
->io_bio
->bi_status
= errno_to_blk_status(error
);
1638 bio_endio(ioend
->io_bio
);
1642 submit_bio(ioend
->io_bio
);
1646 static struct iomap_ioend
*
1647 iomap_alloc_ioend(struct inode
*inode
, struct iomap_writepage_ctx
*wpc
,
1648 loff_t offset
, sector_t sector
, struct writeback_control
*wbc
)
1650 struct iomap_ioend
*ioend
;
1653 bio
= bio_alloc_bioset(wpc
->iomap
.bdev
, BIO_MAX_VECS
,
1654 REQ_OP_WRITE
| wbc_to_write_flags(wbc
),
1655 GFP_NOFS
, &iomap_ioend_bioset
);
1656 bio
->bi_iter
.bi_sector
= sector
;
1657 wbc_init_bio(wbc
, bio
);
1659 ioend
= container_of(bio
, struct iomap_ioend
, io_inline_bio
);
1660 INIT_LIST_HEAD(&ioend
->io_list
);
1661 ioend
->io_type
= wpc
->iomap
.type
;
1662 ioend
->io_flags
= wpc
->iomap
.flags
;
1663 ioend
->io_inode
= inode
;
1665 ioend
->io_folios
= 0;
1666 ioend
->io_offset
= offset
;
1667 ioend
->io_bio
= bio
;
1668 ioend
->io_sector
= sector
;
1673 * Allocate a new bio, and chain the old bio to the new one.
1675 * Note that we have to perform the chaining in this unintuitive order
1676 * so that the bi_private linkage is set up in the right direction for the
1677 * traversal in iomap_finish_ioend().
1680 iomap_chain_bio(struct bio
*prev
)
1684 new = bio_alloc(prev
->bi_bdev
, BIO_MAX_VECS
, prev
->bi_opf
, GFP_NOFS
);
1685 bio_clone_blkg_association(new, prev
);
1686 new->bi_iter
.bi_sector
= bio_end_sector(prev
);
1688 bio_chain(prev
, new);
1689 bio_get(prev
); /* for iomap_finish_ioend */
1695 iomap_can_add_to_ioend(struct iomap_writepage_ctx
*wpc
, loff_t offset
,
1698 if ((wpc
->iomap
.flags
& IOMAP_F_SHARED
) !=
1699 (wpc
->ioend
->io_flags
& IOMAP_F_SHARED
))
1701 if (wpc
->iomap
.type
!= wpc
->ioend
->io_type
)
1703 if (offset
!= wpc
->ioend
->io_offset
+ wpc
->ioend
->io_size
)
1705 if (sector
!= bio_end_sector(wpc
->ioend
->io_bio
))
1708 * Limit ioend bio chain lengths to minimise IO completion latency. This
1709 * also prevents long tight loops ending page writeback on all the
1710 * folios in the ioend.
1712 if (wpc
->ioend
->io_folios
>= IOEND_BATCH_SIZE
)
1718 * Test to see if we have an existing ioend structure that we could append to
1719 * first; otherwise finish off the current ioend and start another.
1722 iomap_add_to_ioend(struct inode
*inode
, loff_t pos
, struct folio
*folio
,
1723 struct iomap_folio_state
*ifs
, struct iomap_writepage_ctx
*wpc
,
1724 struct writeback_control
*wbc
, struct list_head
*iolist
)
1726 sector_t sector
= iomap_sector(&wpc
->iomap
, pos
);
1727 unsigned len
= i_blocksize(inode
);
1728 size_t poff
= offset_in_folio(folio
, pos
);
1730 if (!wpc
->ioend
|| !iomap_can_add_to_ioend(wpc
, pos
, sector
)) {
1732 list_add(&wpc
->ioend
->io_list
, iolist
);
1733 wpc
->ioend
= iomap_alloc_ioend(inode
, wpc
, pos
, sector
, wbc
);
1736 if (!bio_add_folio(wpc
->ioend
->io_bio
, folio
, len
, poff
)) {
1737 wpc
->ioend
->io_bio
= iomap_chain_bio(wpc
->ioend
->io_bio
);
1738 bio_add_folio_nofail(wpc
->ioend
->io_bio
, folio
, len
, poff
);
1742 atomic_add(len
, &ifs
->write_bytes_pending
);
1743 wpc
->ioend
->io_size
+= len
;
1744 wbc_account_cgroup_owner(wbc
, &folio
->page
, len
);
1748 * We implement an immediate ioend submission policy here to avoid needing to
1749 * chain multiple ioends and hence nest mempool allocations which can violate
1750 * the forward progress guarantees we need to provide. The current ioend we're
1751 * adding blocks to is cached in the writepage context, and if the new block
1752 * doesn't append to the cached ioend, it will create a new ioend and cache that
1755 * If a new ioend is created and cached, the old ioend is returned and queued
1756 * locally for submission once the entire page is processed or an error has been
1757 * detected. While ioends are submitted immediately after they are completed,
1758 * batching optimisations are provided by higher level block plugging.
1760 * At the end of a writeback pass, there will be a cached ioend remaining on the
1761 * writepage context that the caller will need to submit.
1764 iomap_writepage_map(struct iomap_writepage_ctx
*wpc
,
1765 struct writeback_control
*wbc
, struct inode
*inode
,
1766 struct folio
*folio
, u64 end_pos
)
1768 struct iomap_folio_state
*ifs
= folio
->private;
1769 struct iomap_ioend
*ioend
, *next
;
1770 unsigned len
= i_blocksize(inode
);
1771 unsigned nblocks
= i_blocks_per_folio(inode
, folio
);
1772 u64 pos
= folio_pos(folio
);
1773 int error
= 0, count
= 0, i
;
1774 LIST_HEAD(submit_list
);
1776 WARN_ON_ONCE(end_pos
<= pos
);
1778 if (!ifs
&& nblocks
> 1) {
1779 ifs
= ifs_alloc(inode
, folio
, 0);
1780 iomap_set_range_dirty(folio
, 0, end_pos
- pos
);
1783 WARN_ON_ONCE(ifs
&& atomic_read(&ifs
->write_bytes_pending
) != 0);
1786 * Walk through the folio to find areas to write back. If we
1787 * run off the end of the current map or find the current map
1788 * invalid, grab a new one.
1790 for (i
= 0; i
< nblocks
&& pos
< end_pos
; i
++, pos
+= len
) {
1791 if (ifs
&& !ifs_block_is_dirty(folio
, ifs
, i
))
1794 error
= wpc
->ops
->map_blocks(wpc
, inode
, pos
);
1797 trace_iomap_writepage_map(inode
, &wpc
->iomap
);
1798 if (WARN_ON_ONCE(wpc
->iomap
.type
== IOMAP_INLINE
))
1800 if (wpc
->iomap
.type
== IOMAP_HOLE
)
1802 iomap_add_to_ioend(inode
, pos
, folio
, ifs
, wpc
, wbc
,
1807 wpc
->ioend
->io_folios
++;
1809 WARN_ON_ONCE(!wpc
->ioend
&& !list_empty(&submit_list
));
1810 WARN_ON_ONCE(!folio_test_locked(folio
));
1811 WARN_ON_ONCE(folio_test_writeback(folio
));
1812 WARN_ON_ONCE(folio_test_dirty(folio
));
1815 * We cannot cancel the ioend directly here on error. We may have
1816 * already set other pages under writeback and hence we have to run I/O
1817 * completion to mark the error state of the pages under writeback
1820 if (unlikely(error
)) {
1822 * Let the filesystem know what portion of the current page
1823 * failed to map. If the page hasn't been added to ioend, it
1824 * won't be affected by I/O completion and we must unlock it
1827 if (wpc
->ops
->discard_folio
)
1828 wpc
->ops
->discard_folio(folio
, pos
);
1830 folio_unlock(folio
);
1836 * We can have dirty bits set past end of file in page_mkwrite path
1837 * while mapping the last partial folio. Hence it's better to clear
1838 * all the dirty bits in the folio here.
1840 iomap_clear_range_dirty(folio
, 0, folio_size(folio
));
1841 folio_start_writeback(folio
);
1842 folio_unlock(folio
);
1845 * Preserve the original error if there was one; catch
1846 * submission errors here and propagate into subsequent ioend
1849 list_for_each_entry_safe(ioend
, next
, &submit_list
, io_list
) {
1852 list_del_init(&ioend
->io_list
);
1853 error2
= iomap_submit_ioend(wpc
, ioend
, error
);
1854 if (error2
&& !error
)
1859 * We can end up here with no error and nothing to write only if we race
1860 * with a partial page truncate on a sub-page block sized filesystem.
1863 folio_end_writeback(folio
);
1865 mapping_set_error(inode
->i_mapping
, error
);
1870 * Write out a dirty page.
1872 * For delalloc space on the page, we need to allocate space and flush it.
1873 * For unwritten space on the page, we need to start the conversion to
1874 * regular allocated space.
1876 static int iomap_do_writepage(struct folio
*folio
,
1877 struct writeback_control
*wbc
, void *data
)
1879 struct iomap_writepage_ctx
*wpc
= data
;
1880 struct inode
*inode
= folio
->mapping
->host
;
1883 trace_iomap_writepage(inode
, folio_pos(folio
), folio_size(folio
));
1886 * Refuse to write the folio out if we're called from reclaim context.
1888 * This avoids stack overflows when called from deeply used stacks in
1889 * random callers for direct reclaim or memcg reclaim. We explicitly
1890 * allow reclaim from kswapd as the stack usage there is relatively low.
1892 * This should never happen except in the case of a VM regression so
1895 if (WARN_ON_ONCE((current
->flags
& (PF_MEMALLOC
|PF_KSWAPD
)) ==
1900 * Is this folio beyond the end of the file?
1902 * The folio index is less than the end_index, adjust the end_pos
1903 * to the highest offset that this folio should represent.
1904 * -----------------------------------------------------
1905 * | file mapping | <EOF> |
1906 * -----------------------------------------------------
1907 * | Page ... | Page N-2 | Page N-1 | Page N | |
1908 * ^--------------------------------^----------|--------
1909 * | desired writeback range | see else |
1910 * ---------------------------------^------------------|
1912 isize
= i_size_read(inode
);
1913 end_pos
= folio_pos(folio
) + folio_size(folio
);
1914 if (end_pos
> isize
) {
1916 * Check whether the page to write out is beyond or straddles
1918 * -------------------------------------------------------
1919 * | file mapping | <EOF> |
1920 * -------------------------------------------------------
1921 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
1922 * ^--------------------------------^-----------|---------
1924 * ---------------------------------^-----------|--------|
1926 size_t poff
= offset_in_folio(folio
, isize
);
1927 pgoff_t end_index
= isize
>> PAGE_SHIFT
;
1930 * Skip the page if it's fully outside i_size, e.g.
1931 * due to a truncate operation that's in progress. We've
1932 * cleaned this page and truncate will finish things off for
1935 * Note that the end_index is unsigned long. If the given
1936 * offset is greater than 16TB on a 32-bit system then if we
1937 * checked if the page is fully outside i_size with
1938 * "if (page->index >= end_index + 1)", "end_index + 1" would
1939 * overflow and evaluate to 0. Hence this page would be
1940 * redirtied and written out repeatedly, which would result in
1941 * an infinite loop; the user program performing this operation
1942 * would hang. Instead, we can detect this situation by
1943 * checking if the page is totally beyond i_size or if its
1944 * offset is just equal to the EOF.
1946 if (folio
->index
> end_index
||
1947 (folio
->index
== end_index
&& poff
== 0))
1951 * The page straddles i_size. It must be zeroed out on each
1952 * and every writepage invocation because it may be mmapped.
1953 * "A file is mapped in multiples of the page size. For a file
1954 * that is not a multiple of the page size, the remaining
1955 * memory is zeroed when mapped, and writes to that region are
1956 * not written out to the file."
1958 folio_zero_segment(folio
, poff
, folio_size(folio
));
1962 return iomap_writepage_map(wpc
, wbc
, inode
, folio
, end_pos
);
1965 folio_redirty_for_writepage(wbc
, folio
);
1967 folio_unlock(folio
);
1972 iomap_writepages(struct address_space
*mapping
, struct writeback_control
*wbc
,
1973 struct iomap_writepage_ctx
*wpc
,
1974 const struct iomap_writeback_ops
*ops
)
1979 ret
= write_cache_pages(mapping
, wbc
, iomap_do_writepage
, wpc
);
1982 return iomap_submit_ioend(wpc
, wpc
->ioend
, ret
);
1984 EXPORT_SYMBOL_GPL(iomap_writepages
);
1986 static int __init
iomap_init(void)
1988 return bioset_init(&iomap_ioend_bioset
, 4 * (PAGE_SIZE
/ SECTOR_SIZE
),
1989 offsetof(struct iomap_ioend
, io_inline_bio
),
1992 fs_initcall(iomap_init
);