1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4 * Copyright (c) 2016-2018 Christoph Hellwig.
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_trans.h"
15 #include "xfs_inode_item.h"
16 #include "xfs_alloc.h"
17 #include "xfs_error.h"
18 #include "xfs_iomap.h"
19 #include "xfs_trace.h"
21 #include "xfs_bmap_util.h"
22 #include "xfs_bmap_btree.h"
23 #include "xfs_reflink.h"
24 #include <linux/writeback.h>
27 * structure owned by writepages passed to individual writepage calls
29 struct xfs_writepage_ctx
{
30 struct xfs_bmbt_irec imap
;
32 unsigned int data_seq
;
34 struct xfs_ioend
*ioend
;
38 xfs_find_bdev_for_inode(
41 struct xfs_inode
*ip
= XFS_I(inode
);
42 struct xfs_mount
*mp
= ip
->i_mount
;
44 if (XFS_IS_REALTIME_INODE(ip
))
45 return mp
->m_rtdev_targp
->bt_bdev
;
47 return mp
->m_ddev_targp
->bt_bdev
;
51 xfs_find_daxdev_for_inode(
54 struct xfs_inode
*ip
= XFS_I(inode
);
55 struct xfs_mount
*mp
= ip
->i_mount
;
57 if (XFS_IS_REALTIME_INODE(ip
))
58 return mp
->m_rtdev_targp
->bt_daxdev
;
60 return mp
->m_ddev_targp
->bt_daxdev
;
64 xfs_finish_page_writeback(
69 struct iomap_page
*iop
= to_iomap_page(bvec
->bv_page
);
72 SetPageError(bvec
->bv_page
);
73 mapping_set_error(inode
->i_mapping
, -EIO
);
76 ASSERT(iop
|| i_blocksize(inode
) == PAGE_SIZE
);
77 ASSERT(!iop
|| atomic_read(&iop
->write_count
) > 0);
79 if (!iop
|| atomic_dec_and_test(&iop
->write_count
))
80 end_page_writeback(bvec
->bv_page
);
84 * We're now finished for good with this ioend structure. Update the page
85 * state, release holds on bios, and finally free up memory. Do not use the
90 struct xfs_ioend
*ioend
,
93 struct inode
*inode
= ioend
->io_inode
;
94 struct bio
*bio
= &ioend
->io_inline_bio
;
95 struct bio
*last
= ioend
->io_bio
, *next
;
96 u64 start
= bio
->bi_iter
.bi_sector
;
97 bool quiet
= bio_flagged(bio
, BIO_QUIET
);
99 for (bio
= &ioend
->io_inline_bio
; bio
; bio
= next
) {
100 struct bio_vec
*bvec
;
101 struct bvec_iter_all iter_all
;
104 * For the last bio, bi_private points to the ioend, so we
105 * need to explicitly end the iteration here.
110 next
= bio
->bi_private
;
112 /* walk each page on bio, ending page IO on them */
113 bio_for_each_segment_all(bvec
, bio
, iter_all
)
114 xfs_finish_page_writeback(inode
, bvec
, error
);
118 if (unlikely(error
&& !quiet
)) {
119 xfs_err_ratelimited(XFS_I(inode
)->i_mount
,
120 "writeback error on sector %llu", start
);
125 * Fast and loose check if this write could update the on-disk inode size.
127 static inline bool xfs_ioend_is_append(struct xfs_ioend
*ioend
)
129 return ioend
->io_offset
+ ioend
->io_size
>
130 XFS_I(ioend
->io_inode
)->i_d
.di_size
;
134 xfs_setfilesize_trans_alloc(
135 struct xfs_ioend
*ioend
)
137 struct xfs_mount
*mp
= XFS_I(ioend
->io_inode
)->i_mount
;
138 struct xfs_trans
*tp
;
141 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_fsyncts
, 0, 0,
142 XFS_TRANS_NOFS
, &tp
);
146 ioend
->io_append_trans
= tp
;
149 * We may pass freeze protection with a transaction. So tell lockdep
152 __sb_writers_release(ioend
->io_inode
->i_sb
, SB_FREEZE_FS
);
154 * We hand off the transaction to the completion thread now, so
155 * clear the flag here.
157 current_restore_flags_nested(&tp
->t_pflags
, PF_MEMALLOC_NOFS
);
162 * Update on-disk file size now that data has been written to disk.
166 struct xfs_inode
*ip
,
167 struct xfs_trans
*tp
,
173 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
174 isize
= xfs_new_eof(ip
, offset
+ size
);
176 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
177 xfs_trans_cancel(tp
);
181 trace_xfs_setfilesize(ip
, offset
, size
);
183 ip
->i_d
.di_size
= isize
;
184 xfs_trans_ijoin(tp
, ip
, XFS_ILOCK_EXCL
);
185 xfs_trans_log_inode(tp
, ip
, XFS_ILOG_CORE
);
187 return xfs_trans_commit(tp
);
192 struct xfs_inode
*ip
,
196 struct xfs_mount
*mp
= ip
->i_mount
;
197 struct xfs_trans
*tp
;
200 error
= xfs_trans_alloc(mp
, &M_RES(mp
)->tr_fsyncts
, 0, 0, 0, &tp
);
204 return __xfs_setfilesize(ip
, tp
, offset
, size
);
208 xfs_setfilesize_ioend(
209 struct xfs_ioend
*ioend
,
212 struct xfs_inode
*ip
= XFS_I(ioend
->io_inode
);
213 struct xfs_trans
*tp
= ioend
->io_append_trans
;
216 * The transaction may have been allocated in the I/O submission thread,
217 * thus we need to mark ourselves as being in a transaction manually.
218 * Similarly for freeze protection.
220 current_set_flags_nested(&tp
->t_pflags
, PF_MEMALLOC_NOFS
);
221 __sb_writers_acquired(VFS_I(ip
)->i_sb
, SB_FREEZE_FS
);
223 /* we abort the update if there was an IO error */
225 xfs_trans_cancel(tp
);
229 return __xfs_setfilesize(ip
, tp
, ioend
->io_offset
, ioend
->io_size
);
233 * IO write completion.
237 struct xfs_ioend
*ioend
)
239 struct list_head ioend_list
;
240 struct xfs_inode
*ip
= XFS_I(ioend
->io_inode
);
241 xfs_off_t offset
= ioend
->io_offset
;
242 size_t size
= ioend
->io_size
;
246 * Just clean up the in-memory strutures if the fs has been shut down.
248 if (XFS_FORCED_SHUTDOWN(ip
->i_mount
)) {
254 * Clean up any COW blocks on an I/O error.
256 error
= blk_status_to_errno(ioend
->io_bio
->bi_status
);
257 if (unlikely(error
)) {
258 if (ioend
->io_fork
== XFS_COW_FORK
)
259 xfs_reflink_cancel_cow_range(ip
, offset
, size
, true);
264 * Success: commit the COW or unwritten blocks if needed.
266 if (ioend
->io_fork
== XFS_COW_FORK
)
267 error
= xfs_reflink_end_cow(ip
, offset
, size
);
268 else if (ioend
->io_state
== XFS_EXT_UNWRITTEN
)
269 error
= xfs_iomap_write_unwritten(ip
, offset
, size
, false);
271 ASSERT(!xfs_ioend_is_append(ioend
) || ioend
->io_append_trans
);
274 if (ioend
->io_append_trans
)
275 error
= xfs_setfilesize_ioend(ioend
, error
);
276 list_replace_init(&ioend
->io_list
, &ioend_list
);
277 xfs_destroy_ioend(ioend
, error
);
279 while (!list_empty(&ioend_list
)) {
280 ioend
= list_first_entry(&ioend_list
, struct xfs_ioend
,
282 list_del_init(&ioend
->io_list
);
283 xfs_destroy_ioend(ioend
, error
);
288 * We can merge two adjacent ioends if they have the same set of work to do.
292 struct xfs_ioend
*ioend
,
294 struct xfs_ioend
*next
)
298 next_error
= blk_status_to_errno(next
->io_bio
->bi_status
);
299 if (ioend_error
!= next_error
)
301 if ((ioend
->io_fork
== XFS_COW_FORK
) ^ (next
->io_fork
== XFS_COW_FORK
))
303 if ((ioend
->io_state
== XFS_EXT_UNWRITTEN
) ^
304 (next
->io_state
== XFS_EXT_UNWRITTEN
))
306 if (ioend
->io_offset
+ ioend
->io_size
!= next
->io_offset
)
308 if (xfs_ioend_is_append(ioend
) != xfs_ioend_is_append(next
))
313 /* Try to merge adjacent completions. */
316 struct xfs_ioend
*ioend
,
317 struct list_head
*more_ioends
)
319 struct xfs_ioend
*next_ioend
;
323 if (list_empty(more_ioends
))
326 ioend_error
= blk_status_to_errno(ioend
->io_bio
->bi_status
);
328 while (!list_empty(more_ioends
)) {
329 next_ioend
= list_first_entry(more_ioends
, struct xfs_ioend
,
331 if (!xfs_ioend_can_merge(ioend
, ioend_error
, next_ioend
))
333 list_move_tail(&next_ioend
->io_list
, &ioend
->io_list
);
334 ioend
->io_size
+= next_ioend
->io_size
;
335 if (ioend
->io_append_trans
) {
336 error
= xfs_setfilesize_ioend(next_ioend
, 1);
342 /* list_sort compare function for ioends */
349 struct xfs_ioend
*ia
;
350 struct xfs_ioend
*ib
;
352 ia
= container_of(a
, struct xfs_ioend
, io_list
);
353 ib
= container_of(b
, struct xfs_ioend
, io_list
);
354 if (ia
->io_offset
< ib
->io_offset
)
356 else if (ia
->io_offset
> ib
->io_offset
)
361 /* Finish all pending io completions. */
364 struct work_struct
*work
)
366 struct xfs_inode
*ip
;
367 struct xfs_ioend
*ioend
;
368 struct list_head completion_list
;
371 ip
= container_of(work
, struct xfs_inode
, i_ioend_work
);
373 spin_lock_irqsave(&ip
->i_ioend_lock
, flags
);
374 list_replace_init(&ip
->i_ioend_list
, &completion_list
);
375 spin_unlock_irqrestore(&ip
->i_ioend_lock
, flags
);
377 list_sort(NULL
, &completion_list
, xfs_ioend_compare
);
379 while (!list_empty(&completion_list
)) {
380 ioend
= list_first_entry(&completion_list
, struct xfs_ioend
,
382 list_del_init(&ioend
->io_list
);
383 xfs_ioend_try_merge(ioend
, &completion_list
);
384 xfs_end_ioend(ioend
);
392 struct xfs_ioend
*ioend
= bio
->bi_private
;
393 struct xfs_inode
*ip
= XFS_I(ioend
->io_inode
);
394 struct xfs_mount
*mp
= ip
->i_mount
;
397 if (ioend
->io_fork
== XFS_COW_FORK
||
398 ioend
->io_state
== XFS_EXT_UNWRITTEN
||
399 ioend
->io_append_trans
!= NULL
) {
400 spin_lock_irqsave(&ip
->i_ioend_lock
, flags
);
401 if (list_empty(&ip
->i_ioend_list
))
402 WARN_ON_ONCE(!queue_work(mp
->m_unwritten_workqueue
,
404 list_add_tail(&ioend
->io_list
, &ip
->i_ioend_list
);
405 spin_unlock_irqrestore(&ip
->i_ioend_lock
, flags
);
407 xfs_destroy_ioend(ioend
, blk_status_to_errno(bio
->bi_status
));
411 * Fast revalidation of the cached writeback mapping. Return true if the current
412 * mapping is valid, false otherwise.
416 struct xfs_writepage_ctx
*wpc
,
417 struct xfs_inode
*ip
,
418 xfs_fileoff_t offset_fsb
)
420 if (offset_fsb
< wpc
->imap
.br_startoff
||
421 offset_fsb
>= wpc
->imap
.br_startoff
+ wpc
->imap
.br_blockcount
)
424 * If this is a COW mapping, it is sufficient to check that the mapping
425 * covers the offset. Be careful to check this first because the caller
426 * can revalidate a COW mapping without updating the data seqno.
428 if (wpc
->fork
== XFS_COW_FORK
)
432 * This is not a COW mapping. Check the sequence number of the data fork
433 * because concurrent changes could have invalidated the extent. Check
434 * the COW fork because concurrent changes since the last time we
435 * checked (and found nothing at this offset) could have added
436 * overlapping blocks.
438 if (wpc
->data_seq
!= READ_ONCE(ip
->i_df
.if_seq
))
440 if (xfs_inode_has_cow_data(ip
) &&
441 wpc
->cow_seq
!= READ_ONCE(ip
->i_cowfp
->if_seq
))
447 * Pass in a dellalloc extent and convert it to real extents, return the real
448 * extent that maps offset_fsb in wpc->imap.
450 * The current page is held locked so nothing could have removed the block
451 * backing offset_fsb, although it could have moved from the COW to the data
452 * fork by another thread.
456 struct xfs_writepage_ctx
*wpc
,
457 struct xfs_inode
*ip
,
458 xfs_fileoff_t offset_fsb
)
463 * Attempt to allocate whatever delalloc extent currently backs
464 * offset_fsb and put the result into wpc->imap. Allocate in a loop
465 * because it may take several attempts to allocate real blocks for a
466 * contiguous delalloc extent if free space is sufficiently fragmented.
469 error
= xfs_bmapi_convert_delalloc(ip
, wpc
->fork
, offset_fsb
,
470 &wpc
->imap
, wpc
->fork
== XFS_COW_FORK
?
471 &wpc
->cow_seq
: &wpc
->data_seq
);
474 } while (wpc
->imap
.br_startoff
+ wpc
->imap
.br_blockcount
<= offset_fsb
);
481 struct xfs_writepage_ctx
*wpc
,
485 struct xfs_inode
*ip
= XFS_I(inode
);
486 struct xfs_mount
*mp
= ip
->i_mount
;
487 ssize_t count
= i_blocksize(inode
);
488 xfs_fileoff_t offset_fsb
= XFS_B_TO_FSBT(mp
, offset
);
489 xfs_fileoff_t end_fsb
= XFS_B_TO_FSB(mp
, offset
+ count
);
490 xfs_fileoff_t cow_fsb
= NULLFILEOFF
;
491 struct xfs_bmbt_irec imap
;
492 struct xfs_iext_cursor icur
;
496 if (XFS_FORCED_SHUTDOWN(mp
))
500 * COW fork blocks can overlap data fork blocks even if the blocks
501 * aren't shared. COW I/O always takes precedent, so we must always
502 * check for overlap on reflink inodes unless the mapping is already a
503 * COW one, or the COW fork hasn't changed from the last time we looked
506 * It's safe to check the COW fork if_seq here without the ILOCK because
507 * we've indirectly protected against concurrent updates: writeback has
508 * the page locked, which prevents concurrent invalidations by reflink
509 * and directio and prevents concurrent buffered writes to the same
510 * page. Changes to if_seq always happen under i_lock, which protects
511 * against concurrent updates and provides a memory barrier on the way
512 * out that ensures that we always see the current value.
514 if (xfs_imap_valid(wpc
, ip
, offset_fsb
))
518 * If we don't have a valid map, now it's time to get a new one for this
519 * offset. This will convert delayed allocations (including COW ones)
520 * into real extents. If we return without a valid map, it means we
521 * landed in a hole and we skip the block.
524 xfs_ilock(ip
, XFS_ILOCK_SHARED
);
525 ASSERT(ip
->i_d
.di_format
!= XFS_DINODE_FMT_BTREE
||
526 (ip
->i_df
.if_flags
& XFS_IFEXTENTS
));
529 * Check if this is offset is covered by a COW extents, and if yes use
530 * it directly instead of looking up anything in the data fork.
532 if (xfs_inode_has_cow_data(ip
) &&
533 xfs_iext_lookup_extent(ip
, ip
->i_cowfp
, offset_fsb
, &icur
, &imap
))
534 cow_fsb
= imap
.br_startoff
;
535 if (cow_fsb
!= NULLFILEOFF
&& cow_fsb
<= offset_fsb
) {
536 wpc
->cow_seq
= READ_ONCE(ip
->i_cowfp
->if_seq
);
537 xfs_iunlock(ip
, XFS_ILOCK_SHARED
);
539 wpc
->fork
= XFS_COW_FORK
;
540 goto allocate_blocks
;
544 * No COW extent overlap. Revalidate now that we may have updated
545 * ->cow_seq. If the data mapping is still valid, we're done.
547 if (xfs_imap_valid(wpc
, ip
, offset_fsb
)) {
548 xfs_iunlock(ip
, XFS_ILOCK_SHARED
);
553 * If we don't have a valid map, now it's time to get a new one for this
554 * offset. This will convert delayed allocations (including COW ones)
557 if (!xfs_iext_lookup_extent(ip
, &ip
->i_df
, offset_fsb
, &icur
, &imap
))
558 imap
.br_startoff
= end_fsb
; /* fake a hole past EOF */
559 wpc
->data_seq
= READ_ONCE(ip
->i_df
.if_seq
);
560 xfs_iunlock(ip
, XFS_ILOCK_SHARED
);
562 wpc
->fork
= XFS_DATA_FORK
;
564 /* landed in a hole or beyond EOF? */
565 if (imap
.br_startoff
> offset_fsb
) {
566 imap
.br_blockcount
= imap
.br_startoff
- offset_fsb
;
567 imap
.br_startoff
= offset_fsb
;
568 imap
.br_startblock
= HOLESTARTBLOCK
;
569 imap
.br_state
= XFS_EXT_NORM
;
573 * Truncate to the next COW extent if there is one. This is the only
574 * opportunity to do this because we can skip COW fork lookups for the
575 * subsequent blocks in the mapping; however, the requirement to treat
576 * the COW range separately remains.
578 if (cow_fsb
!= NULLFILEOFF
&&
579 cow_fsb
< imap
.br_startoff
+ imap
.br_blockcount
)
580 imap
.br_blockcount
= cow_fsb
- imap
.br_startoff
;
582 /* got a delalloc extent? */
583 if (imap
.br_startblock
!= HOLESTARTBLOCK
&&
584 isnullstartblock(imap
.br_startblock
))
585 goto allocate_blocks
;
588 trace_xfs_map_blocks_found(ip
, offset
, count
, wpc
->fork
, &imap
);
591 error
= xfs_convert_blocks(wpc
, ip
, offset_fsb
);
594 * If we failed to find the extent in the COW fork we might have
595 * raced with a COW to data fork conversion or truncate.
596 * Restart the lookup to catch the extent in the data fork for
597 * the former case, but prevent additional retries to avoid
598 * looping forever for the latter case.
600 if (error
== -EAGAIN
&& wpc
->fork
== XFS_COW_FORK
&& !retries
++)
602 ASSERT(error
!= -EAGAIN
);
607 * Due to merging the return real extent might be larger than the
608 * original delalloc one. Trim the return extent to the next COW
609 * boundary again to force a re-lookup.
611 if (wpc
->fork
!= XFS_COW_FORK
&& cow_fsb
!= NULLFILEOFF
&&
612 cow_fsb
< wpc
->imap
.br_startoff
+ wpc
->imap
.br_blockcount
)
613 wpc
->imap
.br_blockcount
= cow_fsb
- wpc
->imap
.br_startoff
;
615 ASSERT(wpc
->imap
.br_startoff
<= offset_fsb
);
616 ASSERT(wpc
->imap
.br_startoff
+ wpc
->imap
.br_blockcount
> offset_fsb
);
617 trace_xfs_map_blocks_alloc(ip
, offset
, count
, wpc
->fork
, &imap
);
622 * Submit the bio for an ioend. We are passed an ioend with a bio attached to
623 * it, and we submit that bio. The ioend may be used for multiple bio
624 * submissions, so we only want to allocate an append transaction for the ioend
625 * once. In the case of multiple bio submission, each bio will take an IO
626 * reference to the ioend to ensure that the ioend completion is only done once
627 * all bios have been submitted and the ioend is really done.
629 * If @fail is non-zero, it means that we have a situation where some part of
630 * the submission process has failed after we have marked paged for writeback
631 * and unlocked them. In this situation, we need to fail the bio and ioend
632 * rather than submit it to IO. This typically only happens on a filesystem
637 struct writeback_control
*wbc
,
638 struct xfs_ioend
*ioend
,
641 /* Convert CoW extents to regular */
642 if (!status
&& ioend
->io_fork
== XFS_COW_FORK
) {
644 * Yuk. This can do memory allocation, but is not a
645 * transactional operation so everything is done in GFP_KERNEL
646 * context. That can deadlock, because we hold pages in
647 * writeback state and GFP_KERNEL allocations can block on them.
648 * Hence we must operate in nofs conditions here.
652 nofs_flag
= memalloc_nofs_save();
653 status
= xfs_reflink_convert_cow(XFS_I(ioend
->io_inode
),
654 ioend
->io_offset
, ioend
->io_size
);
655 memalloc_nofs_restore(nofs_flag
);
658 /* Reserve log space if we might write beyond the on-disk inode size. */
660 (ioend
->io_fork
== XFS_COW_FORK
||
661 ioend
->io_state
!= XFS_EXT_UNWRITTEN
) &&
662 xfs_ioend_is_append(ioend
) &&
663 !ioend
->io_append_trans
)
664 status
= xfs_setfilesize_trans_alloc(ioend
);
666 ioend
->io_bio
->bi_private
= ioend
;
667 ioend
->io_bio
->bi_end_io
= xfs_end_bio
;
668 ioend
->io_bio
->bi_opf
= REQ_OP_WRITE
| wbc_to_write_flags(wbc
);
671 * If we are failing the IO now, just mark the ioend with an
672 * error and finish it. This will run IO completion immediately
673 * as there is only one reference to the ioend at this point in
677 ioend
->io_bio
->bi_status
= errno_to_blk_status(status
);
678 bio_endio(ioend
->io_bio
);
682 ioend
->io_bio
->bi_write_hint
= ioend
->io_inode
->i_write_hint
;
683 submit_bio(ioend
->io_bio
);
687 static struct xfs_ioend
*
693 struct block_device
*bdev
,
696 struct xfs_ioend
*ioend
;
699 bio
= bio_alloc_bioset(GFP_NOFS
, BIO_MAX_PAGES
, &xfs_ioend_bioset
);
700 bio_set_dev(bio
, bdev
);
701 bio
->bi_iter
.bi_sector
= sector
;
703 ioend
= container_of(bio
, struct xfs_ioend
, io_inline_bio
);
704 INIT_LIST_HEAD(&ioend
->io_list
);
705 ioend
->io_fork
= fork
;
706 ioend
->io_state
= state
;
707 ioend
->io_inode
= inode
;
709 ioend
->io_offset
= offset
;
710 ioend
->io_append_trans
= NULL
;
716 * Allocate a new bio, and chain the old bio to the new one.
718 * Note that we have to do perform the chaining in this unintuitive order
719 * so that the bi_private linkage is set up in the right direction for the
720 * traversal in xfs_destroy_ioend().
724 struct xfs_ioend
*ioend
,
725 struct writeback_control
*wbc
,
726 struct block_device
*bdev
,
731 new = bio_alloc(GFP_NOFS
, BIO_MAX_PAGES
);
732 bio_set_dev(new, bdev
);
733 new->bi_iter
.bi_sector
= sector
;
734 bio_chain(ioend
->io_bio
, new);
735 bio_get(ioend
->io_bio
); /* for xfs_destroy_ioend */
736 ioend
->io_bio
->bi_opf
= REQ_OP_WRITE
| wbc_to_write_flags(wbc
);
737 ioend
->io_bio
->bi_write_hint
= ioend
->io_inode
->i_write_hint
;
738 submit_bio(ioend
->io_bio
);
743 * Test to see if we have an existing ioend structure that we could append to
744 * first, otherwise finish off the current ioend and start another.
751 struct iomap_page
*iop
,
752 struct xfs_writepage_ctx
*wpc
,
753 struct writeback_control
*wbc
,
754 struct list_head
*iolist
)
756 struct xfs_inode
*ip
= XFS_I(inode
);
757 struct xfs_mount
*mp
= ip
->i_mount
;
758 struct block_device
*bdev
= xfs_find_bdev_for_inode(inode
);
759 unsigned len
= i_blocksize(inode
);
760 unsigned poff
= offset
& (PAGE_SIZE
- 1);
763 sector
= xfs_fsb_to_db(ip
, wpc
->imap
.br_startblock
) +
764 ((offset
- XFS_FSB_TO_B(mp
, wpc
->imap
.br_startoff
)) >> 9);
767 wpc
->fork
!= wpc
->ioend
->io_fork
||
768 wpc
->imap
.br_state
!= wpc
->ioend
->io_state
||
769 sector
!= bio_end_sector(wpc
->ioend
->io_bio
) ||
770 offset
!= wpc
->ioend
->io_offset
+ wpc
->ioend
->io_size
) {
772 list_add(&wpc
->ioend
->io_list
, iolist
);
773 wpc
->ioend
= xfs_alloc_ioend(inode
, wpc
->fork
,
774 wpc
->imap
.br_state
, offset
, bdev
, sector
);
777 if (!__bio_try_merge_page(wpc
->ioend
->io_bio
, page
, len
, poff
, true)) {
779 atomic_inc(&iop
->write_count
);
780 if (bio_full(wpc
->ioend
->io_bio
))
781 xfs_chain_bio(wpc
->ioend
, wbc
, bdev
, sector
);
782 bio_add_page(wpc
->ioend
->io_bio
, page
, len
, poff
);
785 wpc
->ioend
->io_size
+= len
;
789 xfs_vm_invalidatepage(
794 trace_xfs_invalidatepage(page
->mapping
->host
, page
, offset
, length
);
795 iomap_invalidatepage(page
, offset
, length
);
799 * If the page has delalloc blocks on it, we need to punch them out before we
800 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
801 * inode that can trip up a later direct I/O read operation on the same region.
803 * We prevent this by truncating away the delalloc regions on the page. Because
804 * they are delalloc, we can do this without needing a transaction. Indeed - if
805 * we get ENOSPC errors, we have to be able to do this truncation without a
806 * transaction as there is no space left for block reservation (typically why we
807 * see a ENOSPC in writeback).
810 xfs_aops_discard_page(
813 struct inode
*inode
= page
->mapping
->host
;
814 struct xfs_inode
*ip
= XFS_I(inode
);
815 struct xfs_mount
*mp
= ip
->i_mount
;
816 loff_t offset
= page_offset(page
);
817 xfs_fileoff_t start_fsb
= XFS_B_TO_FSBT(mp
, offset
);
820 if (XFS_FORCED_SHUTDOWN(mp
))
824 "page discard on page "PTR_FMT
", inode 0x%llx, offset %llu.",
825 page
, ip
->i_ino
, offset
);
827 error
= xfs_bmap_punch_delalloc_range(ip
, start_fsb
,
828 PAGE_SIZE
/ i_blocksize(inode
));
829 if (error
&& !XFS_FORCED_SHUTDOWN(mp
))
830 xfs_alert(mp
, "page discard unable to remove delalloc mapping.");
832 xfs_vm_invalidatepage(page
, 0, PAGE_SIZE
);
836 * We implement an immediate ioend submission policy here to avoid needing to
837 * chain multiple ioends and hence nest mempool allocations which can violate
838 * forward progress guarantees we need to provide. The current ioend we are
839 * adding blocks to is cached on the writepage context, and if the new block
840 * does not append to the cached ioend it will create a new ioend and cache that
843 * If a new ioend is created and cached, the old ioend is returned and queued
844 * locally for submission once the entire page is processed or an error has been
845 * detected. While ioends are submitted immediately after they are completed,
846 * batching optimisations are provided by higher level block plugging.
848 * At the end of a writeback pass, there will be a cached ioend remaining on the
849 * writepage context that the caller will need to submit.
853 struct xfs_writepage_ctx
*wpc
,
854 struct writeback_control
*wbc
,
859 LIST_HEAD(submit_list
);
860 struct iomap_page
*iop
= to_iomap_page(page
);
861 unsigned len
= i_blocksize(inode
);
862 struct xfs_ioend
*ioend
, *next
;
863 uint64_t file_offset
; /* file offset of page */
864 int error
= 0, count
= 0, i
;
866 ASSERT(iop
|| i_blocksize(inode
) == PAGE_SIZE
);
867 ASSERT(!iop
|| atomic_read(&iop
->write_count
) == 0);
870 * Walk through the page to find areas to write back. If we run off the
871 * end of the current map or find the current map invalid, grab a new
874 for (i
= 0, file_offset
= page_offset(page
);
875 i
< (PAGE_SIZE
>> inode
->i_blkbits
) && file_offset
< end_offset
;
876 i
++, file_offset
+= len
) {
877 if (iop
&& !test_bit(i
, iop
->uptodate
))
880 error
= xfs_map_blocks(wpc
, inode
, file_offset
);
883 if (wpc
->imap
.br_startblock
== HOLESTARTBLOCK
)
885 xfs_add_to_ioend(inode
, file_offset
, page
, iop
, wpc
, wbc
,
890 ASSERT(wpc
->ioend
|| list_empty(&submit_list
));
891 ASSERT(PageLocked(page
));
892 ASSERT(!PageWriteback(page
));
895 * On error, we have to fail the ioend here because we may have set
896 * pages under writeback, we have to make sure we run IO completion to
897 * mark the error state of the IO appropriately, so we can't cancel the
898 * ioend directly here. That means we have to mark this page as under
899 * writeback if we included any blocks from it in the ioend chain so
900 * that completion treats it correctly.
902 * If we didn't include the page in the ioend, the on error we can
903 * simply discard and unlock it as there are no other users of the page
904 * now. The caller will still need to trigger submission of outstanding
905 * ioends on the writepage context so they are treated correctly on
908 if (unlikely(error
)) {
910 xfs_aops_discard_page(page
);
911 ClearPageUptodate(page
);
917 * If the page was not fully cleaned, we need to ensure that the
918 * higher layers come back to it correctly. That means we need
919 * to keep the page dirty, and for WB_SYNC_ALL writeback we need
920 * to ensure the PAGECACHE_TAG_TOWRITE index mark is not removed
921 * so another attempt to write this page in this writeback sweep
924 set_page_writeback_keepwrite(page
);
926 clear_page_dirty_for_io(page
);
927 set_page_writeback(page
);
933 * Preserve the original error if there was one, otherwise catch
934 * submission errors here and propagate into subsequent ioend
937 list_for_each_entry_safe(ioend
, next
, &submit_list
, io_list
) {
940 list_del_init(&ioend
->io_list
);
941 error2
= xfs_submit_ioend(wbc
, ioend
, error
);
942 if (error2
&& !error
)
947 * We can end up here with no error and nothing to write only if we race
948 * with a partial page truncate on a sub-page block sized filesystem.
951 end_page_writeback(page
);
953 mapping_set_error(page
->mapping
, error
);
958 * Write out a dirty page.
960 * For delalloc space on the page we need to allocate space and flush it.
961 * For unwritten space on the page we need to start the conversion to
962 * regular allocated space.
967 struct writeback_control
*wbc
,
970 struct xfs_writepage_ctx
*wpc
= data
;
971 struct inode
*inode
= page
->mapping
->host
;
976 trace_xfs_writepage(inode
, page
, 0, 0);
979 * Refuse to write the page out if we are called from reclaim context.
981 * This avoids stack overflows when called from deeply used stacks in
982 * random callers for direct reclaim or memcg reclaim. We explicitly
983 * allow reclaim from kswapd as the stack usage there is relatively low.
985 * This should never happen except in the case of a VM regression so
988 if (WARN_ON_ONCE((current
->flags
& (PF_MEMALLOC
|PF_KSWAPD
)) ==
993 * Given that we do not allow direct reclaim to call us, we should
994 * never be called while in a filesystem transaction.
996 if (WARN_ON_ONCE(current
->flags
& PF_MEMALLOC_NOFS
))
1000 * Is this page beyond the end of the file?
1002 * The page index is less than the end_index, adjust the end_offset
1003 * to the highest offset that this page should represent.
1004 * -----------------------------------------------------
1005 * | file mapping | <EOF> |
1006 * -----------------------------------------------------
1007 * | Page ... | Page N-2 | Page N-1 | Page N | |
1008 * ^--------------------------------^----------|--------
1009 * | desired writeback range | see else |
1010 * ---------------------------------^------------------|
1012 offset
= i_size_read(inode
);
1013 end_index
= offset
>> PAGE_SHIFT
;
1014 if (page
->index
< end_index
)
1015 end_offset
= (xfs_off_t
)(page
->index
+ 1) << PAGE_SHIFT
;
1018 * Check whether the page to write out is beyond or straddles
1020 * -------------------------------------------------------
1021 * | file mapping | <EOF> |
1022 * -------------------------------------------------------
1023 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
1024 * ^--------------------------------^-----------|---------
1026 * ---------------------------------^-----------|--------|
1028 unsigned offset_into_page
= offset
& (PAGE_SIZE
- 1);
1031 * Skip the page if it is fully outside i_size, e.g. due to a
1032 * truncate operation that is in progress. We must redirty the
1033 * page so that reclaim stops reclaiming it. Otherwise
1034 * xfs_vm_releasepage() is called on it and gets confused.
1036 * Note that the end_index is unsigned long, it would overflow
1037 * if the given offset is greater than 16TB on 32-bit system
1038 * and if we do check the page is fully outside i_size or not
1039 * via "if (page->index >= end_index + 1)" as "end_index + 1"
1040 * will be evaluated to 0. Hence this page will be redirtied
1041 * and be written out repeatedly which would result in an
1042 * infinite loop, the user program that perform this operation
1043 * will hang. Instead, we can verify this situation by checking
1044 * if the page to write is totally beyond the i_size or if it's
1045 * offset is just equal to the EOF.
1047 if (page
->index
> end_index
||
1048 (page
->index
== end_index
&& offset_into_page
== 0))
1052 * The page straddles i_size. It must be zeroed out on each
1053 * and every writepage invocation because it may be mmapped.
1054 * "A file is mapped in multiples of the page size. For a file
1055 * that is not a multiple of the page size, the remaining
1056 * memory is zeroed when mapped, and writes to that region are
1057 * not written out to the file."
1059 zero_user_segment(page
, offset_into_page
, PAGE_SIZE
);
1061 /* Adjust the end_offset to the end of file */
1062 end_offset
= offset
;
1065 return xfs_writepage_map(wpc
, wbc
, inode
, page
, end_offset
);
1068 redirty_page_for_writepage(wbc
, page
);
1076 struct writeback_control
*wbc
)
1078 struct xfs_writepage_ctx wpc
= { };
1081 ret
= xfs_do_writepage(page
, wbc
, &wpc
);
1083 ret
= xfs_submit_ioend(wbc
, wpc
.ioend
, ret
);
1089 struct address_space
*mapping
,
1090 struct writeback_control
*wbc
)
1092 struct xfs_writepage_ctx wpc
= { };
1095 xfs_iflags_clear(XFS_I(mapping
->host
), XFS_ITRUNCATED
);
1096 ret
= write_cache_pages(mapping
, wbc
, xfs_do_writepage
, &wpc
);
1098 ret
= xfs_submit_ioend(wbc
, wpc
.ioend
, ret
);
1104 struct address_space
*mapping
,
1105 struct writeback_control
*wbc
)
1107 xfs_iflags_clear(XFS_I(mapping
->host
), XFS_ITRUNCATED
);
1108 return dax_writeback_mapping_range(mapping
,
1109 xfs_find_bdev_for_inode(mapping
->host
), wbc
);
1117 trace_xfs_releasepage(page
->mapping
->host
, page
, 0, 0);
1118 return iomap_releasepage(page
, gfp_mask
);
1123 struct address_space
*mapping
,
1126 struct xfs_inode
*ip
= XFS_I(mapping
->host
);
1128 trace_xfs_vm_bmap(ip
);
1131 * The swap code (ab-)uses ->bmap to get a block mapping and then
1132 * bypasses the file system for actual I/O. We really can't allow
1133 * that on reflinks inodes, so we have to skip out here. And yes,
1134 * 0 is the magic code for a bmap error.
1136 * Since we don't pass back blockdev info, we can't return bmap
1137 * information for rt files either.
1139 if (xfs_is_cow_inode(ip
) || XFS_IS_REALTIME_INODE(ip
))
1141 return iomap_bmap(mapping
, block
, &xfs_iomap_ops
);
1146 struct file
*unused
,
1149 trace_xfs_vm_readpage(page
->mapping
->host
, 1);
1150 return iomap_readpage(page
, &xfs_iomap_ops
);
1155 struct file
*unused
,
1156 struct address_space
*mapping
,
1157 struct list_head
*pages
,
1160 trace_xfs_vm_readpages(mapping
->host
, nr_pages
);
1161 return iomap_readpages(mapping
, pages
, nr_pages
, &xfs_iomap_ops
);
1165 xfs_iomap_swapfile_activate(
1166 struct swap_info_struct
*sis
,
1167 struct file
*swap_file
,
1170 sis
->bdev
= xfs_find_bdev_for_inode(file_inode(swap_file
));
1171 return iomap_swapfile_activate(sis
, swap_file
, span
, &xfs_iomap_ops
);
1174 const struct address_space_operations xfs_address_space_operations
= {
1175 .readpage
= xfs_vm_readpage
,
1176 .readpages
= xfs_vm_readpages
,
1177 .writepage
= xfs_vm_writepage
,
1178 .writepages
= xfs_vm_writepages
,
1179 .set_page_dirty
= iomap_set_page_dirty
,
1180 .releasepage
= xfs_vm_releasepage
,
1181 .invalidatepage
= xfs_vm_invalidatepage
,
1182 .bmap
= xfs_vm_bmap
,
1183 .direct_IO
= noop_direct_IO
,
1184 .migratepage
= iomap_migrate_page
,
1185 .is_partially_uptodate
= iomap_is_partially_uptodate
,
1186 .error_remove_page
= generic_error_remove_page
,
1187 .swap_activate
= xfs_iomap_swapfile_activate
,
1190 const struct address_space_operations xfs_dax_aops
= {
1191 .writepages
= xfs_dax_writepages
,
1192 .direct_IO
= noop_direct_IO
,
1193 .set_page_dirty
= noop_set_page_dirty
,
1194 .invalidatepage
= noop_invalidatepage
,
1195 .swap_activate
= xfs_iomap_swapfile_activate
,