[thirdparty/kernel/linux.git] / fs / ext4 / page-io.c

// SPDX-License-Identifier: GPL-2.0
/*
 * linux/fs/ext4/page-io.c
 *
 * This contains the new page_io functions for ext4
 *
 * Written by Theodore Ts'o, 2010.
 */

#include <linux/fs.h>
#include <linux/time.h>
#include <linux/highuid.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/string.h>
#include <linux/buffer_head.h>
#include <linux/writeback.h>
#include <linux/pagevec.h>
#include <linux/mpage.h>
#include <linux/namei.h>
#include <linux/uio.h>
#include <linux/bio.h>
#include <linux/workqueue.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/backing-dev.h>

#include "ext4_jbd2.h"
#include "xattr.h"
#include "acl.h"

static struct kmem_cache *io_end_cachep;

int __init ext4_init_pageio(void)
{
	io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT);
	if (io_end_cachep == NULL)
		return -ENOMEM;
	return 0;
}

void ext4_exit_pageio(void)
{
	kmem_cache_destroy(io_end_cachep);
}

/*
 * Print an buffer I/O error compatible with the fs/buffer.c.  This
 * provides compatibility with dmesg scrapers that look for a specific
 * buffer I/O error message.  We really need a unified error reporting
 * structure to userspace ala Digital Unix's uerf system, but it's
 * probably not going to happen in my lifetime, due to LKML politics...
 */
static void buffer_io_error(struct buffer_head *bh)
{
	printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n",
		       bh->b_bdev,
			(unsigned long long)bh->b_blocknr);
}

static void ext4_finish_bio(struct bio *bio)
{
	int i;
	struct bio_vec *bvec;
	struct bvec_iter_all iter_all;

	bio_for_each_segment_all(bvec, bio, i, iter_all) {
		struct page *page = bvec->bv_page;
#ifdef CONFIG_EXT4_FS_ENCRYPTION
		struct page *data_page = NULL;
#endif
		struct buffer_head *bh, *head;
		unsigned bio_start = bvec->bv_offset;
		unsigned bio_end = bio_start + bvec->bv_len;
		unsigned under_io = 0;
		unsigned long flags;

		if (!page)
			continue;

#ifdef CONFIG_EXT4_FS_ENCRYPTION
		if (!page->mapping) {
			/* The bounce data pages are unmapped. */
			data_page = page;
			fscrypt_pullback_bio_page(&page, false);
		}
#endif

		if (bio->bi_status) {
			SetPageError(page);
			mapping_set_error(page->mapping, -EIO);
		}
		bh = head = page_buffers(page);
		/*
		 * We check all buffers in the page under BH_Uptodate_Lock
		 * to avoid races with other end io clearing async_write flags
		 */
		local_irq_save(flags);
		bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
		do {
			if (bh_offset(bh) < bio_start ||
			    bh_offset(bh) + bh->b_size > bio_end) {
				if (buffer_async_write(bh))
					under_io++;
				continue;
			}
			clear_buffer_async_write(bh);
			if (bio->bi_status)
				buffer_io_error(bh);
		} while ((bh = bh->b_this_page) != head);
		bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
		local_irq_restore(flags);
		if (!under_io) {
#ifdef CONFIG_EXT4_FS_ENCRYPTION
			if (data_page)
				fscrypt_restore_control_page(data_page);
#endif
			end_page_writeback(page);
		}
	}
}

static void ext4_release_io_end(ext4_io_end_t *io_end)
{
	struct bio *bio, *next_bio;

	BUG_ON(!list_empty(&io_end->list));
	BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
	WARN_ON(io_end->handle);

	for (bio = io_end->bio; bio; bio = next_bio) {
		next_bio = bio->bi_private;
		ext4_finish_bio(bio);
		bio_put(bio);
	}
	kmem_cache_free(io_end_cachep, io_end);
}

/*
 * Check a range of space and convert unwritten extents to written. Note that
 * we are protected from truncate touching same part of extent tree by the
 * fact that truncate code waits for all DIO to finish (thus exclusion from
 * direct IO is achieved) and also waits for PageWriteback bits. Thus we
 * cannot get to ext4_ext_truncate() before all IOs overlapping that range are
 * completed (happens from ext4_free_ioend()).
 */
static int ext4_end_io(ext4_io_end_t *io)
{
	struct inode *inode = io->inode;
	loff_t offset = io->offset;
	ssize_t size = io->size;
	handle_t *handle = io->handle;
	int ret = 0;

	ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p,"
		   "list->prev 0x%p\n",
		   io, inode->i_ino, io->list.next, io->list.prev);

	io->handle = NULL;	/* Following call will use up the handle */
	ret = ext4_convert_unwritten_extents(handle, inode, offset, size);
	if (ret < 0 && !ext4_forced_shutdown(EXT4_SB(inode->i_sb))) {
		ext4_msg(inode->i_sb, KERN_EMERG,
			 "failed to convert unwritten extents to written "
			 "extents -- potential data loss!  "
			 "(inode %lu, offset %llu, size %zd, error %d)",
			 inode->i_ino, offset, size, ret);
	}
	ext4_clear_io_unwritten_flag(io);
	ext4_release_io_end(io);
	return ret;
}

static void dump_completed_IO(struct inode *inode, struct list_head *head)
{
#ifdef	EXT4FS_DEBUG
	struct list_head *cur, *before, *after;
	ext4_io_end_t *io, *io0, *io1;

	if (list_empty(head))
		return;

	ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
	list_for_each_entry(io, head, list) {
		cur = &io->list;
		before = cur->prev;
		io0 = container_of(before, ext4_io_end_t, list);
		after = cur->next;
		io1 = container_of(after, ext4_io_end_t, list);

		ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
			    io, inode->i_ino, io0, io1);
	}
#endif
}

/* Add the io_end to per-inode completed end_io list. */
static void ext4_add_complete_io(ext4_io_end_t *io_end)
{
	struct ext4_inode_info *ei = EXT4_I(io_end->inode);
	struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);
	struct workqueue_struct *wq;
	unsigned long flags;

	/* Only reserved conversions from writeback should enter here */
	WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
	WARN_ON(!io_end->handle && sbi->s_journal);
	spin_lock_irqsave(&ei->i_completed_io_lock, flags);
	wq = sbi->rsv_conversion_wq;
	if (list_empty(&ei->i_rsv_conversion_list))
		queue_work(wq, &ei->i_rsv_conversion_work);
	list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
	spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
}

static int ext4_do_flush_completed_IO(struct inode *inode,
				      struct list_head *head)
{
	ext4_io_end_t *io;
	struct list_head unwritten;
	unsigned long flags;
	struct ext4_inode_info *ei = EXT4_I(inode);
	int err, ret = 0;

	spin_lock_irqsave(&ei->i_completed_io_lock, flags);
	dump_completed_IO(inode, head);
	list_replace_init(head, &unwritten);
	spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);

	while (!list_empty(&unwritten)) {
		io = list_entry(unwritten.next, ext4_io_end_t, list);
		BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN));
		list_del_init(&io->list);

		err = ext4_end_io(io);
		if (unlikely(!ret && err))
			ret = err;
	}
	return ret;
}

/*
 * work on completed IO, to convert unwritten extents to extents
 */
void ext4_end_io_rsv_work(struct work_struct *work)
{
	struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
						  i_rsv_conversion_work);
	ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
}

ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags)
{
	ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags);
	if (io) {
		io->inode = inode;
		INIT_LIST_HEAD(&io->list);
		atomic_set(&io->count, 1);
	}
	return io;
}

void ext4_put_io_end_defer(ext4_io_end_t *io_end)
{
	if (atomic_dec_and_test(&io_end->count)) {
		if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) || !io_end->size) {
			ext4_release_io_end(io_end);
			return;
		}
		ext4_add_complete_io(io_end);
	}
}

int ext4_put_io_end(ext4_io_end_t *io_end)
{
	int err = 0;

	if (atomic_dec_and_test(&io_end->count)) {
		if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
			err = ext4_convert_unwritten_extents(io_end->handle,
						io_end->inode, io_end->offset,
						io_end->size);
			io_end->handle = NULL;
			ext4_clear_io_unwritten_flag(io_end);
		}
		ext4_release_io_end(io_end);
	}
	return err;
}

ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
{
	atomic_inc(&io_end->count);
	return io_end;
}

/* BIO completion function for page writeback */
static void ext4_end_bio(struct bio *bio)
{
	ext4_io_end_t *io_end = bio->bi_private;
	sector_t bi_sector = bio->bi_iter.bi_sector;
	char b[BDEVNAME_SIZE];

	if (WARN_ONCE(!io_end, "io_end is NULL: %s: sector %Lu len %u err %d\n",
		      bio_devname(bio, b),
		      (long long) bio->bi_iter.bi_sector,
		      (unsigned) bio_sectors(bio),
		      bio->bi_status)) {
		ext4_finish_bio(bio);
		bio_put(bio);
		return;
	}
	bio->bi_end_io = NULL;

	if (bio->bi_status) {
		struct inode *inode = io_end->inode;

		ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
			     "(offset %llu size %ld starting block %llu)",
			     bio->bi_status, inode->i_ino,
			     (unsigned long long) io_end->offset,
			     (long) io_end->size,
			     (unsigned long long)
			     bi_sector >> (inode->i_blkbits - 9));
		mapping_set_error(inode->i_mapping,
				blk_status_to_errno(bio->bi_status));
	}

	if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
		/*
		 * Link bio into list hanging from io_end. We have to do it
		 * atomically as bio completions can be racing against each
		 * other.
		 */
		bio->bi_private = xchg(&io_end->bio, bio);
		ext4_put_io_end_defer(io_end);
	} else {
		/*
		 * Drop io_end reference early. Inode can get freed once
		 * we finish the bio.
		 */
		ext4_put_io_end_defer(io_end);
		ext4_finish_bio(bio);
		bio_put(bio);
	}
}

void ext4_io_submit(struct ext4_io_submit *io)
{
	struct bio *bio = io->io_bio;

	if (bio) {
		int io_op_flags = io->io_wbc->sync_mode == WB_SYNC_ALL ?
				  REQ_SYNC : 0;
		io->io_bio->bi_write_hint = io->io_end->inode->i_write_hint;
		bio_set_op_attrs(io->io_bio, REQ_OP_WRITE, io_op_flags);
		submit_bio(io->io_bio);
	}
	io->io_bio = NULL;
}

void ext4_io_submit_init(struct ext4_io_submit *io,
			 struct writeback_control *wbc)
{
	io->io_wbc = wbc;
	io->io_bio = NULL;
	io->io_end = NULL;
}

static int io_submit_init_bio(struct ext4_io_submit *io,
			      struct buffer_head *bh)
{
	struct bio *bio;

	bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
	if (!bio)
		return -ENOMEM;
	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
	bio_set_dev(bio, bh->b_bdev);
	bio->bi_end_io = ext4_end_bio;
	bio->bi_private = ext4_get_io_end(io->io_end);
	io->io_bio = bio;
	io->io_next_block = bh->b_blocknr;
	wbc_init_bio(io->io_wbc, bio);
	return 0;
}

static int io_submit_add_bh(struct ext4_io_submit *io,
			    struct inode *inode,
			    struct page *page,
			    struct buffer_head *bh)
{
	int ret;

	if (io->io_bio && bh->b_blocknr != io->io_next_block) {
submit_and_retry:
		ext4_io_submit(io);
	}
	if (io->io_bio == NULL) {
		ret = io_submit_init_bio(io, bh);
		if (ret)
			return ret;
		io->io_bio->bi_write_hint = inode->i_write_hint;
	}
	ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh));
	if (ret != bh->b_size)
		goto submit_and_retry;
	wbc_account_io(io->io_wbc, page, bh->b_size);
	io->io_next_block++;
	return 0;
}

int ext4_bio_write_page(struct ext4_io_submit *io,
			struct page *page,
			int len,
			struct writeback_control *wbc,
			bool keep_towrite)
{
	struct page *data_page = NULL;
	struct inode *inode = page->mapping->host;
	unsigned block_start;
	struct buffer_head *bh, *head;
	int ret = 0;
	int nr_submitted = 0;
	int nr_to_submit = 0;

	BUG_ON(!PageLocked(page));
	BUG_ON(PageWriteback(page));

	if (keep_towrite)
		set_page_writeback_keepwrite(page);
	else
		set_page_writeback(page);
	ClearPageError(page);

	/*
	 * Comments copied from block_write_full_page:
	 *
	 * The page straddles i_size.  It must be zeroed out on each and every
	 * writepage invocation because it may be mmapped.  "A file is mapped
	 * in multiples of the page size.  For a file that is not a multiple of
	 * the page size, the remaining memory is zeroed when mapped, and
	 * writes to that region are not written out to the file."
	 */
	if (len < PAGE_SIZE)
		zero_user_segment(page, len, PAGE_SIZE);
	/*
	 * In the first loop we prepare and mark buffers to submit. We have to
	 * mark all buffers in the page before submitting so that
	 * end_page_writeback() cannot be called from ext4_bio_end_io() when IO
	 * on the first buffer finishes and we are still working on submitting
	 * the second buffer.
	 */
	bh = head = page_buffers(page);
	do {
		block_start = bh_offset(bh);
		if (block_start >= len) {
			clear_buffer_dirty(bh);
			set_buffer_uptodate(bh);
			continue;
		}
		if (!buffer_dirty(bh) || buffer_delay(bh) ||
		    !buffer_mapped(bh) || buffer_unwritten(bh)) {
			/* A hole? We can safely clear the dirty bit */
			if (!buffer_mapped(bh))
				clear_buffer_dirty(bh);
			if (io->io_bio)
				ext4_io_submit(io);
			continue;
		}
		if (buffer_new(bh)) {
			clear_buffer_new(bh);
			clean_bdev_bh_alias(bh);
		}
		set_buffer_async_write(bh);
		nr_to_submit++;
	} while ((bh = bh->b_this_page) != head);

	bh = head = page_buffers(page);

	if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode) &&
	    nr_to_submit) {
		gfp_t gfp_flags = GFP_NOFS;

	retry_encrypt:
		data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0,
						page->index, gfp_flags);
		if (IS_ERR(data_page)) {
			ret = PTR_ERR(data_page);
			if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) {
				if (io->io_bio) {
					ext4_io_submit(io);
					congestion_wait(BLK_RW_ASYNC, HZ/50);
				}
				gfp_flags |= __GFP_NOFAIL;
				goto retry_encrypt;
			}
			data_page = NULL;
			goto out;
		}
	}

	/* Now submit buffers to write */
	do {
		if (!buffer_async_write(bh))
			continue;
		ret = io_submit_add_bh(io, inode,
				       data_page ? data_page : page, bh);
		if (ret) {
			/*
			 * We only get here on ENOMEM.  Not much else
			 * we can do but mark the page as dirty, and
			 * better luck next time.
			 */
			break;
		}
		nr_submitted++;
		clear_buffer_dirty(bh);
	} while ((bh = bh->b_this_page) != head);

	/* Error stopped previous loop? Clean up buffers... */
	if (ret) {
	out:
		if (data_page)
			fscrypt_restore_control_page(data_page);
		printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
		redirty_page_for_writepage(wbc, page);
		do {
			clear_buffer_async_write(bh);
			bh = bh->b_this_page;
		} while (bh != head);
	}
	unlock_page(page);
	/* Nothing submitted - we have to end page writeback */
	if (!nr_submitted)
		end_page_writeback(page);
	return ret;
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
bd2d0210 TT	2	/*
	3	* linux/fs/ext4/page-io.c
	4	*
	5	* This contains the new page_io functions for ext4
	6	*
	7	* Written by Theodore Ts'o, 2010.
	8	*/
	9
bd2d0210 TT	10	#include <linux/fs.h>
bd2d0210 TT	11	#include <linux/time.h>
bd2d0210 TT	12	#include <linux/highuid.h>
	13	#include <linux/pagemap.h>
	14	#include <linux/quotaops.h>
	15	#include <linux/string.h>
	16	#include <linux/buffer_head.h>
	17	#include <linux/writeback.h>
	18	#include <linux/pagevec.h>
	19	#include <linux/mpage.h>
	20	#include <linux/namei.h>
	21	#include <linux/uio.h>
	22	#include <linux/bio.h>
	23	#include <linux/workqueue.h>
	24	#include <linux/kernel.h>
	25	#include <linux/slab.h>
1ae48a63	26	#include <linux/mm.h>
c9af28fd	27	#include <linux/backing-dev.h>
bd2d0210 TT	28
	29	#include "ext4_jbd2.h"
	30	#include "xattr.h"
	31	#include "acl.h"
bd2d0210	32
0058f965	33	static struct kmem_cache *io_end_cachep;
bd2d0210	34
5dabfc78	35	int __init ext4_init_pageio(void)
bd2d0210	36	{
bd2d0210	37	io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT);
0058f965	38	if (io_end_cachep == NULL)
bd2d0210	39	return -ENOMEM;
bd2d0210 TT	40	return 0;
	41	}
	42
5dabfc78	43	void ext4_exit_pageio(void)
bd2d0210 TT	44	{
bd2d0210 TT	45	kmem_cache_destroy(io_end_cachep);
bd2d0210 TT	46	}
bd2d0210 TT	47
b0857d30 JK	48	/*
	49	* Print an buffer I/O error compatible with the fs/buffer.c. This
	50	* provides compatibility with dmesg scrapers that look for a specific
	51	* buffer I/O error message. We really need a unified error reporting
	52	* structure to userspace ala Digital Unix's uerf system, but it's
	53	* probably not going to happen in my lifetime, due to LKML politics...
	54	*/
	55	static void buffer_io_error(struct buffer_head *bh)
	56	{
a1c6f057 DM	57	printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n",
a1c6f057 DM	58	bh->b_bdev,
b0857d30 JK	59	(unsigned long long)bh->b_blocknr);
	60	}
	61
	62	static void ext4_finish_bio(struct bio *bio)
	63	{
	64	int i;
2c30c71b	65	struct bio_vec *bvec;
6dc4f100	66	struct bvec_iter_all iter_all;
b0857d30	67
6dc4f100	68	bio_for_each_segment_all(bvec, bio, i, iter_all) {
b0857d30	69	struct page *page = bvec->bv_page;
2058f83a MH	70	#ifdef CONFIG_EXT4_FS_ENCRYPTION
2058f83a MH	71	struct page *data_page = NULL;
2058f83a	72	#endif
b0857d30 JK	73	struct buffer_head bh, head;
	74	unsigned bio_start = bvec->bv_offset;
	75	unsigned bio_end = bio_start + bvec->bv_len;
	76	unsigned under_io = 0;
	77	unsigned long flags;
	78
	79	if (!page)
	80	continue;
	81
2058f83a MH	82	#ifdef CONFIG_EXT4_FS_ENCRYPTION
	83	if (!page->mapping) {
	84	/* The bounce data pages are unmapped. */
	85	data_page = page;
a7550b30	86	fscrypt_pullback_bio_page(&page, false);
2058f83a MH	87	}
	88	#endif
	89
4e4cbee9	90	if (bio->bi_status) {
b0857d30	91	SetPageError(page);
5114a97a	92	mapping_set_error(page->mapping, -EIO);
b0857d30 JK	93	}
	94	bh = head = page_buffers(page);
	95	/*
	96	* We check all buffers in the page under BH_Uptodate_Lock
	97	* to avoid races with other end io clearing async_write flags
	98	*/
	99	local_irq_save(flags);
	100	bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
	101	do {
	102	if (bh_offset(bh) < bio_start \|\|
	103	bh_offset(bh) + bh->b_size > bio_end) {
	104	if (buffer_async_write(bh))
	105	under_io++;
	106	continue;
	107	}
	108	clear_buffer_async_write(bh);
4e4cbee9	109	if (bio->bi_status)
b0857d30 JK	110	buffer_io_error(bh);
	111	} while ((bh = bh->b_this_page) != head);
	112	bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
	113	local_irq_restore(flags);
2058f83a MH	114	if (!under_io) {
2058f83a MH	115	#ifdef CONFIG_EXT4_FS_ENCRYPTION
a7550b30 JK	116	if (data_page)
a7550b30 JK	117	fscrypt_restore_control_page(data_page);
2058f83a	118	#endif
b0857d30	119	end_page_writeback(page);
2058f83a	120	}
b0857d30 JK	121	}
	122	}
	123
97a851ed	124	static void ext4_release_io_end(ext4_io_end_t *io_end)
bd2d0210	125	{
b0857d30 JK	126	struct bio bio, next_bio;
b0857d30 JK	127
97a851ed JK	128	BUG_ON(!list_empty(&io_end->list));
97a851ed JK	129	BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
6b523df4	130	WARN_ON(io_end->handle);
97a851ed	131
b0857d30 JK	132	for (bio = io_end->bio; bio; bio = next_bio) {
	133	next_bio = bio->bi_private;
	134	ext4_finish_bio(bio);
	135	bio_put(bio);
	136	}
97a851ed JK	137	kmem_cache_free(io_end_cachep, io_end);
	138	}
	139
a115f749 JK	140	/*
	141	* Check a range of space and convert unwritten extents to written. Note that
	142	* we are protected from truncate touching same part of extent tree by the
	143	* fact that truncate code waits for all DIO to finish (thus exclusion from
	144	* direct IO is achieved) and also waits for PageWriteback bits. Thus we
	145	* cannot get to ext4_ext_truncate() before all IOs overlapping that range are
	146	* completed (happens from ext4_free_ioend()).
	147	*/
28a535f9	148	static int ext4_end_io(ext4_io_end_t *io)
bd2d0210 TT	149	{
	150	struct inode *inode = io->inode;
	151	loff_t offset = io->offset;
	152	ssize_t size = io->size;
6b523df4	153	handle_t *handle = io->handle;
bd2d0210 TT	154	int ret = 0;
	155
	156	ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p,"
	157	"list->prev 0x%p\n",
	158	io, inode->i_ino, io->list.next, io->list.prev);
	159
6b523df4 JK	160	io->handle = NULL; /* Following call will use up the handle */
6b523df4 JK	161	ret = ext4_convert_unwritten_extents(handle, inode, offset, size);
0db1ff22	162	if (ret < 0 && !ext4_forced_shutdown(EXT4_SB(inode->i_sb))) {
b82e384c TT	163	ext4_msg(inode->i_sb, KERN_EMERG,
	164	"failed to convert unwritten extents to written "
	165	"extents -- potential data loss! "
	166	"(inode %lu, offset %llu, size %zd, error %d)",
	167	inode->i_ino, offset, size, ret);
bd2d0210	168	}
97a851ed JK	169	ext4_clear_io_unwritten_flag(io);
97a851ed JK	170	ext4_release_io_end(io);
bd2d0210 TT	171	return ret;
	172	}
	173
2e8fa54e	174	static void dump_completed_IO(struct inode inode, struct list_head head)
28a535f9 DM	175	{
	176	#ifdef EXT4FS_DEBUG
	177	struct list_head cur, before, *after;
	178	ext4_io_end_t io, io0, *io1;
28a535f9	179
2e8fa54e	180	if (list_empty(head))
28a535f9	181	return;
28a535f9	182
2e8fa54e JK	183	ext4_debug("Dump inode %lu completed io list\n", inode->i_ino);
2e8fa54e JK	184	list_for_each_entry(io, head, list) {
28a535f9 DM	185	cur = &io->list;
	186	before = cur->prev;
	187	io0 = container_of(before, ext4_io_end_t, list);
	188	after = cur->next;
	189	io1 = container_of(after, ext4_io_end_t, list);
	190
	191	ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
	192	io, inode->i_ino, io0, io1);
	193	}
	194	#endif
	195	}
	196
	197	/* Add the io_end to per-inode completed end_io list. */
97a851ed	198	static void ext4_add_complete_io(ext4_io_end_t *io_end)
bd2d0210	199	{
28a535f9	200	struct ext4_inode_info *ei = EXT4_I(io_end->inode);
78371a45	201	struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);
28a535f9 DM	202	struct workqueue_struct *wq;
	203	unsigned long flags;
	204
7b7a8665 CH	205	/* Only reserved conversions from writeback should enter here */
7b7a8665 CH	206	WARN_ON(!(io_end->flag & EXT4_IO_END_UNWRITTEN));
78371a45	207	WARN_ON(!io_end->handle && sbi->s_journal);
d73d5046	208	spin_lock_irqsave(&ei->i_completed_io_lock, flags);
78371a45	209	wq = sbi->rsv_conversion_wq;
7b7a8665 CH	210	if (list_empty(&ei->i_rsv_conversion_list))
	211	queue_work(wq, &ei->i_rsv_conversion_work);
	212	list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);
28a535f9 DM	213	spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
28a535f9 DM	214	}
d73d5046	215
2e8fa54e JK	216	static int ext4_do_flush_completed_IO(struct inode *inode,
2e8fa54e JK	217	struct list_head *head)
28a535f9 DM	218	{
28a535f9 DM	219	ext4_io_end_t *io;
002bd7fa	220	struct list_head unwritten;
28a535f9 DM	221	unsigned long flags;
	222	struct ext4_inode_info *ei = EXT4_I(inode);
	223	int err, ret = 0;
	224
28a535f9	225	spin_lock_irqsave(&ei->i_completed_io_lock, flags);
2e8fa54e JK	226	dump_completed_IO(inode, head);
2e8fa54e JK	227	list_replace_init(head, &unwritten);
28a535f9 DM	228	spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
	229
	230	while (!list_empty(&unwritten)) {
	231	io = list_entry(unwritten.next, ext4_io_end_t, list);
	232	BUG_ON(!(io->flag & EXT4_IO_END_UNWRITTEN));
	233	list_del_init(&io->list);
	234
	235	err = ext4_end_io(io);
	236	if (unlikely(!ret && err))
	237	ret = err;
28a535f9 DM	238	}
	239	return ret;
	240	}
	241
	242	/*
2e8fa54e	243	* work on completed IO, to convert unwritten extents to extents
28a535f9	244	*/
2e8fa54e JK	245	void ext4_end_io_rsv_work(struct work_struct *work)
	246	{
	247	struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info,
	248	i_rsv_conversion_work);
	249	ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list);
	250	}
	251
bd2d0210 TT	252	ext4_io_end_t ext4_init_io_end(struct inode inode, gfp_t flags)
bd2d0210 TT	253	{
b17b35ec	254	ext4_io_end_t *io = kmem_cache_zalloc(io_end_cachep, flags);
bd2d0210	255	if (io) {
f7ad6d2e	256	io->inode = inode;
bd2d0210	257	INIT_LIST_HEAD(&io->list);
97a851ed	258	atomic_set(&io->count, 1);
bd2d0210 TT	259	}
	260	return io;
	261	}
	262
97a851ed JK	263	void ext4_put_io_end_defer(ext4_io_end_t *io_end)
	264	{
	265	if (atomic_dec_and_test(&io_end->count)) {
	266	if (!(io_end->flag & EXT4_IO_END_UNWRITTEN) \|\| !io_end->size) {
	267	ext4_release_io_end(io_end);
	268	return;
	269	}
	270	ext4_add_complete_io(io_end);
	271	}
	272	}
	273
	274	int ext4_put_io_end(ext4_io_end_t *io_end)
	275	{
	276	int err = 0;
	277
	278	if (atomic_dec_and_test(&io_end->count)) {
	279	if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
6b523df4 JK	280	err = ext4_convert_unwritten_extents(io_end->handle,
	281	io_end->inode, io_end->offset,
	282	io_end->size);
	283	io_end->handle = NULL;
97a851ed JK	284	ext4_clear_io_unwritten_flag(io_end);
	285	}
	286	ext4_release_io_end(io_end);
	287	}
	288	return err;
	289	}
	290
	291	ext4_io_end_t ext4_get_io_end(ext4_io_end_t io_end)
	292	{
	293	atomic_inc(&io_end->count);
	294	return io_end;
	295	}
	296
822dbba3	297	/* BIO completion function for page writeback */
4246a0b6	298	static void ext4_end_bio(struct bio *bio)
bd2d0210 TT	299	{
bd2d0210 TT	300	ext4_io_end_t *io_end = bio->bi_private;
4f024f37	301	sector_t bi_sector = bio->bi_iter.bi_sector;
72d622b4	302	char b[BDEVNAME_SIZE];
bd2d0210	303
72d622b4	304	if (WARN_ONCE(!io_end, "io_end is NULL: %s: sector %Lu len %u err %d\n",
74d46992	305	bio_devname(bio, b),
72d622b4 TT	306	(long long) bio->bi_iter.bi_sector,
72d622b4 TT	307	(unsigned) bio_sectors(bio),
4e4cbee9	308	bio->bi_status)) {
72d622b4 TT	309	ext4_finish_bio(bio);
	310	bio_put(bio);
	311	return;
	312	}
bd2d0210	313	bio->bi_end_io = NULL;
0058f965	314
4e4cbee9	315	if (bio->bi_status) {
b0857d30 JK	316	struct inode *inode = io_end->inode;
b0857d30 JK	317
9503c67c	318	ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
f7ad6d2e	319	"(offset %llu size %ld starting block %llu)",
4e4cbee9	320	bio->bi_status, inode->i_ino,
f7ad6d2e TT	321	(unsigned long long) io_end->offset,
	322	(long) io_end->size,
	323	(unsigned long long)
d50bdd5a	324	bi_sector >> (inode->i_blkbits - 9));
4e4cbee9 CH	325	mapping_set_error(inode->i_mapping,
4e4cbee9 CH	326	blk_status_to_errno(bio->bi_status));
f7ad6d2e	327	}
822dbba3 JK	328
	329	if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
	330	/*
	331	* Link bio into list hanging from io_end. We have to do it
	332	* atomically as bio completions can be racing against each
	333	* other.
	334	*/
	335	bio->bi_private = xchg(&io_end->bio, bio);
	336	ext4_put_io_end_defer(io_end);
	337	} else {
	338	/*
	339	* Drop io_end reference early. Inode can get freed once
	340	* we finish the bio.
	341	*/
	342	ext4_put_io_end_defer(io_end);
	343	ext4_finish_bio(bio);
	344	bio_put(bio);
	345	}
bd2d0210 TT	346	}
	347
	348	void ext4_io_submit(struct ext4_io_submit *io)
	349	{
	350	struct bio *bio = io->io_bio;
	351
	352	if (bio) {
95fe6c1a	353	int io_op_flags = io->io_wbc->sync_mode == WB_SYNC_ALL ?
70fd7614	354	REQ_SYNC : 0;
0127251c	355	io->io_bio->bi_write_hint = io->io_end->inode->i_write_hint;
95fe6c1a	356	bio_set_op_attrs(io->io_bio, REQ_OP_WRITE, io_op_flags);
4e49ea4a	357	submit_bio(io->io_bio);
bd2d0210	358	}
7dc57615	359	io->io_bio = NULL;
97a851ed JK	360	}
	361
	362	void ext4_io_submit_init(struct ext4_io_submit *io,
	363	struct writeback_control *wbc)
	364	{
5a33911f	365	io->io_wbc = wbc;
97a851ed	366	io->io_bio = NULL;
7dc57615	367	io->io_end = NULL;
bd2d0210 TT	368	}
bd2d0210 TT	369
97a851ed JK	370	static int io_submit_init_bio(struct ext4_io_submit *io,
97a851ed JK	371	struct buffer_head *bh)
bd2d0210	372	{
bd2d0210 TT	373	struct bio *bio;
bd2d0210 TT	374
b54ffb73	375	bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
a1d8d9a7 TT	376	if (!bio)
a1d8d9a7 TT	377	return -ENOMEM;
4f024f37	378	bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
74d46992	379	bio_set_dev(bio, bh->b_bdev);
bd2d0210	380	bio->bi_end_io = ext4_end_bio;
97a851ed	381	bio->bi_private = ext4_get_io_end(io->io_end);
bd2d0210	382	io->io_bio = bio;
bd2d0210	383	io->io_next_block = bh->b_blocknr;
fd42df30	384	wbc_init_bio(io->io_wbc, bio);
bd2d0210 TT	385	return 0;
	386	}
	387
	388	static int io_submit_add_bh(struct ext4_io_submit *io,
bd2d0210	389	struct inode *inode,
2058f83a	390	struct page *page,
bd2d0210 TT	391	struct buffer_head *bh)
bd2d0210 TT	392	{
bd2d0210 TT	393	int ret;
bd2d0210 TT	394
bd2d0210 TT	395	if (io->io_bio && bh->b_blocknr != io->io_next_block) {
	396	submit_and_retry:
	397	ext4_io_submit(io);
	398	}
	399	if (io->io_bio == NULL) {
97a851ed	400	ret = io_submit_init_bio(io, bh);
bd2d0210 TT	401	if (ret)
bd2d0210 TT	402	return ret;
0127251c	403	io->io_bio->bi_write_hint = inode->i_write_hint;
bd2d0210	404	}
2058f83a	405	ret = bio_add_page(io->io_bio, page, bh->b_size, bh_offset(bh));
97a851ed JK	406	if (ret != bh->b_size)
97a851ed JK	407	goto submit_and_retry;
001e4a87	408	wbc_account_io(io->io_wbc, page, bh->b_size);
bd2d0210	409	io->io_next_block++;
bd2d0210 TT	410	return 0;
	411	}
	412
	413	int ext4_bio_write_page(struct ext4_io_submit *io,
	414	struct page *page,
	415	int len,
1c8349a1 NJ	416	struct writeback_control *wbc,
1c8349a1 NJ	417	bool keep_towrite)
bd2d0210	418	{
2058f83a	419	struct page *data_page = NULL;
bd2d0210	420	struct inode *inode = page->mapping->host;
18017479	421	unsigned block_start;
bd2d0210 TT	422	struct buffer_head bh, head;
bd2d0210 TT	423	int ret = 0;
0058f965	424	int nr_submitted = 0;
937d7b84	425	int nr_to_submit = 0;
bd2d0210	426
d50bdd5a	427	BUG_ON(!PageLocked(page));
bd2d0210	428	BUG_ON(PageWriteback(page));
bd2d0210	429
1c8349a1 NJ	430	if (keep_towrite)
	431	set_page_writeback_keepwrite(page);
	432	else
	433	set_page_writeback(page);
a54aa761	434	ClearPageError(page);
bd2d0210	435
eeece469	436	/*
f8409abd	437	* Comments copied from block_write_full_page:
eeece469 JK	438	*
	439	* The page straddles i_size. It must be zeroed out on each and every
	440	* writepage invocation because it may be mmapped. "A file is mapped
	441	* in multiples of the page size. For a file that is not a multiple of
	442	* the page size, the remaining memory is zeroed when mapped, and
	443	* writes to that region are not written out to the file."
	444	*/
09cbfeaf KS	445	if (len < PAGE_SIZE)
09cbfeaf KS	446	zero_user_segment(page, len, PAGE_SIZE);
0058f965 JK	447	/*
	448	* In the first loop we prepare and mark buffers to submit. We have to
	449	* mark all buffers in the page before submitting so that
	450	* end_page_writeback() cannot be called from ext4_bio_end_io() when IO
	451	* on the first buffer finishes and we are still working on submitting
	452	* the second buffer.
	453	*/
	454	bh = head = page_buffers(page);
	455	do {
	456	block_start = bh_offset(bh);
bd2d0210 TT	457	if (block_start >= len) {
	458	clear_buffer_dirty(bh);
	459	set_buffer_uptodate(bh);
	460	continue;
	461	}
8a850c3f JK	462	if (!buffer_dirty(bh) \|\| buffer_delay(bh) \|\|
	463	!buffer_mapped(bh) \|\| buffer_unwritten(bh)) {
	464	/* A hole? We can safely clear the dirty bit */
	465	if (!buffer_mapped(bh))
	466	clear_buffer_dirty(bh);
	467	if (io->io_bio)
	468	ext4_io_submit(io);
	469	continue;
	470	}
0058f965 JK	471	if (buffer_new(bh)) {
0058f965 JK	472	clear_buffer_new(bh);
e64855c6	473	clean_bdev_bh_alias(bh);
0058f965 JK	474	}
0058f965 JK	475	set_buffer_async_write(bh);
937d7b84	476	nr_to_submit++;
0058f965 JK	477	} while ((bh = bh->b_this_page) != head);
0058f965 JK	478
0058f965	479	bh = head = page_buffers(page);
2058f83a	480
937d7b84 TT	481	if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode) &&
937d7b84 TT	482	nr_to_submit) {
c9af28fd TT	483	gfp_t gfp_flags = GFP_NOFS;
	484
	485	retry_encrypt:
9c4bb8a3 DG	486	data_page = fscrypt_encrypt_page(inode, page, PAGE_SIZE, 0,
9c4bb8a3 DG	487	page->index, gfp_flags);
2058f83a MH	488	if (IS_ERR(data_page)) {
2058f83a MH	489	ret = PTR_ERR(data_page);
4762cc3f	490	if (ret == -ENOMEM && wbc->sync_mode == WB_SYNC_ALL) {
c9af28fd TT	491	if (io->io_bio) {
	492	ext4_io_submit(io);
	493	congestion_wait(BLK_RW_ASYNC, HZ/50);
	494	}
	495	gfp_flags \|= __GFP_NOFAIL;
	496	goto retry_encrypt;
	497	}
2058f83a MH	498	data_page = NULL;
	499	goto out;
	500	}
	501	}
	502
	503	/* Now submit buffers to write */
0058f965 JK	504	do {
	505	if (!buffer_async_write(bh))
	506	continue;
2058f83a MH	507	ret = io_submit_add_bh(io, inode,
2058f83a MH	508	data_page ? data_page : page, bh);
bd2d0210 TT	509	if (ret) {
	510	/*
	511	* We only get here on ENOMEM. Not much else
	512	* we can do but mark the page as dirty, and
	513	* better luck next time.
	514	*/
bd2d0210 TT	515	break;
bd2d0210 TT	516	}
0058f965	517	nr_submitted++;
1ae48a63	518	clear_buffer_dirty(bh);
0058f965 JK	519	} while ((bh = bh->b_this_page) != head);
	520
	521	/* Error stopped previous loop? Clean up buffers... */
	522	if (ret) {
2058f83a MH	523	out:
2058f83a MH	524	if (data_page)
a7550b30	525	fscrypt_restore_control_page(data_page);
2058f83a MH	526	printk_ratelimited(KERN_ERR "%s: ret = %d\n", __func__, ret);
2058f83a MH	527	redirty_page_for_writepage(wbc, page);
0058f965 JK	528	do {
	529	clear_buffer_async_write(bh);
	530	bh = bh->b_this_page;
	531	} while (bh != head);
bd2d0210 TT	532	}
bd2d0210 TT	533	unlock_page(page);
0058f965 JK	534	/* Nothing submitted - we have to end page writeback */
	535	if (!nr_submitted)
	536	end_page_writeback(page);
bd2d0210 TT	537	return ret;
bd2d0210 TT	538	}