#include <linux/iomap.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/fserror.h>
+#include <linux/init.h>
#include "internal.h"
#include "trace.h"
}
EXPORT_SYMBOL_GPL(__iomap_dio_rw);
+struct iomap_dio_simple {
+ struct kiocb *iocb;
+ size_t size;
+ unsigned int dio_flags;
+ struct work_struct work;
+ /*
+ * Align @bio to a cacheline boundary so that, combined with the
+ * front_pad passed to bioset_init(), the bio sits at the start of
+ * a cacheline in memory returned by the (HWCACHE-aligned) bio
+ * slab. This keeps the hot fields block layer touches on submit
+ * and completion (bi_iter, bi_status, ...) within a single line.
+ */
+ struct bio bio ____cacheline_aligned_in_smp;
+};
+
+static struct bio_set iomap_dio_simple_pool;
+
+static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr)
+{
+ struct bio *bio = &sr->bio;
+ struct kiocb *iocb = sr->iocb;
+ struct inode *inode = file_inode(iocb->ki_filp);
+ ssize_t ret;
+
+ if (unlikely(bio->bi_status)) {
+ ret = blk_status_to_errno(bio->bi_status);
+ if (should_report_dio_fserror(ret))
+ fserror_report_io(inode, FSERR_DIRECTIO_READ,
+ iocb->ki_pos, sr->size, ret,
+ GFP_NOFS);
+ } else {
+ ret = sr->size;
+ iocb->ki_pos += ret;
+ }
+
+ if (sr->dio_flags & IOMAP_DIO_USER_BACKED) {
+ bio_check_pages_dirty(bio);
+ } else {
+ bio_release_pages(bio, false);
+ bio_put(bio);
+ }
+ inode_dio_end(inode);
+ trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret);
+ return ret;
+}
+
+static void iomap_dio_simple_complete_work(struct work_struct *work)
+{
+ struct iomap_dio_simple *sr =
+ container_of(work, struct iomap_dio_simple, work);
+ struct kiocb *iocb = sr->iocb;
+
+ WRITE_ONCE(iocb->private, NULL);
+ iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
+}
+
+static void iomap_dio_simple_end_io(struct bio *bio)
+{
+ struct iomap_dio_simple *sr =
+ container_of(bio, struct iomap_dio_simple, bio);
+ struct kiocb *iocb = sr->iocb;
+
+ if (unlikely(sr->bio.bi_status)) {
+ struct inode *inode = file_inode(iocb->ki_filp);
+
+ INIT_WORK(&sr->work, iomap_dio_simple_complete_work);
+ queue_work(inode->i_sb->s_dio_done_wq, &sr->work);
+ return;
+ }
+
+ WRITE_ONCE(iocb->private, NULL);
+ iocb->ki_complete(iocb, iomap_dio_simple_complete(sr));
+}
+
+static inline bool
+iomap_dio_simple_supported(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_dio_ops *dops,
+ unsigned int dio_flags, size_t done_before)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ size_t count = iov_iter_count(iter);
+
+ if (dops || done_before)
+ return false;
+ if (iov_iter_rw(iter) != READ)
+ return false;
+ if (!count)
+ return false;
+ /*
+ * Simple dio is an optimization for small IO. Filter out large IO
+ * early as it's the most common case to fail for typical direct IO
+ * workloads.
+ */
+ if (count > inode->i_sb->s_blocksize)
+ return false;
+ if (dio_flags & (IOMAP_DIO_FORCE_WAIT | IOMAP_DIO_PARTIAL |
+ IOMAP_DIO_BOUNCE))
+ return false;
+ if (iocb->ki_pos + count > i_size_read(inode))
+ return false;
+ if (IS_ENCRYPTED(inode))
+ return false;
+
+ return true;
+}
+
+/*
+ * Fast path for small, block-aligned direct I/Os that map to a single
+ * contiguous on-disk extent.
+ *
+ * iomap_dio_simple_supported() enforces the cheap up-front constraints before
+ * entering this path.
+ *
+ * @dops must be NULL: a non-NULL @dops means the caller wants its
+ * ->end_io / ->submit_io hooks invoked, and in particular wants its bios to be
+ * allocated from the filesystem-private @dops->bio_set (whose front_pad sizes a
+ * filesystem-private wrapper around the bio). The fast path instead allocates
+ * from the shared iomap_dio_simple_pool, whose front_pad matches struct
+ * iomap_dio_simple; the two wrappers are not interchangeable, so we must fall
+ * back to __iomap_dio_rw() in that case.
+ *
+ * @done_before must be zero: a non-zero caller-accumulated residual cannot be
+ * carried through a single-bio inline completion.
+ *
+ * @iter must describe a non-empty READ no larger than the inode block size:
+ * writes, zero-length I/O, and larger requests need the generic iomap direct
+ * I/O path.
+ *
+ * @dio_flags must not request IOMAP_DIO_FORCE_WAIT, IOMAP_DIO_PARTIAL, or
+ * IOMAP_DIO_BOUNCE: this path does not support forced waiting, partial direct
+ * I/O, or bouncing. The range must also stay within i_size and encrypted
+ * inodes must use the generic iomap direct I/O path.
+ *
+ * -ENOTBLK is the private sentinel returned by iomap_dio_simple() when it
+ * decides the request does not fit the fast path. In that case we proceed to
+ * the generic __iomap_dio_rw() slow path. Any other errno is a real result and
+ * is propagated as-is, in particular -EAGAIN for IOCB_NOWAIT must reach the
+ * caller.
+ */
+static ssize_t
+iomap_dio_simple(struct kiocb *iocb, struct iov_iter *iter,
+ const struct iomap_ops *ops, void *private,
+ unsigned int dio_flags)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ size_t count = iov_iter_count(iter);
+ bool wait_for_completion = is_sync_kiocb(iocb);
+ struct iomap_iter iomi = {
+ .inode = inode,
+ .pos = iocb->ki_pos,
+ .len = count,
+ .flags = IOMAP_DIRECT,
+ .private = private,
+ };
+ struct iomap_dio_simple *sr;
+ unsigned int alignment;
+ struct bio *bio;
+ ssize_t ret;
+
+ if (iocb->ki_flags & IOCB_NOWAIT)
+ iomi.flags |= IOMAP_NOWAIT;
+
+ ret = kiocb_write_and_wait(iocb, count);
+ if (ret)
+ return ret;
+
+ inode_dio_begin(inode);
+
+ ret = ops->iomap_begin(inode, iomi.pos, count, iomi.flags,
+ &iomi.iomap, &iomi.srcmap);
+ if (ret) {
+ inode_dio_end(inode);
+ return ret;
+ }
+
+ if (iomi.iomap.type != IOMAP_MAPPED ||
+ iomi.iomap.offset + iomi.iomap.length < iomi.pos + count ||
+ (iomi.iomap.flags & IOMAP_F_INTEGRITY)) {
+ ret = -ENOTBLK;
+ goto out_iomap_end;
+ }
+
+ alignment = iomap_dio_alignment(inode, iomi.iomap.bdev, dio_flags);
+ if ((iomi.pos | count) & (alignment - 1)) {
+ ret = -EINVAL;
+ goto out_iomap_end;
+ }
+
+ if (!wait_for_completion && unlikely(!inode->i_sb->s_dio_done_wq)) {
+ ret = sb_init_dio_done_wq(inode->i_sb);
+ if (ret < 0)
+ goto out_iomap_end;
+ }
+
+ trace_iomap_dio_rw_begin(iocb, iter, dio_flags, 0);
+
+ if (user_backed_iter(iter))
+ dio_flags |= IOMAP_DIO_USER_BACKED;
+
+ bio = bio_alloc_bioset(iomi.iomap.bdev,
+ bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS),
+ REQ_OP_READ, GFP_KERNEL, &iomap_dio_simple_pool);
+ sr = container_of(bio, struct iomap_dio_simple, bio);
+ sr->iocb = iocb;
+ sr->dio_flags = dio_flags;
+
+ bio->bi_iter.bi_sector = iomap_sector(&iomi.iomap, iomi.pos);
+ bio->bi_ioprio = iocb->ki_ioprio;
+
+ ret = bio_iov_iter_get_pages(bio, iter, alignment - 1);
+ if (unlikely(ret))
+ goto out_bio_put;
+
+ if (bio->bi_iter.bi_size != count) {
+ iov_iter_revert(iter, bio->bi_iter.bi_size);
+ ret = -ENOTBLK;
+ goto out_bio_release_pages;
+ }
+
+ sr->size = bio->bi_iter.bi_size;
+
+ if (dio_flags & IOMAP_DIO_USER_BACKED)
+ bio_set_pages_dirty(bio);
+
+ if (iocb->ki_flags & IOCB_NOWAIT)
+ bio->bi_opf |= REQ_NOWAIT;
+ if ((iocb->ki_flags & IOCB_HIPRI) && !wait_for_completion) {
+ bio->bi_opf |= REQ_POLLED;
+ WRITE_ONCE(iocb->private, bio);
+ }
+
+ if (ops->iomap_end)
+ ops->iomap_end(inode, iomi.pos, count, count, iomi.flags,
+ &iomi.iomap);
+
+ if (!wait_for_completion) {
+ bio->bi_end_io = iomap_dio_simple_end_io;
+ submit_bio(bio);
+ trace_iomap_dio_rw_queued(inode, iomi.pos, count);
+ return -EIOCBQUEUED;
+ }
+
+ submit_bio_wait(bio);
+ return iomap_dio_simple_complete(sr);
+
+out_bio_release_pages:
+ bio_release_pages(bio, false);
+out_bio_put:
+ bio_put(bio);
+out_iomap_end:
+ if (ops->iomap_end)
+ ops->iomap_end(inode, iomi.pos, count, 0, iomi.flags,
+ &iomi.iomap);
+ inode_dio_end(inode);
+ return ret;
+}
+
ssize_t
iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
unsigned int dio_flags, void *private, size_t done_before)
{
struct iomap_dio *dio;
+ ssize_t ret;
+
+ if (iomap_dio_simple_supported(iocb, iter, dops, dio_flags,
+ done_before)) {
+ ret = iomap_dio_simple(iocb, iter, ops, private, dio_flags);
+ if (ret != -ENOTBLK)
+ return ret;
+ }
dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
done_before);
return iomap_dio_complete(dio);
}
EXPORT_SYMBOL_GPL(iomap_dio_rw);
+
+static int __init iomap_dio_init(void)
+{
+ return bioset_init(&iomap_dio_simple_pool, 4,
+ offsetof(struct iomap_dio_simple, bio),
+ BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE);
+}
+fs_initcall(iomap_dio_init);
projects / thirdparty / linux.git / commitdiff
