fuse: support copying large folios

Currently, all folios associated with fuse are one page size. As part of
the work to enable large folios, this commit adds support for copying
to/from folios larger than one page size.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
Reviewed-by: Jeff Layton <jlayton@kernel.org>
Reviewed-by: Bernd Schubert <bschubert@ddn.com>
Signed-off-by: Miklos Szeredi <mszeredi@redhat.com>

diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 07e08362d5b911c66199f0797f9f1f522e67a128..f8fd24df173c873bdb2d883955db77ae78123f19 100644 (file)
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -956,10 +956,10 @@ static int fuse_check_folio(struct folio *folio)
  * folio that was originally in @pagep will lose a reference and the new
  * folio returned in @pagep will carry a reference.
  */
-static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
+static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop)
 {
        int err;
-       struct folio *oldfolio = page_folio(*pagep);
+       struct folio *oldfolio = *foliop;
        struct folio *newfolio;
        struct pipe_buffer *buf = cs->pipebufs;
 
@@ -980,7 +980,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
        cs->pipebufs++;
        cs->nr_segs--;
 
-       if (cs->len != PAGE_SIZE)
+       if (cs->len != folio_size(oldfolio))
                goto out_fallback;
 
        if (!pipe_buf_try_steal(cs->pipe, buf))
@@ -1026,7 +1026,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
        if (test_bit(FR_ABORTED, &cs->req->flags))
                err = -ENOENT;
        else
-               *pagep = &newfolio->page;
+               *foliop = newfolio;
        spin_unlock(&cs->req->waitq.lock);
 
        if (err) {
@@ -1059,8 +1059,8 @@ out_fallback:
        goto out_put_old;
 }
 
-static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
-                        unsigned offset, unsigned count)
+static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
+                         unsigned offset, unsigned count)
 {
        struct pipe_buffer *buf;
        int err;
@@ -1068,17 +1068,17 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
        if (cs->nr_segs >= cs->pipe->max_usage)
                return -EIO;
 
-       get_page(page);
+       folio_get(folio);
        err = unlock_request(cs->req);
        if (err) {
-               put_page(page);
+               folio_put(folio);
                return err;
        }
 
        fuse_copy_finish(cs);
 
        buf = cs->pipebufs;
-       buf->page = page;
+       buf->page = &folio->page;
        buf->offset = offset;
        buf->len = count;
 
@@ -1090,20 +1090,24 @@ static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
 }
 
 /*
- * Copy a page in the request to/from the userspace buffer.  Must be
+ * Copy a folio in the request to/from the userspace buffer.  Must be
  * done atomically
  */
-static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
-                         unsigned offset, unsigned count, int zeroing)
+static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
+                          unsigned offset, unsigned count, int zeroing)
 {
        int err;
-       struct page *page = *pagep;
+       struct folio *folio = *foliop;
+       size_t size;
 
-       if (page && zeroing && count < PAGE_SIZE)
-               clear_highpage(page);
+       if (folio) {
+               size = folio_size(folio);
+               if (zeroing && count < size)
+                       folio_zero_range(folio, 0, size);
+       }
 
        while (count) {
-               if (cs->write && cs->pipebufs && page) {
+               if (cs->write && cs->pipebufs && folio) {
                        /*
                         * Can't control lifetime of pipe buffers, so always
                         * copy user pages.
@@ -1113,12 +1117,12 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
                                if (err)
                                        return err;
                        } else {
-                               return fuse_ref_page(cs, page, offset, count);
+                               return fuse_ref_folio(cs, folio, offset, count);
                        }
                } else if (!cs->len) {
-                       if (cs->move_pages && page &&
-                           offset == 0 && count == PAGE_SIZE) {
-                               err = fuse_try_move_page(cs, pagep);
+                       if (cs->move_folios && folio &&
+                           offset == 0 && count == size) {
+                               err = fuse_try_move_folio(cs, foliop);
                                if (err <= 0)
                                        return err;
                        } else {
@@ -1127,22 +1131,30 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
                                        return err;
                        }
                }
-               if (page) {
-                       void *mapaddr = kmap_local_page(page);
-                       void *buf = mapaddr + offset;
-                       offset += fuse_copy_do(cs, &buf, &count);
+               if (folio) {
+                       void *mapaddr = kmap_local_folio(folio, offset);
+                       void *buf = mapaddr;
+                       unsigned int copy = count;
+                       unsigned int bytes_copied;
+
+                       if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset))
+                               copy = PAGE_SIZE - offset_in_page(offset);
+
+                       bytes_copied = fuse_copy_do(cs, &buf, &copy);
                        kunmap_local(mapaddr);
+                       offset += bytes_copied;
+                       count -= bytes_copied;
                } else
                        offset += fuse_copy_do(cs, NULL, &count);
        }
-       if (page && !cs->write)
-               flush_dcache_page(page);
+       if (folio && !cs->write)
+               flush_dcache_folio(folio);
        return 0;
 }
 
-/* Copy pages in the request to/from userspace buffer */
-static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
-                          int zeroing)
+/* Copy folios in the request to/from userspace buffer */
+static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes,
+                           int zeroing)
 {
        unsigned i;
        struct fuse_req *req = cs->req;
@@ -1152,23 +1164,12 @@ static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
                int err;
                unsigned int offset = ap->descs[i].offset;
                unsigned int count = min(nbytes, ap->descs[i].length);
-               struct page *orig, *pagep;
 
-               orig = pagep = &ap->folios[i]->page;
-
-               err = fuse_copy_page(cs, &pagep, offset, count, zeroing);
+               err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing);
                if (err)
                        return err;
 
                nbytes -= count;
-
-               /*
-                *  fuse_copy_page may have moved a page from a pipe instead of
-                *  copying into our given page, so update the folios if it was
-                *  replaced.
-                */
-               if (pagep != orig)
-                       ap->folios[i] = page_folio(pagep);
        }
        return 0;
 }
@@ -1198,7 +1199,7 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
        for (i = 0; !err && i < numargs; i++)  {
                struct fuse_arg *arg = &args[i];
                if (i == numargs - 1 && argpages)
-                       err = fuse_copy_pages(cs, arg->size, zeroing);
+                       err = fuse_copy_folios(cs, arg->size, zeroing);
                else
                        err = fuse_copy_one(cs, arg->value, arg->size);
        }
@@ -1787,7 +1788,6 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
        num = outarg.size;
        while (num) {
                struct folio *folio;
-               struct page *page;
                unsigned int this_num;
 
                folio = filemap_grab_folio(mapping, index);
@@ -1795,9 +1795,8 @@ static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
                if (IS_ERR(folio))
                        goto out_iput;
 
-               page = &folio->page;
                this_num = min_t(unsigned, num, folio_size(folio) - offset);
-               err = fuse_copy_page(cs, &page, offset, this_num, 0);
+               err = fuse_copy_folio(cs, &folio, offset, this_num, 0);
                if (!folio_test_uptodate(folio) && !err && offset == 0 &&
                    (this_num == folio_size(folio) || file_size == end)) {
                        folio_zero_segment(folio, this_num, folio_size(folio));
@@ -2038,8 +2037,8 @@ static int fuse_notify_inc_epoch(struct fuse_conn *fc)
 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
                       unsigned int size, struct fuse_copy_state *cs)
 {
-       /* Don't try to move pages (yet) */
-       cs->move_pages = false;
+       /* Don't try to move folios (yet) */
+       cs->move_folios = false;
 
        switch (code) {
        case FUSE_NOTIFY_POLL:
@@ -2190,7 +2189,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
        spin_unlock(&fpq->lock);
        cs->req = req;
        if (!req->args->page_replace)
-               cs->move_pages = false;
+               cs->move_folios = false;
 
        if (oh.error)
                err = nbytes != sizeof(oh) ? -EINVAL : 0;
@@ -2308,7 +2307,7 @@ static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
        cs.pipe = pipe;
 
        if (flags & SPLICE_F_MOVE)
-               cs.move_pages = true;
+               cs.move_folios = true;
 
        ret = fuse_dev_do_write(fud, &cs, len);
 

diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h
index db136e045925f1823e74f8e75c53676c781e5acb..5a9bd771a3193dbe51c82179a628f1eb65d2253a 100644 (file)
--- a/fs/fuse/fuse_dev_i.h
+++ b/fs/fuse/fuse_dev_i.h
@@ -30,7 +30,7 @@ struct fuse_copy_state {
        unsigned int len;
        unsigned int offset;
        bool write:1;
-       bool move_pages:1;
+       bool move_folios:1;
        bool is_uring:1;
        struct {
                unsigned int copied_sz; /* copied size into the user buffer */