pipe: cache 2 pages instead of 1

User data is kept in a circular buffer backed by pages allocated as
needed. Only having space for one spare is still prone to having to
resort to allocation / freeing.

In my testing this decreases page allocs by 60% during a kernel build.

Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
Link: https://lore.kernel.org/r/20250303230409.452687-3-mjguzik@gmail.com
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/fs/pipe.c b/fs/pipe.c
index b60487b650cb730db22fc6a62c998b094f806e00..f2d5427fba8ec1c1324400a0cb11080f857490e2 100644 (file)
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -112,20 +112,40 @@ void pipe_double_lock(struct pipe_inode_info *pipe1,
        pipe_lock(pipe2);
 }
 
+static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
+{
+       for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
+               if (pipe->tmp_page[i]) {
+                       struct page *page = pipe->tmp_page[i];
+                       pipe->tmp_page[i] = NULL;
+                       return page;
+               }
+       }
+
+       return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
+}
+
+static void anon_pipe_put_page(struct pipe_inode_info *pipe,
+                              struct page *page)
+{
+       if (page_count(page) == 1) {
+               for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
+                       if (!pipe->tmp_page[i]) {
+                               pipe->tmp_page[i] = page;
+                               return;
+                       }
+               }
+       }
+
+       put_page(page);
+}
+
 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
                                  struct pipe_buffer *buf)
 {
        struct page *page = buf->page;
 
-       /*
-        * If nobody else uses this page, and we don't already have a
-        * temporary page, let's keep track of it as a one-deep
-        * allocation cache. (Otherwise just release our reference to it)
-        */
-       if (page_count(page) == 1 && !pipe->tmp_page)
-               pipe->tmp_page = page;
-       else
-               put_page(page);
+       anon_pipe_put_page(pipe, page);
 }
 
 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
@@ -493,27 +513,25 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
                if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
                        unsigned int mask = pipe->ring_size - 1;
                        struct pipe_buffer *buf;
-                       struct page *page = pipe->tmp_page;
+                       struct page *page;
                        int copied;
 
-                       if (!page) {
-                               page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
-                               if (unlikely(!page)) {
-                                       ret = ret ? : -ENOMEM;
-                                       break;
-                               }
-                               pipe->tmp_page = page;
+                       page = anon_pipe_get_page(pipe);
+                       if (unlikely(!page)) {
+                               if (!ret)
+                                       ret = -ENOMEM;
+                               break;
                        }
 
                        copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
                        if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
+                               anon_pipe_put_page(pipe, page);
                                if (!ret)
                                        ret = -EFAULT;
                                break;
                        }
 
                        pipe->head = head + 1;
-                       pipe->tmp_page = NULL;
                        /* Insert it into the buffer array */
                        buf = &pipe->bufs[head & mask];
                        buf->page = page;
@@ -846,8 +864,10 @@ void free_pipe_info(struct pipe_inode_info *pipe)
        if (pipe->watch_queue)
                put_watch_queue(pipe->watch_queue);
 #endif
-       if (pipe->tmp_page)
-               __free_page(pipe->tmp_page);
+       for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
+               if (pipe->tmp_page[i])
+                       __free_page(pipe->tmp_page[i]);
+       }
        kfree(pipe->bufs);
        kfree(pipe);
 }

diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h
index 8ff23bf5a81972f9280d09dbc277f4e33dd38683..eb7994a1ff932078a218a60392cc510f6a5d4421 100644 (file)
--- a/include/linux/pipe_fs_i.h
+++ b/include/linux/pipe_fs_i.h
@@ -72,7 +72,7 @@ struct pipe_inode_info {
 #ifdef CONFIG_WATCH_QUEUE
        bool note_loss;
 #endif
-       struct page *tmp_page;
+       struct page *tmp_page[2];
        struct fasync_struct *fasync_readers;
        struct fasync_struct *fasync_writers;
        struct pipe_buffer *bufs;