erofs: support unencoded inodes for page cache share

This patch adds inode page cache sharing functionality for unencoded
files.

I conducted experiments in the container environment. Below is the
memory usage for reading all files in two different minor versions
of container images:

+-------------------+------------------+-------------+---------------+
|       Image       | Page Cache Share | Memory (MB) |    Memory     |
|                   |                  |             | Reduction (%) |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     241     |       -       |
|       redis       +------------------+-------------+---------------+
|   7.2.4 & 7.2.5   |        Yes       |     163     |      33%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     872     |       -       |
|      postgres     +------------------+-------------+---------------+
|    16.1 & 16.2    |        Yes       |     630     |      28%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     2771    |       -       |
|     tensorflow    +------------------+-------------+---------------+
|  2.11.0 & 2.11.1  |        Yes       |     2340    |      16%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     926     |       -       |
|       mysql       +------------------+-------------+---------------+
|  8.0.11 & 8.0.12  |        Yes       |     735     |      21%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     390     |       -       |
|       nginx       +------------------+-------------+---------------+
|   7.2.4 & 7.2.5   |        Yes       |     219     |      44%      |
+-------------------+------------------+-------------+---------------+
|       tomcat      |        No        |     924     |       -       |
| 10.1.25 & 10.1.26 +------------------+-------------+---------------+
|                   |        Yes       |     474     |      49%      |
+-------------------+------------------+-------------+---------------+

Additionally, the table below shows the runtime memory usage of the
container:

+-------------------+------------------+-------------+---------------+
|       Image       | Page Cache Share | Memory (MB) |    Memory     |
|                   |                  |             | Reduction (%) |
+-------------------+------------------+-------------+---------------+
|                   |        No        |      35     |       -       |
|       redis       +------------------+-------------+---------------+
|   7.2.4 & 7.2.5   |        Yes       |      28     |      20%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     149     |       -       |
|      postgres     +------------------+-------------+---------------+
|    16.1 & 16.2    |        Yes       |      95     |      37%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     1028    |       -       |
|     tensorflow    +------------------+-------------+---------------+
|  2.11.0 & 2.11.1  |        Yes       |     930     |      10%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |     155     |       -       |
|       mysql       +------------------+-------------+---------------+
|  8.0.11 & 8.0.12  |        Yes       |     132     |      15%      |
+-------------------+------------------+-------------+---------------+
|                   |        No        |      25     |       -       |
|       nginx       +------------------+-------------+---------------+
|   7.2.4 & 7.2.5   |        Yes       |      20     |      20%      |
+-------------------+------------------+-------------+---------------+
|       tomcat      |        No        |     186     |       -       |
| 10.1.25 & 10.1.26 +------------------+-------------+---------------+
|                   |        Yes       |      98     |      48%      |
+-------------------+------------------+-------------+---------------+

Co-developed-by: Hongzhen Luo <hongzhen@linux.alibaba.com>
Signed-off-by: Hongzhen Luo <hongzhen@linux.alibaba.com>
Signed-off-by: Hongbo Li <lihongbo22@huawei.com>
Reviewed-by: Gao Xiang <hsiangkao@linux.alibaba.com>
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>

diff --git a/fs/erofs/data.c b/fs/erofs/data.c
index ea198defb531023d69f29e886c23a2e979e02d29..3a4eb0dececd96c603746e57907e863256e7317c 100644 (file)
--- a/fs/erofs/data.c
+++ b/fs/erofs/data.c
@@ -269,6 +269,7 @@ void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty)
 struct erofs_iomap_iter_ctx {
        struct page *page;
        void *base;
+       struct inode *realinode;
 };
 
 static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
@@ -276,14 +277,15 @@ static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
 {
        struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap);
        struct erofs_iomap_iter_ctx *ctx = iter->private;
-       struct super_block *sb = inode->i_sb;
+       struct inode *realinode = ctx ? ctx->realinode : inode;
+       struct super_block *sb = realinode->i_sb;
        struct erofs_map_blocks map;
        struct erofs_map_dev mdev;
        int ret;
 
        map.m_la = offset;
        map.m_llen = length;
-       ret = erofs_map_blocks(inode, &map);
+       ret = erofs_map_blocks(realinode, &map);
        if (ret < 0)
                return ret;
 
@@ -296,7 +298,7 @@ static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
                return 0;
        }
 
-       if (!(map.m_flags & EROFS_MAP_META) || !erofs_inode_in_metabox(inode)) {
+       if (!(map.m_flags & EROFS_MAP_META) || !erofs_inode_in_metabox(realinode)) {
                mdev = (struct erofs_map_dev) {
                        .m_deviceid = map.m_deviceid,
                        .m_pa = map.m_pa,
@@ -322,7 +324,7 @@ static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
                        void *ptr;
 
                        ptr = erofs_read_metabuf(&buf, sb, map.m_pa,
-                                                erofs_inode_in_metabox(inode));
+                                                erofs_inode_in_metabox(realinode));
                        if (IS_ERR(ptr))
                                return PTR_ERR(ptr);
                        iomap->inline_data = ptr;
@@ -383,10 +385,15 @@ static int erofs_read_folio(struct file *file, struct folio *folio)
                .ops            = &iomap_bio_read_ops,
                .cur_folio      = folio,
        };
-       struct erofs_iomap_iter_ctx iter_ctx = {};
+       bool need_iput;
+       struct erofs_iomap_iter_ctx iter_ctx = {
+               .realinode = erofs_real_inode(folio_inode(folio), &need_iput),
+       };
 
-       trace_erofs_read_folio(folio_inode(folio), folio, true);
+       trace_erofs_read_folio(iter_ctx.realinode, folio, true);
        iomap_read_folio(&erofs_iomap_ops, &read_ctx, &iter_ctx);
+       if (need_iput)
+               iput(iter_ctx.realinode);
        return 0;
 }
 
@@ -396,11 +403,16 @@ static void erofs_readahead(struct readahead_control *rac)
                .ops            = &iomap_bio_read_ops,
                .rac            = rac,
        };
-       struct erofs_iomap_iter_ctx iter_ctx = {};
+       bool need_iput;
+       struct erofs_iomap_iter_ctx iter_ctx = {
+               .realinode = erofs_real_inode(rac->mapping->host, &need_iput),
+       };
 
-       trace_erofs_readahead(rac->mapping->host, readahead_index(rac),
+       trace_erofs_readahead(iter_ctx.realinode, readahead_index(rac),
                              readahead_count(rac), true);
        iomap_readahead(&erofs_iomap_ops, &read_ctx, &iter_ctx);
+       if (need_iput)
+               iput(iter_ctx.realinode);
 }
 
 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
@@ -421,7 +433,9 @@ static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
                return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
 #endif
        if ((iocb->ki_flags & IOCB_DIRECT) && inode->i_sb->s_bdev) {
-               struct erofs_iomap_iter_ctx iter_ctx = {};
+               struct erofs_iomap_iter_ctx iter_ctx = {
+                       .realinode = inode,
+               };
 
                return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
                                    NULL, 0, &iter_ctx, 0);

diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c
index d07dc248d264c6f4178a326009a309896c08c339..c1d0081609dc2670ee97501b282224909f0a32e1 100644 (file)
--- a/fs/erofs/fileio.c
+++ b/fs/erofs/fileio.c
@@ -88,9 +88,9 @@ void erofs_fileio_submit_bio(struct bio *bio)
                                                   bio));
 }
 
-static int erofs_fileio_scan_folio(struct erofs_fileio *io, struct folio *folio)
+static int erofs_fileio_scan_folio(struct erofs_fileio *io,
+                                  struct inode *inode, struct folio *folio)
 {
-       struct inode *inode = folio_inode(folio);
        struct erofs_map_blocks *map = &io->map;
        unsigned int cur = 0, end = folio_size(folio), len, attached = 0;
        loff_t pos = folio_pos(folio), ofs;
@@ -158,31 +158,38 @@ io_retry:
 
 static int erofs_fileio_read_folio(struct file *file, struct folio *folio)
 {
+       bool need_iput;
+       struct inode *realinode = erofs_real_inode(folio_inode(folio), &need_iput);
        struct erofs_fileio io = {};
        int err;
 
-       trace_erofs_read_folio(folio_inode(folio), folio, true);
-       err = erofs_fileio_scan_folio(&io, folio);
+       trace_erofs_read_folio(realinode, folio, true);
+       err = erofs_fileio_scan_folio(&io, realinode, folio);
        erofs_fileio_rq_submit(io.rq);
+       if (need_iput)
+               iput(realinode);
        return err;
 }
 
 static void erofs_fileio_readahead(struct readahead_control *rac)
 {
-       struct inode *inode = rac->mapping->host;
+       bool need_iput;
+       struct inode *realinode = erofs_real_inode(rac->mapping->host, &need_iput);
        struct erofs_fileio io = {};
        struct folio *folio;
        int err;
 
-       trace_erofs_readahead(inode, readahead_index(rac),
+       trace_erofs_readahead(realinode, readahead_index(rac),
                              readahead_count(rac), true);
        while ((folio = readahead_folio(rac))) {
-               err = erofs_fileio_scan_folio(&io, folio);
+               err = erofs_fileio_scan_folio(&io, realinode, folio);
                if (err && err != -EINTR)
-                       erofs_err(inode->i_sb, "readahead error at folio %lu @ nid %llu",
-                                 folio->index, EROFS_I(inode)->nid);
+                       erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu",
+                                 folio->index, EROFS_I(realinode)->nid);
        }
        erofs_fileio_rq_submit(io.rq);
+       if (need_iput)
+               iput(realinode);
 }
 
 const struct address_space_operations erofs_fileio_aops = {

diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c
index 202cbbb4eadaa4faed84f257fbc5519e18c70cdf..d33816cff813972a8c28bb1966cadec74de7c714 100644 (file)
--- a/fs/erofs/inode.c
+++ b/fs/erofs/inode.c
@@ -213,7 +213,8 @@ static int erofs_fill_inode(struct inode *inode)
        switch (inode->i_mode & S_IFMT) {
        case S_IFREG:
                inode->i_op = &erofs_generic_iops;
-               inode->i_fop = &erofs_file_fops;
+               inode->i_fop = erofs_ishare_fill_inode(inode) ?
+                              &erofs_ishare_fops : &erofs_file_fops;
                break;
        case S_IFDIR:
                inode->i_op = &erofs_dir_iops;

diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 367a9a9f05424cccdf372f3e54b1215090a44221..3001bfec4e04556f406d76acb10a5e7b466b05da 100644 (file)
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -584,11 +584,17 @@ int __init erofs_init_ishare(void);
 void erofs_exit_ishare(void);
 bool erofs_ishare_fill_inode(struct inode *inode);
 void erofs_ishare_free_inode(struct inode *inode);
+struct inode *erofs_real_inode(struct inode *inode, bool *need_iput);
 #else
 static inline int erofs_init_ishare(void) { return 0; }
 static inline void erofs_exit_ishare(void) {}
 static inline bool erofs_ishare_fill_inode(struct inode *inode) { return false; }
 static inline void erofs_ishare_free_inode(struct inode *inode) {}
+static inline struct inode *erofs_real_inode(struct inode *inode, bool *need_iput)
+{
+       *need_iput = false;
+       return inode;
+}
 #endif
 
 long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);

diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c
index 3d26b2826710e5794b8af1b9a6fd19565cbc0589..ab459fb62473da81d229ef494b1a270f7d10c696 100644 (file)
--- a/fs/erofs/ishare.c
+++ b/fs/erofs/ishare.c
@@ -11,6 +11,12 @@
 
 static struct vfsmount *erofs_ishare_mnt;
 
+static inline bool erofs_is_ishare_inode(struct inode *inode)
+{
+       /* assumed FS_ONDEMAND is excluded with FS_PAGE_CACHE_SHARE feature */
+       return inode->i_sb->s_type == &erofs_anon_fs_type;
+}
+
 static int erofs_ishare_iget5_eq(struct inode *inode, void *data)
 {
        struct erofs_inode_fingerprint *fp1 = &EROFS_I(inode)->fingerprint;
@@ -38,6 +44,8 @@ bool erofs_ishare_fill_inode(struct inode *inode)
        struct inode *sharedinode;
        unsigned long hash;
 
+       if (erofs_inode_is_data_compressed(vi->datalayout))
+               return false;
        if (erofs_xattr_fill_inode_fingerprint(&fp, inode, sbi->domain_id))
                return false;
        hash = xxh32(fp.opaque, fp.size, 0);
@@ -155,6 +163,32 @@ const struct file_operations erofs_ishare_fops = {
        .splice_read    = filemap_splice_read,
 };
 
+struct inode *erofs_real_inode(struct inode *inode, bool *need_iput)
+{
+       struct erofs_inode *vi, *vi_share;
+       struct inode *realinode;
+
+       *need_iput = false;
+       if (!erofs_is_ishare_inode(inode))
+               return inode;
+
+       vi_share = EROFS_I(inode);
+       spin_lock(&vi_share->ishare_lock);
+       /* fetch any one as real inode */
+       DBG_BUGON(list_empty(&vi_share->ishare_list));
+       list_for_each_entry(vi, &vi_share->ishare_list, ishare_list) {
+               realinode = igrab(&vi->vfs_inode);
+               if (realinode) {
+                       *need_iput = true;
+                       break;
+               }
+       }
+       spin_unlock(&vi_share->ishare_lock);
+
+       DBG_BUGON(!realinode);
+       return realinode;
+}
+
 int __init erofs_init_ishare(void)
 {
        erofs_ishare_mnt = kern_mount(&erofs_anon_fs_type);