vfs: support caching symlink lengths in inodes

When utilized it dodges strlen() in vfs_readlink(), giving about 1.5%
speed up when issuing readlink on /initrd.img on ext4.

Filesystems opt in by calling inode_set_cached_link() when creating an
inode.

The size is stored in a new union utilizing the same space as i_devices,
thus avoiding growing the struct or taking up any more space.

Churn-wise the current readlink_copy() helper is patched to accept the
size instead of calculating it.

Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
Link: https://lore.kernel.org/r/20241120112037.822078-2-mjguzik@gmail.com
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/fs/namei.c b/fs/namei.c
index 9d30c7aa9aa6e760b5ad4bda7e458ad823c2b3ef..e56c29a22d262291741d9f06566a52225b2e3b3e 100644 (file)
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -5272,19 +5272,16 @@ SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newna
                                getname(newname), 0);
 }
 
-int readlink_copy(char __user *buffer, int buflen, const char *link)
+int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen)
 {
-       int len = PTR_ERR(link);
-       if (IS_ERR(link))
-               goto out;
+       int copylen;
 
-       len = strlen(link);
-       if (len > (unsigned) buflen)
-               len = buflen;
-       if (copy_to_user(buffer, link, len))
-               len = -EFAULT;
-out:
-       return len;
+       copylen = linklen;
+       if (unlikely(copylen > (unsigned) buflen))
+               copylen = buflen;
+       if (copy_to_user(buffer, link, copylen))
+               copylen = -EFAULT;
+       return copylen;
 }
 
 /**
@@ -5304,6 +5301,9 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
        const char *link;
        int res;
 
+       if (inode->i_opflags & IOP_CACHED_LINK)
+               return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen);
+
        if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
                if (unlikely(inode->i_op->readlink))
                        return inode->i_op->readlink(dentry, buffer, buflen);
@@ -5322,7 +5322,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
                if (IS_ERR(link))
                        return PTR_ERR(link);
        }
-       res = readlink_copy(buffer, buflen, link);
+       res = readlink_copy(buffer, buflen, link, strlen(link));
        do_delayed_call(&done);
        return res;
 }
@@ -5391,10 +5391,14 @@ EXPORT_SYMBOL(page_put_link);
 
 int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
 {
+       const char *link;
+       int res;
+
        DEFINE_DELAYED_CALL(done);
-       int res = readlink_copy(buffer, buflen,
-                               page_get_link(dentry, d_inode(dentry),
-                                             &done));
+       link = page_get_link(dentry, d_inode(dentry), &done);
+       res = PTR_ERR(link);
+       if (!IS_ERR(link))
+               res = readlink_copy(buffer, buflen, link, strlen(link));
        do_delayed_call(&done);
        return res;
 }

diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c
index 8e159fc78c0a12499536f00b40587b3c4c9c56b0..c610224faf109daff698004c4146d5e4e3db7399 100644 (file)
--- a/fs/proc/namespaces.c
+++ b/fs/proc/namespaces.c
@@ -83,7 +83,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl
        if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
                res = ns_get_name(name, sizeof(name), task, ns_ops);
                if (res >= 0)
-                       res = readlink_copy(buffer, buflen, name);
+                       res = readlink_copy(buffer, buflen, name, strlen(name));
        }
        put_task_struct(task);
        return res;

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 7e29433c5ecce2d495ace23a7c4570ac7ad9e2b0..2cc98de5af43839f8724e29b0926905cc08f0a69 100644 (file)
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -626,6 +626,7 @@ is_uncached_acl(struct posix_acl *acl)
 #define IOP_XATTR      0x0008
 #define IOP_DEFAULT_READLINK   0x0010
 #define IOP_MGTIME     0x0020
+#define IOP_CACHED_LINK        0x0040
 
 /*
  * Keep mostly read-only and often accessed (especially for
@@ -723,7 +724,10 @@ struct inode {
        };
        struct file_lock_context        *i_flctx;
        struct address_space    i_data;
-       struct list_head        i_devices;
+       union {
+               struct list_head        i_devices;
+               int                     i_linklen;
+       };
        union {
                struct pipe_inode_info  *i_pipe;
                struct cdev             *i_cdev;
@@ -749,6 +753,13 @@ struct inode {
        void                    *i_private; /* fs or device private pointer */
 } __randomize_layout;
 
+static inline void inode_set_cached_link(struct inode *inode, char *link, int linklen)
+{
+       inode->i_link = link;
+       inode->i_linklen = linklen;
+       inode->i_opflags |= IOP_CACHED_LINK;
+}
+
 /*
  * Get bit address from inode->i_state to use with wait_var_event()
  * infrastructre.
@@ -3351,7 +3362,7 @@ extern const struct file_operations generic_ro_fops;
 
 #define special_file(m) (S_ISCHR(m)||S_ISBLK(m)||S_ISFIFO(m)||S_ISSOCK(m))
 
-extern int readlink_copy(char __user *, int, const char *);
+extern int readlink_copy(char __user *, int, const char *, int);
 extern int page_readlink(struct dentry *, char __user *, int);
 extern const char *page_get_link(struct dentry *, struct inode *,
                                 struct delayed_call *);

diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 2c0185ebc900daa9b40edf58dcef84905c86e52e..c07d150685d7c784e26b1ba42e6fbe946bb5bedb 100644 (file)
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -2612,7 +2612,7 @@ static int policy_readlink(struct dentry *dentry, char __user *buffer,
        res = snprintf(name, sizeof(name), "%s:[%lu]", AAFS_NAME,
                       d_inode(dentry)->i_ino);
        if (res > 0 && res < sizeof(name))
-               res = readlink_copy(buffer, buflen, name);
+               res = readlink_copy(buffer, buflen, name, strlen(name));
        else
                res = -ENOENT;