fs: inline step_into() and walk_component()

The primary consumer is link_path_walk(), calling walk_component() every
time which in turn calls step_into().

Inlining these saves overhead of 2 function calls per path component,
along with allowing the compiler to do better job optimizing them in place.

step_into() had absolutely atrocious assembly to facilitate the
slowpath. In order to lessen the burden at the callsite all the hard
work is moved into step_into_slowpath() and instead an inline-able
fastpath is implemented for rcu-walk.

The new fastpath is a stripped down step_into() RCU handling with a
d_managed() check from handle_mounts().

Benchmarked as follows on Sapphire Rapids:
1. the "before" was a kernel with not-yet-merged optimizations (notably
   elision of calls to security_inode_permission() and marking ext4
   inodes as not having acls as applicable)
2. "after" is the same + the prep patch + this patch
3. benchmark consists of issuing 205 calls to access(2) in a loop with
   pathnames lifted out of gcc and the linker building real code, most
   of which have several path components and 118 of which fail with
   -ENOENT.

Result in terms of ops/s:
before:	21619
after:	22536 (+4%)

profile before:
  20.25%  [kernel]                  [k] __d_lookup_rcu
  10.54%  [kernel]                  [k] link_path_walk
  10.22%  [kernel]                  [k] entry_SYSCALL_64
   6.50%  libc.so.6                 [.] __GI___access
   6.35%  [kernel]                  [k] strncpy_from_user
   4.87%  [kernel]                  [k] step_into
   3.68%  [kernel]                  [k] kmem_cache_alloc_noprof
   2.88%  [kernel]                  [k] walk_component
   2.86%  [kernel]                  [k] kmem_cache_free
   2.14%  [kernel]                  [k] set_root
   2.08%  [kernel]                  [k] lookup_fast

after:
  23.38%  [kernel]                  [k] __d_lookup_rcu
  11.27%  [kernel]                  [k] entry_SYSCALL_64
  10.89%  [kernel]                  [k] link_path_walk
   7.00%  libc.so.6                 [.] __GI___access
   6.88%  [kernel]                  [k] strncpy_from_user
   3.50%  [kernel]                  [k] kmem_cache_alloc_noprof
   2.01%  [kernel]                  [k] kmem_cache_free
   2.00%  [kernel]                  [k] set_root
   1.99%  [kernel]                  [k] lookup_fast
   1.81%  [kernel]                  [k] do_syscall_64
   1.69%  [kernel]                  [k] entry_SYSCALL_64_safe_stack

While walk_component() and step_into() of course disappear from the
profile, the link_path_walk() barely gets more overhead despite the
inlining thanks to the fast path added and while completing more walks
per second.

I did not investigate why overhead grew a lot on __d_lookup_rcu().

Signed-off-by: Mateusz Guzik <mjguzik@gmail.com>
Link: https://patch.msgid.link/20251120003803.2979978-2-mjguzik@gmail.com
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/fs/namei.c b/fs/namei.c
index 5fee8afa510ede3f0492d4074e7376203c5dd6e5..8281dfe5047f4459e2d24cecb4b6e7bba04f1099 100644 (file)
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1951,7 +1951,7 @@ static noinline const char *pick_link(struct nameidata *nd, struct path *link,
        int error;
 
        if (nd->flags & LOOKUP_RCU) {
-               /* make sure that d_is_symlink from step_into() matches the inode */
+               /* make sure that d_is_symlink from step_into_slowpath() matches the inode */
                if (read_seqcount_retry(&link->dentry->d_seq, nd->next_seq))
                        return ERR_PTR(-ECHILD);
        } else {
@@ -2033,7 +2033,7 @@ all_done: // pure jump
  *
  * NOTE: dentry must be what nd->next_seq had been sampled from.
  */
-static const char *step_into(struct nameidata *nd, int flags,
+static noinline const char *step_into_slowpath(struct nameidata *nd, int flags,
                     struct dentry *dentry)
 {
        struct path path;
@@ -2066,6 +2066,31 @@ static const char *step_into(struct nameidata *nd, int flags,
        return pick_link(nd, &path, inode, flags);
 }
 
+static __always_inline const char *step_into(struct nameidata *nd, int flags,
+                    struct dentry *dentry)
+{
+       /*
+        * In the common case we are in rcu-walk and traversing over a non-mounted on
+        * directory (as opposed to e.g., a symlink).
+        *
+        * We can handle that and negative entries with the checks below.
+        */
+       if (likely((nd->flags & LOOKUP_RCU) &&
+           !d_managed(dentry) && !d_is_symlink(dentry))) {
+               struct inode *inode = dentry->d_inode;
+               if (read_seqcount_retry(&dentry->d_seq, nd->next_seq))
+                       return ERR_PTR(-ECHILD);
+               if (unlikely(!inode))
+                       return ERR_PTR(-ENOENT);
+               nd->path.dentry = dentry;
+               /* nd->path.mnt is retained on purpose */
+               nd->inode = inode;
+               nd->seq = nd->next_seq;
+               return NULL;
+       }
+       return step_into_slowpath(nd, flags, dentry);
+}
+
 static struct dentry *follow_dotdot_rcu(struct nameidata *nd)
 {
        struct dentry *parent, *old;
@@ -2176,7 +2201,7 @@ static const char *handle_dots(struct nameidata *nd, int type)
        return NULL;
 }
 
-static const char *walk_component(struct nameidata *nd, int flags)
+static __always_inline const char *walk_component(struct nameidata *nd, int flags)
 {
        struct dentry *dentry;
        /*