proc/kcore: use percpu_rw_semaphore for kclist_lock

The list of memory ranges for /proc/kcore is protected by a
rw_semaphore. We lock it for reading on every read from /proc/kcore.
This is very heavy, especially since it is rarely locked for writing.
Since we want to strongly favor read lock performance, convert it to a
percpu_rw_semaphore. I also experimented with percpu_ref and SRCU, but
this change was the simplest and the fastest.

In my benchmark, this reduces the time per read by yet another 20
nanoseconds on top of the previous two changes, from 195 nanoseconds per
read to 175.

Link: https://github.com/osandov/drgn/issues/106
Signed-off-by: Omar Sandoval <osandov@fb.com>
Link: https://lore.kernel.org/r/83a3b235b4bcc3b8aef7c533e0657f4d7d5d35ae.1731115587.git.osandov@fb.com
Signed-off-by: Christian Brauner <brauner@kernel.org>

diff --git a/fs/proc/kcore.c b/fs/proc/kcore.c
index 3070cf3e2f5cf918573cb0294ec16a5ae810f3b9..1cb33771bf9fcbbea273522fee5fe6361a794993 100644 (file)
--- a/fs/proc/kcore.c
+++ b/fs/proc/kcore.c
@@ -69,7 +69,7 @@ static int kcore_nphdr;
 static size_t kcore_phdrs_len;
 static size_t kcore_notes_len;
 static size_t kcore_data_offset;
-static DECLARE_RWSEM(kclist_lock);
+DEFINE_STATIC_PERCPU_RWSEM(kclist_lock);
 static int kcore_need_update = 1;
 
 /*
@@ -276,7 +276,7 @@ static int kcore_update_ram(void)
        struct kcore_list *tmp, *pos;
        int ret = 0;
 
-       down_write(&kclist_lock);
+       percpu_down_write(&kclist_lock);
        if (!xchg(&kcore_need_update, 0))
                goto out;
 
@@ -297,7 +297,7 @@ static int kcore_update_ram(void)
        update_kcore_size();
 
 out:
-       up_write(&kclist_lock);
+       percpu_up_write(&kclist_lock);
        list_for_each_entry_safe(pos, tmp, &garbage, list) {
                list_del(&pos->list);
                kfree(pos);
@@ -335,7 +335,7 @@ static ssize_t read_kcore_iter(struct kiocb *iocb, struct iov_iter *iter)
        size_t orig_buflen = buflen;
        int ret = 0;
 
-       down_read(&kclist_lock);
+       percpu_down_read(&kclist_lock);
        /*
         * Don't race against drivers that set PageOffline() and expect no
         * further page access.
@@ -626,7 +626,7 @@ skip:
 
 out:
        page_offline_thaw();
-       up_read(&kclist_lock);
+       percpu_up_read(&kclist_lock);
        if (ret)
                return ret;
        return orig_buflen - buflen;