cgroup: replace global cgroup_file_kn_lock with per-cgroup_file lock

Replace the global cgroup_file_kn_lock with a per-cgroup_file spinlock
to eliminate cross-cgroup contention as it is not really protecting
data shared between different cgroups.

The lock is initialized in cgroup_add_file() alongside timer_setup().
No lock acquisition is needed during initialization since the cgroup
directory is being populated under cgroup_mutex and no concurrent
accessors exist at that point.

Reported-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index bb92f5c169ca2dcfc9354623e34968f6449854b1..ba26b5d05ce309ea2c37ba27195a132726eb300b 100644 (file)
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -167,6 +167,7 @@ struct cgroup_file {
        struct kernfs_node *kn;
        unsigned long notified_at;
        struct timer_list notify_timer;
+       spinlock_t lock;
 };
 
 /*

diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index d161bcaa68f13ace95cebf99797a3e61c20054a5..6f58efeb901668793b5eaee6cec7277340fbbbaf 100644 (file)
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -99,12 +99,6 @@ static bool cgroup_debug __read_mostly;
  */
 static DEFINE_SPINLOCK(cgroup_idr_lock);
 
-/*
- * Protects cgroup_file->kn for !self csses.  It synchronizes notifications
- * against file removal/re-creation across css hiding.
- */
-static DEFINE_SPINLOCK(cgroup_file_kn_lock);
-
 DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem);
 
 #define cgroup_assert_mutex_or_rcu_locked()                            \
@@ -1693,9 +1687,9 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
                struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss);
                struct cgroup_file *cfile = (void *)css + cft->file_offset;
 
-               spin_lock_irq(&cgroup_file_kn_lock);
+               spin_lock_irq(&cfile->lock);
                WRITE_ONCE(cfile->kn, NULL);
-               spin_unlock_irq(&cgroup_file_kn_lock);
+               spin_unlock_irq(&cfile->lock);
 
                timer_delete_sync(&cfile->notify_timer);
        }
@@ -4373,10 +4367,8 @@ static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,
                struct cgroup_file *cfile = (void *)css + cft->file_offset;
 
                timer_setup(&cfile->notify_timer, cgroup_file_notify_timer, 0);
-
-               spin_lock_irq(&cgroup_file_kn_lock);
-               WRITE_ONCE(cfile->kn, kn);
-               spin_unlock_irq(&cgroup_file_kn_lock);
+               spin_lock_init(&cfile->lock);
+               cfile->kn = kn;
        }
 
        return 0;
@@ -4645,13 +4637,13 @@ void cgroup_file_notify(struct cgroup_file *cfile)
                        return;
        }
 
-       spin_lock_irqsave(&cgroup_file_kn_lock, flags);
+       spin_lock_irqsave(&cfile->lock, flags);
        if (cfile->kn) {
                kn = cfile->kn;
                kernfs_get(kn);
                WRITE_ONCE(cfile->notified_at, jiffies);
        }
-       spin_unlock_irqrestore(&cgroup_file_kn_lock, flags);
+       spin_unlock_irqrestore(&cfile->lock, flags);
 
        if (kn) {
                kernfs_notify(kn);
@@ -4669,10 +4661,10 @@ void cgroup_file_show(struct cgroup_file *cfile, bool show)
 {
        struct kernfs_node *kn;
 
-       spin_lock_irq(&cgroup_file_kn_lock);
+       spin_lock_irq(&cfile->lock);
        kn = cfile->kn;
        kernfs_get(kn);
-       spin_unlock_irq(&cgroup_file_kn_lock);
+       spin_unlock_irq(&cfile->lock);
 
        if (kn)
                kernfs_show(kn, show);