cgroup: support to enable nmi-safe css_rstat_updated

Add necessary infrastructure to enable the nmi-safe execution of
css_rstat_updated(). Currently css_rstat_updated() takes a per-cpu
per-css raw spinlock to add the given css in the per-cpu per-css update
tree. However the kernel can not spin in nmi context, so we need to
remove the spinning on the raw spinlock in css_rstat_updated().

To support lockless css_rstat_updated(), let's add necessary data
structures in the css and ss structures.

Signed-off-by: Shakeel Butt <shakeel.butt@linux.dev>
Tested-by: JP Kobryn <inwardvessel@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index cd7f093e34cd7808a11de86d069a754fcd7f5d11..04191d99228c3d9138d5baf6bc36ef363805f690 100644 (file)
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -384,6 +384,9 @@ struct css_rstat_cpu {
         */
        struct cgroup_subsys_state *updated_children;
        struct cgroup_subsys_state *updated_next;       /* NULL if not on the list */
+
+       struct llist_node lnode;                /* lockless list for update */
+       struct cgroup_subsys_state *owner;      /* back pointer */
 };
 
 /*
@@ -822,6 +825,7 @@ struct cgroup_subsys {
 
        spinlock_t rstat_ss_lock;
        raw_spinlock_t __percpu *rstat_ss_cpu_lock;
+       struct llist_head __percpu *lhead; /* lockless update list head */
 };
 
 extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;

diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index ce4752ab9e09b0a1b799ba25d8b79746b26d5313..bfa6366d2325b5b5c822fa0d0f37224aa329c91d 100644 (file)
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -11,6 +11,7 @@
 
 static DEFINE_SPINLOCK(rstat_base_lock);
 static DEFINE_PER_CPU(raw_spinlock_t, rstat_base_cpu_lock);
+static DEFINE_PER_CPU(struct llist_head, rstat_backlog_list);
 
 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
 
@@ -45,6 +46,13 @@ static spinlock_t *ss_rstat_lock(struct cgroup_subsys *ss)
        return &rstat_base_lock;
 }
 
+static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)
+{
+       if (ss)
+               return per_cpu_ptr(ss->lhead, cpu);
+       return per_cpu_ptr(&rstat_backlog_list, cpu);
+}
+
 static raw_spinlock_t *ss_rstat_cpu_lock(struct cgroup_subsys *ss, int cpu)
 {
        if (ss)
@@ -456,7 +464,8 @@ int css_rstat_init(struct cgroup_subsys_state *css)
        for_each_possible_cpu(cpu) {
                struct css_rstat_cpu *rstatc = css_rstat_cpu(css, cpu);
 
-               rstatc->updated_children = css;
+               rstatc->owner = rstatc->updated_children = css;
+               init_llist_node(&rstatc->lnode);
 
                if (is_self) {
                        struct cgroup_rstat_base_cpu *rstatbc;
@@ -525,9 +534,19 @@ int __init ss_rstat_init(struct cgroup_subsys *ss)
        }
 #endif
 
+       if (ss) {
+               ss->lhead = alloc_percpu(struct llist_head);
+               if (!ss->lhead) {
+                       free_percpu(ss->rstat_ss_cpu_lock);
+                       return -ENOMEM;
+               }
+       }
+
        spin_lock_init(ss_rstat_lock(ss));
-       for_each_possible_cpu(cpu)
+       for_each_possible_cpu(cpu) {
                raw_spin_lock_init(ss_rstat_cpu_lock(ss, cpu));
+               init_llist_head(ss_lhead_cpu(ss, cpu));
+       }
 
        return 0;
 }