mm: memcontrol: return root object cgroup for root memory cgroup

Memory cgroup functions such as get_mem_cgroup_from_folio() and
get_mem_cgroup_from_mm() return a valid memory cgroup pointer, even for
the root memory cgroup.  In contrast, the situation for object cgroups has
been different.

Previously, the root object cgroup couldn't be returned because it didn't
exist.  Now that a valid root object cgroup exists, for the sake of
consistency, it's necessary to align the behavior of object-cgroup-related
operations with that of memory cgroup APIs.

Link: https://lore.kernel.org/e9c3f40ba7681d9753372d4ee2ac7a0216848b95.1772711148.git.zhengqi.arch@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Cc: Allen Pais <apais@linux.microsoft.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: Chen Ridong <chenridong@huawei.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Hamza Mahfooz <hamzamahfooz@linux.microsoft.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Imran Khan <imran.f.khan@oracle.com>
Cc: Kamalesh Babulal <kamalesh.babulal@oracle.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <Liam.Howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yosry Ahmed <yosry@kernel.org>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 6e88288e90d8245f17859b374a5ecd0af909790d..9a015258a2ffe48bbaf3433ad16a179a2d5f51ea 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -332,6 +332,7 @@ struct mem_cgroup {
 #define MEMCG_CHARGE_BATCH 64U
 
 extern struct mem_cgroup *root_mem_cgroup;
+extern struct obj_cgroup *root_obj_cgroup;
 
 enum page_memcg_data_flags {
        /* page->memcg_data is a pointer to an slabobj_ext vector */
@@ -548,6 +549,11 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
        return (memcg == root_mem_cgroup);
 }
 
+static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg)
+{
+       return objcg == root_obj_cgroup;
+}
+
 static inline bool mem_cgroup_disabled(void)
 {
        return !cgroup_subsys_enabled(memory_cgrp_subsys);
@@ -774,23 +780,26 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
 
 static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg)
 {
+       if (obj_cgroup_is_root(objcg))
+               return true;
        return percpu_ref_tryget(&objcg->refcnt);
 }
 
-static inline void obj_cgroup_get(struct obj_cgroup *objcg)
+static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
+                                      unsigned long nr)
 {
-       percpu_ref_get(&objcg->refcnt);
+       if (!obj_cgroup_is_root(objcg))
+               percpu_ref_get_many(&objcg->refcnt, nr);
 }
 
-static inline void obj_cgroup_get_many(struct obj_cgroup *objcg,
-                                      unsigned long nr)
+static inline void obj_cgroup_get(struct obj_cgroup *objcg)
 {
-       percpu_ref_get_many(&objcg->refcnt, nr);
+       obj_cgroup_get_many(objcg, 1);
 }
 
 static inline void obj_cgroup_put(struct obj_cgroup *objcg)
 {
-       if (objcg)
+       if (objcg && !obj_cgroup_is_root(objcg))
                percpu_ref_put(&objcg->refcnt);
 }
 
@@ -1087,6 +1096,11 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
        return true;
 }
 
+static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg)
+{
+       return true;
+}
+
 static inline bool mem_cgroup_disabled(void)
 {
        return true;

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 2cb2d66579d369ce13da476b81f1fc99d767d2fb..e7022adcea7fafce95f0504c3f6a9d6fcb0bb3c3 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -83,6 +83,8 @@ EXPORT_SYMBOL(memory_cgrp_subsys);
 struct mem_cgroup *root_mem_cgroup __read_mostly;
 EXPORT_SYMBOL(root_mem_cgroup);
 
+struct obj_cgroup *root_obj_cgroup __read_mostly;
+
 /* Active memory cgroup to use from an interrupt context */
 DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
 EXPORT_PER_CPU_SYMBOL_GPL(int_active_memcg);
@@ -2693,15 +2695,14 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p)
 
 static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)
 {
-       struct obj_cgroup *objcg = NULL;
+       for (; memcg; memcg = parent_mem_cgroup(memcg)) {
+               struct obj_cgroup *objcg = rcu_dereference(memcg->objcg);
 
-       for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
-               objcg = rcu_dereference(memcg->objcg);
                if (likely(objcg && obj_cgroup_tryget(objcg)))
-                       break;
-               objcg = NULL;
+                       return objcg;
        }
-       return objcg;
+
+       return NULL;
 }
 
 static struct obj_cgroup *current_objcg_update(void)
@@ -2775,18 +2776,17 @@ __always_inline struct obj_cgroup *current_obj_cgroup(void)
                 * Objcg reference is kept by the task, so it's safe
                 * to use the objcg by the current task.
                 */
-               return objcg;
+               return objcg ? : root_obj_cgroup;
        }
 
        memcg = this_cpu_read(int_active_memcg);
        if (unlikely(memcg))
                goto from_memcg;
 
-       return NULL;
+       return root_obj_cgroup;
 
 from_memcg:
-       objcg = NULL;
-       for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
+       for (; memcg; memcg = parent_mem_cgroup(memcg)) {
                /*
                 * Memcg pointer is protected by scope (see set_active_memcg())
                 * and is pinning the corresponding objcg, so objcg can't go
@@ -2795,10 +2795,10 @@ from_memcg:
                 */
                objcg = rcu_dereference_check(memcg->objcg, 1);
                if (likely(objcg))
-                       break;
+                       return objcg;
        }
 
-       return objcg;
+       return root_obj_cgroup;
 }
 
 struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
@@ -2812,14 +2812,8 @@ struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio)
                objcg = __folio_objcg(folio);
                obj_cgroup_get(objcg);
        } else {
-               struct mem_cgroup *memcg;
-
                rcu_read_lock();
-               memcg = __folio_memcg(folio);
-               if (memcg)
-                       objcg = __get_obj_cgroup_from_memcg(memcg);
-               else
-                       objcg = NULL;
+               objcg = __get_obj_cgroup_from_memcg(__folio_memcg(folio));
                rcu_read_unlock();
        }
        return objcg;
@@ -2922,7 +2916,7 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
        int ret = 0;
 
        objcg = current_obj_cgroup();
-       if (objcg) {
+       if (objcg && !obj_cgroup_is_root(objcg)) {
                ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order);
                if (!ret) {
                        obj_cgroup_get(objcg);
@@ -3251,7 +3245,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
         * obj_cgroup_get() is used to get a permanent reference.
         */
        objcg = current_obj_cgroup();
-       if (!objcg)
+       if (!objcg || obj_cgroup_is_root(objcg))
                return true;
 
        /*
@@ -3927,6 +3921,9 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
        if (!objcg)
                goto free_shrinker;
 
+       if (unlikely(mem_cgroup_is_root(memcg)))
+               root_obj_cgroup = objcg;
+
        objcg->memcg = memcg;
        rcu_assign_pointer(memcg->objcg, objcg);
        obj_cgroup_get(objcg);
@@ -5551,6 +5548,9 @@ void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size)
        if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
                return;
 
+       if (obj_cgroup_is_root(objcg))
+               return;
+
        VM_WARN_ON_ONCE(!(current->flags & PF_MEMALLOC));
 
        /* PF_MEMALLOC context, charging must succeed */
@@ -5580,6 +5580,9 @@ void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size)
        if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
                return;
 
+       if (obj_cgroup_is_root(objcg))
+               return;
+
        obj_cgroup_uncharge(objcg, size);
 
        rcu_read_lock();

diff --git a/mm/percpu.c b/mm/percpu.c
index a2107bdebf0b5b7a21b59d27948535919b7fe72c..b0676b8054ed09e266bf81b4217871bc45aaf931 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1622,7 +1622,7 @@ static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp,
                return true;
 
        objcg = current_obj_cgroup();
-       if (!objcg)
+       if (!objcg || obj_cgroup_is_root(objcg))
                return true;
 
        if (obj_cgroup_charge(objcg, gfp, pcpu_obj_full_size(size)))