mm: memcontrol: allocate object cgroup for non-kmem case

To allow LRU page reparenting, the objcg infrastructure is no longer
solely applicable to the kmem case.  In this patch, we extend the scope of
the objcg infrastructure beyond the kmem case, enabling LRU folios to
reuse it for folio charging purposes.

It should be noted that LRU folios are not accounted for at the root
level, yet the folio->memcg_data points to the root_mem_cgroup.  Hence,
the folio->memcg_data of LRU folios always points to a valid pointer.
However, the root_mem_cgroup does not possess an object cgroup.
Therefore, we also allocate an object cgroup for the root_mem_cgroup.

Link: https://lore.kernel.org/b77274aa8e3f37c419bedf4782943fd5885dda82.1772711148.git.zhengqi.arch@bytedance.com
Signed-off-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Chen Ridong <chenridong@huawei.com>
Cc: Allen Pais <apais@linux.microsoft.com>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Chengming Zhou <chengming.zhou@linux.dev>
Cc: David Hildenbrand <david@kernel.org>
Cc: Hamza Mahfooz <hamzamahfooz@linux.microsoft.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Imran Khan <imran.f.khan@oracle.com>
Cc: Kamalesh Babulal <kamalesh.babulal@oracle.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam Howlett <Liam.Howlett@oracle.com>
Cc: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Michal Koutný <mkoutny@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Usama Arif <usamaarif642@gmail.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Wei Xu <weixugc@google.com>
Cc: Yosry Ahmed <yosry@kernel.org>
Cc: Yuanchu Xie <yuanchu@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 4efa56a9144799f2f1b2efe40d7be1de32d61ce4..2cb2d66579d369ce13da476b81f1fc99d767d2fb 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -206,10 +206,10 @@ static struct obj_cgroup *obj_cgroup_alloc(void)
        return objcg;
 }
 
-static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
-                                 struct mem_cgroup *parent)
+static void memcg_reparent_objcgs(struct mem_cgroup *memcg)
 {
        struct obj_cgroup *objcg, *iter;
+       struct mem_cgroup *parent = parent_mem_cgroup(memcg);
 
        objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
 
@@ -3386,30 +3386,17 @@ void folio_split_memcg_refs(struct folio *folio, unsigned old_order,
        css_get_many(&__folio_memcg(folio)->css, new_refs);
 }
 
-static int memcg_online_kmem(struct mem_cgroup *memcg)
+static void memcg_online_kmem(struct mem_cgroup *memcg)
 {
-       struct obj_cgroup *objcg;
-
        if (mem_cgroup_kmem_disabled())
-               return 0;
+               return;
 
        if (unlikely(mem_cgroup_is_root(memcg)))
-               return 0;
-
-       objcg = obj_cgroup_alloc();
-       if (!objcg)
-               return -ENOMEM;
-
-       objcg->memcg = memcg;
-       rcu_assign_pointer(memcg->objcg, objcg);
-       obj_cgroup_get(objcg);
-       memcg->orig_objcg = objcg;
+               return;
 
        static_branch_enable(&memcg_kmem_online_key);
 
        memcg->kmemcg_id = memcg->id.id;
-
-       return 0;
 }
 
 static void memcg_offline_kmem(struct mem_cgroup *memcg)
@@ -3424,12 +3411,6 @@ static void memcg_offline_kmem(struct mem_cgroup *memcg)
 
        parent = parent_mem_cgroup(memcg);
        memcg_reparent_list_lrus(memcg, parent);
-
-       /*
-        * Objcg's reparenting must be after list_lru's, make sure list_lru
-        * helpers won't use parent's list_lru until child is drained.
-        */
-       memcg_reparent_objcgs(memcg, parent);
 }
 
 #ifdef CONFIG_CGROUP_WRITEBACK
@@ -3930,9 +3911,9 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
 {
        struct mem_cgroup *memcg = mem_cgroup_from_css(css);
+       struct obj_cgroup *objcg;
 
-       if (memcg_online_kmem(memcg))
-               goto remove_id;
+       memcg_online_kmem(memcg);
 
        /*
         * A memcg must be visible for expand_shrinker_info()
@@ -3942,6 +3923,15 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
        if (alloc_shrinker_info(memcg))
                goto offline_kmem;
 
+       objcg = obj_cgroup_alloc();
+       if (!objcg)
+               goto free_shrinker;
+
+       objcg->memcg = memcg;
+       rcu_assign_pointer(memcg->objcg, objcg);
+       obj_cgroup_get(objcg);
+       memcg->orig_objcg = objcg;
+
        if (unlikely(mem_cgroup_is_root(memcg)) && !mem_cgroup_disabled())
                queue_delayed_work(system_dfl_wq, &stats_flush_dwork,
                                   FLUSH_TIME);
@@ -3964,9 +3954,10 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
        xa_store(&mem_cgroup_private_ids, memcg->id.id, memcg, GFP_KERNEL);
 
        return 0;
+free_shrinker:
+       free_shrinker_info(memcg);
 offline_kmem:
        memcg_offline_kmem(memcg);
-remove_id:
        mem_cgroup_private_id_remove(memcg);
        return -ENOMEM;
 }
@@ -3984,6 +3975,12 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
 
        memcg_offline_kmem(memcg);
        reparent_deferred_split_queue(memcg);
+       /*
+        * The reparenting of objcg must be after the reparenting of the
+        * list_lru and deferred_split_queue above, which ensures that they will
+        * not mistakenly get the parent list_lru and deferred_split_queue.
+        */
+       memcg_reparent_objcgs(memcg);
        reparent_shrinker_deferred(memcg);
        wb_memcg_offline(memcg);
        lru_gen_offline_memcg(memcg);