mm: list_lru: introduce folio_memcg_list_lru_alloc()

memcg_list_lru_alloc() is called every time an object that may end up on
the list_lru is created.  It needs to quickly check if the list_lru heads
for the memcg already exist, and allocate them when they don't.

Doing this with folio objects is tricky: folio_memcg() is not stable and
requires either RCU protection or pinning the cgroup.  But it's desirable
to make the existence check lightweight under RCU, and only pin the memcg
when we need to allocate list_lru heads and may block.

In preparation for switching the THP shrinker to list_lru, add a helper
function for allocating list_lru heads coming from a folio.

Link: https://lore.kernel.org/20260527204757.2544958-8-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: David Hildenbrand (Arm) <david@kernel.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Reviewed-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Kairui Song <ryncsn@gmail.com>
Cc: Lance Yang <lance.yang@linux.dev>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Mikhail Zaslonko <zaslonko@linux.ibm.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nico Pache <npache@redhat.com>
Cc: Roman Gushchin <roman.gushchin@linux.dev>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Usama Arif <usama.arif@linux.dev>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h
index 134cb3e5652a22aff398a534561cc1ec0f9daa50..a450fffe1550be7016997ae90854a90346d8a31b 100644 (file)
--- a/include/linux/list_lru.h
+++ b/include/linux/list_lru.h
@@ -81,6 +81,33 @@ static inline int list_lru_init_memcg_key(struct list_lru *lru, struct shrinker
 
 int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
                         gfp_t gfp);
+
+#ifdef CONFIG_MEMCG
+/**
+ * folio_memcg_list_lru_alloc - allocate list_lru heads for shrinkable folio
+ * @folio: the newly allocated & charged folio
+ * @lru: the list_lru this might be queued on
+ * @gfp: gfp mask
+ *
+ * Allocate list_lru heads (per-memcg, per-node) needed to queue this
+ * particular folio down the line.
+ *
+ * This does memcg_list_lru_alloc(), but on the memcg that @folio is
+ * associated with. Handles folio_memcg() access rules in the fast
+ * path (list_lru heads allocated) and the allocation slowpath.
+ *
+ * Returns 0 on success, a negative error value otherwise.
+ */
+int folio_memcg_list_lru_alloc(struct folio *folio, struct list_lru *lru,
+                              gfp_t gfp);
+#else
+static inline int folio_memcg_list_lru_alloc(struct folio *folio,
+                                            struct list_lru *lru, gfp_t gfp)
+{
+       return 0;
+}
+#endif
+
 void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent);
 
 /**

diff --git a/mm/list_lru.c b/mm/list_lru.c
index 402bb028114d96b72965124993c16f9278824119..41a8119660639ba12cd8596350529a025610f773 100644 (file)
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -568,17 +568,14 @@ static inline bool memcg_list_lru_allocated(struct mem_cgroup *memcg,
        return idx < 0 || xa_load(&lru->xa, idx);
 }
 
-int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
-                        gfp_t gfp)
+static int __memcg_list_lru_alloc(struct mem_cgroup *memcg,
+                                 struct list_lru *lru, gfp_t gfp)
 {
        unsigned long flags;
        struct list_lru_memcg *mlru = NULL;
        struct mem_cgroup *pos, *parent;
        XA_STATE(xas, &lru->xa, 0);
 
-       if (!list_lru_memcg_aware(lru) || memcg_list_lru_allocated(memcg, lru))
-               return 0;
-
        gfp &= GFP_RECLAIM_MASK;
        /*
         * Because the list_lru can be reparented to the parent cgroup's
@@ -619,6 +616,38 @@ int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
 
        return xas_error(&xas);
 }
+
+int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru,
+                        gfp_t gfp)
+{
+       if (!list_lru_memcg_aware(lru) || memcg_list_lru_allocated(memcg, lru))
+               return 0;
+       return __memcg_list_lru_alloc(memcg, lru, gfp);
+}
+
+int folio_memcg_list_lru_alloc(struct folio *folio, struct list_lru *lru,
+                              gfp_t gfp)
+{
+       struct mem_cgroup *memcg;
+       int res;
+
+       if (!list_lru_memcg_aware(lru))
+               return 0;
+
+       /* Fast path when list_lru heads already exist */
+       rcu_read_lock();
+       memcg = folio_memcg(folio);
+       res = memcg_list_lru_allocated(memcg, lru);
+       rcu_read_unlock();
+       if (likely(res))
+               return 0;
+
+       /* Allocation may block, pin the memcg */
+       memcg = get_mem_cgroup_from_folio(folio);
+       res = __memcg_list_lru_alloc(memcg, lru, gfp);
+       mem_cgroup_put(memcg);
+       return res;
+}
 #else
 static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware)
 {