mm/slab: pass alloc_flags through slab_post_alloc_hook() chain

Convert the whole following call stack to pass either slab_alloc_context
(thus including alloc_flags) or just alloc_flags as necessary:

slab_post_alloc_hook()
  alloc_tagging_slab_alloc_hook()
    __alloc_tagging_slab_alloc_hook()
      prepare_slab_obj_exts_hook()
        alloc_slab_obj_exts()
  memcg_slab_post_alloc_hook()
    __memcg_slab_post_alloc_hook()
      alloc_slab_obj_exts()

Converting all these at once avoids unnecessary churn and is mostly
mechanical.

This ultimately allows to decide if spinning is allowed using
alloc_flags in alloc_slab_obj_exts(), as well as slab_post_alloc_hook().
Aside from alloc_from_pcs_bulk() (to be handled next) there is nothing
else in slab itself relying on gfpflags_allow_spinning() which can
be false even if not called from kmalloc_nolock().

A followup change will also use the alloc_flags availability in the call
stack above to remove the __GFP_NO_OBJ_EXT flag.

For alloc_slab_obj_exts(), also replace the suboptimal "bool new_slab"
parameter with a SLAB_ALLOC_NEW_SLAB flag with identical functionality.

To further reduce the number of parameters of slab_post_alloc_hook(),
also make 'struct list_lru *lru' (which is NULL for most callers) a new
field of slab_alloc_context.

Link: https://patch.msgid.link/20260610-slab_alloc_flags-v2-9-7190909db118@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c03d4787d466803db49cdaa90e6d6ba426b7afe2..29390ba13baa28dbd659a240a92c65a0642aeadf 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3424,7 +3424,8 @@ static inline size_t obj_full_size(struct kmem_cache *s)
 }
 
 bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
-                                 gfp_t flags, size_t size, void **p)
+                                 gfp_t flags, unsigned int slab_alloc_flags,
+                                 size_t size, void **p)
 {
        size_t obj_size = obj_full_size(s);
        struct obj_cgroup *objcg;
@@ -3472,7 +3473,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
                slab = virt_to_slab(p[i]);
 
                if (!slab_obj_exts(slab) &&
-                   alloc_slab_obj_exts(slab, s, flags, false)) {
+                   alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) {
                        continue;
                }
 

diff --git a/mm/slab.h b/mm/slab.h
index f1246f0c9f74157049d1e004b2294b6450394032..d86203131f588ef64c9bcdbebf966bf53a3da16e 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -19,6 +19,7 @@
 /* slab's alloc_flags definitions */
 #define SLAB_ALLOC_DEFAULT     0x00 /* no flags */
 #define SLAB_ALLOC_NOLOCK      0x01 /* a kmalloc_nolock() allocation */
+#define SLAB_ALLOC_NEW_SLAB    0x02 /* a flag for alloc_slab_obj_exts() */
 
 static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
 {
@@ -612,7 +613,7 @@ static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
 }
 
 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
-                        gfp_t gfp, bool new_slab);
+                       gfp_t gfp, unsigned int alloc_flags);
 
 #else /* CONFIG_SLAB_OBJ_EXT */
 
@@ -642,7 +643,8 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
 
 #ifdef CONFIG_MEMCG
 bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
-                                 gfp_t flags, size_t size, void **p);
+                                 gfp_t flags, unsigned int slab_alloc_flags,
+                                 size_t size, void **p);
 void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
                            void **p, int objects, unsigned long obj_exts);
 #endif

diff --git a/mm/slub.c b/mm/slub.c
index a975a2e727c815805c285280cb333dc8cc0d0786..465eb4db577052eddfedf7d2ea6cdec3f77a073a 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -218,6 +218,7 @@ struct slab_alloc_context {
        unsigned long caller_addr;
        size_t orig_size;
        unsigned int alloc_flags;
+       struct list_lru *lru;
 };
 
 /* Structure holding parameters for get_partial_node_bulk() */
@@ -2155,9 +2156,9 @@ static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
 }
 
 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
-                       gfp_t gfp, bool new_slab)
+                       gfp_t gfp, unsigned int alloc_flags)
 {
-       bool allow_spin = gfpflags_allow_spinning(gfp);
+       const bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
        unsigned int objects = objs_per_slab(s, slab);
        unsigned long new_exts;
        unsigned long old_exts;
@@ -2206,7 +2207,7 @@ retry:
        old_exts = READ_ONCE(slab->obj_exts);
        handle_failed_objexts_alloc(old_exts, vec, objects);
 
-       if (new_slab) {
+       if (alloc_flags & SLAB_ALLOC_NEW_SLAB) {
                /*
                 * If the slab is brand new and nobody can yet access its
                 * obj_exts, no synchronization is required and obj_exts can
@@ -2331,7 +2332,7 @@ static inline void init_slab_obj_exts(struct slab *slab)
 }
 
 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
-                              gfp_t gfp, bool new_slab)
+                              gfp_t gfp, unsigned int alloc_flags)
 {
        return 0;
 }
@@ -2351,10 +2352,10 @@ static inline void alloc_slab_obj_exts_early(struct kmem_cache *s,
 
 static inline unsigned long
 prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
-                          gfp_t flags, void *p)
+                          gfp_t flags, unsigned int alloc_flags, void *p)
 {
        if (!slab_obj_exts(slab) &&
-           alloc_slab_obj_exts(slab, s, flags, false)) {
+           alloc_slab_obj_exts(slab, s, flags, alloc_flags)) {
                pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
                             __func__, s->name);
                return 0;
@@ -2366,7 +2367,8 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
 
 /* Should be called only if mem_alloc_profiling_enabled() */
 static noinline void
-__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+                               unsigned int alloc_flags)
 {
        unsigned long obj_exts;
        struct slabobj_ext *obj_ext;
@@ -2382,7 +2384,7 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
                return;
 
        slab = virt_to_slab(object);
-       obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, object);
+       obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object);
        /*
         * Currently obj_exts is used only for allocation profiling.
         * If other users appear then mem_alloc_profiling_enabled()
@@ -2401,10 +2403,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
 }
 
 static inline void
-alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+                             unsigned int alloc_flags)
 {
        if (mem_alloc_profiling_enabled())
-               __alloc_tagging_slab_alloc_hook(s, object, flags);
+               __alloc_tagging_slab_alloc_hook(s, object, flags, alloc_flags);
 }
 
 /* Should be called only if mem_alloc_profiling_enabled() */
@@ -2443,7 +2446,8 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
 #else /* CONFIG_MEM_ALLOC_PROFILING */
 
 static inline void
-alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
+alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
+                             unsigned int alloc_flags)
 {
 }
 
@@ -2461,8 +2465,9 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
 static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
 
 static __fastpath_inline
-bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
-                               gfp_t flags, size_t size, void **p)
+bool memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
+                               size_t size, void **p,
+                               const struct slab_alloc_context *ac)
 {
        if (likely(!memcg_kmem_online()))
                return true;
@@ -2470,7 +2475,8 @@ bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
        if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
                return true;
 
-       if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p)))
+       if (likely(__memcg_slab_post_alloc_hook(s, ac->lru, flags,
+                                               ac->alloc_flags, size, p)))
                return true;
 
        if (likely(size == 1)) {
@@ -2558,14 +2564,15 @@ bool memcg_slab_post_charge(void *p, gfp_t flags)
                put_slab_obj_exts(obj_exts);
        }
 
-       return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p);
+       return __memcg_slab_post_alloc_hook(s, NULL, flags, SLAB_ALLOC_DEFAULT,
+                                           1, &p);
 }
 
 #else /* CONFIG_MEMCG */
 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
-                                             struct list_lru *lru,
-                                             gfp_t flags, size_t size,
-                                             void **p)
+                                             gfp_t flags,
+                                             size_t size, void **p,
+                                             const struct slab_alloc_context *ac)
 {
        return true;
 }
@@ -3352,12 +3359,14 @@ static inline void init_freelist_randomization(void) { }
 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
 
 static __always_inline void account_slab(struct slab *slab, int order,
-                                        struct kmem_cache *s, gfp_t gfp)
+                                        struct kmem_cache *s, gfp_t gfp,
+                                        unsigned int alloc_flags)
 {
        if (memcg_kmem_online() &&
                        (s->flags & SLAB_ACCOUNT) &&
                        !slab_obj_exts(slab))
-               alloc_slab_obj_exts(slab, s, gfp, true);
+               alloc_slab_obj_exts(slab, s, gfp,
+                                   alloc_flags | SLAB_ALLOC_NEW_SLAB);
 
        mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
                            PAGE_SIZE << order);
@@ -3430,7 +3439,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
         * to prevent the array from being overwritten.
         */
        alloc_slab_obj_exts_early(s, slab);
-       account_slab(slab, oo_order(oo), s, flags);
+       account_slab(slab, oo_order(oo), s, flags, alloc_flags);
 
        return slab;
 }
@@ -4564,9 +4573,8 @@ struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
 }
 
 static __fastpath_inline
-bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
-                         gfp_t flags, size_t size, void **p,
-                         unsigned int orig_size)
+bool slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size,
+                         void **p, const struct slab_alloc_context *ac)
 {
        bool init = slab_want_init_on_alloc(flags, s);
        unsigned int zero_size = s->object_size;
@@ -4585,7 +4593,7 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
         * orig_size if we track it.
         */
        if (slub_debug_orig_size(s))
-               zero_size = orig_size;
+               zero_size = ac->orig_size;
 
        /*
         * ARM64 can set memory tags and zero the memory using a single
@@ -4615,14 +4623,14 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
                if (init && p[i] && !is_kfence_address(p[i]))
                        memset(p[i], 0, zero_size);
 
-               if (gfpflags_allow_spinning(flags))
+               if (alloc_flags_allow_spinning(ac->alloc_flags))
                        kmemleak_alloc_recursive(p[i], s->object_size, 1,
                                                 s->flags, init_flags);
                kmsan_slab_alloc(s, p[i], init_flags);
-               alloc_tagging_slab_alloc_hook(s, p[i], flags);
+               alloc_tagging_slab_alloc_hook(s, p[i], flags, ac->alloc_flags);
        }
 
-       return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
+       return memcg_slab_post_alloc_hook(s, flags, size, p, ac);
 }
 
 /*
@@ -4917,6 +4925,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
 {
        const unsigned int alloc_flags = SLAB_ALLOC_DEFAULT;
        void *object;
+       const struct slab_alloc_context ac = {
+               .caller_addr = addr,
+               .orig_size = orig_size,
+               .alloc_flags = alloc_flags,
+               .lru = lru,
+       };
 
        s = slab_pre_alloc_hook(s, gfpflags);
        if (unlikely(!s))
@@ -4928,14 +4942,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
 
        object = alloc_from_pcs(s, gfpflags, alloc_flags, node);
 
-       if (unlikely(!object)) {
-               const struct slab_alloc_context ac = {
-                       .caller_addr = addr,
-                       .orig_size = orig_size,
-                       .alloc_flags = alloc_flags,
-               };
+       if (unlikely(!object))
                object = __slab_alloc_node(s, gfpflags, node, &ac);
-       }
 
        maybe_wipe_obj_freeptr(s, object);
 
@@ -4944,7 +4952,7 @@ out:
         * In case this fails due to memcg_slab_post_alloc_hook(),
         * object is set to NULL
         */
-       slab_post_alloc_hook(s, lru, gfpflags, 1, &object, orig_size);
+       slab_post_alloc_hook(s, gfpflags, 1, &object, &ac);
 
        return object;
 }
@@ -5239,6 +5247,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
                                   struct slab_sheaf *sheaf)
 {
        void *ret = NULL;
+       const struct slab_alloc_context ac = {
+               .orig_size = s->object_size,
+               .alloc_flags = SLAB_ALLOC_DEFAULT,
+       };
 
        if (sheaf->size == 0)
                goto out;
@@ -5249,7 +5261,7 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
                ret = sheaf->objects[--sheaf->size];
 
        /* add __GFP_NOFAIL to force successful memcg charging */
-       slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, s->object_size);
+       slab_post_alloc_hook(s, gfp | __GFP_NOFAIL, 1, &ret, &ac);
 out:
        trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE);
 
@@ -5435,7 +5447,7 @@ retry:
 
 success:
        maybe_wipe_obj_freeptr(s, ret);
-       slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, orig_size);
+       slab_post_alloc_hook(s, alloc_gfp, 1, &ret, &ac);
 
        ret = kasan_kmalloc(s, ret, orig_size, alloc_gfp);
        return ret;
@@ -7301,6 +7313,10 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
 {
        unsigned int i = 0;
        void *kfence_obj;
+       const struct slab_alloc_context ac = {
+               .orig_size = s->object_size,
+               .alloc_flags = SLAB_ALLOC_DEFAULT,
+       };
 
        if (!size)
                return false;
@@ -7351,7 +7367,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
 
 out:
        /* memcg and kmem_cache debug support and memory initialization */
-       return likely(slab_post_alloc_hook(s, NULL, flags, size, p, s->object_size));
+       return likely(slab_post_alloc_hook(s, flags, size, p, &ac));
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);