mm/slab: remove __GFP_NO_OBJ_EXT usage from alloc_slab_obj_exts()

__GFP_NO_OBJ_EXT has limited scope within the slab allocator itself and
gfp flags are a scarce resource, unlike slab's alloc_flags.

Introduce SLAB_ALLOC_NO_RECURSE alloc flag that has the same intent as
__GFP_NO_OBJ_EXT but a more generic name, meaning that a kmalloc()
family function should not recurse into another kmalloc*() for the
purposes of allocating auxiliary structures (obj_ext arrays or sheaves).

First, replace the __GFP_NO_OBJ_EXT for allocating obj_ext arrays in
alloc_slab_obj_exts(). Make use of the newly added kmalloc_flags()
function, where we can pass alloc_flags with SLAB_ALLOC_NO_RECURSE
added. This will also pass through SLAB_ALLOC_NOLOCK so we don't need
to special case kmalloc_nolock() anymore.

Note that until now the kmalloc_nolock() ignored the incoming gfp flags
and hardcoded __GFP_ZERO | __GFP_NO_OBJ_EXT. But it's correct to pass on
the incoming gfp flags (only augmented with __GFP_ZERO), because if
alloc_flags contain SLAB_ALLOC_NOLOCK, the incoming gfp flags have to
be also compatible with it. However, we might have added __GFP_THISNODE
for opportunistic slab allocation, as pointed out by Hao Li, and
__GFP_COMP by allocate_slab() as pointed out by Shengming Hu. Solve this
by adding both flags to OBJCGS_CLEAR_MASK as it makes sense to strip
them anyway for non-kmalloc_nolock() allocations of sheaves or obj_ext
arrays as well.

To avoid recursion of sheaf -> obj_ext -> sheaf -> ... allocations at
this patch, until the next patch converts sheaves to
SLAB_ALLOC_NO_RECURSE, use both gfp and alloc_flags for obj_ext. The
next patch will remove the gfp part.

Link: https://patch.msgid.link/20260610-slab_alloc_flags-v2-15-7190909db118@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

diff --git a/mm/slab.h b/mm/slab.h
index 482b8e0fe797b033f6adb1d22b64035fed14c88d..281a65233795d67f41a7cdfb1bd0b5227d98da87 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -21,6 +21,7 @@
 #define SLAB_ALLOC_DEFAULT     0x00 /* no flags */
 #define SLAB_ALLOC_NOLOCK      0x01 /* a kmalloc_nolock() allocation */
 #define SLAB_ALLOC_NEW_SLAB    0x02 /* a flag for alloc_slab_obj_exts() */
+#define SLAB_ALLOC_NO_RECURSE  0x04 /* prevent kmalloc() recursion */
 
 static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
 {

diff --git a/mm/slub.c b/mm/slub.c
index 383d39a22561c3e8322167b50fcd91e2d3cf4b74..dc4b4ae874ce77921aed5a178b332830e8c6e3f1 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2047,12 +2047,16 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
 #endif /* CONFIG_SLUB_DEBUG */
 
 /*
- * The allocated objcg pointers array is not accounted directly.
+ * The allocated objcg pointers array or sheaf is not accounted directly.
  * Moreover, it should not come from DMA buffer and is not readily
- * reclaimable. So those GFP bits should be masked off.
+ * reclaimable. Node restriction for the parent allocation also should
+ * not apply to the slab's internal objects, as well as __GFP_COMP used
+ * for new slab allocations.
+ * So those GFP bits should be masked off.
  */
 #define OBJCGS_CLEAR_MASK      (__GFP_DMA | __GFP_RECLAIMABLE | \
-                               __GFP_ACCOUNT | __GFP_NOFAIL)
+                               __GFP_ACCOUNT | __GFP_NOFAIL | \
+                               __GFP_THISNODE | __GFP_COMP)
 
 #ifdef CONFIG_SLAB_OBJ_EXT
 
@@ -2160,6 +2164,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 {
        const bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
        unsigned int objects = objs_per_slab(s, slab);
+       bool new_slab = alloc_flags & SLAB_ALLOC_NEW_SLAB;
        unsigned long new_exts;
        unsigned long old_exts;
        struct slabobj_ext *vec;
@@ -2168,14 +2173,13 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
        gfp &= ~OBJCGS_CLEAR_MASK;
        /* Prevent recursive extension vector allocation */
        gfp |= __GFP_NO_OBJ_EXT;
+       alloc_flags |= SLAB_ALLOC_NO_RECURSE;
+       alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
 
        sz = obj_exts_alloc_size(s, slab, gfp);
 
-       if (unlikely(!allow_spin))
-               vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT,
-                                    slab_nid(slab));
-       else
-               vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab));
+       /* This will use kmalloc_nolock() if alloc_flags say so */
+       vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));
 
        if (!vec) {
                /*
@@ -2201,7 +2205,7 @@ retry:
        old_exts = READ_ONCE(slab->obj_exts);
        handle_failed_objexts_alloc(old_exts, vec, objects);
 
-       if (alloc_flags & SLAB_ALLOC_NEW_SLAB) {
+       if (new_slab) {
                /*
                 * If the slab is brand new and nobody can yet access its
                 * obj_exts, no synchronization is required and obj_exts can
@@ -2251,7 +2255,7 @@ static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)
        }
 
        /*
-        * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its
+        * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its
         * corresponding extension will be NULL. alloc_tag_sub() will throw a
         * warning if slab has extensions but the extension of an object is
         * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
@@ -2374,7 +2378,7 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
        if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
                return;
 
-       if (flags & __GFP_NO_OBJ_EXT)
+       if (alloc_flags & SLAB_ALLOC_NO_RECURSE || flags & __GFP_NO_OBJ_EXT)
                return;
 
        slab = virt_to_slab(object);