mm/slab: introduce alloc_flags and SLAB_ALLOC_NOLOCK

Similarly to the page allocators, introduce slab-allocator specific
alloc flags that internally control allocation behavior in addition to
gfp_flags, without occupying the limited gfp flags space.

Introduce the first flag SLAB_ALLOC_NOLOCK that behaves similarly to
page allocator's ALLOC_TRYLOCK and will be used to reimplement
kmalloc_nolock()'s "!allow_spin" behavior. That currently relies on
gfpflags_allow_spinning() and thus the lack of both __GFP_RECLAIM flags,
importantly __GFP_KSWAPD_RECLAIM. This can give false-positive results
e.g. in early boot with a restricted gfp_allowed_mask.

Also introduce alloc_flags_allow_spinning() to replace the usage of
gfpflags_allow_spinning().

Start using alloc_flags and the new check first in alloc_from_pcs() and
__pcs_replace_empty_main(). This means some slab allocations that were
falsely treated as kmalloc_nolock() due to their gfp flags will now have
higher chances of success, and this will further increase with followup
changes.

Remove a WARN_ON_ONCE() from refill_objects() as it's now legitimate to
reach it from a slab allocation that's not _nolock() and yet lacks
__GFP_KSWAPD_RECLAIM for other reasons.

Link: https://patch.msgid.link/20260610-slab_alloc_flags-v2-5-7190909db118@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

diff --git a/mm/slab.h b/mm/slab.h
index 1bf9c3021ae3dd176b38e72775ddb070431dfef4..f1246f0c9f74157049d1e004b2294b6450394032 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -16,6 +16,15 @@
  * Internal slab definitions
  */
 
+/* slab's alloc_flags definitions */
+#define SLAB_ALLOC_DEFAULT     0x00 /* no flags */
+#define SLAB_ALLOC_NOLOCK      0x01 /* a kmalloc_nolock() allocation */
+
+static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
+{
+       return !(alloc_flags & SLAB_ALLOC_NOLOCK);
+}
+
 #ifdef CONFIG_64BIT
 # ifdef system_has_cmpxchg128
 # define system_has_freelist_aba()     system_has_cmpxchg128()

diff --git a/mm/slub.c b/mm/slub.c
index 8a0c5553876e60f5f719b7f87498b47fccd5d94f..c53592baa0272bd2e763b9d4fe67072b3df75b36 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4641,7 +4641,8 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
  * unlocked.
  */
 static struct slub_percpu_sheaves *
-__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp)
+__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
+                        gfp_t gfp, unsigned int alloc_flags)
 {
        struct slab_sheaf *empty = NULL;
        struct slab_sheaf *full;
@@ -4667,7 +4668,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
                return NULL;
        }
 
-       allow_spin = gfpflags_allow_spinning(gfp);
+       allow_spin = alloc_flags_allow_spinning(alloc_flags);
 
        full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);
 
@@ -4753,7 +4754,7 @@ barn_put:
 }
 
 static __fastpath_inline
-void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
+void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node)
 {
        struct slub_percpu_sheaves *pcs;
        bool node_requested;
@@ -4798,7 +4799,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
        pcs = this_cpu_ptr(s->cpu_sheaves);
 
        if (unlikely(pcs->main->size == 0)) {
-               pcs = __pcs_replace_empty_main(s, pcs, gfp);
+               pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags);
                if (unlikely(!pcs))
                        return NULL;
        }
@@ -4931,7 +4932,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
        if (unlikely(object))
                goto out;
 
-       object = alloc_from_pcs(s, gfpflags, node);
+       object = alloc_from_pcs(s, gfpflags, SLAB_ALLOC_DEFAULT, node);
 
        if (unlikely(!object)) {
                const struct slab_alloc_context ac = {
@@ -5362,6 +5363,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in
 {
        gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags;
        size_t orig_size = size;
+       unsigned int alloc_flags = SLAB_ALLOC_NOLOCK;
        struct kmem_cache *s;
        bool can_retry = true;
        void *ret;
@@ -5404,7 +5406,7 @@ retry:
                 */
                return NULL;
 
-       ret = alloc_from_pcs(s, alloc_gfp, node);
+       ret = alloc_from_pcs(s, alloc_gfp, alloc_flags, node);
        if (ret)
                goto success;
 
@@ -7218,9 +7220,6 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
        unsigned int refilled;
        struct slab *slab;
 
-       if (WARN_ON_ONCE(!gfpflags_allow_spinning(gfp)))
-               return 0;
-
        refilled = __refill_objects_node(s, p, gfp, min, max,
                                         get_node(s, local_node),
                                         /* allow_spin = */ true);