slab: move kfence_alloc() out of internal bulk alloc

SLUB's internal bulk allocation __kmem_cache_alloc_bulk() can currently
allocate some objects from KFENCE, i.e. when refilling a sheaf. It works
but it's conceptually the wrong layer, as KFENCE allocations should only
happen when objects are actually handed out from slab to its users.

Currently for sheaf-enabled caches, slab_alloc_node() can return KFENCE
object via kfence_alloc(), but also via alloc_from_pcs() when a sheaf
was refilled with KFENCE objects. Continuing like this would also
complicate the upcoming sheaf refill changes.

Thus remove KFENCE allocation from __kmem_cache_alloc_bulk() and move it
to the places that return slab objects to users. slab_alloc_node() is
already covered (see above). Add kfence_alloc() to
kmem_cache_alloc_from_sheaf() to handle KFENCE allocations from
prefilled sheafs, with a comment that the caller should not expect the
sheaf size to decrease after every allocation because of this
possibility.

For kmem_cache_alloc_bulk() implement a different strategy to handle
KFENCE upfront and rely on internal batched operations afterwards.
Assume there will be at most once KFENCE allocation per bulk allocation
and then assign its index in the array of objects randomly.

Cc: Alexander Potapenko <glider@google.com>
Cc: Marco Elver <elver@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Link: https://patch.msgid.link/20251105-sheaves-cleanups-v1-2-b8218e1ac7ef@suse.cz
Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slub.c b/mm/slub.c
index 074abe8e79f89bbd84677db226298b3efce22d01..0237a329d4e5639bc6bdcbe3b63ed09f0acd9b9a 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5540,6 +5540,9 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
  *
  * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT
  * memcg charging is forced over limit if necessary, to avoid failure.
+ *
+ * It is possible that the allocation comes from kfence and then the sheaf
+ * size is not decreased.
  */
 void *
 kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
@@ -5551,7 +5554,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
        if (sheaf->size == 0)
                goto out;
 
-       ret = sheaf->objects[--sheaf->size];
+       ret = kfence_alloc(s, s->object_size, gfp);
+
+       if (likely(!ret))
+               ret = sheaf->objects[--sheaf->size];
 
        init = slab_want_init_on_alloc(gfp, s);
 
@@ -7399,14 +7405,8 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        local_lock_irqsave(&s->cpu_slab->lock, irqflags);
 
        for (i = 0; i < size; i++) {
-               void *object = kfence_alloc(s, s->object_size, flags);
-
-               if (unlikely(object)) {
-                       p[i] = object;
-                       continue;
-               }
+               void *object = c->freelist;
 
-               object = c->freelist;
                if (unlikely(!object)) {
                        /*
                         * We may have removed an object from c->freelist using
@@ -7487,6 +7487,7 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
                                 void **p)
 {
        unsigned int i = 0;
+       void *kfence_obj;
 
        if (!size)
                return 0;
@@ -7495,6 +7496,20 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
        if (unlikely(!s))
                return 0;
 
+       /*
+        * to make things simpler, only assume at most once kfence allocated
+        * object per bulk allocation and choose its index randomly
+        */
+       kfence_obj = kfence_alloc(s, s->object_size, flags);
+
+       if (unlikely(kfence_obj)) {
+               if (unlikely(size == 1)) {
+                       p[0] = kfence_obj;
+                       goto out;
+               }
+               size--;
+       }
+
        if (s->cpu_sheaves)
                i = alloc_from_pcs_bulk(s, size, p);
 
@@ -7506,10 +7521,23 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
                if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
                        if (i > 0)
                                __kmem_cache_free_bulk(s, i, p);
+                       if (kfence_obj)
+                               __kfence_free(kfence_obj);
                        return 0;
                }
        }
 
+       if (unlikely(kfence_obj)) {
+               int idx = get_random_u32_below(size + 1);
+
+               if (idx != size)
+                       p[size] = p[idx];
+               p[idx] = kfence_obj;
+
+               size++;
+       }
+
+out:
        /*
         * memcg and kmem_cache debug support and memory initialization.
         * Done outside of the IRQ disabled fastpath loop.