mm/slab: improve kmem_cache_alloc_bulk

The kmem_cache_alloc_bulk return value is weird.  It returns the number
of allocated objects, but that must always be 0 or the requested number
based on the implementations and the handling in the callers, but that
assumption is not actually documented anywhere, which confuses automated
review tools.

Fix this by returning a bool if the allocation succeeded and adding a
kerneldoc comment explaining the API.

[rob.clark@oss.qualcomm.com: fixups in
 msm_iommu_pagetable_prealloc_allocate() ]

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Alexander Lobakin <aleksander.lobakin@intel.com> # skbuff
Link: https://patch.msgid.link/20260528093437.2519248-2-hch@lst.de
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index 7d449e5202c5d6fc70262c151bb2a92feb0658d3..35dd43e3ef436fb6b899009da0c8043a399f4ebc 100644 (file)
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -330,17 +330,20 @@ static int
 msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p)
 {
        struct kmem_cache *pt_cache = get_pt_cache(mmu);
-       int ret;
+
+       if (!p->count) {
+               p->pages = NULL;
+               return 0;
+       }
 
        p->pages = kvmalloc_objs(*p->pages, p->count);
        if (!p->pages)
                return -ENOMEM;
 
-       ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages);
-       if (ret != p->count) {
-               kfree(p->pages);
+       if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages)) {
+               kvfree(p->pages);
                p->pages = NULL;
-               p->count = ret;
+               p->count = 0;
                return -ENOMEM;
        }
 

diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index 75d98dad7b1dd107c447219b76282fdde9014451..b12c641af46c89a66d2991b895fe764dc0a8285b 100644 (file)
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -1274,13 +1274,13 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
                goto err_cleanup;
        }
 
-       ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
-                                   op_ctx->rsvd_page_tables.pages);
-       op_ctx->rsvd_page_tables.count = ret;
-       if (ret != pt_count) {
+       if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+                       op_ctx->rsvd_page_tables.pages)) {
+               op_ctx->rsvd_page_tables.count = 0;
                ret = -ENOMEM;
                goto err_cleanup;
        }
+       op_ctx->rsvd_page_tables.count = pt_count;
 
        /* Insert BO into the extobj list last, when we know nothing can fail. */
        dma_resv_lock(panthor_vm_resv(vm), NULL);
@@ -1328,9 +1328,8 @@ static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
                        goto err_cleanup;
                }
 
-               ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
-                                           op_ctx->rsvd_page_tables.pages);
-               if (ret != pt_count) {
+               if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+                               op_ctx->rsvd_page_tables.pages)) {
                        ret = -ENOMEM;
                        goto err_cleanup;
                }

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 15a60b501b95b6c3f5b7099f4aab67689be312ab..24b244e63ba93c0af726cd2570f41d7c94fd5241 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -815,8 +815,10 @@ kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
  */
 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
 
-int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
-#define kmem_cache_alloc_bulk(...)     alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
+bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
+               size_t size, void **p);
+#define kmem_cache_alloc_bulk(...) \
+       alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
 
 static __always_inline void kfree_bulk(size_t size, void **p)
 {

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4ed998d60c09cb0706d23ce5e9a5c9da5e282607..b46d038400ff699419fb18088b40c036f6af1a15 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -966,29 +966,24 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
 {
        gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO;
        void *reqs[IO_REQ_ALLOC_BATCH];
-       int ret;
-
-       ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
+       int nr_reqs = ARRAY_SIZE(reqs);
 
        /*
-        * Bulk alloc is all-or-nothing. If we fail to get a batch,
-        * retry single alloc to be on the safe side.
+        * Bulk alloc is all-or-nothing. If we fail to get a batch, retry a
+        * single allocation to be on the safe side.
         */
-       if (unlikely(ret <= 0)) {
+       if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) {
                reqs[0] = kmem_cache_alloc(req_cachep, gfp);
                if (!reqs[0])
                        return false;
-               ret = 1;
+               nr_reqs = 1;
        }
 
-       percpu_ref_get_many(&ctx->refs, ret);
-       ctx->nr_req_allocated += ret;
-
-       while (ret--) {
-               struct io_kiocb *req = reqs[ret];
+       percpu_ref_get_many(&ctx->refs, nr_reqs);
+       ctx->nr_req_allocated += nr_reqs;
 
-               io_req_add_to_cache(req, ctx);
-       }
+       while (nr_reqs--)
+               io_req_add_to_cache(reqs[nr_reqs], ctx);
        return true;
 }
 

diff --git a/lib/test_meminit.c b/lib/test_meminit.c
index 6298f66c964bb10a41bc31bfdc7e6dff5b92339f..e106a0c0601a082cfb8e2df671a8c2c18cd89400 100644 (file)
--- a/lib/test_meminit.c
+++ b/lib/test_meminit.c
@@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor,
        for (iter = 0; iter < 10; iter++) {
                /* Do a test of bulk allocations */
                if (!want_rcu && !want_ctor) {
-                       int ret;
-
-                       ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array);
-                       if (!ret) {
+                       if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE,
+                                       bulk_array)) {
                                fail = true;
                        } else {
                                int i;
-                               for (i = 0; i < ret; i++)
+                               for (i = 0; i < BULK_SIZE; i++)
                                        fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero);
-                               kmem_cache_free_bulk(c, ret, bulk_array);
+                               kmem_cache_free_bulk(c, BULK_SIZE, bulk_array);
                        }
                }
 
@@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
 {
        struct kmem_cache *c;
        int i, iter, maxiter = 1024;
-       int num, bytes;
+       int bytes;
        bool fail = false;
        void *objects[10];
 
        c = kmem_cache_create("test_cache", size, size, 0, NULL);
        for (iter = 0; (iter < maxiter) && !fail; iter++) {
-               num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
-                                           objects);
-               for (i = 0; i < num; i++) {
+               if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
+                               objects))
+                       continue;
+
+               for (i = 0; i < ARRAY_SIZE(objects); i++) {
                        bytes = count_nonzero_bytes(objects[i], size);
                        if (bytes)
                                fail = true;
                        fill_with_garbage(objects[i], size);
                }
 
-               if (num)
-                       kmem_cache_free_bulk(c, num, objects);
+               kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects);
        }
        kmem_cache_destroy(c);
        *total_failures += fail;

diff --git a/mm/kasan/kasan_test_c.c b/mm/kasan/kasan_test_c.c
index 32d06cbf6a31069c92ba343ba0b59ba0ab787634..e41ba69592effcf9636106e120463248af9abeef 100644 (file)
--- a/mm/kasan/kasan_test_c.c
+++ b/mm/kasan/kasan_test_c.c
@@ -1215,14 +1215,13 @@ static void kmem_cache_bulk(struct kunit *test)
        struct kmem_cache *cache;
        size_t size = 200;
        char *p[10];
-       bool ret;
        int i;
 
        cache = kmem_cache_create("test_cache", size, 0, 0, NULL);
        KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache);
 
-       ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p);
-       if (!ret) {
+       if (!kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p),
+                       (void **)&p)) {
                kunit_err(test, "Allocation failed: %s\n", __func__);
                kmem_cache_destroy(cache);
                return;

diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c
index 5725a367246d9b52eb056a6de8de949ff7801f95..bac6f2aff10102d1b8eac7840feb643fe7dbe167 100644 (file)
--- a/mm/kfence/kfence_test.c
+++ b/mm/kfence/kfence_test.c
@@ -761,9 +761,10 @@ static void test_memcache_alloc_bulk(struct kunit *test)
        timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
        do {
                void *objects[100];
-               int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
-                                                  objects);
-               if (!num)
+               int i;
+
+               if (!kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC,
+                               ARRAY_SIZE(objects), objects))
                        continue;
                for (i = 0; i < ARRAY_SIZE(objects); i++) {
                        if (is_kfence_address(objects[i])) {
@@ -771,7 +772,7 @@ static void test_memcache_alloc_bulk(struct kunit *test)
                                break;
                        }
                }
-               kmem_cache_free_bulk(test_cache, num, objects);
+               kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
                /*
                 * kmem_cache_alloc_bulk() disables interrupts, and calling it
                 * in a tight loop may not give KFENCE a chance to switch the

diff --git a/mm/slub.c b/mm/slub.c
index 0baa906f39ab840fa79bf7f3744d5deac8891ca8..711df528c9a68b6d4b47f0f87b8b30a5d4bfe2dd 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -4980,8 +4980,8 @@ static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
        return ret;
 }
 
-static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
-                                  size_t size, void **p);
+static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
+               size_t size, void **p);
 
 /*
  * returns a sheaf that has at least the requested size
@@ -5153,9 +5153,8 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
                        return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
 
                if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
-                                            &sheaf->objects[sheaf->size])) {
+                                            &sheaf->objects[sheaf->size]))
                        return -ENOMEM;
-               }
                sheaf->size = sheaf->capacity;
 
                return 0;
@@ -7272,9 +7271,8 @@ out:
        return refilled;
 }
 
-static inline
-int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
-                           void **p)
+static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
+               size_t size, void **p)
 {
        int i;
 
@@ -7295,30 +7293,43 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                stat_add(s, ALLOC_SLOWPATH, i);
        }
 
-       return i;
+       return true;
 
 error:
        __kmem_cache_free_bulk(s, i, p);
-       return 0;
-
+       return false;
 }
 
-/*
- * Note that interrupts must be enabled when calling this function and gfp
- * flags must allow spinning.
+/**
+ * kmem_cache_alloc_bulk - Allocate multiple objects
+ * @s:         The cache to allocate from
+ * @flags:     GFP_* flags. See kmalloc().
+ * @size:      Number of objects to allocate
+ * @p:         Array of allocated objects
+ *
+ * Allocate @size objects from @s and places them into @p.  @size must be larger
+ * than 0.
+ *
+ * Interrupts must be enabled when calling this function and @flags must allow
+ * spinning.
+ *
+ * Unlike alloc_pages_bulk(), this function does not check for already allocated
+ * objects in @p, and thus the caller does not need to zero it.
+ *
+ * Return: %true if the allocation succeeded, or %false if it failed.
  */
-int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
-                                void **p)
+bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
+               size_t size, void **p)
 {
        unsigned int i = 0;
        void *kfence_obj;
 
        if (!size)
-               return 0;
+               return false;
 
        s = slab_pre_alloc_hook(s, flags);
        if (unlikely(!s))
-               return 0;
+               return false;
 
        /*
         * to make things simpler, only assume at most once kfence allocated
@@ -7335,18 +7346,18 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
        }
 
        i = alloc_from_pcs_bulk(s, flags, size, p);
-
        if (i < size) {
                /*
                 * If we ran out of memory, don't bother with freeing back to
                 * the percpu sheaves, we have bigger problems.
                 */
-               if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
+               if (unlikely(!__kmem_cache_alloc_bulk(s, flags, size - i,
+                               p + i))) {
                        if (i > 0)
                                __kmem_cache_free_bulk(s, i, p);
                        if (kfence_obj)
                                __kfence_free(kfence_obj);
-                       return 0;
+                       return false;
                }
        }
 
@@ -7361,16 +7372,9 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
        }
 
 out:
-       /*
-        * memcg and kmem_cache debug support and memory initialization.
-        * Done outside of the IRQ disabled fastpath loop.
-        */
-       if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p,
-                   slab_want_init_on_alloc(flags, s), s->object_size))) {
-               return 0;
-       }
-
-       return size;
+       /* memcg and kmem_cache debug support and memory initialization */
+       return likely(slab_post_alloc_hook(s, NULL, flags, size, p,
+                       slab_want_init_on_alloc(flags, s), s->object_size));
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
 

diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 2bc04feadfabedd4584431905c513af5e4882803..dbf0d8eae8d8966bd3ebaa06a7e017bd1ad4a95b 100644 (file)
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -243,12 +243,11 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
                           struct net_device *dev)
 {
        gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
-       int i, n;
+       int i;
        LIST_HEAD(list);
 
-       n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
-                                 (void **)skbs);
-       if (unlikely(n == 0)) {
+       if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp,
+                                           nframes, (void **)skbs))) {
                for (i = 0; i < nframes; i++)
                        xdp_return_frame(frames[i]);
                return -ENOMEM;

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 7dad68e3b5186cf622a3ed5a6e87c09d46bc3fd6..74f32c581403897a6fc2402be734cd9333ad5fb0 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -288,11 +288,11 @@ static inline struct sk_buff *napi_skb_cache_get(bool alloc)
 
        local_lock_nested_bh(&napi_alloc_cache.bh_lock);
        if (unlikely(!nc->skb_count)) {
-               if (alloc)
-                       nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
-                                               GFP_ATOMIC | __GFP_NOWARN,
-                                               NAPI_SKB_CACHE_BULK,
-                                               nc->skb_cache);
+               if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
+                                                  GFP_ATOMIC | __GFP_NOWARN,
+                                                  NAPI_SKB_CACHE_BULK,
+                                                  nc->skb_cache))
+                       nc->skb_count = NAPI_SKB_CACHE_BULK;
                if (unlikely(!nc->skb_count)) {
                        local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
                        return NULL;
@@ -353,16 +353,18 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n)
 
        /* No enough cached skbs. Try refilling the cache first */
        bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK);
-       nc->skb_count += kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
-                                              GFP_ATOMIC | __GFP_NOWARN, bulk,
-                                              &nc->skb_cache[nc->skb_count]);
+       if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
+                                 GFP_ATOMIC | __GFP_NOWARN, bulk,
+                                 &nc->skb_cache[nc->skb_count]))
+               nc->skb_count += bulk;
        if (likely(nc->skb_count >= n))
                goto get;
 
        /* Still not enough. Bulk-allocate the missing part directly, zeroed */
-       n -= kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
-                                  GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
-                                  n - nc->skb_count, &skbs[nc->skb_count]);
+       if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
+                                 GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
+                                 n - nc->skb_count, &skbs[nc->skb_count]))
+               n = nc->skb_count;
        if (likely(nc->skb_count >= n))
                goto get;
 

diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h
index 6d8e9413d5a4d73b3d4d381cdd543556ce70071d..2e63c2e726aaf90840027f27089d4d7bba9094ac 100644 (file)
--- a/tools/include/linux/slab.h
+++ b/tools/include/linux/slab.h
@@ -183,7 +183,7 @@ __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
                default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
 
 void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
-int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
+bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
                          void **list);
 struct slab_sheaf *
 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size);

diff --git a/tools/testing/shared/linux.c b/tools/testing/shared/linux.c
index 8c72571559583759456c2b469a2abc2611117c13..e0a0693df08f5bf9273ec915aa22141e01937d26 100644 (file)
--- a/tools/testing/shared/linux.c
+++ b/tools/testing/shared/linux.c
@@ -154,7 +154,7 @@ void kmem_cache_shrink(struct kmem_cache *cachep)
 {
 }
 
-int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
+bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
                          void **p)
 {
        size_t i;
@@ -213,7 +213,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
                pthread_mutex_unlock(&cachep->lock);
                if (cachep->callback)
                        cachep->exec_callback = true;
-               return 0;
+               return false;
        }
 
        for (i = 0; i < size; i++) {
@@ -224,7 +224,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
                        printf("Allocating %p from slab\n", p[i]);
        }
 
-       return size;
+       return true;
 }
 
 struct kmem_cache *
@@ -271,8 +271,8 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
 
        sheaf->cache = s;
        sheaf->capacity = capacity;
-       sheaf->size = kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects);
-       if (!sheaf->size) {
+       sheaf->size = size;
+       if (!kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects)) {
                free(sheaf);
                return NULL;
        }
@@ -284,7 +284,6 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
                 struct slab_sheaf **sheafp, unsigned int size)
 {
        struct slab_sheaf *sheaf = *sheafp;
-       int refill;
 
        if (sheaf->size >= size)
                return 0;
@@ -299,12 +298,10 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
                return 0;
        }
 
-       refill = kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
-                                      &sheaf->objects[sheaf->size]);
-       if (!refill)
+       if (!kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
+                       &sheaf->objects[sheaf->size]))
                return -ENOMEM;
-
-       sheaf->size += refill;
+       sheaf->size = size;
        return 0;
 }