From: Vlastimil Babka (SUSE) Date: Fri, 22 May 2026 14:23:20 +0000 (+0200) Subject: mm, slab: add an optimistic __slab_try_return_freelist() X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ba742531260782a2646bc031f9a12cafebc22594;p=thirdparty%2Flinux.git mm, slab: add an optimistic __slab_try_return_freelist() When we end up returning extraneous objects during refill to a slab where we just did a get_freelist_nofreeze(), it is likely no other CPU has freed objects to it meanwhile. We can then reattach the remainder of the freelist without having to walk the (potentially cache cold) freelist for finding its tail to connect slab->freelist to it. Add a __slab_try_return_freelist() function that does that. As suggested by Hao Li, it doesn't need to also return the slab to the partial list, because there's code in __refill_objects_node() that already does that for any slabs where we don't detach the freelist in the first place. So we just put the slab back to the pc.slabs list. It's no longer likely that the list will be empty now, so remove the unlikely() annotation. However, also change that code to add to the tail of the partial list instead of head to match what __slab_free() did and avoid a regression, that was reported for the earlier version by the kernel test robot [1]. This change will also affect slabs which were grabbed from the partial list and not refilled from even partially, but those should be much more rare than a partial refill. [1] https://lore.kernel.org/all/202605112204.9382cecf-lkp@intel.com/ Reviewed-by: Hao Li Tested-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260522-b4-refill-optimistic-return-v3-1-2ba78ec1c6ed@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- diff --git a/mm/slub.c b/mm/slub.c index a7bd929bcb262..60a765b03b2da 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4326,7 +4326,8 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags) * Assumes this is performed only for caches without debugging so we * don't need to worry about adding the slab to the full list. */ -static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab) +static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab, + unsigned int *count) { struct freelist_counters old, new; @@ -4342,6 +4343,7 @@ static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *sla } while (!slab_update_freelist(s, slab, &old, &new, "get_freelist_nofreeze")); + *count = old.objects - old.inuse; return old.freelist; } @@ -5509,6 +5511,34 @@ static noinline void free_to_partial_list( } } +/* + * Try returning (remainder of) the freelist that we just detached from the + * slab. Optimistically assume the slab is still full, so we don't need to find + * the tail of the detached freelist. + * + * Fail if the slab isn't full anymore due to a concurrent free. + */ +static bool __slab_try_return_freelist(struct kmem_cache *s, struct slab *slab, + void *head, int cnt) +{ + struct freelist_counters old, new; + + old.freelist = slab->freelist; + old.counters = slab->counters; + + if (old.freelist) + return false; + + new.freelist = head; + new.counters = old.counters; + new.inuse -= cnt; + + if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist")) + return false; + + return true; +} + /* * Slow path handling. This may still be called frequently since objects * have a longer lifetime than the cpu slabs in most processing loads. @@ -7120,41 +7150,48 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) { + unsigned int count; + list_del(&slab->slab_list); - object = get_freelist_nofreeze(s, slab); + object = get_freelist_nofreeze(s, slab, &count); - while (object && refilled < max) { + while (count && refilled < max) { p[refilled] = object; object = get_freepointer(s, object); maybe_wipe_obj_freeptr(s, p[refilled]); refilled++; + count--; } /* * Freelist had more objects than we can accommodate, we need to - * free them back. We can treat it like a detached freelist, just - * need to find the tail object. + * free them back. First we try to be optimistic and assume the + * slab is still full since we just detached its freelist. + * Otherwise we must find the tail object. */ - if (unlikely(object)) { + if (unlikely(count)) { void *head = object; void *tail; - int cnt = 0; + + if (__slab_try_return_freelist(s, slab, head, count)) { + list_add(&slab->slab_list, &pc.slabs); + break; + } do { tail = object; - cnt++; object = get_freepointer(s, object); } while (object); - __slab_free(s, slab, head, tail, cnt, _RET_IP_); + __slab_free(s, slab, head, tail, count, _RET_IP_); } if (refilled >= max) break; } - if (unlikely(!list_empty(&pc.slabs))) { + if (!list_empty(&pc.slabs)) { spin_lock_irqsave(&n->list_lock, flags); list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) { @@ -7163,7 +7200,7 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi continue; list_del(&slab->slab_list); - add_partial(n, slab, ADD_TO_HEAD); + add_partial(n, slab, ADD_TO_TAIL); } spin_unlock_irqrestore(&n->list_lock, flags);