mm/slub: detach and reattach partial slabs in batch

get_partial_node_bulk() moves each selected slab from the node's
partial list to the local pc->slabs list using a remove_partial() and
list_add() pair. In practice, the loop often detaches several adjacent
slabs. Doing this individually repeatedly manipulates list pointers
while holding n->list_lock, which causes unnecessary churn.

To demonstrate this, the counts below show how often single vs. multiple
consecutive slabs are retrieved during a will-it-scale mmap stress test:

consecutive_slabs_count        frequency
= 1                            277345324
= 2                            335238023
= 3                            175717884
>= 4                           88862337

The data confirms that retrieving multiple contiguous slabs is highly
frequent.

To optimize this, track contiguous runs of matching slabs and move each
run in a single operation using list_bulk_move_tail(). This reduces list
pointer churn inside the lock critical section.

Apply the same optimization to __refill_objects_node() when reattaching
leftover partial slabs back to the node's partial list.

The will-it-scale mmap benchmark shows a 2% ~ 5% performance improvement
after applying this patch.

Signed-off-by: Hao Li <hao.li@linux.dev>
Link: https://patch.msgid.link/20260529035120.81304-3-hao.li@linux.dev
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>

diff --git a/mm/slub.c b/mm/slub.c
index f9a4da5363564f56be6e0bcdeb534b65f77015f2..492128ae3af95e7abc9abff294cfb2e9df043431 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3751,6 +3751,7 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
                                  bool allow_spin)
 {
        struct slab *slab, *slab2;
+       struct slab *first = NULL, *last = NULL;
        unsigned int total_free = 0;
        unsigned long flags;
 
@@ -3769,8 +3770,15 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
                struct freelist_counters flc;
                unsigned int slab_free;
 
-               if (!pfmemalloc_match(slab, pc->flags))
+               if (!pfmemalloc_match(slab, pc->flags)) {
+                       if (first) {
+                               list_bulk_move_tail(&pc->slabs,
+                                                   &first->slab_list,
+                                                   &last->slab_list);
+                               first = NULL;
+                       }
                        continue;
+               }
 
                /*
                 * determine the number of free objects in the slab racily
@@ -3787,15 +3795,20 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
                    && total_free + slab_free > pc->max_objects)
                        break;
 
-               remove_partial(n, slab);
-
-               list_add(&slab->slab_list, &pc->slabs);
+               if (!first)
+                       first = slab;
+               last = slab;
+               clear_node_partial_state(n, slab);
 
                total_free += slab_free;
                if (total_free >= pc->max_objects)
                        break;
        }
 
+       if (first)
+               list_bulk_move_tail(&pc->slabs, &first->slab_list,
+                                   &last->slab_list);
+
        spin_unlock_irqrestore(&n->list_lock, flags);
        return total_free > 0;
 }
@@ -7205,11 +7218,10 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi
        if (!list_empty(&pc.slabs)) {
                spin_lock_irqsave(&n->list_lock, flags);
 
-               list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {
+               list_for_each_entry(slab, &pc.slabs, slab_list)
+                       set_node_partial_state(n, slab);
 
-                       list_del(&slab->slab_list);
-                       add_partial(n, slab, ADD_TO_TAIL);
-               }
+               list_splice_tail(&pc.slabs, &n->partial);
 
                spin_unlock_irqrestore(&n->list_lock, flags);
        }