slab: handle kmalloc sheaves bootstrap

Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
we can simply allow them in calculate_sizes() as they are created later
than KMALLOC_NORMAL caches and can allocate sheaves and barns from
those.

For KMALLOC_NORMAL caches we perform additional step after first
creating them without sheaves. Then bootstrap_cache_sheaves() simply
allocates and initializes barns and sheaves and finally sets
s->sheaf_capacity to make them actually used.

Afterwards the only caches left without sheaves (unless SLUB_TINY or
debugging is enabled) are kmem_cache and kmem_cache_node. These are only
used when creating or destroying other kmem_caches. Thus they are not
performance critical and we can simply leave it that way.

Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slub.c b/mm/slub.c
index 988433435779e187aff213dd3d803f32aaabdba8..cd8d3712b195733ac32b344c16d26f5f4c40cd16 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2593,7 +2593,8 @@ static void *setup_object(struct kmem_cache *s, void *object)
        return object;
 }
 
-static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
+static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
+                                             unsigned int capacity)
 {
        struct slab_sheaf *sheaf;
        size_t sheaf_size;
@@ -2611,7 +2612,7 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
        if (s->flags & SLAB_KMALLOC)
                gfp |= __GFP_NO_OBJ_EXT;
 
-       sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
+       sheaf_size = struct_size(sheaf, objects, capacity);
        sheaf = kzalloc(sheaf_size, gfp);
 
        if (unlikely(!sheaf))
@@ -2624,6 +2625,12 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
        return sheaf;
 }
 
+static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
+                                                  gfp_t gfp)
+{
+       return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);
+}
+
 static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
 {
        kfree(sheaf);
@@ -8157,8 +8164,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
        if (s->flags & SLAB_RECLAIM_ACCOUNT)
                s->allocflags |= __GFP_RECLAIMABLE;
 
-       /* kmalloc caches need extra care to support sheaves */
-       if (!is_kmalloc_cache(s))
+       /*
+        * For KMALLOC_NORMAL caches we enable sheaves later by
+        * bootstrap_kmalloc_sheaves() to avoid recursion
+        */
+       if (!is_kmalloc_normal(s))
                s->sheaf_capacity = calculate_sheaf_capacity(s, args);
 
        /*
@@ -8653,6 +8663,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
        return s;
 }
 
+/*
+ * Finish the sheaves initialization done normally by init_percpu_sheaves() and
+ * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
+ * since sheaves and barns are allocated by kmalloc.
+ */
+static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
+{
+       struct kmem_cache_args empty_args = {};
+       unsigned int capacity;
+       bool failed = false;
+       int node, cpu;
+
+       capacity = calculate_sheaf_capacity(s, &empty_args);
+
+       /* capacity can be 0 due to debugging or SLUB_TINY */
+       if (!capacity)
+               return;
+
+       for_each_node_mask(node, slab_nodes) {
+               struct node_barn *barn;
+
+               barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
+
+               if (!barn) {
+                       failed = true;
+                       goto out;
+               }
+
+               barn_init(barn);
+               get_node(s, node)->barn = barn;
+       }
+
+       for_each_possible_cpu(cpu) {
+               struct slub_percpu_sheaves *pcs;
+
+               pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
+
+               pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
+
+               if (!pcs->main) {
+                       failed = true;
+                       break;
+               }
+       }
+
+out:
+       /*
+        * It's still early in boot so treat this like same as a failure to
+        * create the kmalloc cache in the first place
+        */
+       if (failed)
+               panic("Out of memory when creating kmem_cache %s\n", s->name);
+
+       s->sheaf_capacity = capacity;
+}
+
+static void __init bootstrap_kmalloc_sheaves(void)
+{
+       enum kmalloc_cache_type type;
+
+       for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
+               for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
+                       if (kmalloc_caches[type][idx])
+                               bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
+               }
+       }
+}
+
 void __init kmem_cache_init(void)
 {
        static __initdata struct kmem_cache boot_kmem_cache,
@@ -8696,6 +8774,8 @@ void __init kmem_cache_init(void)
        setup_kmalloc_cache_index_table();
        create_kmalloc_caches();
 
+       bootstrap_kmalloc_sheaves();
+
        /* Setup random freelists for each cache */
        init_freelist_randomization();