slab: remove defer_deactivate_slab()

There are no more cpu slabs so we don't need their deferred
deactivation. The function is now only used from places where we
allocate a new slab but then can't spin on node list_lock to put it on
the partial list. Instead of the deferred action we can free it directly
via __free_slab(), we just need to tell it to use _nolock() freeing of
the underlying pages and take care of the accounting.

Since free_frozen_pages_nolock() variant does not yet exist for code
outside of the page allocator, create it as a trivial wrapper for
__free_frozen_pages(..., FPI_TRYLOCK).

Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/internal.h b/mm/internal.h
index e430da900430a15b9fa61dae6f9a431ba75599df..1f44ccb4badf4493e6e8a59d3a3480bcfd08872e 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -846,6 +846,7 @@ static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int ord
 struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
 #define alloc_frozen_pages_nolock(...) \
        alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))
+void free_frozen_pages_nolock(struct page *page, unsigned int order);
 
 extern void zone_pcp_reset(struct zone *zone);
 extern void zone_pcp_disable(struct zone *zone);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c380f063e8b7bd383186479fc8c2b7a51243278d..0127e9d661ad99d5239d84a614eb742120e6e50d 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2981,6 +2981,11 @@ void free_frozen_pages(struct page *page, unsigned int order)
        __free_frozen_pages(page, order, FPI_NONE);
 }
 
+void free_frozen_pages_nolock(struct page *page, unsigned int order)
+{
+       __free_frozen_pages(page, order, FPI_TRYLOCK);
+}
+
 /*
  * Free a batch of folios
  */

diff --git a/mm/slab.h b/mm/slab.h
index 565279a68fd060d4d4d6cee680be43cca9a690dd..f88568e31268e4a3c2d87d7275fd25ede1ef5480 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -71,13 +71,7 @@ struct slab {
        struct kmem_cache *slab_cache;
        union {
                struct {
-                       union {
-                               struct list_head slab_list;
-                               struct { /* For deferred deactivate_slab() */
-                                       struct llist_node llnode;
-                                       void *flush_freelist;
-                               };
-                       };
+                       struct list_head slab_list;
                        /* Double-word boundary */
                        struct freelist_counters;
                };

diff --git a/mm/slub.c b/mm/slub.c
index 4346ace499bb80b8ae0e05981410f6f7e17beded..883effab47fd395c09ddeb052db9dbe038209567 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3262,7 +3262,7 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
                flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
 }
 
-static void __free_slab(struct kmem_cache *s, struct slab *slab)
+static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)
 {
        struct page *page = slab_page(slab);
        int order = compound_order(page);
@@ -3273,14 +3273,26 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)
        __ClearPageSlab(page);
        mm_account_reclaimed_pages(pages);
        unaccount_slab(slab, order, s);
-       free_frozen_pages(page, order);
+       if (allow_spin)
+               free_frozen_pages(page, order);
+       else
+               free_frozen_pages_nolock(page, order);
+}
+
+static void free_new_slab_nolock(struct kmem_cache *s, struct slab *slab)
+{
+       /*
+        * Since it was just allocated, we can skip the actions in
+        * discard_slab() and free_slab().
+        */
+       __free_slab(s, slab, false);
 }
 
 static void rcu_free_slab(struct rcu_head *h)
 {
        struct slab *slab = container_of(h, struct slab, rcu_head);
 
-       __free_slab(slab->slab_cache, slab);
+       __free_slab(slab->slab_cache, slab, true);
 }
 
 static void free_slab(struct kmem_cache *s, struct slab *slab)
@@ -3296,7 +3308,7 @@ static void free_slab(struct kmem_cache *s, struct slab *slab)
        if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
                call_rcu(&slab->rcu_head, rcu_free_slab);
        else
-               __free_slab(s, slab);
+               __free_slab(s, slab, true);
 }
 
 static void discard_slab(struct kmem_cache *s, struct slab *slab)
@@ -3389,8 +3401,6 @@ static void *alloc_single_from_partial(struct kmem_cache *s,
        return object;
 }
 
-static void defer_deactivate_slab(struct slab *slab, void *flush_freelist);
-
 /*
  * Called only for kmem_cache_debug() caches to allocate from a freshly
  * allocated slab. Allocate a single object instead of whole freelist
@@ -3406,8 +3416,8 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
        void *object;
 
        if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) {
-               /* Unlucky, discard newly allocated slab */
-               defer_deactivate_slab(slab, NULL);
+               /* Unlucky, discard newly allocated slab. */
+               free_new_slab_nolock(s, slab);
                return NULL;
        }
 
@@ -4279,7 +4289,7 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,
 
                if (!spin_trylock_irqsave(&n->list_lock, flags)) {
                        /* Unlucky, discard newly allocated slab */
-                       defer_deactivate_slab(slab, NULL);
+                       free_new_slab_nolock(s, slab);
                        return 0;
                }
        }
@@ -6059,7 +6069,6 @@ flush_remote:
 
 struct defer_free {
        struct llist_head objects;
-       struct llist_head slabs;
        struct irq_work work;
 };
 
@@ -6067,23 +6076,21 @@ static void free_deferred_objects(struct irq_work *work);
 
 static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = {
        .objects = LLIST_HEAD_INIT(objects),
-       .slabs = LLIST_HEAD_INIT(slabs),
        .work = IRQ_WORK_INIT(free_deferred_objects),
 };
 
 /*
  * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe
- * to take sleeping spin_locks from __slab_free() and deactivate_slab().
+ * to take sleeping spin_locks from __slab_free().
  * In !PREEMPT_RT irq_work will run after local_unlock_irqrestore().
  */
 static void free_deferred_objects(struct irq_work *work)
 {
        struct defer_free *df = container_of(work, struct defer_free, work);
        struct llist_head *objs = &df->objects;
-       struct llist_head *slabs = &df->slabs;
        struct llist_node *llnode, *pos, *t;
 
-       if (llist_empty(objs) && llist_empty(slabs))
+       if (llist_empty(objs))
                return;
 
        llnode = llist_del_all(objs);
@@ -6107,16 +6114,6 @@ static void free_deferred_objects(struct irq_work *work)
 
                __slab_free(s, slab, x, x, 1, _THIS_IP_);
        }
-
-       llnode = llist_del_all(slabs);
-       llist_for_each_safe(pos, t, llnode) {
-               struct slab *slab = container_of(pos, struct slab, llnode);
-
-               if (slab->frozen)
-                       deactivate_slab(slab->slab_cache, slab, slab->flush_freelist);
-               else
-                       free_slab(slab->slab_cache, slab);
-       }
 }
 
 static void defer_free(struct kmem_cache *s, void *head)
@@ -6132,19 +6129,6 @@ static void defer_free(struct kmem_cache *s, void *head)
                irq_work_queue(&df->work);
 }
 
-static void defer_deactivate_slab(struct slab *slab, void *flush_freelist)
-{
-       struct defer_free *df;
-
-       slab->flush_freelist = flush_freelist;
-
-       guard(preempt)();
-
-       df = this_cpu_ptr(&defer_free_objects);
-       if (llist_add(&slab->llnode, &df->slabs))
-               irq_work_queue(&df->work);
-}
-
 void defer_free_barrier(void)
 {
        int cpu;