mm/slab: allow freeing kmalloc_nolock()'d objects using kfree[_rcu]()

Slab objects that are allocated with kmalloc_nolock() must be freed
using kfree_nolock() because only a subset of alloc hooks are called,
since kmalloc_nolock() can't spin on a lock during allocation.

This imposes a limitation: such objects cannot be freed with kfree_rcu(),
forcing users to work around this limitation by calling call_rcu()
with a callback that frees the object using kfree_nolock().

Remove this limitation by teaching kmemleak to gracefully ignore cases
when kmemleak_free() or kmemleak_ignore() is called without a prior
kmemleak_alloc().

Unlike kmemleak, kfence already handles this case, because,
due to its design, only a subset of allocations are served from kfence.

With this change, kfree() and kfree_rcu() can be used to free objects
that are allocated using kmalloc_nolock().

Suggested-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
Link: https://patch.msgid.link/20260210044642.139482-2-harry.yoo@oracle.com
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index c5b30054cd018bb96d227e23ee911e26d98e27b3..72ba681360adbe06ca1a5d074303b877d38602b9 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1076,8 +1076,8 @@ static inline void rcu_read_unlock_migrate(void)
  * either fall back to use of call_rcu() or rearrange the structure to
  * position the rcu_head structure into the first 4096 bytes.
  *
- * The object to be freed can be allocated either by kmalloc() or
- * kmem_cache_alloc().
+ * The object to be freed can be allocated either by kmalloc(),
+ * kmalloc_nolock(), or kmem_cache_alloc().
  *
  * Note that the allowable offset might decrease in the future.
  *

diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 1ac56ceb29b6b1baa613cf746e7b23a5d364571c..95ad827fcd6987fd34402e99b865553786b35149 100644 (file)
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -837,13 +837,12 @@ static void delete_object_full(unsigned long ptr, unsigned int objflags)
        struct kmemleak_object *object;
 
        object = find_and_remove_object(ptr, 0, objflags);
-       if (!object) {
-#ifdef DEBUG
-               kmemleak_warn("Freeing unknown object at 0x%08lx\n",
-                             ptr);
-#endif
+       if (!object)
+               /*
+                * kmalloc_nolock() -> kfree() calls kmemleak_free()
+                * without kmemleak_alloc().
+                */
                return;
-       }
        __delete_object(object);
 }
 
@@ -926,13 +925,12 @@ static void paint_ptr(unsigned long ptr, int color, unsigned int objflags)
        struct kmemleak_object *object;
 
        object = __find_and_get_object(ptr, 0, objflags);
-       if (!object) {
-               kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
-                             ptr,
-                             (color == KMEMLEAK_GREY) ? "Grey" :
-                             (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
+       if (!object)
+               /*
+                * kmalloc_nolock() -> kfree_rcu() calls kmemleak_ignore()
+                * without kmemleak_alloc().
+                */
                return;
-       }
        paint_it(object, color);
        put_object(object);
 }

diff --git a/mm/slub.c b/mm/slub.c
index 591e41e5acc49168f7092da3ebc109a77a1af3c1..3f64a6b94571acdd47669aa431466573eba76cfa 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2585,6 +2585,24 @@ struct rcu_delayed_free {
  * Returns true if freeing of the object can proceed, false if its reuse
  * was delayed by CONFIG_SLUB_RCU_DEBUG or KASAN quarantine, or it was returned
  * to KFENCE.
+ *
+ * For objects allocated via kmalloc_nolock(), only a subset of alloc hooks
+ * are invoked, so some free hooks must handle asymmetric hook calls.
+ *
+ * Alloc hooks called for kmalloc_nolock():
+ * - kmsan_slab_alloc()
+ * - kasan_slab_alloc()
+ * - memcg_slab_post_alloc_hook()
+ * - alloc_tagging_slab_alloc_hook()
+ *
+ * Free hooks that must handle missing corresponding alloc hooks:
+ * - kmemleak_free_recursive()
+ * - kfence_free()
+ *
+ * Free hooks that have no alloc hook counterpart, and thus safe to call:
+ * - debug_check_no_locks_freed()
+ * - debug_check_no_obj_freed()
+ * - __kcsan_check_access()
  */
 static __always_inline
 bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
@@ -6394,7 +6412,7 @@ void kvfree_rcu_cb(struct rcu_head *head)
 
 /**
  * kfree - free previously allocated memory
- * @object: pointer returned by kmalloc() or kmem_cache_alloc()
+ * @object: pointer returned by kmalloc(), kmalloc_nolock(), or kmem_cache_alloc()
  *
  * If @object is NULL, no operation is performed.
  */
@@ -6413,6 +6431,7 @@ void kfree(const void *object)
        page = virt_to_page(object);
        slab = page_slab(page);
        if (!slab) {
+               /* kmalloc_nolock() doesn't support large kmalloc */
                free_large_kmalloc(page, (void *)object);
                return;
        }