mm/slab: allow specifying free pointer offset when using constructor

When a slab cache has a constructor, the free pointer is placed after the
object because certain fields must not be overwritten even after the
object is freed.

However, some fields that the constructor does not initialize can safely
be overwritten after free. Allow specifying the free pointer offset within
the object, reducing the overall object size when some fields can be reused
for the free pointer.

Adjust the document accordingly.

Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
Link: https://patch.msgid.link/20260113061845.159790-3-harry.yoo@oracle.com
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 2482992248dc9c48b6d25d2aff8bf196fa3d0e1e..4554c04a9bd79583d5aa2c78907695ba3def5028 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -299,24 +299,26 @@ struct kmem_cache_args {
        unsigned int usersize;
        /**
         * @freeptr_offset: Custom offset for the free pointer
-        * in &SLAB_TYPESAFE_BY_RCU caches
+        * in caches with &SLAB_TYPESAFE_BY_RCU or @ctor
         *
-        * By default &SLAB_TYPESAFE_BY_RCU caches place the free pointer
-        * outside of the object. This might cause the object to grow in size.
-        * Cache creators that have a reason to avoid this can specify a custom
-        * free pointer offset in their struct where the free pointer will be
-        * placed.
+        * By default, &SLAB_TYPESAFE_BY_RCU and @ctor caches place the free
+        * pointer outside of the object. This might cause the object to grow
+        * in size. Cache creators that have a reason to avoid this can specify
+        * a custom free pointer offset in their data structure where the free
+        * pointer will be placed.
         *
-        * Note that placing the free pointer inside the object requires the
-        * caller to ensure that no fields are invalidated that are required to
-        * guard against object recycling (See &SLAB_TYPESAFE_BY_RCU for
-        * details).
+        * For caches with &SLAB_TYPESAFE_BY_RCU, the caller must ensure that
+        * the free pointer does not overlay fields required to guard against
+        * object recycling (See &SLAB_TYPESAFE_BY_RCU for details).
         *
-        * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
-        * is specified, %use_freeptr_offset must be set %true.
+        * For caches with @ctor, the caller must ensure that the free pointer
+        * does not overlay fields initialized by the constructor.
+        *
+        * Currently, only caches with &SLAB_TYPESAFE_BY_RCU or @ctor
+        * may specify @freeptr_offset.
         *
-        * Note that @ctor currently isn't supported with custom free pointers
-        * as a @ctor requires an external free pointer.
+        * Using %0 as a value for @freeptr_offset is valid. If @freeptr_offset
+        * is specified, @use_freeptr_offset must be set %true.
         */
        unsigned int freeptr_offset;
        /**

diff --git a/mm/slab_common.c b/mm/slab_common.c
index b6836f8500b684b0f74c1c3107d1bb07f46a7005..027bf64c2e3515b7e6bbffc1c61dbba174950431 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -239,7 +239,7 @@ static struct kmem_cache *create_cache(const char *name,
        err = -EINVAL;
        if (args->use_freeptr_offset &&
            (args->freeptr_offset >= object_size ||
-            !(flags & SLAB_TYPESAFE_BY_RCU) ||
+            (!(flags & SLAB_TYPESAFE_BY_RCU) && !args->ctor) ||
             !IS_ALIGNED(args->freeptr_offset, __alignof__(freeptr_t))))
                goto out;
 

diff --git a/mm/slub.c b/mm/slub.c
index 2c000dddcf74f22606882d851de34f1936411d07..1b7ed91a2f1575231007caa3ecef2e4528480ab0 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -7998,7 +7998,8 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
        s->inuse = size;
 
        if (((flags & SLAB_TYPESAFE_BY_RCU) && !args->use_freeptr_offset) ||
-           (flags & SLAB_POISON) || s->ctor ||
+           (flags & SLAB_POISON) ||
+           (s->ctor && !args->use_freeptr_offset) ||
            ((flags & SLAB_RED_ZONE) &&
             (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) {
                /*
@@ -8019,7 +8020,8 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
                 */
                s->offset = size;
                size += sizeof(void *);
-       } else if ((flags & SLAB_TYPESAFE_BY_RCU) && args->use_freeptr_offset) {
+       } else if (((flags & SLAB_TYPESAFE_BY_RCU) || s->ctor) &&
+                       args->use_freeptr_offset) {
                s->offset = args->freeptr_offset;
        } else {
                /*