#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */
#define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */
#define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */
+#define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */
static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
{
* KMALLOC_MAX_CACHE_SIZE and the caller must check that.
*/
static inline struct kmem_cache *
-kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token)
+kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token,
+ unsigned int alloc_flags)
{
unsigned int index;
+ enum kmalloc_cache_type type = kmalloc_type(flags, token);
+
+ if (alloc_flags & SLAB_ALLOC_NO_OBJ_EXT)
+ type = KMALLOC_NO_OBJ_EXT;
if (!b)
- b = &kmalloc_caches[kmalloc_type(flags, token)];
+ b = &kmalloc_caches[type];
if (size <= 192)
index = kmalloc_size_index[size_index_elem(size)];
else
{
if (!is_kmalloc_cache(s))
return false;
- return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT));
+
+ return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT));
}
bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj);
kasan_enable_current();
}
+/*
+ * Return true if KMALLOC_NORMAL caches may need obj_exts arrays.
+ *
+ * Memory allocation profiling requires obj_exts for all caches.
+ * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types
+ * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL.
+ */
+static inline bool need_kmalloc_no_objext(void)
+{
+ if (!mem_alloc_profiling_permanently_disabled())
+ return true;
+
+ if (!mem_cgroup_kmem_disabled() &&
+ (KMALLOC_NORMAL == KMALLOC_RECLAIM))
+ return true;
+
+ return false;
+}
+
#ifdef CONFIG_SLAB_OBJ_EXT
/*
size_t kmalloc_size_roundup(size_t size)
{
if (size && size <= KMALLOC_MAX_CACHE_SIZE) {
+ struct kmem_cache *s;
+
/*
* The flags don't matter since size_index is common to all.
* Neither does the caller for just getting ->object_size.
*/
- return kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0))->object_size;
+ s = kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0),
+ SLAB_ALLOC_DEFAULT);
+ return s->object_size;
}
/* Above the smaller buckets, size is a multiple of page size. */
#define KMALLOC_PARTITION_NAME(N, sz)
#endif
+#ifdef CONFIG_SLAB_OBJ_EXT
+#define KMALLOC_NO_OBJ_EXT_NAME(sz) .name[KMALLOC_NO_OBJ_EXT] = "kmalloc-no-objext-" #sz,
+#else
+#define KMALLOC_NO_OBJ_EXT_NAME(sz)
+#endif
+
#define INIT_KMALLOC_INFO(__size, __short_size) \
{ \
.name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
KMALLOC_CGROUP_NAME(__short_size) \
KMALLOC_DMA_NAME(__short_size) \
KMALLOC_PARTITION_NAME(KMALLOC_PARTITION_CACHES_NR, __short_size) \
+ KMALLOC_NO_OBJ_EXT_NAME(__short_size) \
.size = __size, \
}
return;
}
flags |= SLAB_ACCOUNT;
+ } else if (IS_ENABLED(CONFIG_SLAB_OBJ_EXT) && type == KMALLOC_NO_OBJ_EXT) {
+ if (!need_kmalloc_no_objext()) {
+ kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
+ return;
+ }
+ flags |= SLAB_NO_OBJ_EXT | SLAB_NO_MERGE;
} else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
flags |= SLAB_CACHE_DMA;
}
slab->obj_exts = 0;
}
-/*
- * Calculate the allocation size for slabobj_ext array.
- *
- * When memory allocation profiling is enabled, the obj_exts array
- * could be allocated from the same slab cache it's being allocated for.
- * This would prevent the slab from ever being freed because it would
- * always contain at least one allocated object (its own obj_exts array).
- *
- * To avoid this, increase the allocation size when we detect the array
- * may come from the same cache, forcing it to use a different cache.
- */
-static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
- struct slab *slab, gfp_t gfp)
-{
- size_t sz = sizeof(struct slabobj_ext) * slab->objects;
- struct kmem_cache *obj_exts_cache;
-
- if (sz > KMALLOC_MAX_CACHE_SIZE)
- return sz;
-
- if (!is_kmalloc_normal(s))
- return sz;
-
- obj_exts_cache = kmalloc_slab(sz, NULL, gfp, __kmalloc_token(0));
- /*
- * We can't simply compare s with obj_exts_cache, because partitioned kmalloc
- * caches have multiple caches per size, selected by caller address or type.
- * Since caller address or type may differ between kmalloc_slab() and actual
- * allocation, bump size when sizes are equal.
- */
- if (s->object_size == obj_exts_cache->object_size)
- return obj_exts_cache->object_size + 1;
-
- return sz;
-}
-
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, unsigned int alloc_flags)
{
unsigned long new_exts;
unsigned long old_exts;
struct slabobj_ext *vec;
- size_t sz;
+ size_t sz = sizeof(struct slabobj_ext) * slab->objects;
gfp &= ~OBJCGS_CLEAR_MASK;
- /* Prevent recursive extension vector allocation */
- alloc_flags |= SLAB_ALLOC_NO_RECURSE;
- alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
+ /*
+ * In most cases, obj_exts arrays are allocated from normal kmalloc.
+ * However, normal kmalloc caches must allocate them from
+ * KMALLOC_NO_OBJ_EXT caches to prevent recursion.
+ */
+ if (is_kmalloc_normal(s))
+ alloc_flags |= SLAB_ALLOC_NO_OBJ_EXT;
- sz = obj_exts_alloc_size(s, slab, gfp);
+ alloc_flags &= ~SLAB_ALLOC_NEW_SLAB;
/* This will use kmalloc_nolock() if alloc_flags say so */
vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));
return -ENOMEM;
}
- VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL &&
- virt_to_slab(vec)->slab_cache == s);
+ if (IS_ENABLED(CONFIG_DEBUG_VM)) {
+ struct kmem_cache *exts_cache;
+ struct slab *exts_slab;
+
+ exts_slab = virt_to_slab(vec);
+ if (exts_slab) {
+ /*
+ * The vector must be allocated from either normal or
+ * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles.
+ */
+ exts_cache = exts_slab->slab_cache;
+ WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) &&
+ !(exts_cache->flags & SLAB_NO_OBJ_EXT));
+ }
+ }
new_exts = (unsigned long)vec;
#ifdef CONFIG_MEMCG
* assign slabobj_exts in parallel. In this case the existing
* objcg vector should be reused.
*/
- mark_obj_codetag_empty(vec);
if (unlikely(!allow_spin))
kfree_nolock(vec);
else
return;
}
- /*
- * obj_exts was created with SLAB_ALLOC_NO_RECURSE flag, therefore its
- * corresponding extension will be NULL. alloc_tag_sub() will throw a
- * warning if slab has extensions but the extension of an object is
- * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that
- * the extension for obj_exts is expected to be NULL.
- */
- mark_obj_codetag_empty(obj_exts);
if (allow_spin)
kfree(obj_exts);
else
if (unlikely(!size))
return ZERO_SIZE_PTR;
- s = kmalloc_slab(size, b, flags, token);
+ s = kmalloc_slab(size, b, flags, token, ac->alloc_flags);
ret = slab_alloc_node(s, flags, node, ac);
ret = kasan_kmalloc(s, ret, size, flags);
retry:
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return NULL;
- s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token));
+
+ s = kmalloc_slab(size, NULL, gfp_flags, PASS_TOKEN_PARAM(token),
+ ac->alloc_flags);
if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
/*
s->allocflags |= __GFP_RECLAIMABLE;
/*
- * For KMALLOC_NORMAL caches we enable sheaves later by
- * bootstrap_kmalloc_sheaves() to avoid recursion
+ * For kmalloc caches we enable sheaves later by
+ * bootstrap_kmalloc_sheaves() to avoid recursion.
*/
- if (!is_kmalloc_normal(s))
+ if (!is_kmalloc_cache(s))
s->sheaf_capacity = calculate_sheaf_capacity(s, args);
/*
{
enum kmalloc_cache_type type;
- for (type = KMALLOC_NORMAL; type <= KMALLOC_PARTITION_END; type++) {
+ for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
struct kmem_cache *s = kmalloc_caches[type][idx];
projects / thirdparty / linux.git / commitdiff
