* their codetag uninitialized. Track these early PFNs so we can clear
* their codetag refs later to avoid warnings when they are freed.
*
- * Early allocations include:
- * - Base allocations independent of CPU count
- * - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init,
- * such as trace ring buffers, scheduler per-cpu data)
- *
- * For simplicity, we fix the size to 8192.
- * If insufficient, a warning will be triggered to alert the user.
+ * Each page is cast to a pfn_pool: the first few bytes hold metadata
+ * (next pointer and slot count), the remainder stores PFNs.
+ */
+struct pfn_pool {
+ struct pfn_pool *next;
+ atomic_t count;
+ unsigned long pfns[];
+};
+
+#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
+ sizeof(unsigned long))
+
+/*
+ * Skip early PFN recording for a page allocation. Reuses the
+ * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid
+ * recursion when allocating pages for the early PFN tracking list
+ * itself.
*
- * TODO: Replace fixed-size array with dynamic allocation using
- * a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion.
+ * Codetags of the pages allocated with __GFP_NO_CODETAG should be
+ * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
+ * alloc_tag_sub_check() from triggering a warning.
*/
-#define EARLY_ALLOC_PFN_MAX 8192
+#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT
-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX] __initdata;
-static atomic_t early_pfn_count __initdata = ATOMIC_INIT(0);
+static struct pfn_pool *current_pfn_pool __initdata;
static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
{
- int old_idx, new_idx;
+ struct pfn_pool *pool;
+ int idx;
do {
- old_idx = atomic_read(&early_pfn_count);
- if (old_idx >= EARLY_ALLOC_PFN_MAX) {
- pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n",
- EARLY_ALLOC_PFN_MAX);
- return;
+ pool = READ_ONCE(current_pfn_pool);
+ if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
+ struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG);
+ struct pfn_pool *new;
+
+ if (!new_page) {
+ pr_warn_once("early PFN tracking page allocation failed\n");
+ return;
+ }
+ new = page_address(new_page);
+ new->next = pool;
+ atomic_set(&new->count, 0);
+ if (cmpxchg(¤t_pfn_pool, pool, new) != pool) {
+ clear_page_tag_ref(new_page);
+ __free_page(new_page);
+ continue;
+ }
+ pool = new;
}
- new_idx = old_idx + 1;
- } while (!atomic_try_cmpxchg(&early_pfn_count, &old_idx, new_idx));
+ idx = atomic_read(&pool->count);
+ if (idx >= PFN_POOL_SIZE)
+ continue;
+ if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
+ break;
+ } while (1);
- early_pfns[old_idx] = pfn;
+ pool->pfns[idx] = pfn;
}
typedef void alloc_tag_add_func(unsigned long pfn);
static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
RCU_INITIALIZER(__alloc_tag_add_early_pfn);
-void alloc_tag_add_early_pfn(unsigned long pfn)
+void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
{
alloc_tag_add_func *alloc_tag_add;
if (static_key_enabled(&mem_profiling_compressed))
return;
+ /* Skip allocations for the tracking list itself to avoid recursion. */
+ if (gfp_flags & __GFP_NO_CODETAG)
+ return;
+
rcu_read_lock();
alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
if (alloc_tag_add)
static void __init clear_early_alloc_pfn_tag_refs(void)
{
- unsigned int i;
+ struct pfn_pool *pool, *next;
+ struct page *page;
+ int i;
if (static_key_enabled(&mem_profiling_compressed))
return;
/* Make sure we are not racing with __alloc_tag_add_early_pfn() */
synchronize_rcu();
- for (i = 0; i < atomic_read(&early_pfn_count); i++) {
- unsigned long pfn = early_pfns[i];
-
- if (pfn_valid(pfn)) {
- struct page *page = pfn_to_page(pfn);
- union pgtag_ref_handle handle;
- union codetag_ref ref;
-
- if (get_page_tag_ref(page, &ref, &handle)) {
- /*
- * An early-allocated page could be freed and reallocated
- * after its page_ext is initialized but before we clear it.
- * In that case, it already has a valid tag set.
- * We should not overwrite that valid tag with CODETAG_EMPTY.
- *
- * Note: there is still a small race window between checking
- * ref.ct and calling set_codetag_empty(). We accept this
- * race as it's unlikely and the extra complexity of atomic
- * cmpxchg is not worth it for this debug-only code path.
- */
- if (ref.ct) {
+ for (pool = current_pfn_pool; pool; pool = next) {
+ int nr_pfns = atomic_read(&pool->count);
+
+ for (i = 0; i < nr_pfns; i++) {
+ unsigned long pfn = pool->pfns[i];
+
+ if (pfn_valid(pfn)) {
+ union pgtag_ref_handle handle;
+ union codetag_ref ref;
+
+ if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
+ /*
+ * An early-allocated page could be freed and reallocated
+ * after its page_ext is initialized but before we clear it.
+ * In that case, it already has a valid tag set.
+ * We should not overwrite that valid tag
+ * with CODETAG_EMPTY.
+ *
+ * Note: there is still a small race window between checking
+ * ref.ct and calling set_codetag_empty(). We accept this
+ * race as it's unlikely and the extra complexity of atomic
+ * cmpxchg is not worth it for this debug-only code path.
+ */
+ if (ref.ct) {
+ put_page_tag_ref(handle);
+ continue;
+ }
+
+ set_codetag_empty(&ref);
+ update_page_tag_ref(handle, &ref);
put_page_tag_ref(handle);
- continue;
}
-
- set_codetag_empty(&ref);
- update_page_tag_ref(handle, &ref);
- put_page_tag_ref(handle);
}
}
+ next = pool->next;
+ page = virt_to_page(pool);
+ clear_page_tag_ref(page);
+ __free_page(page);
}
}
#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
/* Should be called only if mem_alloc_profiling_enabled() */
static noinline
void __pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr)
+ unsigned int nr, gfp_t gfp_flags)
{
union pgtag_ref_handle handle;
union codetag_ref ref;
* page_ext is not available yet, record the pfn so we can
* clear the tag ref later when page_ext is initialized.
*/
- alloc_tag_add_early_pfn(page_to_pfn(page));
+ alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags);
if (task->alloc_tag)
alloc_tag_set_inaccurate(task->alloc_tag);
}
}
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr)
+ unsigned int nr, gfp_t gfp_flags)
{
if (mem_alloc_profiling_enabled())
- __pgalloc_tag_add(page, task, nr);
+ __pgalloc_tag_add(page, task, nr, gfp_flags);
}
/* Should be called only if mem_alloc_profiling_enabled() */
#else /* CONFIG_MEM_ALLOC_PROFILING */
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr) {}
+ unsigned int nr, gfp_t gfp_flags) {}
static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
set_page_owner(page, order, gfp_flags);
page_table_check_alloc(page, order);
- pgalloc_tag_add(page, current, 1 << order);
+ pgalloc_tag_add(page, current, 1 << order, gfp_flags);
}
static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
projects / thirdparty / kernel / linux.git / commitdiff
