1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h>
4 #include <linux/slab.h>
5 #include <linux/uaccess.h>
6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h>
10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h>
13 #include <linux/memcontrol.h>
14 #include <linux/sched/clock.h>
19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
20 * to use off stack temporal storage
22 #define PAGE_OWNER_STACK_DEPTH (16)
26 short last_migrate_reason
;
28 depot_stack_handle_t handle
;
29 depot_stack_handle_t free_handle
;
32 char comm
[TASK_COMM_LEN
];
37 static bool page_owner_enabled __initdata
;
38 DEFINE_STATIC_KEY_FALSE(page_owner_inited
);
40 static depot_stack_handle_t dummy_handle
;
41 static depot_stack_handle_t failure_handle
;
42 static depot_stack_handle_t early_handle
;
44 static void init_early_allocated_pages(void);
46 static int __init
early_page_owner_param(char *buf
)
48 int ret
= kstrtobool(buf
, &page_owner_enabled
);
50 if (page_owner_enabled
)
51 stack_depot_request_early_init();
55 early_param("page_owner", early_page_owner_param
);
57 static __init
bool need_page_owner(void)
59 return page_owner_enabled
;
62 static __always_inline depot_stack_handle_t
create_dummy_stack(void)
64 unsigned long entries
[4];
65 unsigned int nr_entries
;
67 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 0);
68 return stack_depot_save(entries
, nr_entries
, GFP_KERNEL
);
71 static noinline
void register_dummy_stack(void)
73 dummy_handle
= create_dummy_stack();
76 static noinline
void register_failure_stack(void)
78 failure_handle
= create_dummy_stack();
81 static noinline
void register_early_stack(void)
83 early_handle
= create_dummy_stack();
86 static __init
void init_page_owner(void)
88 if (!page_owner_enabled
)
91 register_dummy_stack();
92 register_failure_stack();
93 register_early_stack();
94 static_branch_enable(&page_owner_inited
);
95 init_early_allocated_pages();
98 struct page_ext_operations page_owner_ops
= {
99 .size
= sizeof(struct page_owner
),
100 .need
= need_page_owner
,
101 .init
= init_page_owner
,
102 .need_shared_flags
= true,
105 static inline struct page_owner
*get_page_owner(struct page_ext
*page_ext
)
107 return page_ext_data(page_ext
, &page_owner_ops
);
110 static noinline depot_stack_handle_t
save_stack(gfp_t flags
)
112 unsigned long entries
[PAGE_OWNER_STACK_DEPTH
];
113 depot_stack_handle_t handle
;
114 unsigned int nr_entries
;
119 * Sometimes page metadata allocation tracking requires more
120 * memory to be allocated:
121 * - when new stack trace is saved to stack depot
122 * - when backtrace itself is calculated (ia64)
124 if (current
->in_page_owner
)
126 current
->in_page_owner
= 1;
128 nr_entries
= stack_trace_save(entries
, ARRAY_SIZE(entries
), 2);
129 handle
= stack_depot_save(entries
, nr_entries
, flags
);
131 handle
= failure_handle
;
133 current
->in_page_owner
= 0;
137 void __reset_page_owner(struct page
*page
, unsigned short order
)
140 struct page_ext
*page_ext
;
141 depot_stack_handle_t handle
;
142 struct page_owner
*page_owner
;
143 u64 free_ts_nsec
= local_clock();
145 page_ext
= page_ext_get(page
);
146 if (unlikely(!page_ext
))
149 handle
= save_stack(GFP_NOWAIT
| __GFP_NOWARN
);
150 for (i
= 0; i
< (1 << order
); i
++) {
151 __clear_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
);
152 page_owner
= get_page_owner(page_ext
);
153 page_owner
->free_handle
= handle
;
154 page_owner
->free_ts_nsec
= free_ts_nsec
;
155 page_ext
= page_ext_next(page_ext
);
157 page_ext_put(page_ext
);
160 static inline void __set_page_owner_handle(struct page_ext
*page_ext
,
161 depot_stack_handle_t handle
,
162 unsigned short order
, gfp_t gfp_mask
)
164 struct page_owner
*page_owner
;
166 u64 ts_nsec
= local_clock();
168 for (i
= 0; i
< (1 << order
); i
++) {
169 page_owner
= get_page_owner(page_ext
);
170 page_owner
->handle
= handle
;
171 page_owner
->order
= order
;
172 page_owner
->gfp_mask
= gfp_mask
;
173 page_owner
->last_migrate_reason
= -1;
174 page_owner
->pid
= current
->pid
;
175 page_owner
->tgid
= current
->tgid
;
176 page_owner
->ts_nsec
= ts_nsec
;
177 strscpy(page_owner
->comm
, current
->comm
,
178 sizeof(page_owner
->comm
));
179 __set_bit(PAGE_EXT_OWNER
, &page_ext
->flags
);
180 __set_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
);
182 page_ext
= page_ext_next(page_ext
);
186 noinline
void __set_page_owner(struct page
*page
, unsigned short order
,
189 struct page_ext
*page_ext
;
190 depot_stack_handle_t handle
;
192 handle
= save_stack(gfp_mask
);
194 page_ext
= page_ext_get(page
);
195 if (unlikely(!page_ext
))
197 __set_page_owner_handle(page_ext
, handle
, order
, gfp_mask
);
198 page_ext_put(page_ext
);
201 void __set_page_owner_migrate_reason(struct page
*page
, int reason
)
203 struct page_ext
*page_ext
= page_ext_get(page
);
204 struct page_owner
*page_owner
;
206 if (unlikely(!page_ext
))
209 page_owner
= get_page_owner(page_ext
);
210 page_owner
->last_migrate_reason
= reason
;
211 page_ext_put(page_ext
);
214 void __split_page_owner(struct page
*page
, unsigned int nr
)
217 struct page_ext
*page_ext
= page_ext_get(page
);
218 struct page_owner
*page_owner
;
220 if (unlikely(!page_ext
))
223 for (i
= 0; i
< nr
; i
++) {
224 page_owner
= get_page_owner(page_ext
);
225 page_owner
->order
= 0;
226 page_ext
= page_ext_next(page_ext
);
228 page_ext_put(page_ext
);
231 void __folio_copy_owner(struct folio
*newfolio
, struct folio
*old
)
233 struct page_ext
*old_ext
;
234 struct page_ext
*new_ext
;
235 struct page_owner
*old_page_owner
, *new_page_owner
;
237 old_ext
= page_ext_get(&old
->page
);
238 if (unlikely(!old_ext
))
241 new_ext
= page_ext_get(&newfolio
->page
);
242 if (unlikely(!new_ext
)) {
243 page_ext_put(old_ext
);
247 old_page_owner
= get_page_owner(old_ext
);
248 new_page_owner
= get_page_owner(new_ext
);
249 new_page_owner
->order
= old_page_owner
->order
;
250 new_page_owner
->gfp_mask
= old_page_owner
->gfp_mask
;
251 new_page_owner
->last_migrate_reason
=
252 old_page_owner
->last_migrate_reason
;
253 new_page_owner
->handle
= old_page_owner
->handle
;
254 new_page_owner
->pid
= old_page_owner
->pid
;
255 new_page_owner
->tgid
= old_page_owner
->tgid
;
256 new_page_owner
->ts_nsec
= old_page_owner
->ts_nsec
;
257 new_page_owner
->free_ts_nsec
= old_page_owner
->ts_nsec
;
258 strcpy(new_page_owner
->comm
, old_page_owner
->comm
);
261 * We don't clear the bit on the old folio as it's going to be freed
262 * after migration. Until then, the info can be useful in case of
263 * a bug, and the overall stats will be off a bit only temporarily.
264 * Also, migrate_misplaced_transhuge_page() can still fail the
265 * migration and then we want the old folio to retain the info. But
266 * in that case we also don't need to explicitly clear the info from
267 * the new page, which will be freed.
269 __set_bit(PAGE_EXT_OWNER
, &new_ext
->flags
);
270 __set_bit(PAGE_EXT_OWNER_ALLOCATED
, &new_ext
->flags
);
271 page_ext_put(new_ext
);
272 page_ext_put(old_ext
);
275 void pagetypeinfo_showmixedcount_print(struct seq_file
*m
,
276 pg_data_t
*pgdat
, struct zone
*zone
)
279 struct page_ext
*page_ext
;
280 struct page_owner
*page_owner
;
281 unsigned long pfn
, block_end_pfn
;
282 unsigned long end_pfn
= zone_end_pfn(zone
);
283 unsigned long count
[MIGRATE_TYPES
] = { 0, };
284 int pageblock_mt
, page_mt
;
287 /* Scan block by block. First and last block may be incomplete */
288 pfn
= zone
->zone_start_pfn
;
291 * Walk the zone in pageblock_nr_pages steps. If a page block spans
292 * a zone boundary, it will be double counted between zones. This does
293 * not matter as the mixed block count will still be correct
295 for (; pfn
< end_pfn
; ) {
296 page
= pfn_to_online_page(pfn
);
298 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
302 block_end_pfn
= pageblock_end_pfn(pfn
);
303 block_end_pfn
= min(block_end_pfn
, end_pfn
);
305 pageblock_mt
= get_pageblock_migratetype(page
);
307 for (; pfn
< block_end_pfn
; pfn
++) {
308 /* The pageblock is online, no need to recheck. */
309 page
= pfn_to_page(pfn
);
311 if (page_zone(page
) != zone
)
314 if (PageBuddy(page
)) {
315 unsigned long freepage_order
;
317 freepage_order
= buddy_order_unsafe(page
);
318 if (freepage_order
<= MAX_ORDER
)
319 pfn
+= (1UL << freepage_order
) - 1;
323 if (PageReserved(page
))
326 page_ext
= page_ext_get(page
);
327 if (unlikely(!page_ext
))
330 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
331 goto ext_put_continue
;
333 page_owner
= get_page_owner(page_ext
);
334 page_mt
= gfp_migratetype(page_owner
->gfp_mask
);
335 if (pageblock_mt
!= page_mt
) {
336 if (is_migrate_cma(pageblock_mt
))
337 count
[MIGRATE_MOVABLE
]++;
339 count
[pageblock_mt
]++;
342 page_ext_put(page_ext
);
345 pfn
+= (1UL << page_owner
->order
) - 1;
347 page_ext_put(page_ext
);
352 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
353 for (i
= 0; i
< MIGRATE_TYPES
; i
++)
354 seq_printf(m
, "%12lu ", count
[i
]);
359 * Looking for memcg information and print it out
361 static inline int print_page_owner_memcg(char *kbuf
, size_t count
, int ret
,
365 unsigned long memcg_data
;
366 struct mem_cgroup
*memcg
;
371 memcg_data
= READ_ONCE(page
->memcg_data
);
375 if (memcg_data
& MEMCG_DATA_OBJCGS
)
376 ret
+= scnprintf(kbuf
+ ret
, count
- ret
,
377 "Slab cache page\n");
379 memcg
= page_memcg_check(page
);
383 online
= (memcg
->css
.flags
& CSS_ONLINE
);
384 cgroup_name(memcg
->css
.cgroup
, name
, sizeof(name
));
385 ret
+= scnprintf(kbuf
+ ret
, count
- ret
,
386 "Charged %sto %smemcg %s\n",
387 PageMemcgKmem(page
) ? "(via objcg) " : "",
388 online
? "" : "offline ",
392 #endif /* CONFIG_MEMCG */
398 print_page_owner(char __user
*buf
, size_t count
, unsigned long pfn
,
399 struct page
*page
, struct page_owner
*page_owner
,
400 depot_stack_handle_t handle
)
402 int ret
, pageblock_mt
, page_mt
;
405 count
= min_t(size_t, count
, PAGE_SIZE
);
406 kbuf
= kmalloc(count
, GFP_KERNEL
);
410 ret
= scnprintf(kbuf
, count
,
411 "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n",
412 page_owner
->order
, page_owner
->gfp_mask
,
413 &page_owner
->gfp_mask
, page_owner
->pid
,
414 page_owner
->tgid
, page_owner
->comm
,
415 page_owner
->ts_nsec
, page_owner
->free_ts_nsec
);
417 /* Print information relevant to grouping pages by mobility */
418 pageblock_mt
= get_pageblock_migratetype(page
);
419 page_mt
= gfp_migratetype(page_owner
->gfp_mask
);
420 ret
+= scnprintf(kbuf
+ ret
, count
- ret
,
421 "PFN 0x%lx type %s Block %lu type %s Flags %pGp\n",
423 migratetype_names
[page_mt
],
424 pfn
>> pageblock_order
,
425 migratetype_names
[pageblock_mt
],
428 ret
+= stack_depot_snprint(handle
, kbuf
+ ret
, count
- ret
, 0);
432 if (page_owner
->last_migrate_reason
!= -1) {
433 ret
+= scnprintf(kbuf
+ ret
, count
- ret
,
434 "Page has been migrated, last migrate reason: %s\n",
435 migrate_reason_names
[page_owner
->last_migrate_reason
]);
438 ret
= print_page_owner_memcg(kbuf
, count
, ret
, page
);
440 ret
+= snprintf(kbuf
+ ret
, count
- ret
, "\n");
444 if (copy_to_user(buf
, kbuf
, ret
))
455 void __dump_page_owner(const struct page
*page
)
457 struct page_ext
*page_ext
= page_ext_get((void *)page
);
458 struct page_owner
*page_owner
;
459 depot_stack_handle_t handle
;
463 if (unlikely(!page_ext
)) {
464 pr_alert("There is not page extension available.\n");
468 page_owner
= get_page_owner(page_ext
);
469 gfp_mask
= page_owner
->gfp_mask
;
470 mt
= gfp_migratetype(gfp_mask
);
472 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
)) {
473 pr_alert("page_owner info is not present (never set?)\n");
474 page_ext_put(page_ext
);
478 if (test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
479 pr_alert("page_owner tracks the page as allocated\n");
481 pr_alert("page_owner tracks the page as freed\n");
483 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n",
484 page_owner
->order
, migratetype_names
[mt
], gfp_mask
, &gfp_mask
,
485 page_owner
->pid
, page_owner
->tgid
, page_owner
->comm
,
486 page_owner
->ts_nsec
, page_owner
->free_ts_nsec
);
488 handle
= READ_ONCE(page_owner
->handle
);
490 pr_alert("page_owner allocation stack trace missing\n");
492 stack_depot_print(handle
);
494 handle
= READ_ONCE(page_owner
->free_handle
);
496 pr_alert("page_owner free stack trace missing\n");
498 pr_alert("page last free stack trace:\n");
499 stack_depot_print(handle
);
502 if (page_owner
->last_migrate_reason
!= -1)
503 pr_alert("page has been migrated, last migrate reason: %s\n",
504 migrate_reason_names
[page_owner
->last_migrate_reason
]);
505 page_ext_put(page_ext
);
509 read_page_owner(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*ppos
)
513 struct page_ext
*page_ext
;
514 struct page_owner
*page_owner
;
515 depot_stack_handle_t handle
;
517 if (!static_branch_unlikely(&page_owner_inited
))
525 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
526 while (!pfn_valid(pfn
) && (pfn
& (MAX_ORDER_NR_PAGES
- 1)) != 0)
529 /* Find an allocated page */
530 for (; pfn
< max_pfn
; pfn
++) {
532 * This temporary page_owner is required so
533 * that we can avoid the context switches while holding
534 * the rcu lock and copying the page owner information to
535 * user through copy_to_user() or GFP_KERNEL allocations.
537 struct page_owner page_owner_tmp
;
540 * If the new page is in a new MAX_ORDER_NR_PAGES area,
541 * validate the area as existing, skip it if not
543 if ((pfn
& (MAX_ORDER_NR_PAGES
- 1)) == 0 && !pfn_valid(pfn
)) {
544 pfn
+= MAX_ORDER_NR_PAGES
- 1;
548 page
= pfn_to_page(pfn
);
549 if (PageBuddy(page
)) {
550 unsigned long freepage_order
= buddy_order_unsafe(page
);
552 if (freepage_order
<= MAX_ORDER
)
553 pfn
+= (1UL << freepage_order
) - 1;
557 page_ext
= page_ext_get(page
);
558 if (unlikely(!page_ext
))
562 * Some pages could be missed by concurrent allocation or free,
563 * because we don't hold the zone lock.
565 if (!test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
566 goto ext_put_continue
;
569 * Although we do have the info about past allocation of free
570 * pages, it's not relevant for current memory usage.
572 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED
, &page_ext
->flags
))
573 goto ext_put_continue
;
575 page_owner
= get_page_owner(page_ext
);
578 * Don't print "tail" pages of high-order allocations as that
579 * would inflate the stats.
581 if (!IS_ALIGNED(pfn
, 1 << page_owner
->order
))
582 goto ext_put_continue
;
585 * Access to page_ext->handle isn't synchronous so we should
586 * be careful to access it.
588 handle
= READ_ONCE(page_owner
->handle
);
590 goto ext_put_continue
;
592 /* Record the next PFN to read in the file offset */
595 page_owner_tmp
= *page_owner
;
596 page_ext_put(page_ext
);
597 return print_page_owner(buf
, count
, pfn
, page
,
598 &page_owner_tmp
, handle
);
600 page_ext_put(page_ext
);
606 static loff_t
lseek_page_owner(struct file
*file
, loff_t offset
, int orig
)
610 file
->f_pos
= offset
;
613 file
->f_pos
+= offset
;
621 static void init_pages_in_zone(pg_data_t
*pgdat
, struct zone
*zone
)
623 unsigned long pfn
= zone
->zone_start_pfn
;
624 unsigned long end_pfn
= zone_end_pfn(zone
);
625 unsigned long count
= 0;
628 * Walk the zone in pageblock_nr_pages steps. If a page block spans
629 * a zone boundary, it will be double counted between zones. This does
630 * not matter as the mixed block count will still be correct
632 for (; pfn
< end_pfn
; ) {
633 unsigned long block_end_pfn
;
635 if (!pfn_valid(pfn
)) {
636 pfn
= ALIGN(pfn
+ 1, MAX_ORDER_NR_PAGES
);
640 block_end_pfn
= pageblock_end_pfn(pfn
);
641 block_end_pfn
= min(block_end_pfn
, end_pfn
);
643 for (; pfn
< block_end_pfn
; pfn
++) {
644 struct page
*page
= pfn_to_page(pfn
);
645 struct page_ext
*page_ext
;
647 if (page_zone(page
) != zone
)
651 * To avoid having to grab zone->lock, be a little
652 * careful when reading buddy page order. The only
653 * danger is that we skip too much and potentially miss
654 * some early allocated pages, which is better than
655 * heavy lock contention.
657 if (PageBuddy(page
)) {
658 unsigned long order
= buddy_order_unsafe(page
);
660 if (order
> 0 && order
<= MAX_ORDER
)
661 pfn
+= (1UL << order
) - 1;
665 if (PageReserved(page
))
668 page_ext
= page_ext_get(page
);
669 if (unlikely(!page_ext
))
672 /* Maybe overlapping zone */
673 if (test_bit(PAGE_EXT_OWNER
, &page_ext
->flags
))
674 goto ext_put_continue
;
676 /* Found early allocated page */
677 __set_page_owner_handle(page_ext
, early_handle
,
681 page_ext_put(page_ext
);
686 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
687 pgdat
->node_id
, zone
->name
, count
);
690 static void init_zones_in_node(pg_data_t
*pgdat
)
693 struct zone
*node_zones
= pgdat
->node_zones
;
695 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
696 if (!populated_zone(zone
))
699 init_pages_in_zone(pgdat
, zone
);
703 static void init_early_allocated_pages(void)
707 for_each_online_pgdat(pgdat
)
708 init_zones_in_node(pgdat
);
711 static const struct file_operations proc_page_owner_operations
= {
712 .read
= read_page_owner
,
713 .llseek
= lseek_page_owner
,
716 static int __init
pageowner_init(void)
718 if (!static_branch_unlikely(&page_owner_inited
)) {
719 pr_info("page_owner is disabled\n");
723 debugfs_create_file("page_owner", 0400, NULL
, NULL
,
724 &proc_page_owner_operations
);
728 late_initcall(pageowner_init
)