1 // SPDX-License-Identifier: GPL-2.0-only
5 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/debugfs.h>
24 #include <linux/sched.h>
25 #include <linux/math64.h>
26 #include <linux/writeback.h>
27 #include <linux/compaction.h>
28 #include <linux/mm_inline.h>
29 #include <linux/page_owner.h>
30 #include <linux/sched/isolation.h>
35 int sysctl_vm_numa_stat
= ENABLE_NUMA_STAT
;
37 /* zero numa counters within a zone */
38 static void zero_zone_numa_counters(struct zone
*zone
)
42 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++) {
43 atomic_long_set(&zone
->vm_numa_event
[item
], 0);
44 for_each_online_cpu(cpu
) {
45 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->vm_numa_event
[item
]
51 /* zero numa counters of all the populated zones */
52 static void zero_zones_numa_counters(void)
56 for_each_populated_zone(zone
)
57 zero_zone_numa_counters(zone
);
60 /* zero global numa counters */
61 static void zero_global_numa_counters(void)
65 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
66 atomic_long_set(&vm_numa_event
[item
], 0);
69 static void invalid_numa_statistics(void)
71 zero_zones_numa_counters();
72 zero_global_numa_counters();
75 static DEFINE_MUTEX(vm_numa_stat_lock
);
77 int sysctl_vm_numa_stat_handler(struct ctl_table
*table
, int write
,
78 void *buffer
, size_t *length
, loff_t
*ppos
)
82 mutex_lock(&vm_numa_stat_lock
);
84 oldval
= sysctl_vm_numa_stat
;
85 ret
= proc_dointvec_minmax(table
, write
, buffer
, length
, ppos
);
89 if (oldval
== sysctl_vm_numa_stat
)
91 else if (sysctl_vm_numa_stat
== ENABLE_NUMA_STAT
) {
92 static_branch_enable(&vm_numa_stat_key
);
93 pr_info("enable numa statistics\n");
95 static_branch_disable(&vm_numa_stat_key
);
96 invalid_numa_statistics();
97 pr_info("disable numa statistics, and clear numa counters\n");
101 mutex_unlock(&vm_numa_stat_lock
);
106 #ifdef CONFIG_VM_EVENT_COUNTERS
107 DEFINE_PER_CPU(struct vm_event_state
, vm_event_states
) = {{0}};
108 EXPORT_PER_CPU_SYMBOL(vm_event_states
);
110 static void sum_vm_events(unsigned long *ret
)
115 memset(ret
, 0, NR_VM_EVENT_ITEMS
* sizeof(unsigned long));
117 for_each_online_cpu(cpu
) {
118 struct vm_event_state
*this = &per_cpu(vm_event_states
, cpu
);
120 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++)
121 ret
[i
] += this->event
[i
];
126 * Accumulate the vm event counters across all CPUs.
127 * The result is unavoidably approximate - it can change
128 * during and after execution of this function.
130 void all_vm_events(unsigned long *ret
)
136 EXPORT_SYMBOL_GPL(all_vm_events
);
139 * Fold the foreign cpu events into our own.
141 * This is adding to the events on one processor
142 * but keeps the global counts constant.
144 void vm_events_fold_cpu(int cpu
)
146 struct vm_event_state
*fold_state
= &per_cpu(vm_event_states
, cpu
);
149 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++) {
150 count_vm_events(i
, fold_state
->event
[i
]);
151 fold_state
->event
[i
] = 0;
155 #endif /* CONFIG_VM_EVENT_COUNTERS */
158 * Manage combined zone based / global counters
160 * vm_stat contains the global counters
162 atomic_long_t vm_zone_stat
[NR_VM_ZONE_STAT_ITEMS
] __cacheline_aligned_in_smp
;
163 atomic_long_t vm_node_stat
[NR_VM_NODE_STAT_ITEMS
] __cacheline_aligned_in_smp
;
164 atomic_long_t vm_numa_event
[NR_VM_NUMA_EVENT_ITEMS
] __cacheline_aligned_in_smp
;
165 EXPORT_SYMBOL(vm_zone_stat
);
166 EXPORT_SYMBOL(vm_node_stat
);
169 static void fold_vm_zone_numa_events(struct zone
*zone
)
171 unsigned long zone_numa_events
[NR_VM_NUMA_EVENT_ITEMS
] = { 0, };
173 enum numa_stat_item item
;
175 for_each_online_cpu(cpu
) {
176 struct per_cpu_zonestat
*pzstats
;
178 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
179 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
180 zone_numa_events
[item
] += xchg(&pzstats
->vm_numa_event
[item
], 0);
183 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
184 zone_numa_event_add(zone_numa_events
[item
], zone
, item
);
187 void fold_vm_numa_events(void)
191 for_each_populated_zone(zone
)
192 fold_vm_zone_numa_events(zone
);
198 int calculate_pressure_threshold(struct zone
*zone
)
201 int watermark_distance
;
204 * As vmstats are not up to date, there is drift between the estimated
205 * and real values. For high thresholds and a high number of CPUs, it
206 * is possible for the min watermark to be breached while the estimated
207 * value looks fine. The pressure threshold is a reduced value such
208 * that even the maximum amount of drift will not accidentally breach
211 watermark_distance
= low_wmark_pages(zone
) - min_wmark_pages(zone
);
212 threshold
= max(1, (int)(watermark_distance
/ num_online_cpus()));
215 * Maximum threshold is 125
217 threshold
= min(125, threshold
);
222 int calculate_normal_threshold(struct zone
*zone
)
225 int mem
; /* memory in 128 MB units */
228 * The threshold scales with the number of processors and the amount
229 * of memory per zone. More memory means that we can defer updates for
230 * longer, more processors could lead to more contention.
231 * fls() is used to have a cheap way of logarithmic scaling.
233 * Some sample thresholds:
235 * Threshold Processors (fls) Zonesize fls(mem)+1
236 * ------------------------------------------------------------------
253 * 125 1024 10 8-16 GB 8
254 * 125 1024 10 16-32 GB 9
257 mem
= zone_managed_pages(zone
) >> (27 - PAGE_SHIFT
);
259 threshold
= 2 * fls(num_online_cpus()) * (1 + fls(mem
));
262 * Maximum threshold is 125
264 threshold
= min(125, threshold
);
270 * Refresh the thresholds for each zone.
272 void refresh_zone_stat_thresholds(void)
274 struct pglist_data
*pgdat
;
279 /* Zero current pgdat thresholds */
280 for_each_online_pgdat(pgdat
) {
281 for_each_online_cpu(cpu
) {
282 per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
= 0;
286 for_each_populated_zone(zone
) {
287 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
288 unsigned long max_drift
, tolerate_drift
;
290 threshold
= calculate_normal_threshold(zone
);
292 for_each_online_cpu(cpu
) {
295 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->stat_threshold
298 /* Base nodestat threshold on the largest populated zone. */
299 pgdat_threshold
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
;
300 per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
301 = max(threshold
, pgdat_threshold
);
305 * Only set percpu_drift_mark if there is a danger that
306 * NR_FREE_PAGES reports the low watermark is ok when in fact
307 * the min watermark could be breached by an allocation
309 tolerate_drift
= low_wmark_pages(zone
) - min_wmark_pages(zone
);
310 max_drift
= num_online_cpus() * threshold
;
311 if (max_drift
> tolerate_drift
)
312 zone
->percpu_drift_mark
= high_wmark_pages(zone
) +
317 void set_pgdat_percpu_threshold(pg_data_t
*pgdat
,
318 int (*calculate_pressure
)(struct zone
*))
325 for (i
= 0; i
< pgdat
->nr_zones
; i
++) {
326 zone
= &pgdat
->node_zones
[i
];
327 if (!zone
->percpu_drift_mark
)
330 threshold
= (*calculate_pressure
)(zone
);
331 for_each_online_cpu(cpu
)
332 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->stat_threshold
338 * For use when we know that interrupts are disabled,
339 * or when we know that preemption is disabled and that
340 * particular counter cannot be updated from interrupt context.
342 void __mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
345 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
346 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
351 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels,
352 * atomicity is provided by IRQs being disabled -- either explicitly
353 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables
354 * CPU migrations and preemption potentially corrupts a counter so
355 * disable preemption.
357 preempt_disable_nested();
359 x
= delta
+ __this_cpu_read(*p
);
361 t
= __this_cpu_read(pcp
->stat_threshold
);
363 if (unlikely(abs(x
) > t
)) {
364 zone_page_state_add(x
, zone
, item
);
367 __this_cpu_write(*p
, x
);
369 preempt_enable_nested();
371 EXPORT_SYMBOL(__mod_zone_page_state
);
373 void __mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
376 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
377 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
381 if (vmstat_item_in_bytes(item
)) {
383 * Only cgroups use subpage accounting right now; at
384 * the global level, these items still change in
385 * multiples of whole pages. Store them as pages
386 * internally to keep the per-cpu counters compact.
388 VM_WARN_ON_ONCE(delta
& (PAGE_SIZE
- 1));
389 delta
>>= PAGE_SHIFT
;
392 /* See __mod_node_page_state */
393 preempt_disable_nested();
395 x
= delta
+ __this_cpu_read(*p
);
397 t
= __this_cpu_read(pcp
->stat_threshold
);
399 if (unlikely(abs(x
) > t
)) {
400 node_page_state_add(x
, pgdat
, item
);
403 __this_cpu_write(*p
, x
);
405 preempt_enable_nested();
407 EXPORT_SYMBOL(__mod_node_page_state
);
410 * Optimized increment and decrement functions.
412 * These are only for a single page and therefore can take a struct page *
413 * argument instead of struct zone *. This allows the inclusion of the code
414 * generated for page_zone(page) into the optimized functions.
416 * No overflow check is necessary and therefore the differential can be
417 * incremented or decremented in place which may allow the compilers to
418 * generate better code.
419 * The increment or decrement is known and therefore one boundary check can
422 * NOTE: These functions are very performance sensitive. Change only
425 * Some processors have inc/dec instructions that are atomic vs an interrupt.
426 * However, the code must first determine the differential location in a zone
427 * based on the processor number and then inc/dec the counter. There is no
428 * guarantee without disabling preemption that the processor will not change
429 * in between and therefore the atomicity vs. interrupt cannot be exploited
430 * in a useful way here.
432 void __inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
434 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
435 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
438 /* See __mod_node_page_state */
439 preempt_disable_nested();
441 v
= __this_cpu_inc_return(*p
);
442 t
= __this_cpu_read(pcp
->stat_threshold
);
443 if (unlikely(v
> t
)) {
444 s8 overstep
= t
>> 1;
446 zone_page_state_add(v
+ overstep
, zone
, item
);
447 __this_cpu_write(*p
, -overstep
);
450 preempt_enable_nested();
453 void __inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
455 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
456 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
459 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
461 /* See __mod_node_page_state */
462 preempt_disable_nested();
464 v
= __this_cpu_inc_return(*p
);
465 t
= __this_cpu_read(pcp
->stat_threshold
);
466 if (unlikely(v
> t
)) {
467 s8 overstep
= t
>> 1;
469 node_page_state_add(v
+ overstep
, pgdat
, item
);
470 __this_cpu_write(*p
, -overstep
);
473 preempt_enable_nested();
476 void __inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
478 __inc_zone_state(page_zone(page
), item
);
480 EXPORT_SYMBOL(__inc_zone_page_state
);
482 void __inc_node_page_state(struct page
*page
, enum node_stat_item item
)
484 __inc_node_state(page_pgdat(page
), item
);
486 EXPORT_SYMBOL(__inc_node_page_state
);
488 void __dec_zone_state(struct zone
*zone
, enum zone_stat_item item
)
490 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
491 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
494 /* See __mod_node_page_state */
495 preempt_disable_nested();
497 v
= __this_cpu_dec_return(*p
);
498 t
= __this_cpu_read(pcp
->stat_threshold
);
499 if (unlikely(v
< - t
)) {
500 s8 overstep
= t
>> 1;
502 zone_page_state_add(v
- overstep
, zone
, item
);
503 __this_cpu_write(*p
, overstep
);
506 preempt_enable_nested();
509 void __dec_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
511 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
512 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
515 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
517 /* See __mod_node_page_state */
518 preempt_disable_nested();
520 v
= __this_cpu_dec_return(*p
);
521 t
= __this_cpu_read(pcp
->stat_threshold
);
522 if (unlikely(v
< - t
)) {
523 s8 overstep
= t
>> 1;
525 node_page_state_add(v
- overstep
, pgdat
, item
);
526 __this_cpu_write(*p
, overstep
);
529 preempt_enable_nested();
532 void __dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
534 __dec_zone_state(page_zone(page
), item
);
536 EXPORT_SYMBOL(__dec_zone_page_state
);
538 void __dec_node_page_state(struct page
*page
, enum node_stat_item item
)
540 __dec_node_state(page_pgdat(page
), item
);
542 EXPORT_SYMBOL(__dec_node_page_state
);
544 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
546 * If we have cmpxchg_local support then we do not need to incur the overhead
547 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
549 * mod_state() modifies the zone counter state through atomic per cpu
552 * Overstep mode specifies how overstep should handled:
554 * 1 Overstepping half of threshold
555 * -1 Overstepping minus half of threshold
557 static inline void mod_zone_state(struct zone
*zone
,
558 enum zone_stat_item item
, long delta
, int overstep_mode
)
560 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
561 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
565 o
= this_cpu_read(*p
);
567 z
= 0; /* overflow to zone counters */
570 * The fetching of the stat_threshold is racy. We may apply
571 * a counter threshold to the wrong the cpu if we get
572 * rescheduled while executing here. However, the next
573 * counter update will apply the threshold again and
574 * therefore bring the counter under the threshold again.
576 * Most of the time the thresholds are the same anyways
577 * for all cpus in a zone.
579 t
= this_cpu_read(pcp
->stat_threshold
);
584 int os
= overstep_mode
* (t
>> 1) ;
586 /* Overflow must be added to zone counters */
590 } while (!this_cpu_try_cmpxchg(*p
, &o
, n
));
593 zone_page_state_add(z
, zone
, item
);
596 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
599 mod_zone_state(zone
, item
, delta
, 0);
601 EXPORT_SYMBOL(mod_zone_page_state
);
603 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
605 mod_zone_state(page_zone(page
), item
, 1, 1);
607 EXPORT_SYMBOL(inc_zone_page_state
);
609 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
611 mod_zone_state(page_zone(page
), item
, -1, -1);
613 EXPORT_SYMBOL(dec_zone_page_state
);
615 static inline void mod_node_state(struct pglist_data
*pgdat
,
616 enum node_stat_item item
, int delta
, int overstep_mode
)
618 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
619 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
623 if (vmstat_item_in_bytes(item
)) {
625 * Only cgroups use subpage accounting right now; at
626 * the global level, these items still change in
627 * multiples of whole pages. Store them as pages
628 * internally to keep the per-cpu counters compact.
630 VM_WARN_ON_ONCE(delta
& (PAGE_SIZE
- 1));
631 delta
>>= PAGE_SHIFT
;
634 o
= this_cpu_read(*p
);
636 z
= 0; /* overflow to node counters */
639 * The fetching of the stat_threshold is racy. We may apply
640 * a counter threshold to the wrong the cpu if we get
641 * rescheduled while executing here. However, the next
642 * counter update will apply the threshold again and
643 * therefore bring the counter under the threshold again.
645 * Most of the time the thresholds are the same anyways
646 * for all cpus in a node.
648 t
= this_cpu_read(pcp
->stat_threshold
);
653 int os
= overstep_mode
* (t
>> 1) ;
655 /* Overflow must be added to node counters */
659 } while (!this_cpu_try_cmpxchg(*p
, &o
, n
));
662 node_page_state_add(z
, pgdat
, item
);
665 void mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
668 mod_node_state(pgdat
, item
, delta
, 0);
670 EXPORT_SYMBOL(mod_node_page_state
);
672 void inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
674 mod_node_state(pgdat
, item
, 1, 1);
677 void inc_node_page_state(struct page
*page
, enum node_stat_item item
)
679 mod_node_state(page_pgdat(page
), item
, 1, 1);
681 EXPORT_SYMBOL(inc_node_page_state
);
683 void dec_node_page_state(struct page
*page
, enum node_stat_item item
)
685 mod_node_state(page_pgdat(page
), item
, -1, -1);
687 EXPORT_SYMBOL(dec_node_page_state
);
690 * Use interrupt disable to serialize counter updates
692 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
697 local_irq_save(flags
);
698 __mod_zone_page_state(zone
, item
, delta
);
699 local_irq_restore(flags
);
701 EXPORT_SYMBOL(mod_zone_page_state
);
703 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
708 zone
= page_zone(page
);
709 local_irq_save(flags
);
710 __inc_zone_state(zone
, item
);
711 local_irq_restore(flags
);
713 EXPORT_SYMBOL(inc_zone_page_state
);
715 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
719 local_irq_save(flags
);
720 __dec_zone_page_state(page
, item
);
721 local_irq_restore(flags
);
723 EXPORT_SYMBOL(dec_zone_page_state
);
725 void inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
729 local_irq_save(flags
);
730 __inc_node_state(pgdat
, item
);
731 local_irq_restore(flags
);
733 EXPORT_SYMBOL(inc_node_state
);
735 void mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
740 local_irq_save(flags
);
741 __mod_node_page_state(pgdat
, item
, delta
);
742 local_irq_restore(flags
);
744 EXPORT_SYMBOL(mod_node_page_state
);
746 void inc_node_page_state(struct page
*page
, enum node_stat_item item
)
749 struct pglist_data
*pgdat
;
751 pgdat
= page_pgdat(page
);
752 local_irq_save(flags
);
753 __inc_node_state(pgdat
, item
);
754 local_irq_restore(flags
);
756 EXPORT_SYMBOL(inc_node_page_state
);
758 void dec_node_page_state(struct page
*page
, enum node_stat_item item
)
762 local_irq_save(flags
);
763 __dec_node_page_state(page
, item
);
764 local_irq_restore(flags
);
766 EXPORT_SYMBOL(dec_node_page_state
);
770 * Fold a differential into the global counters.
771 * Returns the number of counters updated.
773 static int fold_diff(int *zone_diff
, int *node_diff
)
778 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
780 atomic_long_add(zone_diff
[i
], &vm_zone_stat
[i
]);
784 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++)
786 atomic_long_add(node_diff
[i
], &vm_node_stat
[i
]);
793 * Update the zone counters for the current cpu.
795 * Note that refresh_cpu_vm_stats strives to only access
796 * node local memory. The per cpu pagesets on remote zones are placed
797 * in the memory local to the processor using that pageset. So the
798 * loop over all zones will access a series of cachelines local to
801 * The call to zone_page_state_add updates the cachelines with the
802 * statistics in the remote zone struct as well as the global cachelines
803 * with the global counters. These could cause remote node cache line
804 * bouncing and will have to be only done when necessary.
806 * The function returns the number of global counters updated.
808 static int refresh_cpu_vm_stats(bool do_pagesets
)
810 struct pglist_data
*pgdat
;
813 int global_zone_diff
[NR_VM_ZONE_STAT_ITEMS
] = { 0, };
814 int global_node_diff
[NR_VM_NODE_STAT_ITEMS
] = { 0, };
817 for_each_populated_zone(zone
) {
818 struct per_cpu_zonestat __percpu
*pzstats
= zone
->per_cpu_zonestats
;
819 struct per_cpu_pages __percpu
*pcp
= zone
->per_cpu_pageset
;
821 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
824 v
= this_cpu_xchg(pzstats
->vm_stat_diff
[i
], 0);
827 atomic_long_add(v
, &zone
->vm_stat
[i
]);
828 global_zone_diff
[i
] += v
;
830 /* 3 seconds idle till flush */
831 __this_cpu_write(pcp
->expire
, 3);
839 changes
+= decay_pcp_high(zone
, this_cpu_ptr(pcp
));
842 * Deal with draining the remote pageset of this
845 * Check if there are pages remaining in this pageset
846 * if not then there is nothing to expire.
848 if (!__this_cpu_read(pcp
->expire
) ||
849 !__this_cpu_read(pcp
->count
))
853 * We never drain zones local to this processor.
855 if (zone_to_nid(zone
) == numa_node_id()) {
856 __this_cpu_write(pcp
->expire
, 0);
860 if (__this_cpu_dec_return(pcp
->expire
)) {
865 if (__this_cpu_read(pcp
->count
)) {
866 drain_zone_pages(zone
, this_cpu_ptr(pcp
));
873 for_each_online_pgdat(pgdat
) {
874 struct per_cpu_nodestat __percpu
*p
= pgdat
->per_cpu_nodestats
;
876 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
879 v
= this_cpu_xchg(p
->vm_node_stat_diff
[i
], 0);
881 atomic_long_add(v
, &pgdat
->vm_stat
[i
]);
882 global_node_diff
[i
] += v
;
887 changes
+= fold_diff(global_zone_diff
, global_node_diff
);
892 * Fold the data for an offline cpu into the global array.
893 * There cannot be any access by the offline cpu and therefore
894 * synchronization is simplified.
896 void cpu_vm_stats_fold(int cpu
)
898 struct pglist_data
*pgdat
;
901 int global_zone_diff
[NR_VM_ZONE_STAT_ITEMS
] = { 0, };
902 int global_node_diff
[NR_VM_NODE_STAT_ITEMS
] = { 0, };
904 for_each_populated_zone(zone
) {
905 struct per_cpu_zonestat
*pzstats
;
907 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
909 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
910 if (pzstats
->vm_stat_diff
[i
]) {
913 v
= pzstats
->vm_stat_diff
[i
];
914 pzstats
->vm_stat_diff
[i
] = 0;
915 atomic_long_add(v
, &zone
->vm_stat
[i
]);
916 global_zone_diff
[i
] += v
;
920 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++) {
921 if (pzstats
->vm_numa_event
[i
]) {
924 v
= pzstats
->vm_numa_event
[i
];
925 pzstats
->vm_numa_event
[i
] = 0;
926 zone_numa_event_add(v
, zone
, i
);
932 for_each_online_pgdat(pgdat
) {
933 struct per_cpu_nodestat
*p
;
935 p
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
);
937 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++)
938 if (p
->vm_node_stat_diff
[i
]) {
941 v
= p
->vm_node_stat_diff
[i
];
942 p
->vm_node_stat_diff
[i
] = 0;
943 atomic_long_add(v
, &pgdat
->vm_stat
[i
]);
944 global_node_diff
[i
] += v
;
948 fold_diff(global_zone_diff
, global_node_diff
);
952 * this is only called if !populated_zone(zone), which implies no other users of
953 * pset->vm_stat_diff[] exist.
955 void drain_zonestat(struct zone
*zone
, struct per_cpu_zonestat
*pzstats
)
960 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
961 if (pzstats
->vm_stat_diff
[i
]) {
962 v
= pzstats
->vm_stat_diff
[i
];
963 pzstats
->vm_stat_diff
[i
] = 0;
964 zone_page_state_add(v
, zone
, i
);
969 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++) {
970 if (pzstats
->vm_numa_event
[i
]) {
971 v
= pzstats
->vm_numa_event
[i
];
972 pzstats
->vm_numa_event
[i
] = 0;
973 zone_numa_event_add(v
, zone
, i
);
982 * Determine the per node value of a stat item. This function
983 * is called frequently in a NUMA machine, so try to be as
984 * frugal as possible.
986 unsigned long sum_zone_node_page_state(int node
,
987 enum zone_stat_item item
)
989 struct zone
*zones
= NODE_DATA(node
)->node_zones
;
991 unsigned long count
= 0;
993 for (i
= 0; i
< MAX_NR_ZONES
; i
++)
994 count
+= zone_page_state(zones
+ i
, item
);
999 /* Determine the per node value of a numa stat item. */
1000 unsigned long sum_zone_numa_event_state(int node
,
1001 enum numa_stat_item item
)
1003 struct zone
*zones
= NODE_DATA(node
)->node_zones
;
1004 unsigned long count
= 0;
1007 for (i
= 0; i
< MAX_NR_ZONES
; i
++)
1008 count
+= zone_numa_event_state(zones
+ i
, item
);
1014 * Determine the per node value of a stat item.
1016 unsigned long node_page_state_pages(struct pglist_data
*pgdat
,
1017 enum node_stat_item item
)
1019 long x
= atomic_long_read(&pgdat
->vm_stat
[item
]);
1027 unsigned long node_page_state(struct pglist_data
*pgdat
,
1028 enum node_stat_item item
)
1030 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
1032 return node_page_state_pages(pgdat
, item
);
1036 #ifdef CONFIG_COMPACTION
1038 struct contig_page_info
{
1039 unsigned long free_pages
;
1040 unsigned long free_blocks_total
;
1041 unsigned long free_blocks_suitable
;
1045 * Calculate the number of free pages in a zone, how many contiguous
1046 * pages are free and how many are large enough to satisfy an allocation of
1047 * the target size. Note that this function makes no attempt to estimate
1048 * how many suitable free blocks there *might* be if MOVABLE pages were
1049 * migrated. Calculating that is possible, but expensive and can be
1050 * figured out from userspace
1052 static void fill_contig_page_info(struct zone
*zone
,
1053 unsigned int suitable_order
,
1054 struct contig_page_info
*info
)
1058 info
->free_pages
= 0;
1059 info
->free_blocks_total
= 0;
1060 info
->free_blocks_suitable
= 0;
1062 for (order
= 0; order
< NR_PAGE_ORDERS
; order
++) {
1063 unsigned long blocks
;
1066 * Count number of free blocks.
1068 * Access to nr_free is lockless as nr_free is used only for
1069 * diagnostic purposes. Use data_race to avoid KCSAN warning.
1071 blocks
= data_race(zone
->free_area
[order
].nr_free
);
1072 info
->free_blocks_total
+= blocks
;
1074 /* Count free base pages */
1075 info
->free_pages
+= blocks
<< order
;
1077 /* Count the suitable free blocks */
1078 if (order
>= suitable_order
)
1079 info
->free_blocks_suitable
+= blocks
<<
1080 (order
- suitable_order
);
1085 * A fragmentation index only makes sense if an allocation of a requested
1086 * size would fail. If that is true, the fragmentation index indicates
1087 * whether external fragmentation or a lack of memory was the problem.
1088 * The value can be used to determine if page reclaim or compaction
1091 static int __fragmentation_index(unsigned int order
, struct contig_page_info
*info
)
1093 unsigned long requested
= 1UL << order
;
1095 if (WARN_ON_ONCE(order
> MAX_PAGE_ORDER
))
1098 if (!info
->free_blocks_total
)
1101 /* Fragmentation index only makes sense when a request would fail */
1102 if (info
->free_blocks_suitable
)
1106 * Index is between 0 and 1 so return within 3 decimal places
1108 * 0 => allocation would fail due to lack of memory
1109 * 1 => allocation would fail due to fragmentation
1111 return 1000 - div_u64( (1000+(div_u64(info
->free_pages
* 1000ULL, requested
))), info
->free_blocks_total
);
1115 * Calculates external fragmentation within a zone wrt the given order.
1116 * It is defined as the percentage of pages found in blocks of size
1117 * less than 1 << order. It returns values in range [0, 100].
1119 unsigned int extfrag_for_order(struct zone
*zone
, unsigned int order
)
1121 struct contig_page_info info
;
1123 fill_contig_page_info(zone
, order
, &info
);
1124 if (info
.free_pages
== 0)
1127 return div_u64((info
.free_pages
-
1128 (info
.free_blocks_suitable
<< order
)) * 100,
1132 /* Same as __fragmentation index but allocs contig_page_info on stack */
1133 int fragmentation_index(struct zone
*zone
, unsigned int order
)
1135 struct contig_page_info info
;
1137 fill_contig_page_info(zone
, order
, &info
);
1138 return __fragmentation_index(order
, &info
);
1142 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
1143 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
1144 #ifdef CONFIG_ZONE_DMA
1145 #define TEXT_FOR_DMA(xx) xx "_dma",
1147 #define TEXT_FOR_DMA(xx)
1150 #ifdef CONFIG_ZONE_DMA32
1151 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1153 #define TEXT_FOR_DMA32(xx)
1156 #ifdef CONFIG_HIGHMEM
1157 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1159 #define TEXT_FOR_HIGHMEM(xx)
1162 #ifdef CONFIG_ZONE_DEVICE
1163 #define TEXT_FOR_DEVICE(xx) xx "_device",
1165 #define TEXT_FOR_DEVICE(xx)
1168 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1169 TEXT_FOR_HIGHMEM(xx) xx "_movable", \
1172 const char * const vmstat_text
[] = {
1173 /* enum zone_stat_item counters */
1175 "nr_zone_inactive_anon",
1176 "nr_zone_active_anon",
1177 "nr_zone_inactive_file",
1178 "nr_zone_active_file",
1179 "nr_zone_unevictable",
1180 "nr_zone_write_pending",
1183 #if IS_ENABLED(CONFIG_ZSMALLOC)
1187 #ifdef CONFIG_UNACCEPTED_MEMORY
1191 /* enum numa_stat_item counters */
1201 /* enum node_stat_item counters */
1207 "nr_slab_reclaimable",
1208 "nr_slab_unreclaimable",
1212 "workingset_refault_anon",
1213 "workingset_refault_file",
1214 "workingset_activate_anon",
1215 "workingset_activate_file",
1216 "workingset_restore_anon",
1217 "workingset_restore_file",
1218 "workingset_nodereclaim",
1224 "nr_writeback_temp",
1226 "nr_shmem_hugepages",
1227 "nr_shmem_pmdmapped",
1228 "nr_file_hugepages",
1229 "nr_file_pmdmapped",
1230 "nr_anon_transparent_hugepages",
1232 "nr_vmscan_immediate_reclaim",
1235 "nr_throttled_written",
1236 "nr_kernel_misc_reclaimable",
1237 "nr_foll_pin_acquired",
1238 "nr_foll_pin_released",
1240 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1241 "nr_shadow_call_stack",
1243 "nr_page_table_pages",
1244 "nr_sec_page_table_pages",
1248 #ifdef CONFIG_NUMA_BALANCING
1249 "pgpromote_success",
1250 "pgpromote_candidate",
1254 "pgdemote_khugepaged",
1256 /* enum writeback_stat_item counters */
1257 "nr_dirty_threshold",
1258 "nr_dirty_background_threshold",
1260 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
1261 /* enum vm_event_item counters */
1267 TEXTS_FOR_ZONES("pgalloc")
1268 TEXTS_FOR_ZONES("allocstall")
1269 TEXTS_FOR_ZONES("pgskip")
1284 "pgsteal_khugepaged",
1287 "pgscan_khugepaged",
1288 "pgscan_direct_throttle",
1295 "zone_reclaim_failed",
1299 "kswapd_inodesteal",
1300 "kswapd_low_wmark_hit_quickly",
1301 "kswapd_high_wmark_hit_quickly",
1310 #ifdef CONFIG_NUMA_BALANCING
1312 "numa_huge_pte_updates",
1314 "numa_hint_faults_local",
1315 "numa_pages_migrated",
1317 #ifdef CONFIG_MIGRATION
1318 "pgmigrate_success",
1320 "thp_migration_success",
1321 "thp_migration_fail",
1322 "thp_migration_split",
1324 #ifdef CONFIG_COMPACTION
1325 "compact_migrate_scanned",
1326 "compact_free_scanned",
1331 "compact_daemon_wake",
1332 "compact_daemon_migrate_scanned",
1333 "compact_daemon_free_scanned",
1336 #ifdef CONFIG_HUGETLB_PAGE
1337 "htlb_buddy_alloc_success",
1338 "htlb_buddy_alloc_fail",
1341 "cma_alloc_success",
1344 "unevictable_pgs_culled",
1345 "unevictable_pgs_scanned",
1346 "unevictable_pgs_rescued",
1347 "unevictable_pgs_mlocked",
1348 "unevictable_pgs_munlocked",
1349 "unevictable_pgs_cleared",
1350 "unevictable_pgs_stranded",
1352 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1354 "thp_fault_fallback",
1355 "thp_fault_fallback_charge",
1356 "thp_collapse_alloc",
1357 "thp_collapse_alloc_failed",
1359 "thp_file_fallback",
1360 "thp_file_fallback_charge",
1363 "thp_split_page_failed",
1364 "thp_deferred_split_page",
1366 "thp_scan_exceed_none_pte",
1367 "thp_scan_exceed_swap_pte",
1368 "thp_scan_exceed_share_pte",
1369 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1372 "thp_zero_page_alloc",
1373 "thp_zero_page_alloc_failed",
1375 "thp_swpout_fallback",
1377 #ifdef CONFIG_MEMORY_BALLOON
1380 #ifdef CONFIG_BALLOON_COMPACTION
1383 #endif /* CONFIG_MEMORY_BALLOON */
1384 #ifdef CONFIG_DEBUG_TLBFLUSH
1385 "nr_tlb_remote_flush",
1386 "nr_tlb_remote_flush_received",
1387 "nr_tlb_local_flush_all",
1388 "nr_tlb_local_flush_one",
1389 #endif /* CONFIG_DEBUG_TLBFLUSH */
1407 "direct_map_level2_splits",
1408 "direct_map_level3_splits",
1410 #ifdef CONFIG_PER_VMA_LOCK_STATS
1416 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
1418 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
1420 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1421 defined(CONFIG_PROC_FS)
1422 static void *frag_start(struct seq_file
*m
, loff_t
*pos
)
1427 for (pgdat
= first_online_pgdat();
1429 pgdat
= next_online_pgdat(pgdat
))
1435 static void *frag_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
1437 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1440 return next_online_pgdat(pgdat
);
1443 static void frag_stop(struct seq_file
*m
, void *arg
)
1448 * Walk zones in a node and print using a callback.
1449 * If @assert_populated is true, only use callback for zones that are populated.
1451 static void walk_zones_in_node(struct seq_file
*m
, pg_data_t
*pgdat
,
1452 bool assert_populated
, bool nolock
,
1453 void (*print
)(struct seq_file
*m
, pg_data_t
*, struct zone
*))
1456 struct zone
*node_zones
= pgdat
->node_zones
;
1457 unsigned long flags
;
1459 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
1460 if (assert_populated
&& !populated_zone(zone
))
1464 spin_lock_irqsave(&zone
->lock
, flags
);
1465 print(m
, pgdat
, zone
);
1467 spin_unlock_irqrestore(&zone
->lock
, flags
);
1472 #ifdef CONFIG_PROC_FS
1473 static void frag_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
1478 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
1479 for (order
= 0; order
< NR_PAGE_ORDERS
; ++order
)
1481 * Access to nr_free is lockless as nr_free is used only for
1482 * printing purposes. Use data_race to avoid KCSAN warning.
1484 seq_printf(m
, "%6lu ", data_race(zone
->free_area
[order
].nr_free
));
1489 * This walks the free areas for each zone.
1491 static int frag_show(struct seq_file
*m
, void *arg
)
1493 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1494 walk_zones_in_node(m
, pgdat
, true, false, frag_show_print
);
1498 static void pagetypeinfo_showfree_print(struct seq_file
*m
,
1499 pg_data_t
*pgdat
, struct zone
*zone
)
1503 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++) {
1504 seq_printf(m
, "Node %4d, zone %8s, type %12s ",
1507 migratetype_names
[mtype
]);
1508 for (order
= 0; order
< NR_PAGE_ORDERS
; ++order
) {
1509 unsigned long freecount
= 0;
1510 struct free_area
*area
;
1511 struct list_head
*curr
;
1512 bool overflow
= false;
1514 area
= &(zone
->free_area
[order
]);
1516 list_for_each(curr
, &area
->free_list
[mtype
]) {
1518 * Cap the free_list iteration because it might
1519 * be really large and we are under a spinlock
1520 * so a long time spent here could trigger a
1521 * hard lockup detector. Anyway this is a
1522 * debugging tool so knowing there is a handful
1523 * of pages of this order should be more than
1526 if (++freecount
>= 100000) {
1531 seq_printf(m
, "%s%6lu ", overflow
? ">" : "", freecount
);
1532 spin_unlock_irq(&zone
->lock
);
1534 spin_lock_irq(&zone
->lock
);
1540 /* Print out the free pages at each order for each migatetype */
1541 static void pagetypeinfo_showfree(struct seq_file
*m
, void *arg
)
1544 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1547 seq_printf(m
, "%-43s ", "Free pages count per migrate type at order");
1548 for (order
= 0; order
< NR_PAGE_ORDERS
; ++order
)
1549 seq_printf(m
, "%6d ", order
);
1552 walk_zones_in_node(m
, pgdat
, true, false, pagetypeinfo_showfree_print
);
1555 static void pagetypeinfo_showblockcount_print(struct seq_file
*m
,
1556 pg_data_t
*pgdat
, struct zone
*zone
)
1560 unsigned long start_pfn
= zone
->zone_start_pfn
;
1561 unsigned long end_pfn
= zone_end_pfn(zone
);
1562 unsigned long count
[MIGRATE_TYPES
] = { 0, };
1564 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pageblock_nr_pages
) {
1567 page
= pfn_to_online_page(pfn
);
1571 if (page_zone(page
) != zone
)
1574 mtype
= get_pageblock_migratetype(page
);
1576 if (mtype
< MIGRATE_TYPES
)
1581 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
1582 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1583 seq_printf(m
, "%12lu ", count
[mtype
]);
1587 /* Print out the number of pageblocks for each migratetype */
1588 static void pagetypeinfo_showblockcount(struct seq_file
*m
, void *arg
)
1591 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1593 seq_printf(m
, "\n%-23s", "Number of blocks type ");
1594 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1595 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
1597 walk_zones_in_node(m
, pgdat
, true, false,
1598 pagetypeinfo_showblockcount_print
);
1602 * Print out the number of pageblocks for each migratetype that contain pages
1603 * of other types. This gives an indication of how well fallbacks are being
1604 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1605 * to determine what is going on
1607 static void pagetypeinfo_showmixedcount(struct seq_file
*m
, pg_data_t
*pgdat
)
1609 #ifdef CONFIG_PAGE_OWNER
1612 if (!static_branch_unlikely(&page_owner_inited
))
1615 drain_all_pages(NULL
);
1617 seq_printf(m
, "\n%-23s", "Number of mixed blocks ");
1618 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1619 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
1622 walk_zones_in_node(m
, pgdat
, true, true,
1623 pagetypeinfo_showmixedcount_print
);
1624 #endif /* CONFIG_PAGE_OWNER */
1628 * This prints out statistics in relation to grouping pages by mobility.
1629 * It is expensive to collect so do not constantly read the file.
1631 static int pagetypeinfo_show(struct seq_file
*m
, void *arg
)
1633 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1635 /* check memoryless node */
1636 if (!node_state(pgdat
->node_id
, N_MEMORY
))
1639 seq_printf(m
, "Page block order: %d\n", pageblock_order
);
1640 seq_printf(m
, "Pages per block: %lu\n", pageblock_nr_pages
);
1642 pagetypeinfo_showfree(m
, pgdat
);
1643 pagetypeinfo_showblockcount(m
, pgdat
);
1644 pagetypeinfo_showmixedcount(m
, pgdat
);
1649 static const struct seq_operations fragmentation_op
= {
1650 .start
= frag_start
,
1656 static const struct seq_operations pagetypeinfo_op
= {
1657 .start
= frag_start
,
1660 .show
= pagetypeinfo_show
,
1663 static bool is_zone_first_populated(pg_data_t
*pgdat
, struct zone
*zone
)
1667 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
1668 struct zone
*compare
= &pgdat
->node_zones
[zid
];
1670 if (populated_zone(compare
))
1671 return zone
== compare
;
1677 static void zoneinfo_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
1681 seq_printf(m
, "Node %d, zone %8s", pgdat
->node_id
, zone
->name
);
1682 if (is_zone_first_populated(pgdat
, zone
)) {
1683 seq_printf(m
, "\n per-node stats");
1684 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1685 unsigned long pages
= node_page_state_pages(pgdat
, i
);
1687 if (vmstat_item_print_in_thp(i
))
1688 pages
/= HPAGE_PMD_NR
;
1689 seq_printf(m
, "\n %-12s %lu", node_stat_name(i
),
1703 zone_page_state(zone
, NR_FREE_PAGES
),
1704 zone
->watermark_boost
,
1705 min_wmark_pages(zone
),
1706 low_wmark_pages(zone
),
1707 high_wmark_pages(zone
),
1708 zone
->spanned_pages
,
1709 zone
->present_pages
,
1710 zone_managed_pages(zone
),
1711 zone_cma_pages(zone
));
1714 "\n protection: (%ld",
1715 zone
->lowmem_reserve
[0]);
1716 for (i
= 1; i
< ARRAY_SIZE(zone
->lowmem_reserve
); i
++)
1717 seq_printf(m
, ", %ld", zone
->lowmem_reserve
[i
]);
1720 /* If unpopulated, no other information is useful */
1721 if (!populated_zone(zone
)) {
1726 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
1727 seq_printf(m
, "\n %-12s %lu", zone_stat_name(i
),
1728 zone_page_state(zone
, i
));
1731 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++)
1732 seq_printf(m
, "\n %-12s %lu", numa_stat_name(i
),
1733 zone_numa_event_state(zone
, i
));
1736 seq_printf(m
, "\n pagesets");
1737 for_each_online_cpu(i
) {
1738 struct per_cpu_pages
*pcp
;
1739 struct per_cpu_zonestat __maybe_unused
*pzstats
;
1741 pcp
= per_cpu_ptr(zone
->per_cpu_pageset
, i
);
1752 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, i
);
1753 seq_printf(m
, "\n vm stats threshold: %d",
1754 pzstats
->stat_threshold
);
1758 "\n node_unreclaimable: %u"
1759 "\n start_pfn: %lu",
1760 pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
,
1761 zone
->zone_start_pfn
);
1766 * Output information about zones in @pgdat. All zones are printed regardless
1767 * of whether they are populated or not: lowmem_reserve_ratio operates on the
1768 * set of all zones and userspace would not be aware of such zones if they are
1769 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
1771 static int zoneinfo_show(struct seq_file
*m
, void *arg
)
1773 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1774 walk_zones_in_node(m
, pgdat
, false, false, zoneinfo_show_print
);
1778 static const struct seq_operations zoneinfo_op
= {
1779 .start
= frag_start
, /* iterate over all zones. The same as in
1783 .show
= zoneinfo_show
,
1786 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1787 NR_VM_NUMA_EVENT_ITEMS + \
1788 NR_VM_NODE_STAT_ITEMS + \
1789 NR_VM_WRITEBACK_STAT_ITEMS + \
1790 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1791 NR_VM_EVENT_ITEMS : 0))
1793 static void *vmstat_start(struct seq_file
*m
, loff_t
*pos
)
1798 if (*pos
>= NR_VMSTAT_ITEMS
)
1801 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text
) < NR_VMSTAT_ITEMS
);
1802 fold_vm_numa_events();
1803 v
= kmalloc_array(NR_VMSTAT_ITEMS
, sizeof(unsigned long), GFP_KERNEL
);
1806 return ERR_PTR(-ENOMEM
);
1807 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
1808 v
[i
] = global_zone_page_state(i
);
1809 v
+= NR_VM_ZONE_STAT_ITEMS
;
1812 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++)
1813 v
[i
] = global_numa_event_state(i
);
1814 v
+= NR_VM_NUMA_EVENT_ITEMS
;
1817 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1818 v
[i
] = global_node_page_state_pages(i
);
1819 if (vmstat_item_print_in_thp(i
))
1820 v
[i
] /= HPAGE_PMD_NR
;
1822 v
+= NR_VM_NODE_STAT_ITEMS
;
1824 global_dirty_limits(v
+ NR_DIRTY_BG_THRESHOLD
,
1825 v
+ NR_DIRTY_THRESHOLD
);
1826 v
+= NR_VM_WRITEBACK_STAT_ITEMS
;
1828 #ifdef CONFIG_VM_EVENT_COUNTERS
1830 v
[PGPGIN
] /= 2; /* sectors -> kbytes */
1833 return (unsigned long *)m
->private + *pos
;
1836 static void *vmstat_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
1839 if (*pos
>= NR_VMSTAT_ITEMS
)
1841 return (unsigned long *)m
->private + *pos
;
1844 static int vmstat_show(struct seq_file
*m
, void *arg
)
1846 unsigned long *l
= arg
;
1847 unsigned long off
= l
- (unsigned long *)m
->private;
1849 seq_puts(m
, vmstat_text
[off
]);
1850 seq_put_decimal_ull(m
, " ", *l
);
1853 if (off
== NR_VMSTAT_ITEMS
- 1) {
1855 * We've come to the end - add any deprecated counters to avoid
1856 * breaking userspace which might depend on them being present.
1858 seq_puts(m
, "nr_unstable 0\n");
1863 static void vmstat_stop(struct seq_file
*m
, void *arg
)
1869 static const struct seq_operations vmstat_op
= {
1870 .start
= vmstat_start
,
1871 .next
= vmstat_next
,
1872 .stop
= vmstat_stop
,
1873 .show
= vmstat_show
,
1875 #endif /* CONFIG_PROC_FS */
1878 static DEFINE_PER_CPU(struct delayed_work
, vmstat_work
);
1879 int sysctl_stat_interval __read_mostly
= HZ
;
1881 #ifdef CONFIG_PROC_FS
1882 static void refresh_vm_stats(struct work_struct
*work
)
1884 refresh_cpu_vm_stats(true);
1887 int vmstat_refresh(struct ctl_table
*table
, int write
,
1888 void *buffer
, size_t *lenp
, loff_t
*ppos
)
1895 * The regular update, every sysctl_stat_interval, may come later
1896 * than expected: leaving a significant amount in per_cpu buckets.
1897 * This is particularly misleading when checking a quantity of HUGE
1898 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1899 * which can equally be echo'ed to or cat'ted from (by root),
1900 * can be used to update the stats just before reading them.
1902 * Oh, and since global_zone_page_state() etc. are so careful to hide
1903 * transiently negative values, report an error here if any of
1904 * the stats is negative, so we know to go looking for imbalance.
1906 err
= schedule_on_each_cpu(refresh_vm_stats
);
1909 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
1911 * Skip checking stats known to go negative occasionally.
1914 case NR_ZONE_WRITE_PENDING
:
1915 case NR_FREE_CMA_PAGES
:
1918 val
= atomic_long_read(&vm_zone_stat
[i
]);
1920 pr_warn("%s: %s %ld\n",
1921 __func__
, zone_stat_name(i
), val
);
1924 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1926 * Skip checking stats known to go negative occasionally.
1932 val
= atomic_long_read(&vm_node_stat
[i
]);
1934 pr_warn("%s: %s %ld\n",
1935 __func__
, node_stat_name(i
), val
);
1944 #endif /* CONFIG_PROC_FS */
1946 static void vmstat_update(struct work_struct
*w
)
1948 if (refresh_cpu_vm_stats(true)) {
1950 * Counters were updated so we expect more updates
1951 * to occur in the future. Keep on running the
1952 * update worker thread.
1954 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq
,
1955 this_cpu_ptr(&vmstat_work
),
1956 round_jiffies_relative(sysctl_stat_interval
));
1961 * Check if the diffs for a certain cpu indicate that
1962 * an update is needed.
1964 static bool need_update(int cpu
)
1966 pg_data_t
*last_pgdat
= NULL
;
1969 for_each_populated_zone(zone
) {
1970 struct per_cpu_zonestat
*pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
1971 struct per_cpu_nodestat
*n
;
1974 * The fast way of checking if there are any vmstat diffs.
1976 if (memchr_inv(pzstats
->vm_stat_diff
, 0, sizeof(pzstats
->vm_stat_diff
)))
1979 if (last_pgdat
== zone
->zone_pgdat
)
1981 last_pgdat
= zone
->zone_pgdat
;
1982 n
= per_cpu_ptr(zone
->zone_pgdat
->per_cpu_nodestats
, cpu
);
1983 if (memchr_inv(n
->vm_node_stat_diff
, 0, sizeof(n
->vm_node_stat_diff
)))
1990 * Switch off vmstat processing and then fold all the remaining differentials
1991 * until the diffs stay at zero. The function is used by NOHZ and can only be
1992 * invoked when tick processing is not active.
1994 void quiet_vmstat(void)
1996 if (system_state
!= SYSTEM_RUNNING
)
1999 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work
)))
2002 if (!need_update(smp_processor_id()))
2006 * Just refresh counters and do not care about the pending delayed
2007 * vmstat_update. It doesn't fire that often to matter and canceling
2008 * it would be too expensive from this path.
2009 * vmstat_shepherd will take care about that for us.
2011 refresh_cpu_vm_stats(false);
2015 * Shepherd worker thread that checks the
2016 * differentials of processors that have their worker
2017 * threads for vm statistics updates disabled because of
2020 static void vmstat_shepherd(struct work_struct
*w
);
2022 static DECLARE_DEFERRABLE_WORK(shepherd
, vmstat_shepherd
);
2024 static void vmstat_shepherd(struct work_struct
*w
)
2029 /* Check processors whose vmstat worker threads have been disabled */
2030 for_each_online_cpu(cpu
) {
2031 struct delayed_work
*dw
= &per_cpu(vmstat_work
, cpu
);
2034 * In kernel users of vmstat counters either require the precise value and
2035 * they are using zone_page_state_snapshot interface or they can live with
2036 * an imprecision as the regular flushing can happen at arbitrary time and
2037 * cumulative error can grow (see calculate_normal_threshold).
2039 * From that POV the regular flushing can be postponed for CPUs that have
2040 * been isolated from the kernel interference without critical
2041 * infrastructure ever noticing. Skip regular flushing from vmstat_shepherd
2042 * for all isolated CPUs to avoid interference with the isolated workload.
2044 if (cpu_is_isolated(cpu
))
2047 if (!delayed_work_pending(dw
) && need_update(cpu
))
2048 queue_delayed_work_on(cpu
, mm_percpu_wq
, dw
, 0);
2054 schedule_delayed_work(&shepherd
,
2055 round_jiffies_relative(sysctl_stat_interval
));
2058 static void __init
start_shepherd_timer(void)
2062 for_each_possible_cpu(cpu
)
2063 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work
, cpu
),
2066 schedule_delayed_work(&shepherd
,
2067 round_jiffies_relative(sysctl_stat_interval
));
2070 static void __init
init_cpu_node_state(void)
2074 for_each_online_node(node
) {
2075 if (!cpumask_empty(cpumask_of_node(node
)))
2076 node_set_state(node
, N_CPU
);
2080 static int vmstat_cpu_online(unsigned int cpu
)
2082 refresh_zone_stat_thresholds();
2084 if (!node_state(cpu_to_node(cpu
), N_CPU
)) {
2085 node_set_state(cpu_to_node(cpu
), N_CPU
);
2091 static int vmstat_cpu_down_prep(unsigned int cpu
)
2093 cancel_delayed_work_sync(&per_cpu(vmstat_work
, cpu
));
2097 static int vmstat_cpu_dead(unsigned int cpu
)
2099 const struct cpumask
*node_cpus
;
2102 node
= cpu_to_node(cpu
);
2104 refresh_zone_stat_thresholds();
2105 node_cpus
= cpumask_of_node(node
);
2106 if (!cpumask_empty(node_cpus
))
2109 node_clear_state(node
, N_CPU
);
2116 struct workqueue_struct
*mm_percpu_wq
;
2118 void __init
init_mm_internals(void)
2120 int ret __maybe_unused
;
2122 mm_percpu_wq
= alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM
, 0);
2125 ret
= cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD
, "mm/vmstat:dead",
2126 NULL
, vmstat_cpu_dead
);
2128 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2130 ret
= cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN
, "mm/vmstat:online",
2132 vmstat_cpu_down_prep
);
2134 pr_err("vmstat: failed to register 'online' hotplug state\n");
2137 init_cpu_node_state();
2140 start_shepherd_timer();
2142 #ifdef CONFIG_PROC_FS
2143 proc_create_seq("buddyinfo", 0444, NULL
, &fragmentation_op
);
2144 proc_create_seq("pagetypeinfo", 0400, NULL
, &pagetypeinfo_op
);
2145 proc_create_seq("vmstat", 0444, NULL
, &vmstat_op
);
2146 proc_create_seq("zoneinfo", 0444, NULL
, &zoneinfo_op
);
2150 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2153 * Return an index indicating how much of the available free memory is
2154 * unusable for an allocation of the requested size.
2156 static int unusable_free_index(unsigned int order
,
2157 struct contig_page_info
*info
)
2159 /* No free memory is interpreted as all free memory is unusable */
2160 if (info
->free_pages
== 0)
2164 * Index should be a value between 0 and 1. Return a value to 3
2167 * 0 => no fragmentation
2168 * 1 => high fragmentation
2170 return div_u64((info
->free_pages
- (info
->free_blocks_suitable
<< order
)) * 1000ULL, info
->free_pages
);
2174 static void unusable_show_print(struct seq_file
*m
,
2175 pg_data_t
*pgdat
, struct zone
*zone
)
2179 struct contig_page_info info
;
2181 seq_printf(m
, "Node %d, zone %8s ",
2184 for (order
= 0; order
< NR_PAGE_ORDERS
; ++order
) {
2185 fill_contig_page_info(zone
, order
, &info
);
2186 index
= unusable_free_index(order
, &info
);
2187 seq_printf(m
, "%d.%03d ", index
/ 1000, index
% 1000);
2194 * Display unusable free space index
2196 * The unusable free space index measures how much of the available free
2197 * memory cannot be used to satisfy an allocation of a given size and is a
2198 * value between 0 and 1. The higher the value, the more of free memory is
2199 * unusable and by implication, the worse the external fragmentation is. This
2200 * can be expressed as a percentage by multiplying by 100.
2202 static int unusable_show(struct seq_file
*m
, void *arg
)
2204 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
2206 /* check memoryless node */
2207 if (!node_state(pgdat
->node_id
, N_MEMORY
))
2210 walk_zones_in_node(m
, pgdat
, true, false, unusable_show_print
);
2215 static const struct seq_operations unusable_sops
= {
2216 .start
= frag_start
,
2219 .show
= unusable_show
,
2222 DEFINE_SEQ_ATTRIBUTE(unusable
);
2224 static void extfrag_show_print(struct seq_file
*m
,
2225 pg_data_t
*pgdat
, struct zone
*zone
)
2230 /* Alloc on stack as interrupts are disabled for zone walk */
2231 struct contig_page_info info
;
2233 seq_printf(m
, "Node %d, zone %8s ",
2236 for (order
= 0; order
< NR_PAGE_ORDERS
; ++order
) {
2237 fill_contig_page_info(zone
, order
, &info
);
2238 index
= __fragmentation_index(order
, &info
);
2239 seq_printf(m
, "%2d.%03d ", index
/ 1000, index
% 1000);
2246 * Display fragmentation index for orders that allocations would fail for
2248 static int extfrag_show(struct seq_file
*m
, void *arg
)
2250 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
2252 walk_zones_in_node(m
, pgdat
, true, false, extfrag_show_print
);
2257 static const struct seq_operations extfrag_sops
= {
2258 .start
= frag_start
,
2261 .show
= extfrag_show
,
2264 DEFINE_SEQ_ATTRIBUTE(extfrag
);
2266 static int __init
extfrag_debug_init(void)
2268 struct dentry
*extfrag_debug_root
;
2270 extfrag_debug_root
= debugfs_create_dir("extfrag", NULL
);
2272 debugfs_create_file("unusable_index", 0444, extfrag_debug_root
, NULL
,
2275 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root
, NULL
,
2281 module_init(extfrag_debug_init
);