1 // SPDX-License-Identifier: GPL-2.0-only
5 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/debugfs.h>
24 #include <linux/sched.h>
25 #include <linux/math64.h>
26 #include <linux/writeback.h>
27 #include <linux/compaction.h>
28 #include <linux/mm_inline.h>
29 #include <linux/page_ext.h>
30 #include <linux/page_owner.h>
35 int sysctl_vm_numa_stat
= ENABLE_NUMA_STAT
;
37 /* zero numa counters within a zone */
38 static void zero_zone_numa_counters(struct zone
*zone
)
42 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++) {
43 atomic_long_set(&zone
->vm_numa_event
[item
], 0);
44 for_each_online_cpu(cpu
) {
45 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->vm_numa_event
[item
]
51 /* zero numa counters of all the populated zones */
52 static void zero_zones_numa_counters(void)
56 for_each_populated_zone(zone
)
57 zero_zone_numa_counters(zone
);
60 /* zero global numa counters */
61 static void zero_global_numa_counters(void)
65 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
66 atomic_long_set(&vm_numa_event
[item
], 0);
69 static void invalid_numa_statistics(void)
71 zero_zones_numa_counters();
72 zero_global_numa_counters();
75 static DEFINE_MUTEX(vm_numa_stat_lock
);
77 int sysctl_vm_numa_stat_handler(struct ctl_table
*table
, int write
,
78 void *buffer
, size_t *length
, loff_t
*ppos
)
82 mutex_lock(&vm_numa_stat_lock
);
84 oldval
= sysctl_vm_numa_stat
;
85 ret
= proc_dointvec_minmax(table
, write
, buffer
, length
, ppos
);
89 if (oldval
== sysctl_vm_numa_stat
)
91 else if (sysctl_vm_numa_stat
== ENABLE_NUMA_STAT
) {
92 static_branch_enable(&vm_numa_stat_key
);
93 pr_info("enable numa statistics\n");
95 static_branch_disable(&vm_numa_stat_key
);
96 invalid_numa_statistics();
97 pr_info("disable numa statistics, and clear numa counters\n");
101 mutex_unlock(&vm_numa_stat_lock
);
106 #ifdef CONFIG_VM_EVENT_COUNTERS
107 DEFINE_PER_CPU(struct vm_event_state
, vm_event_states
) = {{0}};
108 EXPORT_PER_CPU_SYMBOL(vm_event_states
);
110 static void sum_vm_events(unsigned long *ret
)
115 memset(ret
, 0, NR_VM_EVENT_ITEMS
* sizeof(unsigned long));
117 for_each_online_cpu(cpu
) {
118 struct vm_event_state
*this = &per_cpu(vm_event_states
, cpu
);
120 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++)
121 ret
[i
] += this->event
[i
];
126 * Accumulate the vm event counters across all CPUs.
127 * The result is unavoidably approximate - it can change
128 * during and after execution of this function.
130 void all_vm_events(unsigned long *ret
)
136 EXPORT_SYMBOL_GPL(all_vm_events
);
139 * Fold the foreign cpu events into our own.
141 * This is adding to the events on one processor
142 * but keeps the global counts constant.
144 void vm_events_fold_cpu(int cpu
)
146 struct vm_event_state
*fold_state
= &per_cpu(vm_event_states
, cpu
);
149 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++) {
150 count_vm_events(i
, fold_state
->event
[i
]);
151 fold_state
->event
[i
] = 0;
155 #endif /* CONFIG_VM_EVENT_COUNTERS */
158 * Manage combined zone based / global counters
160 * vm_stat contains the global counters
162 atomic_long_t vm_zone_stat
[NR_VM_ZONE_STAT_ITEMS
] __cacheline_aligned_in_smp
;
163 atomic_long_t vm_node_stat
[NR_VM_NODE_STAT_ITEMS
] __cacheline_aligned_in_smp
;
164 atomic_long_t vm_numa_event
[NR_VM_NUMA_EVENT_ITEMS
] __cacheline_aligned_in_smp
;
165 EXPORT_SYMBOL(vm_zone_stat
);
166 EXPORT_SYMBOL(vm_node_stat
);
169 static void fold_vm_zone_numa_events(struct zone
*zone
)
171 unsigned long zone_numa_events
[NR_VM_NUMA_EVENT_ITEMS
] = { 0, };
173 enum numa_stat_item item
;
175 for_each_online_cpu(cpu
) {
176 struct per_cpu_zonestat
*pzstats
;
178 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
179 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
180 zone_numa_events
[item
] += xchg(&pzstats
->vm_numa_event
[item
], 0);
183 for (item
= 0; item
< NR_VM_NUMA_EVENT_ITEMS
; item
++)
184 zone_numa_event_add(zone_numa_events
[item
], zone
, item
);
187 void fold_vm_numa_events(void)
191 for_each_populated_zone(zone
)
192 fold_vm_zone_numa_events(zone
);
198 int calculate_pressure_threshold(struct zone
*zone
)
201 int watermark_distance
;
204 * As vmstats are not up to date, there is drift between the estimated
205 * and real values. For high thresholds and a high number of CPUs, it
206 * is possible for the min watermark to be breached while the estimated
207 * value looks fine. The pressure threshold is a reduced value such
208 * that even the maximum amount of drift will not accidentally breach
211 watermark_distance
= low_wmark_pages(zone
) - min_wmark_pages(zone
);
212 threshold
= max(1, (int)(watermark_distance
/ num_online_cpus()));
215 * Maximum threshold is 125
217 threshold
= min(125, threshold
);
222 int calculate_normal_threshold(struct zone
*zone
)
225 int mem
; /* memory in 128 MB units */
228 * The threshold scales with the number of processors and the amount
229 * of memory per zone. More memory means that we can defer updates for
230 * longer, more processors could lead to more contention.
231 * fls() is used to have a cheap way of logarithmic scaling.
233 * Some sample thresholds:
235 * Threshold Processors (fls) Zonesize fls(mem)+1
236 * ------------------------------------------------------------------
253 * 125 1024 10 8-16 GB 8
254 * 125 1024 10 16-32 GB 9
257 mem
= zone_managed_pages(zone
) >> (27 - PAGE_SHIFT
);
259 threshold
= 2 * fls(num_online_cpus()) * (1 + fls(mem
));
262 * Maximum threshold is 125
264 threshold
= min(125, threshold
);
270 * Refresh the thresholds for each zone.
272 void refresh_zone_stat_thresholds(void)
274 struct pglist_data
*pgdat
;
279 /* Zero current pgdat thresholds */
280 for_each_online_pgdat(pgdat
) {
281 for_each_online_cpu(cpu
) {
282 per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
= 0;
286 for_each_populated_zone(zone
) {
287 struct pglist_data
*pgdat
= zone
->zone_pgdat
;
288 unsigned long max_drift
, tolerate_drift
;
290 threshold
= calculate_normal_threshold(zone
);
292 for_each_online_cpu(cpu
) {
295 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->stat_threshold
298 /* Base nodestat threshold on the largest populated zone. */
299 pgdat_threshold
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
;
300 per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
)->stat_threshold
301 = max(threshold
, pgdat_threshold
);
305 * Only set percpu_drift_mark if there is a danger that
306 * NR_FREE_PAGES reports the low watermark is ok when in fact
307 * the min watermark could be breached by an allocation
309 tolerate_drift
= low_wmark_pages(zone
) - min_wmark_pages(zone
);
310 max_drift
= num_online_cpus() * threshold
;
311 if (max_drift
> tolerate_drift
)
312 zone
->percpu_drift_mark
= high_wmark_pages(zone
) +
317 void set_pgdat_percpu_threshold(pg_data_t
*pgdat
,
318 int (*calculate_pressure
)(struct zone
*))
325 for (i
= 0; i
< pgdat
->nr_zones
; i
++) {
326 zone
= &pgdat
->node_zones
[i
];
327 if (!zone
->percpu_drift_mark
)
330 threshold
= (*calculate_pressure
)(zone
);
331 for_each_online_cpu(cpu
)
332 per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
)->stat_threshold
338 * For use when we know that interrupts are disabled,
339 * or when we know that preemption is disabled and that
340 * particular counter cannot be updated from interrupt context.
342 void __mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
345 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
346 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
351 * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels,
352 * atomicity is provided by IRQs being disabled -- either explicitly
353 * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables
354 * CPU migrations and preemption potentially corrupts a counter so
355 * disable preemption.
357 preempt_disable_nested();
359 x
= delta
+ __this_cpu_read(*p
);
361 t
= __this_cpu_read(pcp
->stat_threshold
);
363 if (unlikely(abs(x
) > t
)) {
364 zone_page_state_add(x
, zone
, item
);
367 __this_cpu_write(*p
, x
);
369 preempt_enable_nested();
371 EXPORT_SYMBOL(__mod_zone_page_state
);
373 void __mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
376 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
377 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
381 if (vmstat_item_in_bytes(item
)) {
383 * Only cgroups use subpage accounting right now; at
384 * the global level, these items still change in
385 * multiples of whole pages. Store them as pages
386 * internally to keep the per-cpu counters compact.
388 VM_WARN_ON_ONCE(delta
& (PAGE_SIZE
- 1));
389 delta
>>= PAGE_SHIFT
;
392 /* See __mod_node_page_state */
393 preempt_disable_nested();
395 x
= delta
+ __this_cpu_read(*p
);
397 t
= __this_cpu_read(pcp
->stat_threshold
);
399 if (unlikely(abs(x
) > t
)) {
400 node_page_state_add(x
, pgdat
, item
);
403 __this_cpu_write(*p
, x
);
405 preempt_enable_nested();
407 EXPORT_SYMBOL(__mod_node_page_state
);
410 * Optimized increment and decrement functions.
412 * These are only for a single page and therefore can take a struct page *
413 * argument instead of struct zone *. This allows the inclusion of the code
414 * generated for page_zone(page) into the optimized functions.
416 * No overflow check is necessary and therefore the differential can be
417 * incremented or decremented in place which may allow the compilers to
418 * generate better code.
419 * The increment or decrement is known and therefore one boundary check can
422 * NOTE: These functions are very performance sensitive. Change only
425 * Some processors have inc/dec instructions that are atomic vs an interrupt.
426 * However, the code must first determine the differential location in a zone
427 * based on the processor number and then inc/dec the counter. There is no
428 * guarantee without disabling preemption that the processor will not change
429 * in between and therefore the atomicity vs. interrupt cannot be exploited
430 * in a useful way here.
432 void __inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
434 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
435 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
438 /* See __mod_node_page_state */
439 preempt_disable_nested();
441 v
= __this_cpu_inc_return(*p
);
442 t
= __this_cpu_read(pcp
->stat_threshold
);
443 if (unlikely(v
> t
)) {
444 s8 overstep
= t
>> 1;
446 zone_page_state_add(v
+ overstep
, zone
, item
);
447 __this_cpu_write(*p
, -overstep
);
450 preempt_enable_nested();
453 void __inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
455 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
456 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
459 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
461 /* See __mod_node_page_state */
462 preempt_disable_nested();
464 v
= __this_cpu_inc_return(*p
);
465 t
= __this_cpu_read(pcp
->stat_threshold
);
466 if (unlikely(v
> t
)) {
467 s8 overstep
= t
>> 1;
469 node_page_state_add(v
+ overstep
, pgdat
, item
);
470 __this_cpu_write(*p
, -overstep
);
473 preempt_enable_nested();
476 void __inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
478 __inc_zone_state(page_zone(page
), item
);
480 EXPORT_SYMBOL(__inc_zone_page_state
);
482 void __inc_node_page_state(struct page
*page
, enum node_stat_item item
)
484 __inc_node_state(page_pgdat(page
), item
);
486 EXPORT_SYMBOL(__inc_node_page_state
);
488 void __dec_zone_state(struct zone
*zone
, enum zone_stat_item item
)
490 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
491 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
494 /* See __mod_node_page_state */
495 preempt_disable_nested();
497 v
= __this_cpu_dec_return(*p
);
498 t
= __this_cpu_read(pcp
->stat_threshold
);
499 if (unlikely(v
< - t
)) {
500 s8 overstep
= t
>> 1;
502 zone_page_state_add(v
- overstep
, zone
, item
);
503 __this_cpu_write(*p
, overstep
);
506 preempt_enable_nested();
509 void __dec_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
511 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
512 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
515 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
517 /* See __mod_node_page_state */
518 preempt_disable_nested();
520 v
= __this_cpu_dec_return(*p
);
521 t
= __this_cpu_read(pcp
->stat_threshold
);
522 if (unlikely(v
< - t
)) {
523 s8 overstep
= t
>> 1;
525 node_page_state_add(v
- overstep
, pgdat
, item
);
526 __this_cpu_write(*p
, overstep
);
529 preempt_enable_nested();
532 void __dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
534 __dec_zone_state(page_zone(page
), item
);
536 EXPORT_SYMBOL(__dec_zone_page_state
);
538 void __dec_node_page_state(struct page
*page
, enum node_stat_item item
)
540 __dec_node_state(page_pgdat(page
), item
);
542 EXPORT_SYMBOL(__dec_node_page_state
);
544 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
546 * If we have cmpxchg_local support then we do not need to incur the overhead
547 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
549 * mod_state() modifies the zone counter state through atomic per cpu
552 * Overstep mode specifies how overstep should handled:
554 * 1 Overstepping half of threshold
555 * -1 Overstepping minus half of threshold
557 static inline void mod_zone_state(struct zone
*zone
,
558 enum zone_stat_item item
, long delta
, int overstep_mode
)
560 struct per_cpu_zonestat __percpu
*pcp
= zone
->per_cpu_zonestats
;
561 s8 __percpu
*p
= pcp
->vm_stat_diff
+ item
;
565 z
= 0; /* overflow to zone counters */
568 * The fetching of the stat_threshold is racy. We may apply
569 * a counter threshold to the wrong the cpu if we get
570 * rescheduled while executing here. However, the next
571 * counter update will apply the threshold again and
572 * therefore bring the counter under the threshold again.
574 * Most of the time the thresholds are the same anyways
575 * for all cpus in a zone.
577 t
= this_cpu_read(pcp
->stat_threshold
);
579 o
= this_cpu_read(*p
);
583 int os
= overstep_mode
* (t
>> 1) ;
585 /* Overflow must be added to zone counters */
589 } while (this_cpu_cmpxchg(*p
, o
, n
) != o
);
592 zone_page_state_add(z
, zone
, item
);
595 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
598 mod_zone_state(zone
, item
, delta
, 0);
600 EXPORT_SYMBOL(mod_zone_page_state
);
602 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
604 mod_zone_state(page_zone(page
), item
, 1, 1);
606 EXPORT_SYMBOL(inc_zone_page_state
);
608 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
610 mod_zone_state(page_zone(page
), item
, -1, -1);
612 EXPORT_SYMBOL(dec_zone_page_state
);
614 static inline void mod_node_state(struct pglist_data
*pgdat
,
615 enum node_stat_item item
, int delta
, int overstep_mode
)
617 struct per_cpu_nodestat __percpu
*pcp
= pgdat
->per_cpu_nodestats
;
618 s8 __percpu
*p
= pcp
->vm_node_stat_diff
+ item
;
621 if (vmstat_item_in_bytes(item
)) {
623 * Only cgroups use subpage accounting right now; at
624 * the global level, these items still change in
625 * multiples of whole pages. Store them as pages
626 * internally to keep the per-cpu counters compact.
628 VM_WARN_ON_ONCE(delta
& (PAGE_SIZE
- 1));
629 delta
>>= PAGE_SHIFT
;
633 z
= 0; /* overflow to node counters */
636 * The fetching of the stat_threshold is racy. We may apply
637 * a counter threshold to the wrong the cpu if we get
638 * rescheduled while executing here. However, the next
639 * counter update will apply the threshold again and
640 * therefore bring the counter under the threshold again.
642 * Most of the time the thresholds are the same anyways
643 * for all cpus in a node.
645 t
= this_cpu_read(pcp
->stat_threshold
);
647 o
= this_cpu_read(*p
);
651 int os
= overstep_mode
* (t
>> 1) ;
653 /* Overflow must be added to node counters */
657 } while (this_cpu_cmpxchg(*p
, o
, n
) != o
);
660 node_page_state_add(z
, pgdat
, item
);
663 void mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
666 mod_node_state(pgdat
, item
, delta
, 0);
668 EXPORT_SYMBOL(mod_node_page_state
);
670 void inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
672 mod_node_state(pgdat
, item
, 1, 1);
675 void inc_node_page_state(struct page
*page
, enum node_stat_item item
)
677 mod_node_state(page_pgdat(page
), item
, 1, 1);
679 EXPORT_SYMBOL(inc_node_page_state
);
681 void dec_node_page_state(struct page
*page
, enum node_stat_item item
)
683 mod_node_state(page_pgdat(page
), item
, -1, -1);
685 EXPORT_SYMBOL(dec_node_page_state
);
688 * Use interrupt disable to serialize counter updates
690 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
695 local_irq_save(flags
);
696 __mod_zone_page_state(zone
, item
, delta
);
697 local_irq_restore(flags
);
699 EXPORT_SYMBOL(mod_zone_page_state
);
701 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
706 zone
= page_zone(page
);
707 local_irq_save(flags
);
708 __inc_zone_state(zone
, item
);
709 local_irq_restore(flags
);
711 EXPORT_SYMBOL(inc_zone_page_state
);
713 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
717 local_irq_save(flags
);
718 __dec_zone_page_state(page
, item
);
719 local_irq_restore(flags
);
721 EXPORT_SYMBOL(dec_zone_page_state
);
723 void inc_node_state(struct pglist_data
*pgdat
, enum node_stat_item item
)
727 local_irq_save(flags
);
728 __inc_node_state(pgdat
, item
);
729 local_irq_restore(flags
);
731 EXPORT_SYMBOL(inc_node_state
);
733 void mod_node_page_state(struct pglist_data
*pgdat
, enum node_stat_item item
,
738 local_irq_save(flags
);
739 __mod_node_page_state(pgdat
, item
, delta
);
740 local_irq_restore(flags
);
742 EXPORT_SYMBOL(mod_node_page_state
);
744 void inc_node_page_state(struct page
*page
, enum node_stat_item item
)
747 struct pglist_data
*pgdat
;
749 pgdat
= page_pgdat(page
);
750 local_irq_save(flags
);
751 __inc_node_state(pgdat
, item
);
752 local_irq_restore(flags
);
754 EXPORT_SYMBOL(inc_node_page_state
);
756 void dec_node_page_state(struct page
*page
, enum node_stat_item item
)
760 local_irq_save(flags
);
761 __dec_node_page_state(page
, item
);
762 local_irq_restore(flags
);
764 EXPORT_SYMBOL(dec_node_page_state
);
768 * Fold a differential into the global counters.
769 * Returns the number of counters updated.
771 static int fold_diff(int *zone_diff
, int *node_diff
)
776 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
778 atomic_long_add(zone_diff
[i
], &vm_zone_stat
[i
]);
782 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++)
784 atomic_long_add(node_diff
[i
], &vm_node_stat
[i
]);
791 * Update the zone counters for the current cpu.
793 * Note that refresh_cpu_vm_stats strives to only access
794 * node local memory. The per cpu pagesets on remote zones are placed
795 * in the memory local to the processor using that pageset. So the
796 * loop over all zones will access a series of cachelines local to
799 * The call to zone_page_state_add updates the cachelines with the
800 * statistics in the remote zone struct as well as the global cachelines
801 * with the global counters. These could cause remote node cache line
802 * bouncing and will have to be only done when necessary.
804 * The function returns the number of global counters updated.
806 static int refresh_cpu_vm_stats(bool do_pagesets
)
808 struct pglist_data
*pgdat
;
811 int global_zone_diff
[NR_VM_ZONE_STAT_ITEMS
] = { 0, };
812 int global_node_diff
[NR_VM_NODE_STAT_ITEMS
] = { 0, };
815 for_each_populated_zone(zone
) {
816 struct per_cpu_zonestat __percpu
*pzstats
= zone
->per_cpu_zonestats
;
818 struct per_cpu_pages __percpu
*pcp
= zone
->per_cpu_pageset
;
821 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
824 v
= this_cpu_xchg(pzstats
->vm_stat_diff
[i
], 0);
827 atomic_long_add(v
, &zone
->vm_stat
[i
]);
828 global_zone_diff
[i
] += v
;
830 /* 3 seconds idle till flush */
831 __this_cpu_write(pcp
->expire
, 3);
840 * Deal with draining the remote pageset of this
843 * Check if there are pages remaining in this pageset
844 * if not then there is nothing to expire.
846 if (!__this_cpu_read(pcp
->expire
) ||
847 !__this_cpu_read(pcp
->count
))
851 * We never drain zones local to this processor.
853 if (zone_to_nid(zone
) == numa_node_id()) {
854 __this_cpu_write(pcp
->expire
, 0);
858 if (__this_cpu_dec_return(pcp
->expire
))
861 if (__this_cpu_read(pcp
->count
)) {
862 drain_zone_pages(zone
, this_cpu_ptr(pcp
));
869 for_each_online_pgdat(pgdat
) {
870 struct per_cpu_nodestat __percpu
*p
= pgdat
->per_cpu_nodestats
;
872 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
875 v
= this_cpu_xchg(p
->vm_node_stat_diff
[i
], 0);
877 atomic_long_add(v
, &pgdat
->vm_stat
[i
]);
878 global_node_diff
[i
] += v
;
883 changes
+= fold_diff(global_zone_diff
, global_node_diff
);
888 * Fold the data for an offline cpu into the global array.
889 * There cannot be any access by the offline cpu and therefore
890 * synchronization is simplified.
892 void cpu_vm_stats_fold(int cpu
)
894 struct pglist_data
*pgdat
;
897 int global_zone_diff
[NR_VM_ZONE_STAT_ITEMS
] = { 0, };
898 int global_node_diff
[NR_VM_NODE_STAT_ITEMS
] = { 0, };
900 for_each_populated_zone(zone
) {
901 struct per_cpu_zonestat
*pzstats
;
903 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
905 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
906 if (pzstats
->vm_stat_diff
[i
]) {
909 v
= pzstats
->vm_stat_diff
[i
];
910 pzstats
->vm_stat_diff
[i
] = 0;
911 atomic_long_add(v
, &zone
->vm_stat
[i
]);
912 global_zone_diff
[i
] += v
;
916 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++) {
917 if (pzstats
->vm_numa_event
[i
]) {
920 v
= pzstats
->vm_numa_event
[i
];
921 pzstats
->vm_numa_event
[i
] = 0;
922 zone_numa_event_add(v
, zone
, i
);
928 for_each_online_pgdat(pgdat
) {
929 struct per_cpu_nodestat
*p
;
931 p
= per_cpu_ptr(pgdat
->per_cpu_nodestats
, cpu
);
933 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++)
934 if (p
->vm_node_stat_diff
[i
]) {
937 v
= p
->vm_node_stat_diff
[i
];
938 p
->vm_node_stat_diff
[i
] = 0;
939 atomic_long_add(v
, &pgdat
->vm_stat
[i
]);
940 global_node_diff
[i
] += v
;
944 fold_diff(global_zone_diff
, global_node_diff
);
948 * this is only called if !populated_zone(zone), which implies no other users of
949 * pset->vm_stat_diff[] exist.
951 void drain_zonestat(struct zone
*zone
, struct per_cpu_zonestat
*pzstats
)
956 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
957 if (pzstats
->vm_stat_diff
[i
]) {
958 v
= pzstats
->vm_stat_diff
[i
];
959 pzstats
->vm_stat_diff
[i
] = 0;
960 zone_page_state_add(v
, zone
, i
);
965 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++) {
966 if (pzstats
->vm_numa_event
[i
]) {
967 v
= pzstats
->vm_numa_event
[i
];
968 pzstats
->vm_numa_event
[i
] = 0;
969 zone_numa_event_add(v
, zone
, i
);
978 * Determine the per node value of a stat item. This function
979 * is called frequently in a NUMA machine, so try to be as
980 * frugal as possible.
982 unsigned long sum_zone_node_page_state(int node
,
983 enum zone_stat_item item
)
985 struct zone
*zones
= NODE_DATA(node
)->node_zones
;
987 unsigned long count
= 0;
989 for (i
= 0; i
< MAX_NR_ZONES
; i
++)
990 count
+= zone_page_state(zones
+ i
, item
);
995 /* Determine the per node value of a numa stat item. */
996 unsigned long sum_zone_numa_event_state(int node
,
997 enum numa_stat_item item
)
999 struct zone
*zones
= NODE_DATA(node
)->node_zones
;
1000 unsigned long count
= 0;
1003 for (i
= 0; i
< MAX_NR_ZONES
; i
++)
1004 count
+= zone_numa_event_state(zones
+ i
, item
);
1010 * Determine the per node value of a stat item.
1012 unsigned long node_page_state_pages(struct pglist_data
*pgdat
,
1013 enum node_stat_item item
)
1015 long x
= atomic_long_read(&pgdat
->vm_stat
[item
]);
1023 unsigned long node_page_state(struct pglist_data
*pgdat
,
1024 enum node_stat_item item
)
1026 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item
));
1028 return node_page_state_pages(pgdat
, item
);
1032 #ifdef CONFIG_COMPACTION
1034 struct contig_page_info
{
1035 unsigned long free_pages
;
1036 unsigned long free_blocks_total
;
1037 unsigned long free_blocks_suitable
;
1041 * Calculate the number of free pages in a zone, how many contiguous
1042 * pages are free and how many are large enough to satisfy an allocation of
1043 * the target size. Note that this function makes no attempt to estimate
1044 * how many suitable free blocks there *might* be if MOVABLE pages were
1045 * migrated. Calculating that is possible, but expensive and can be
1046 * figured out from userspace
1048 static void fill_contig_page_info(struct zone
*zone
,
1049 unsigned int suitable_order
,
1050 struct contig_page_info
*info
)
1054 info
->free_pages
= 0;
1055 info
->free_blocks_total
= 0;
1056 info
->free_blocks_suitable
= 0;
1058 for (order
= 0; order
< MAX_ORDER
; order
++) {
1059 unsigned long blocks
;
1062 * Count number of free blocks.
1064 * Access to nr_free is lockless as nr_free is used only for
1065 * diagnostic purposes. Use data_race to avoid KCSAN warning.
1067 blocks
= data_race(zone
->free_area
[order
].nr_free
);
1068 info
->free_blocks_total
+= blocks
;
1070 /* Count free base pages */
1071 info
->free_pages
+= blocks
<< order
;
1073 /* Count the suitable free blocks */
1074 if (order
>= suitable_order
)
1075 info
->free_blocks_suitable
+= blocks
<<
1076 (order
- suitable_order
);
1081 * A fragmentation index only makes sense if an allocation of a requested
1082 * size would fail. If that is true, the fragmentation index indicates
1083 * whether external fragmentation or a lack of memory was the problem.
1084 * The value can be used to determine if page reclaim or compaction
1087 static int __fragmentation_index(unsigned int order
, struct contig_page_info
*info
)
1089 unsigned long requested
= 1UL << order
;
1091 if (WARN_ON_ONCE(order
>= MAX_ORDER
))
1094 if (!info
->free_blocks_total
)
1097 /* Fragmentation index only makes sense when a request would fail */
1098 if (info
->free_blocks_suitable
)
1102 * Index is between 0 and 1 so return within 3 decimal places
1104 * 0 => allocation would fail due to lack of memory
1105 * 1 => allocation would fail due to fragmentation
1107 return 1000 - div_u64( (1000+(div_u64(info
->free_pages
* 1000ULL, requested
))), info
->free_blocks_total
);
1111 * Calculates external fragmentation within a zone wrt the given order.
1112 * It is defined as the percentage of pages found in blocks of size
1113 * less than 1 << order. It returns values in range [0, 100].
1115 unsigned int extfrag_for_order(struct zone
*zone
, unsigned int order
)
1117 struct contig_page_info info
;
1119 fill_contig_page_info(zone
, order
, &info
);
1120 if (info
.free_pages
== 0)
1123 return div_u64((info
.free_pages
-
1124 (info
.free_blocks_suitable
<< order
)) * 100,
1128 /* Same as __fragmentation index but allocs contig_page_info on stack */
1129 int fragmentation_index(struct zone
*zone
, unsigned int order
)
1131 struct contig_page_info info
;
1133 fill_contig_page_info(zone
, order
, &info
);
1134 return __fragmentation_index(order
, &info
);
1138 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
1139 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
1140 #ifdef CONFIG_ZONE_DMA
1141 #define TEXT_FOR_DMA(xx) xx "_dma",
1143 #define TEXT_FOR_DMA(xx)
1146 #ifdef CONFIG_ZONE_DMA32
1147 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1149 #define TEXT_FOR_DMA32(xx)
1152 #ifdef CONFIG_HIGHMEM
1153 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1155 #define TEXT_FOR_HIGHMEM(xx)
1158 #ifdef CONFIG_ZONE_DEVICE
1159 #define TEXT_FOR_DEVICE(xx) xx "_device",
1161 #define TEXT_FOR_DEVICE(xx)
1164 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1165 TEXT_FOR_HIGHMEM(xx) xx "_movable", \
1168 const char * const vmstat_text
[] = {
1169 /* enum zone_stat_item counters */
1171 "nr_zone_inactive_anon",
1172 "nr_zone_active_anon",
1173 "nr_zone_inactive_file",
1174 "nr_zone_active_file",
1175 "nr_zone_unevictable",
1176 "nr_zone_write_pending",
1179 #if IS_ENABLED(CONFIG_ZSMALLOC)
1184 /* enum numa_stat_item counters */
1194 /* enum node_stat_item counters */
1200 "nr_slab_reclaimable",
1201 "nr_slab_unreclaimable",
1205 "workingset_refault_anon",
1206 "workingset_refault_file",
1207 "workingset_activate_anon",
1208 "workingset_activate_file",
1209 "workingset_restore_anon",
1210 "workingset_restore_file",
1211 "workingset_nodereclaim",
1217 "nr_writeback_temp",
1219 "nr_shmem_hugepages",
1220 "nr_shmem_pmdmapped",
1221 "nr_file_hugepages",
1222 "nr_file_pmdmapped",
1223 "nr_anon_transparent_hugepages",
1225 "nr_vmscan_immediate_reclaim",
1228 "nr_throttled_written",
1229 "nr_kernel_misc_reclaimable",
1230 "nr_foll_pin_acquired",
1231 "nr_foll_pin_released",
1233 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1234 "nr_shadow_call_stack",
1236 "nr_page_table_pages",
1237 "nr_sec_page_table_pages",
1241 #ifdef CONFIG_NUMA_BALANCING
1242 "pgpromote_success",
1243 "pgpromote_candidate",
1246 /* enum writeback_stat_item counters */
1247 "nr_dirty_threshold",
1248 "nr_dirty_background_threshold",
1250 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
1251 /* enum vm_event_item counters */
1257 TEXTS_FOR_ZONES("pgalloc")
1258 TEXTS_FOR_ZONES("allocstall")
1259 TEXTS_FOR_ZONES("pgskip")
1278 "pgscan_direct_throttle",
1285 "zone_reclaim_failed",
1289 "kswapd_inodesteal",
1290 "kswapd_low_wmark_hit_quickly",
1291 "kswapd_high_wmark_hit_quickly",
1300 #ifdef CONFIG_NUMA_BALANCING
1302 "numa_huge_pte_updates",
1304 "numa_hint_faults_local",
1305 "numa_pages_migrated",
1307 #ifdef CONFIG_MIGRATION
1308 "pgmigrate_success",
1310 "thp_migration_success",
1311 "thp_migration_fail",
1312 "thp_migration_split",
1314 #ifdef CONFIG_COMPACTION
1315 "compact_migrate_scanned",
1316 "compact_free_scanned",
1321 "compact_daemon_wake",
1322 "compact_daemon_migrate_scanned",
1323 "compact_daemon_free_scanned",
1326 #ifdef CONFIG_HUGETLB_PAGE
1327 "htlb_buddy_alloc_success",
1328 "htlb_buddy_alloc_fail",
1331 "cma_alloc_success",
1334 "unevictable_pgs_culled",
1335 "unevictable_pgs_scanned",
1336 "unevictable_pgs_rescued",
1337 "unevictable_pgs_mlocked",
1338 "unevictable_pgs_munlocked",
1339 "unevictable_pgs_cleared",
1340 "unevictable_pgs_stranded",
1342 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1344 "thp_fault_fallback",
1345 "thp_fault_fallback_charge",
1346 "thp_collapse_alloc",
1347 "thp_collapse_alloc_failed",
1349 "thp_file_fallback",
1350 "thp_file_fallback_charge",
1353 "thp_split_page_failed",
1354 "thp_deferred_split_page",
1356 "thp_scan_exceed_none_pte",
1357 "thp_scan_exceed_swap_pte",
1358 "thp_scan_exceed_share_pte",
1359 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1362 "thp_zero_page_alloc",
1363 "thp_zero_page_alloc_failed",
1365 "thp_swpout_fallback",
1367 #ifdef CONFIG_MEMORY_BALLOON
1370 #ifdef CONFIG_BALLOON_COMPACTION
1373 #endif /* CONFIG_MEMORY_BALLOON */
1374 #ifdef CONFIG_DEBUG_TLBFLUSH
1375 "nr_tlb_remote_flush",
1376 "nr_tlb_remote_flush_received",
1377 "nr_tlb_local_flush_all",
1378 "nr_tlb_local_flush_one",
1379 #endif /* CONFIG_DEBUG_TLBFLUSH */
1396 "direct_map_level2_splits",
1397 "direct_map_level3_splits",
1399 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
1401 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
1403 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1404 defined(CONFIG_PROC_FS)
1405 static void *frag_start(struct seq_file
*m
, loff_t
*pos
)
1410 for (pgdat
= first_online_pgdat();
1412 pgdat
= next_online_pgdat(pgdat
))
1418 static void *frag_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
1420 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1423 return next_online_pgdat(pgdat
);
1426 static void frag_stop(struct seq_file
*m
, void *arg
)
1431 * Walk zones in a node and print using a callback.
1432 * If @assert_populated is true, only use callback for zones that are populated.
1434 static void walk_zones_in_node(struct seq_file
*m
, pg_data_t
*pgdat
,
1435 bool assert_populated
, bool nolock
,
1436 void (*print
)(struct seq_file
*m
, pg_data_t
*, struct zone
*))
1439 struct zone
*node_zones
= pgdat
->node_zones
;
1440 unsigned long flags
;
1442 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
1443 if (assert_populated
&& !populated_zone(zone
))
1447 spin_lock_irqsave(&zone
->lock
, flags
);
1448 print(m
, pgdat
, zone
);
1450 spin_unlock_irqrestore(&zone
->lock
, flags
);
1455 #ifdef CONFIG_PROC_FS
1456 static void frag_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
1461 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
1462 for (order
= 0; order
< MAX_ORDER
; ++order
)
1464 * Access to nr_free is lockless as nr_free is used only for
1465 * printing purposes. Use data_race to avoid KCSAN warning.
1467 seq_printf(m
, "%6lu ", data_race(zone
->free_area
[order
].nr_free
));
1472 * This walks the free areas for each zone.
1474 static int frag_show(struct seq_file
*m
, void *arg
)
1476 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1477 walk_zones_in_node(m
, pgdat
, true, false, frag_show_print
);
1481 static void pagetypeinfo_showfree_print(struct seq_file
*m
,
1482 pg_data_t
*pgdat
, struct zone
*zone
)
1486 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++) {
1487 seq_printf(m
, "Node %4d, zone %8s, type %12s ",
1490 migratetype_names
[mtype
]);
1491 for (order
= 0; order
< MAX_ORDER
; ++order
) {
1492 unsigned long freecount
= 0;
1493 struct free_area
*area
;
1494 struct list_head
*curr
;
1495 bool overflow
= false;
1497 area
= &(zone
->free_area
[order
]);
1499 list_for_each(curr
, &area
->free_list
[mtype
]) {
1501 * Cap the free_list iteration because it might
1502 * be really large and we are under a spinlock
1503 * so a long time spent here could trigger a
1504 * hard lockup detector. Anyway this is a
1505 * debugging tool so knowing there is a handful
1506 * of pages of this order should be more than
1509 if (++freecount
>= 100000) {
1514 seq_printf(m
, "%s%6lu ", overflow
? ">" : "", freecount
);
1515 spin_unlock_irq(&zone
->lock
);
1517 spin_lock_irq(&zone
->lock
);
1523 /* Print out the free pages at each order for each migatetype */
1524 static void pagetypeinfo_showfree(struct seq_file
*m
, void *arg
)
1527 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1530 seq_printf(m
, "%-43s ", "Free pages count per migrate type at order");
1531 for (order
= 0; order
< MAX_ORDER
; ++order
)
1532 seq_printf(m
, "%6d ", order
);
1535 walk_zones_in_node(m
, pgdat
, true, false, pagetypeinfo_showfree_print
);
1538 static void pagetypeinfo_showblockcount_print(struct seq_file
*m
,
1539 pg_data_t
*pgdat
, struct zone
*zone
)
1543 unsigned long start_pfn
= zone
->zone_start_pfn
;
1544 unsigned long end_pfn
= zone_end_pfn(zone
);
1545 unsigned long count
[MIGRATE_TYPES
] = { 0, };
1547 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pageblock_nr_pages
) {
1550 page
= pfn_to_online_page(pfn
);
1554 if (page_zone(page
) != zone
)
1557 mtype
= get_pageblock_migratetype(page
);
1559 if (mtype
< MIGRATE_TYPES
)
1564 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
1565 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1566 seq_printf(m
, "%12lu ", count
[mtype
]);
1570 /* Print out the number of pageblocks for each migratetype */
1571 static void pagetypeinfo_showblockcount(struct seq_file
*m
, void *arg
)
1574 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1576 seq_printf(m
, "\n%-23s", "Number of blocks type ");
1577 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1578 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
1580 walk_zones_in_node(m
, pgdat
, true, false,
1581 pagetypeinfo_showblockcount_print
);
1585 * Print out the number of pageblocks for each migratetype that contain pages
1586 * of other types. This gives an indication of how well fallbacks are being
1587 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1588 * to determine what is going on
1590 static void pagetypeinfo_showmixedcount(struct seq_file
*m
, pg_data_t
*pgdat
)
1592 #ifdef CONFIG_PAGE_OWNER
1595 if (!static_branch_unlikely(&page_owner_inited
))
1598 drain_all_pages(NULL
);
1600 seq_printf(m
, "\n%-23s", "Number of mixed blocks ");
1601 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
1602 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
1605 walk_zones_in_node(m
, pgdat
, true, true,
1606 pagetypeinfo_showmixedcount_print
);
1607 #endif /* CONFIG_PAGE_OWNER */
1611 * This prints out statistics in relation to grouping pages by mobility.
1612 * It is expensive to collect so do not constantly read the file.
1614 static int pagetypeinfo_show(struct seq_file
*m
, void *arg
)
1616 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1618 /* check memoryless node */
1619 if (!node_state(pgdat
->node_id
, N_MEMORY
))
1622 seq_printf(m
, "Page block order: %d\n", pageblock_order
);
1623 seq_printf(m
, "Pages per block: %lu\n", pageblock_nr_pages
);
1625 pagetypeinfo_showfree(m
, pgdat
);
1626 pagetypeinfo_showblockcount(m
, pgdat
);
1627 pagetypeinfo_showmixedcount(m
, pgdat
);
1632 static const struct seq_operations fragmentation_op
= {
1633 .start
= frag_start
,
1639 static const struct seq_operations pagetypeinfo_op
= {
1640 .start
= frag_start
,
1643 .show
= pagetypeinfo_show
,
1646 static bool is_zone_first_populated(pg_data_t
*pgdat
, struct zone
*zone
)
1650 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
1651 struct zone
*compare
= &pgdat
->node_zones
[zid
];
1653 if (populated_zone(compare
))
1654 return zone
== compare
;
1660 static void zoneinfo_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
1664 seq_printf(m
, "Node %d, zone %8s", pgdat
->node_id
, zone
->name
);
1665 if (is_zone_first_populated(pgdat
, zone
)) {
1666 seq_printf(m
, "\n per-node stats");
1667 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1668 unsigned long pages
= node_page_state_pages(pgdat
, i
);
1670 if (vmstat_item_print_in_thp(i
))
1671 pages
/= HPAGE_PMD_NR
;
1672 seq_printf(m
, "\n %-12s %lu", node_stat_name(i
),
1686 zone_page_state(zone
, NR_FREE_PAGES
),
1687 zone
->watermark_boost
,
1688 min_wmark_pages(zone
),
1689 low_wmark_pages(zone
),
1690 high_wmark_pages(zone
),
1691 zone
->spanned_pages
,
1692 zone
->present_pages
,
1693 zone_managed_pages(zone
),
1694 zone_cma_pages(zone
));
1697 "\n protection: (%ld",
1698 zone
->lowmem_reserve
[0]);
1699 for (i
= 1; i
< ARRAY_SIZE(zone
->lowmem_reserve
); i
++)
1700 seq_printf(m
, ", %ld", zone
->lowmem_reserve
[i
]);
1703 /* If unpopulated, no other information is useful */
1704 if (!populated_zone(zone
)) {
1709 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
1710 seq_printf(m
, "\n %-12s %lu", zone_stat_name(i
),
1711 zone_page_state(zone
, i
));
1714 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++)
1715 seq_printf(m
, "\n %-12s %lu", numa_stat_name(i
),
1716 zone_numa_event_state(zone
, i
));
1719 seq_printf(m
, "\n pagesets");
1720 for_each_online_cpu(i
) {
1721 struct per_cpu_pages
*pcp
;
1722 struct per_cpu_zonestat __maybe_unused
*pzstats
;
1724 pcp
= per_cpu_ptr(zone
->per_cpu_pageset
, i
);
1735 pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, i
);
1736 seq_printf(m
, "\n vm stats threshold: %d",
1737 pzstats
->stat_threshold
);
1741 "\n node_unreclaimable: %u"
1742 "\n start_pfn: %lu",
1743 pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
,
1744 zone
->zone_start_pfn
);
1749 * Output information about zones in @pgdat. All zones are printed regardless
1750 * of whether they are populated or not: lowmem_reserve_ratio operates on the
1751 * set of all zones and userspace would not be aware of such zones if they are
1752 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
1754 static int zoneinfo_show(struct seq_file
*m
, void *arg
)
1756 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
1757 walk_zones_in_node(m
, pgdat
, false, false, zoneinfo_show_print
);
1761 static const struct seq_operations zoneinfo_op
= {
1762 .start
= frag_start
, /* iterate over all zones. The same as in
1766 .show
= zoneinfo_show
,
1769 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1770 NR_VM_NUMA_EVENT_ITEMS + \
1771 NR_VM_NODE_STAT_ITEMS + \
1772 NR_VM_WRITEBACK_STAT_ITEMS + \
1773 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1774 NR_VM_EVENT_ITEMS : 0))
1776 static void *vmstat_start(struct seq_file
*m
, loff_t
*pos
)
1781 if (*pos
>= NR_VMSTAT_ITEMS
)
1784 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text
) < NR_VMSTAT_ITEMS
);
1785 fold_vm_numa_events();
1786 v
= kmalloc_array(NR_VMSTAT_ITEMS
, sizeof(unsigned long), GFP_KERNEL
);
1789 return ERR_PTR(-ENOMEM
);
1790 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
1791 v
[i
] = global_zone_page_state(i
);
1792 v
+= NR_VM_ZONE_STAT_ITEMS
;
1795 for (i
= 0; i
< NR_VM_NUMA_EVENT_ITEMS
; i
++)
1796 v
[i
] = global_numa_event_state(i
);
1797 v
+= NR_VM_NUMA_EVENT_ITEMS
;
1800 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1801 v
[i
] = global_node_page_state_pages(i
);
1802 if (vmstat_item_print_in_thp(i
))
1803 v
[i
] /= HPAGE_PMD_NR
;
1805 v
+= NR_VM_NODE_STAT_ITEMS
;
1807 global_dirty_limits(v
+ NR_DIRTY_BG_THRESHOLD
,
1808 v
+ NR_DIRTY_THRESHOLD
);
1809 v
+= NR_VM_WRITEBACK_STAT_ITEMS
;
1811 #ifdef CONFIG_VM_EVENT_COUNTERS
1813 v
[PGPGIN
] /= 2; /* sectors -> kbytes */
1816 return (unsigned long *)m
->private + *pos
;
1819 static void *vmstat_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
1822 if (*pos
>= NR_VMSTAT_ITEMS
)
1824 return (unsigned long *)m
->private + *pos
;
1827 static int vmstat_show(struct seq_file
*m
, void *arg
)
1829 unsigned long *l
= arg
;
1830 unsigned long off
= l
- (unsigned long *)m
->private;
1832 seq_puts(m
, vmstat_text
[off
]);
1833 seq_put_decimal_ull(m
, " ", *l
);
1836 if (off
== NR_VMSTAT_ITEMS
- 1) {
1838 * We've come to the end - add any deprecated counters to avoid
1839 * breaking userspace which might depend on them being present.
1841 seq_puts(m
, "nr_unstable 0\n");
1846 static void vmstat_stop(struct seq_file
*m
, void *arg
)
1852 static const struct seq_operations vmstat_op
= {
1853 .start
= vmstat_start
,
1854 .next
= vmstat_next
,
1855 .stop
= vmstat_stop
,
1856 .show
= vmstat_show
,
1858 #endif /* CONFIG_PROC_FS */
1861 static DEFINE_PER_CPU(struct delayed_work
, vmstat_work
);
1862 int sysctl_stat_interval __read_mostly
= HZ
;
1864 #ifdef CONFIG_PROC_FS
1865 static void refresh_vm_stats(struct work_struct
*work
)
1867 refresh_cpu_vm_stats(true);
1870 int vmstat_refresh(struct ctl_table
*table
, int write
,
1871 void *buffer
, size_t *lenp
, loff_t
*ppos
)
1878 * The regular update, every sysctl_stat_interval, may come later
1879 * than expected: leaving a significant amount in per_cpu buckets.
1880 * This is particularly misleading when checking a quantity of HUGE
1881 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1882 * which can equally be echo'ed to or cat'ted from (by root),
1883 * can be used to update the stats just before reading them.
1885 * Oh, and since global_zone_page_state() etc. are so careful to hide
1886 * transiently negative values, report an error here if any of
1887 * the stats is negative, so we know to go looking for imbalance.
1889 err
= schedule_on_each_cpu(refresh_vm_stats
);
1892 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++) {
1894 * Skip checking stats known to go negative occasionally.
1897 case NR_ZONE_WRITE_PENDING
:
1898 case NR_FREE_CMA_PAGES
:
1901 val
= atomic_long_read(&vm_zone_stat
[i
]);
1903 pr_warn("%s: %s %ld\n",
1904 __func__
, zone_stat_name(i
), val
);
1907 for (i
= 0; i
< NR_VM_NODE_STAT_ITEMS
; i
++) {
1909 * Skip checking stats known to go negative occasionally.
1915 val
= atomic_long_read(&vm_node_stat
[i
]);
1917 pr_warn("%s: %s %ld\n",
1918 __func__
, node_stat_name(i
), val
);
1927 #endif /* CONFIG_PROC_FS */
1929 static void vmstat_update(struct work_struct
*w
)
1931 if (refresh_cpu_vm_stats(true)) {
1933 * Counters were updated so we expect more updates
1934 * to occur in the future. Keep on running the
1935 * update worker thread.
1937 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq
,
1938 this_cpu_ptr(&vmstat_work
),
1939 round_jiffies_relative(sysctl_stat_interval
));
1944 * Check if the diffs for a certain cpu indicate that
1945 * an update is needed.
1947 static bool need_update(int cpu
)
1949 pg_data_t
*last_pgdat
= NULL
;
1952 for_each_populated_zone(zone
) {
1953 struct per_cpu_zonestat
*pzstats
= per_cpu_ptr(zone
->per_cpu_zonestats
, cpu
);
1954 struct per_cpu_nodestat
*n
;
1957 * The fast way of checking if there are any vmstat diffs.
1959 if (memchr_inv(pzstats
->vm_stat_diff
, 0, sizeof(pzstats
->vm_stat_diff
)))
1962 if (last_pgdat
== zone
->zone_pgdat
)
1964 last_pgdat
= zone
->zone_pgdat
;
1965 n
= per_cpu_ptr(zone
->zone_pgdat
->per_cpu_nodestats
, cpu
);
1966 if (memchr_inv(n
->vm_node_stat_diff
, 0, sizeof(n
->vm_node_stat_diff
)))
1973 * Switch off vmstat processing and then fold all the remaining differentials
1974 * until the diffs stay at zero. The function is used by NOHZ and can only be
1975 * invoked when tick processing is not active.
1977 void quiet_vmstat(void)
1979 if (system_state
!= SYSTEM_RUNNING
)
1982 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work
)))
1985 if (!need_update(smp_processor_id()))
1989 * Just refresh counters and do not care about the pending delayed
1990 * vmstat_update. It doesn't fire that often to matter and canceling
1991 * it would be too expensive from this path.
1992 * vmstat_shepherd will take care about that for us.
1994 refresh_cpu_vm_stats(false);
1998 * Shepherd worker thread that checks the
1999 * differentials of processors that have their worker
2000 * threads for vm statistics updates disabled because of
2003 static void vmstat_shepherd(struct work_struct
*w
);
2005 static DECLARE_DEFERRABLE_WORK(shepherd
, vmstat_shepherd
);
2007 static void vmstat_shepherd(struct work_struct
*w
)
2012 /* Check processors whose vmstat worker threads have been disabled */
2013 for_each_online_cpu(cpu
) {
2014 struct delayed_work
*dw
= &per_cpu(vmstat_work
, cpu
);
2016 if (!delayed_work_pending(dw
) && need_update(cpu
))
2017 queue_delayed_work_on(cpu
, mm_percpu_wq
, dw
, 0);
2023 schedule_delayed_work(&shepherd
,
2024 round_jiffies_relative(sysctl_stat_interval
));
2027 static void __init
start_shepherd_timer(void)
2031 for_each_possible_cpu(cpu
)
2032 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work
, cpu
),
2035 schedule_delayed_work(&shepherd
,
2036 round_jiffies_relative(sysctl_stat_interval
));
2039 static void __init
init_cpu_node_state(void)
2043 for_each_online_node(node
) {
2044 if (!cpumask_empty(cpumask_of_node(node
)))
2045 node_set_state(node
, N_CPU
);
2049 static int vmstat_cpu_online(unsigned int cpu
)
2051 refresh_zone_stat_thresholds();
2053 if (!node_state(cpu_to_node(cpu
), N_CPU
)) {
2054 node_set_state(cpu_to_node(cpu
), N_CPU
);
2060 static int vmstat_cpu_down_prep(unsigned int cpu
)
2062 cancel_delayed_work_sync(&per_cpu(vmstat_work
, cpu
));
2066 static int vmstat_cpu_dead(unsigned int cpu
)
2068 const struct cpumask
*node_cpus
;
2071 node
= cpu_to_node(cpu
);
2073 refresh_zone_stat_thresholds();
2074 node_cpus
= cpumask_of_node(node
);
2075 if (!cpumask_empty(node_cpus
))
2078 node_clear_state(node
, N_CPU
);
2085 struct workqueue_struct
*mm_percpu_wq
;
2087 void __init
init_mm_internals(void)
2089 int ret __maybe_unused
;
2091 mm_percpu_wq
= alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM
, 0);
2094 ret
= cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD
, "mm/vmstat:dead",
2095 NULL
, vmstat_cpu_dead
);
2097 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2099 ret
= cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN
, "mm/vmstat:online",
2101 vmstat_cpu_down_prep
);
2103 pr_err("vmstat: failed to register 'online' hotplug state\n");
2106 init_cpu_node_state();
2109 start_shepherd_timer();
2111 #ifdef CONFIG_PROC_FS
2112 proc_create_seq("buddyinfo", 0444, NULL
, &fragmentation_op
);
2113 proc_create_seq("pagetypeinfo", 0400, NULL
, &pagetypeinfo_op
);
2114 proc_create_seq("vmstat", 0444, NULL
, &vmstat_op
);
2115 proc_create_seq("zoneinfo", 0444, NULL
, &zoneinfo_op
);
2119 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2122 * Return an index indicating how much of the available free memory is
2123 * unusable for an allocation of the requested size.
2125 static int unusable_free_index(unsigned int order
,
2126 struct contig_page_info
*info
)
2128 /* No free memory is interpreted as all free memory is unusable */
2129 if (info
->free_pages
== 0)
2133 * Index should be a value between 0 and 1. Return a value to 3
2136 * 0 => no fragmentation
2137 * 1 => high fragmentation
2139 return div_u64((info
->free_pages
- (info
->free_blocks_suitable
<< order
)) * 1000ULL, info
->free_pages
);
2143 static void unusable_show_print(struct seq_file
*m
,
2144 pg_data_t
*pgdat
, struct zone
*zone
)
2148 struct contig_page_info info
;
2150 seq_printf(m
, "Node %d, zone %8s ",
2153 for (order
= 0; order
< MAX_ORDER
; ++order
) {
2154 fill_contig_page_info(zone
, order
, &info
);
2155 index
= unusable_free_index(order
, &info
);
2156 seq_printf(m
, "%d.%03d ", index
/ 1000, index
% 1000);
2163 * Display unusable free space index
2165 * The unusable free space index measures how much of the available free
2166 * memory cannot be used to satisfy an allocation of a given size and is a
2167 * value between 0 and 1. The higher the value, the more of free memory is
2168 * unusable and by implication, the worse the external fragmentation is. This
2169 * can be expressed as a percentage by multiplying by 100.
2171 static int unusable_show(struct seq_file
*m
, void *arg
)
2173 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
2175 /* check memoryless node */
2176 if (!node_state(pgdat
->node_id
, N_MEMORY
))
2179 walk_zones_in_node(m
, pgdat
, true, false, unusable_show_print
);
2184 static const struct seq_operations unusable_sops
= {
2185 .start
= frag_start
,
2188 .show
= unusable_show
,
2191 DEFINE_SEQ_ATTRIBUTE(unusable
);
2193 static void extfrag_show_print(struct seq_file
*m
,
2194 pg_data_t
*pgdat
, struct zone
*zone
)
2199 /* Alloc on stack as interrupts are disabled for zone walk */
2200 struct contig_page_info info
;
2202 seq_printf(m
, "Node %d, zone %8s ",
2205 for (order
= 0; order
< MAX_ORDER
; ++order
) {
2206 fill_contig_page_info(zone
, order
, &info
);
2207 index
= __fragmentation_index(order
, &info
);
2208 seq_printf(m
, "%2d.%03d ", index
/ 1000, index
% 1000);
2215 * Display fragmentation index for orders that allocations would fail for
2217 static int extfrag_show(struct seq_file
*m
, void *arg
)
2219 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
2221 walk_zones_in_node(m
, pgdat
, true, false, extfrag_show_print
);
2226 static const struct seq_operations extfrag_sops
= {
2227 .start
= frag_start
,
2230 .show
= extfrag_show
,
2233 DEFINE_SEQ_ATTRIBUTE(extfrag
);
2235 static int __init
extfrag_debug_init(void)
2237 struct dentry
*extfrag_debug_root
;
2239 extfrag_debug_root
= debugfs_create_dir("extfrag", NULL
);
2241 debugfs_create_file("unusable_index", 0444, extfrag_debug_root
, NULL
,
2244 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root
, NULL
,
2250 module_init(extfrag_debug_init
);