mm/vmpressure: skip socket pressure for costly order reclaim

When reclaim is triggered by high order allocations on a fragmented
system, vmpressure() can report poor reclaim efficiency even though the
system has plenty of free memory.  This is because many pages are scanned,
but few are found to actually reclaim - the pages are actively in use and
don't need to be freed.  The resulting scan:reclaim ratio causes
vmpressure() to assert socket pressure, throttling TCP throughput
unnecessarily.

Costly order allocations (above PAGE_ALLOC_COSTLY_ORDER) rely heavily on
compaction to succeed, so poor reclaim efficiency at these orders does not
necessarily indicate memory pressure.  The kernel already treats this
order as the boundary where reclaim is no longer expected to succeed and
compaction may take over.

Make vmpressure() order-aware through an additional parameter sourced from
scan_control at existing call sites.  Socket pressure is now only asserted
when order <= PAGE_ALLOC_COSTLY_ORDER.

Memcg reclaim is unaffected since try_to_free_mem_cgroup_pages() always
uses order 0, which passes the filter unconditionally.  Similarly,
vmpressure_prio() now passes order 0 internally when calling vmpressure(),
ensuring critical pressure from low reclaim priority is not suppressed by
the order filter.

The patch was motivated by a case of impacted net throughput in
production.  On one affected host, the memory state at the time showed
~15GB available, zero cgroup pressure, and the following buddyinfo state:

Order FreePages
0:    133,970
1:    29,230
2:    17,351
3:    18,984
7+:   0

Using bpf, it was found that 94% of vmpressure calls on this host were
from order-7 kswapd reclaim.

TCP minimum recv window is rcv_ssthresh:19712.

Before patch:
723 out of 3,843 (19%) TCP connections stuck at minimum recv window

After live-patching and ~30min elapsed:
0 out of 3,470 TCP connections stuck at minimum recv window

Link: https://lore.kernel.org/20260406195014.112521-1-jp.kobryn@linux.dev
Signed-off-by: JP Kobryn (Meta) <jp.kobryn@linux.dev>
Reviewed-by: Rik van Riel <riel@surriel.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Shakeel Butt <shakeel.butt@linux.dev>
Acked-by: Jakub Kicinski <kuba@kernel.org>
Reviewed-by: Barry Song <baohua@kernel.org>
Acked-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Kairui Song <kasong@tencent.com>
Cc: Liam Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Qi Zheng <qi.zheng@linux.dev>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Wei Xu <weixugc@google.com>
Cc: Yuanchu Xie <yuanchu@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h
index 6a2f51ebbfd354bda2116ef41bc3199ac28fed9c..faecd55224017991f4a87d3c089d9b5f13abff56 100644 (file)
--- a/include/linux/vmpressure.h
+++ b/include/linux/vmpressure.h
@@ -30,8 +30,8 @@ struct vmpressure {
 struct mem_cgroup;
 
 #ifdef CONFIG_MEMCG
-extern void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
-                      unsigned long scanned, unsigned long reclaimed);
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
+               unsigned long scanned, unsigned long reclaimed);
 extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
 
 extern void vmpressure_init(struct vmpressure *vmpr);
@@ -44,8 +44,9 @@ extern int vmpressure_register_event(struct mem_cgroup *memcg,
 extern void vmpressure_unregister_event(struct mem_cgroup *memcg,
                                        struct eventfd_ctx *eventfd);
 #else
-static inline void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
-                             unsigned long scanned, unsigned long reclaimed) {}
+static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg,
+                             bool tree, unsigned long scanned,
+                             unsigned long reclaimed) {}
 static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg,
                                   int prio) {}
 #endif /* CONFIG_MEMCG */

diff --git a/mm/vmpressure.c b/mm/vmpressure.c
index 3fbb86996c4d26b2c6c1192f5b7719b629945471..f053554e5826429f79189be84afa906de933c49e 100644 (file)
--- a/mm/vmpressure.c
+++ b/mm/vmpressure.c
@@ -218,6 +218,7 @@ static void vmpressure_work_fn(struct work_struct *work)
 /**
  * vmpressure() - Account memory pressure through scanned/reclaimed ratio
  * @gfp:       reclaimer's gfp mask
+ * @order:     allocation order being reclaimed for
  * @memcg:     cgroup memory controller handle
  * @tree:      legacy subtree mode
  * @scanned:   number of pages scanned
@@ -236,7 +237,7 @@ static void vmpressure_work_fn(struct work_struct *work)
  *
  * This function does not return any value.
  */
-void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
                unsigned long scanned, unsigned long reclaimed)
 {
        struct vmpressure *vmpr;
@@ -307,7 +308,15 @@ void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
 
                level = vmpressure_calc_level(scanned, reclaimed);
 
-               if (level > VMPRESSURE_LOW) {
+               /*
+                * Once we go above COSTLY_ORDER, reclaim relies heavily on
+                * compaction to make progress. Reclaim efficiency was never a
+                * great proxy for pressure to begin with, but it's outright
+                * misleading with these high orders. Don't throttle sockets
+                * because somebody is attempting something crazy like an order-7
+                * and predictably struggling.
+                */
+               if (level > VMPRESSURE_LOW && order <= PAGE_ALLOC_COSTLY_ORDER) {
                        /*
                         * Let the socket buffer allocator know that
                         * we are having trouble reclaiming LRU pages.
@@ -348,7 +357,7 @@ void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
         * to the vmpressure() basically means that we signal 'critical'
         * level.
         */
-       vmpressure(gfp, memcg, true, vmpressure_win, 0);
+       vmpressure(gfp, 0, memcg, true, vmpressure_win, 0);
 }
 
 #define MAX_VMPRESSURE_ARGS_LEN        (strlen("critical") + strlen("hierarchy") + 2)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index a9fd43b23a58ba598a9ecb827553e9bbaf0bf83b..4b09843876583abc6624d5d2ba972b5abd079af2 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -5071,8 +5071,8 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
        shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
 
        if (!sc->proactive)
-               vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
-                          sc->nr_reclaimed - reclaimed);
+               vmpressure(sc->gfp_mask, sc->order, memcg, false,
+                          sc->nr_scanned - scanned, sc->nr_reclaimed - reclaimed);
 
        flush_reclaim_state(sc);
 
@@ -6175,7 +6175,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
 
                /* Record the group's reclaim efficiency */
                if (!sc->proactive)
-                       vmpressure(sc->gfp_mask, memcg, false,
+                       vmpressure(sc->gfp_mask, sc->order, memcg, false,
                                   sc->nr_scanned - scanned,
                                   sc->nr_reclaimed - reclaimed);
 
@@ -6220,7 +6220,7 @@ again:
 
        /* Record the subtree's reclaim efficiency */
        if (!sc->proactive)
-               vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
+               vmpressure(sc->gfp_mask, sc->order, sc->target_mem_cgroup, true,
                           sc->nr_scanned - nr_scanned, nr_node_reclaimed);
 
        if (nr_node_reclaimed)