tcp: cache RTAX_QUICKACK metric in a hot cache line

[ Upstream commit 15492700ac41459b54a6683490adcee350ab11e3 ]

tcp_in_quickack_mode() is called from input path for small packets.

It calls __sk_dst_get() which reads sk->sk_dst_cache which has been
put in sock_read_tx group (for good reasons).

Then dst_metric(dst, RTAX_QUICKACK) also needs extra cache line misses.

Cache RTAX_QUICKACK in icsk->icsk_ack.dst_quick_ack to no longer pull
these cache lines for the cases a delayed ACK is scheduled.

After this patch TCP receive path does not longer access sock_read_tx
group.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Reviewed-by: Neal Cardwell <ncardwell@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://patch.msgid.link/20250312083907.1931644-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Stable-dep-of: 833d4313bc1e ("mptcp: reset blackhole on success with non-loopback ifaces")
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index 4bd93571e6c1b543605a4cfc324bde1f238b7c87..bcc138ff087bd60ce71556dac2ee4db5b24a82fa 100644 (file)
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -116,7 +116,8 @@ struct inet_connection_sock {
                #define ATO_BITS 8
                __u32             ato:ATO_BITS,  /* Predicted tick of soft clock           */
                                  lrcv_flowlabel:20, /* last received ipv6 flowlabel       */
-                                 unused:4;
+                                 dst_quick_ack:1, /* cache dst RTAX_QUICKACK              */
+                                 unused:3;
                unsigned long     timeout;       /* Currently scheduled timeout            */
                __u32             lrcvtime;      /* timestamp of last received data packet */
                __u16             last_seg_size; /* Size of last incoming segment          */

diff --git a/net/core/sock.c b/net/core/sock.c
index d392cb37a864f784a5b9a1956b879b8f62a93ead..b5723adab4ebf740cec7adf1ab50eda94e0a1684 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2547,8 +2547,12 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
        u32 max_segs = 1;
 
        sk->sk_route_caps = dst->dev->features;
-       if (sk_is_tcp(sk))
+       if (sk_is_tcp(sk)) {
+               struct inet_connection_sock *icsk = inet_csk(sk);
+
                sk->sk_route_caps |= NETIF_F_GSO;
+               icsk->icsk_ack.dst_quick_ack = dst_metric(dst, RTAX_QUICKACK);
+       }
        if (sk->sk_route_caps & NETIF_F_GSO)
                sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
        if (unlikely(sk->sk_gso_disabled))

diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 4c8d84fc27ca35b64b87037b3c41af5db1ff4fc0..1d9e93a04930b85e6957d1be5e3faff3067a937e 100644 (file)
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -331,9 +331,8 @@ static void tcp_enter_quickack_mode(struct sock *sk, unsigned int max_quickacks)
 static bool tcp_in_quickack_mode(struct sock *sk)
 {
        const struct inet_connection_sock *icsk = inet_csk(sk);
-       const struct dst_entry *dst = __sk_dst_get(sk);
 
-       return (dst && dst_metric(dst, RTAX_QUICKACK)) ||
+       return icsk->icsk_ack.dst_quick_ack ||
                (icsk->icsk_ack.quick && !inet_csk_in_pingpong_mode(sk));
 }