tcp: split tcp_check_space() in two parts

tcp_check_space() is fat and not inlined.

Move its slow path in (out of line) __tcp_check_space()
and make tcp_check_space() an inline function for better TCP performance.

$ scripts/bloat-o-meter -t vmlinux.old vmlinux.new
add/remove: 2/2 grow/shrink: 4/0 up/down: 708/-582 (126)
Function                                     old     new   delta
__tcp_check_space                              -     521    +521
tcp_rcv_established                         1860    1916     +56
tcp_rcv_state_process                       3342    3384     +42
tcp_event_new_data_sent                      248     286     +38
tcp_data_snd_check                            71     106     +35
__pfx___tcp_check_space                        -      16     +16
__pfx_tcp_check_space                         16       -     -16
tcp_check_space                              566       -    -566
Total: Before=24896373, After=24896499, chg +0.00%

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20260203050932.3522221-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 6c12be2cdd4d49360a6d43caa56f2fb327163033..8f9f52f3408c2e87adaad3e26ffc7162776309f1 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -763,7 +763,15 @@ void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
 void tcp_done_with_error(struct sock *sk, int err);
 void tcp_reset(struct sock *sk, struct sk_buff *skb);
 void tcp_fin(struct sock *sk);
-void tcp_check_space(struct sock *sk);
+void __tcp_check_space(struct sock *sk);
+static inline void tcp_check_space(struct sock *sk)
+{
+       /* pairs with tcp_poll() */
+       smp_mb();
+
+       if (sk->sk_socket && test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
+               __tcp_check_space(sk);
+}
 void tcp_sack_compress_send_ack(struct sock *sk);
 
 static inline void tcp_cleanup_skb(struct sk_buff *skb)

diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 366c786d51bedfa7445ebf3d5b6e07109eaba7fa..e7b41abb82aad33d8cab4fcfa989cc4771149b41 100644 (file)
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6118,16 +6118,11 @@ static void tcp_new_space(struct sock *sk)
  *    small enough that tcp_stream_memory_free() decides it
  *    is time to generate EPOLLOUT.
  */
-void tcp_check_space(struct sock *sk)
+void __tcp_check_space(struct sock *sk)
 {
-       /* pairs with tcp_poll() */
-       smp_mb();
-       if (sk->sk_socket &&
-           test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
-               tcp_new_space(sk);
-               if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
-                       tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
-       }
+       tcp_new_space(sk);
+       if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
+               tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
 }
 
 static inline void tcp_data_snd_check(struct sock *sk)