net: use ktime_t in struct scm_timestamping_internal

Instead of using struct timespec64 in scm_timestamping_internal,
use ktime_t, saving 24 bytes in kernel stack.

This makes tcp_update_recv_tstamps() small enough to be inlined.

The ktime_t -> timespec64 conversions happen after socket lock
has been released in tcp_recvmsg(), and only if the application
requested them.

$ scripts/bloat-o-meter -t vmlinux.0 vmlinux
add/remove: 0/2 grow/shrink: 5/4 up/down: 146/-277 (-131)
Function                                     old     new   delta
tcp_zerocopy_receive                        2383    2425     +42
mptcp_recvmsg                               1565    1607     +42
tcp_recvmsg_locked                          3797    3823     +26
put_cmsg_scm_timestamping64                  131     149     +18
put_cmsg_scm_timestamping                    131     149     +18
__pfx_tcp_update_recv_tstamps                 16       -     -16
do_tcp_getsockopt                           4024    4006     -18
tcp_recv_timestamp                           474     430     -44
tcp_zc_handle_leftover                       417     371     -46
__sock_recv_timestamp                       1087    1031     -56
tcp_update_recv_tstamps                       97       -     -97
Total: Before=25223788, After=25223657, chg -0.00%

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Link: https://patch.msgid.link/20260304012747.881644-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/socket.h b/include/linux/socket.h
index ec715ad4bf25f5f759d2cab3c6b796fed84df932..ec4a0a0257939a5363c55bed3ccb20182965b2e3 100644 (file)
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -415,7 +415,7 @@ struct __kernel_timespec;
 struct old_timespec32;
 
 struct scm_timestamping_internal {
-       struct timespec64 ts[3];
+       ktime_t ts[3];
 };
 
 extern void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_internal *tss);

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 9cf8785ef0b42460a2389b4aa75570b3aa31cbb4..fea6081cf6c798fe5f3f3b3ab3335619d7324f6e 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -503,8 +503,15 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                int flags);
 int tcp_set_rcvlowat(struct sock *sk, int val);
 int tcp_set_window_clamp(struct sock *sk, int val);
-void tcp_update_recv_tstamps(struct sk_buff *skb,
-                            struct scm_timestamping_internal *tss);
+
+static inline void
+tcp_update_recv_tstamps(struct sk_buff *skb,
+                       struct scm_timestamping_internal *tss)
+{
+       tss->ts[0] = skb->tstamp;
+       tss->ts[2] = skb_hwtstamps(skb)->hwtstamp;
+}
+
 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
                        struct scm_timestamping_internal *tss);
 void tcp_data_ready(struct sock *sk);

diff --git a/net/core/scm.c b/net/core/scm.c
index a29aa8fb8065665f8a8f2255ec3e55df973a6b0a..eec13f50ecaf6fa7caf9316bf17ab401d98d0efc 100644 (file)
--- a/net/core/scm.c
+++ b/net/core/scm.c
@@ -318,8 +318,10 @@ void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_int
        int i;
 
        for (i = 0; i < ARRAY_SIZE(tss.ts); i++) {
-               tss.ts[i].tv_sec = tss_internal->ts[i].tv_sec;
-               tss.ts[i].tv_nsec = tss_internal->ts[i].tv_nsec;
+               struct timespec64 tv = ktime_to_timespec64(tss_internal->ts[i]);
+
+               tss.ts[i].tv_sec = tv.tv_sec;
+               tss.ts[i].tv_nsec = tv.tv_nsec;
        }
 
        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPING_NEW, sizeof(tss), &tss);
@@ -332,8 +334,10 @@ void put_cmsg_scm_timestamping(struct msghdr *msg, struct scm_timestamping_inter
        int i;
 
        for (i = 0; i < ARRAY_SIZE(tss.ts); i++) {
-               tss.ts[i].tv_sec = tss_internal->ts[i].tv_sec;
-               tss.ts[i].tv_nsec = tss_internal->ts[i].tv_nsec;
+               struct timespec64 tv = ktime_to_timespec64(tss_internal->ts[i]);
+
+               tss.ts[i].tv_sec = tv.tv_sec;
+               tss.ts[i].tv_nsec = tv.tv_nsec;
        }
 
        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPING_OLD, sizeof(tss), &tss);

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 5997e0fb7a45fe74b692009f7275bec7671b5341..1c8be22a361e5b3da55235c509e43a3804fa6b2f 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1871,20 +1871,6 @@ int tcp_set_rcvlowat(struct sock *sk, int val)
 }
 EXPORT_IPV6_MOD(tcp_set_rcvlowat);
 
-void tcp_update_recv_tstamps(struct sk_buff *skb,
-                            struct scm_timestamping_internal *tss)
-{
-       if (skb->tstamp)
-               tss->ts[0] = ktime_to_timespec64(skb->tstamp);
-       else
-               tss->ts[0] = (struct timespec64) {0};
-
-       if (skb_hwtstamps(skb)->hwtstamp)
-               tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
-       else
-               tss->ts[2] = (struct timespec64) {0};
-}
-
 #ifdef CONFIG_MMU
 static const struct vm_operations_struct tcp_vm_ops = {
 };
@@ -2376,22 +2362,23 @@ void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
 {
        int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
        u32 tsflags = READ_ONCE(sk->sk_tsflags);
-       bool has_timestamping = false;
 
-       if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
+       if (tss->ts[0]) {
                if (sock_flag(sk, SOCK_RCVTSTAMP)) {
+                       struct timespec64 tv = ktime_to_timespec64(tss->ts[0]);
+
                        if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
                                if (new_tstamp) {
                                        struct __kernel_timespec kts = {
-                                               .tv_sec = tss->ts[0].tv_sec,
-                                               .tv_nsec = tss->ts[0].tv_nsec,
+                                               .tv_sec = tv.tv_sec,
+                                               .tv_nsec = tv.tv_nsec,
                                        };
                                        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
                                                 sizeof(kts), &kts);
                                } else {
                                        struct __kernel_old_timespec ts_old = {
-                                               .tv_sec = tss->ts[0].tv_sec,
-                                               .tv_nsec = tss->ts[0].tv_nsec,
+                                               .tv_sec = tv.tv_sec,
+                                               .tv_nsec = tv.tv_nsec,
                                        };
                                        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
                                                 sizeof(ts_old), &ts_old);
@@ -2399,41 +2386,37 @@ void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
                        } else {
                                if (new_tstamp) {
                                        struct __kernel_sock_timeval stv = {
-                                               .tv_sec = tss->ts[0].tv_sec,
-                                               .tv_usec = tss->ts[0].tv_nsec / 1000,
+                                               .tv_sec = tv.tv_sec,
+                                               .tv_usec = tv.tv_nsec / 1000,
                                        };
                                        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
                                                 sizeof(stv), &stv);
                                } else {
-                                       struct __kernel_old_timeval tv = {
-                                               .tv_sec = tss->ts[0].tv_sec,
-                                               .tv_usec = tss->ts[0].tv_nsec / 1000,
+                                       struct __kernel_old_timeval otv = {
+                                               .tv_sec = tv.tv_sec,
+                                               .tv_usec = tv.tv_nsec / 1000,
                                        };
                                        put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
-                                                sizeof(tv), &tv);
+                                                sizeof(otv), &otv);
                                }
                        }
                }
 
-               if (tsflags & SOF_TIMESTAMPING_SOFTWARE &&
+               if (!(tsflags & SOF_TIMESTAMPING_SOFTWARE &&
                    (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
-                    !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
-                       has_timestamping = true;
-               else
-                       tss->ts[0] = (struct timespec64) {0};
+                    !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))))
+                       tss->ts[0] = 0;
        }
 
-       if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
-               if (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
+       if (tss->ts[2]) {
+               if (!(tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
                    (tsflags & SOF_TIMESTAMPING_RX_HARDWARE ||
-                    !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
-                       has_timestamping = true;
-               else
-                       tss->ts[2] = (struct timespec64) {0};
+                    !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))))
+                       tss->ts[2] = 0;
        }
 
-       if (has_timestamping) {
-               tss->ts[1] = (struct timespec64) {0};
+       if (tss->ts[0] | tss->ts[2]) {
+               tss->ts[1] = 0;
                if (sock_flag(sk, SOCK_TSTAMP_NEW))
                        put_cmsg_scm_timestamping64(msg, tss);
                else

diff --git a/net/socket.c b/net/socket.c
index 05952188127f5bce41494e99e86418249e9022f2..68829d09bcf1468dee49864c5e4ea52362c3c6c7 100644 (file)
--- a/net/socket.c
+++ b/net/socket.c
@@ -912,11 +912,10 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
 {
        int need_software_tstamp = sock_flag(sk, SOCK_RCVTSTAMP);
        int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
-       struct scm_timestamping_internal tss;
-       int empty = 1, false_tstamp = 0;
        struct skb_shared_hwtstamps *shhwtstamps =
                skb_hwtstamps(skb);
-       int if_index;
+       struct scm_timestamping_internal tss;
+       int if_index, false_tstamp = 0;
        ktime_t hwtstamp;
        u32 tsflags;
 
@@ -961,12 +960,12 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
 
        memset(&tss, 0, sizeof(tss));
        tsflags = READ_ONCE(sk->sk_tsflags);
-       if ((tsflags & SOF_TIMESTAMPING_SOFTWARE &&
-            (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
-             skb_is_err_queue(skb) ||
-             !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER))) &&
-           ktime_to_timespec64_cond(skb->tstamp, tss.ts + 0))
-               empty = 0;
+       if (tsflags & SOF_TIMESTAMPING_SOFTWARE &&
+           (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE ||
+           skb_is_err_queue(skb) ||
+           !(tsflags & SOF_TIMESTAMPING_OPT_RX_FILTER)))
+               tss.ts[0] = skb->tstamp;
+
        if (shhwtstamps &&
            (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE &&
             (tsflags & SOF_TIMESTAMPING_RX_HARDWARE ||
@@ -983,15 +982,15 @@ void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
                        hwtstamp = ptp_convert_timestamp(&hwtstamp,
                                                         READ_ONCE(sk->sk_bind_phc));
 
-               if (ktime_to_timespec64_cond(hwtstamp, tss.ts + 2)) {
-                       empty = 0;
+               if (hwtstamp) {
+                       tss.ts[2] = hwtstamp;
 
                        if ((tsflags & SOF_TIMESTAMPING_OPT_PKTINFO) &&
                            !skb_is_err_queue(skb))
                                put_ts_pktinfo(msg, skb, if_index);
                }
        }
-       if (!empty) {
+       if (tss.ts[0] | tss.ts[2]) {
                if (sock_flag(sk, SOCK_TSTAMP_NEW))
                        put_cmsg_scm_timestamping64(msg, &tss);
                else