udp: udplite is unlikely

Add some unlikely() annotations to speed up the fast path,
at least with clang compiler.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20260105101719.2378881-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/udp.h b/include/linux/udp.h
index 58795688a18636ea79aa1f5d06eacc676a2e7849..1cbf6b4d3aab3bf4bd542b3ec6cd17d7cbd750f0 100644 (file)
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -236,7 +236,7 @@ static inline void udp_allow_gso(struct sock *sk)
        hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
 #endif
 
-#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)
+#define IS_UDPLITE(__sk) (unlikely(__sk->sk_protocol == IPPROTO_UDPLITE))
 
 static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
 {

diff --git a/include/net/udp.h b/include/net/udp.h
index a061d1b22ddc2fcf8ac62eccdf22512d63848838..700dbedcb15f7d1dbf1d370193776725e3982aa8 100644 (file)
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -527,18 +527,18 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off,
  *     SNMP statistics for UDP and UDP-Lite
  */
 #define UDP_INC_STATS(net, field, is_udplite)                do { \
-       if (is_udplite) SNMP_INC_STATS((net)->mib.udplite_statistics, field);       \
+       if (unlikely(is_udplite)) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \
        else            SNMP_INC_STATS((net)->mib.udp_statistics, field);  }  while(0)
 #define __UDP_INC_STATS(net, field, is_udplite)              do { \
-       if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_statistics, field);         \
+       if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_statistics, field);       \
        else            __SNMP_INC_STATS((net)->mib.udp_statistics, field);    }  while(0)
 
 #define __UDP6_INC_STATS(net, field, is_udplite)           do { \
-       if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);\
+       if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);        \
        else            __SNMP_INC_STATS((net)->mib.udp_stats_in6, field);  \
 } while(0)
 #define UDP6_INC_STATS(net, field, __lite)                 do { \
-       if (__lite) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);  \
+       if (unlikely(__lite)) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);      \
        else        SNMP_INC_STATS((net)->mib.udp_stats_in6, field);      \
 } while(0)
 

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 591527ae811cc37e303cef81b61f96d59db3c4ba..9c87067c74bc4aa675e20538a9b81520c3cd7635 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1193,7 +1193,7 @@ csum_partial:
 
 send:
        err = ip_send_skb(sock_net(sk), skb);
-       if (err) {
+       if (unlikely(err)) {
                if (err == -ENOBUFS &&
                    !inet_test_bit(RECVERR, sk)) {
                        UDP_INC_STATS(sock_net(sk),
@@ -2429,7 +2429,8 @@ static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
        /*
         *      UDP-Lite specific tests, ignored on UDP sockets
         */
-       if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
+       if (unlikely(udp_test_bit(UDPLITE_RECV_CC, sk) &&
+                    UDP_SKB_CB(skb)->partial_cov)) {
                u16 pcrlen = READ_ONCE(up->pcrlen);
 
                /*

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 794c13674e8ab8aa552bdbd90203f9591016cb66..010b909275dd01583fa5825eaba49a1013dbf974 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -875,7 +875,8 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
        /*
         * UDP-Lite specific tests, ignored on UDP sockets (see net/ipv4/udp.c).
         */
-       if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
+       if (unlikely(udp_test_bit(UDPLITE_RECV_CC, sk) &&
+                    UDP_SKB_CB(skb)->partial_cov)) {
                u16 pcrlen = READ_ONCE(up->pcrlen);
 
                if (pcrlen == 0) {          /* full coverage was set  */
@@ -1439,7 +1440,7 @@ csum_partial:
 
 send:
        err = ip6_send_skb(skb);
-       if (err) {
+       if (unlikely(err)) {
                if (err == -ENOBUFS && !inet6_test_bit(RECVERR6, sk)) {
                        UDP6_INC_STATS(sock_net(sk),
                                       UDP_MIB_SNDBUFERRORS, is_udplite);