udp: add drop_counters to udp socket

When a packet flood hits one or more UDP sockets, many cpus
have to update sk->sk_drops.

This slows down other cpus, because currently
sk_drops is in sock_write_rx group.

Add a socket_drop_counters structure to udp sockets.

Using dedicated cache lines to hold drop counters
makes sure that consumers no longer suffer from
false sharing if/when producers only change sk->sk_drops.

This adds 128 bytes per UDP socket.

Tested with the following stress test, sending about 11 Mpps
to a dual socket AMD EPYC 7B13 64-Core.

super_netperf 20 -t UDP_STREAM -H DUT -l10 -- -n -P,1000 -m 120
Note: due to socket lookup, only one UDP socket is receiving
packets on DUT.

Then measure receiver (DUT) behavior. We can see both
consumer and BH handlers can process more packets per second.

Before:

nstat -n ; sleep 1 ; nstat | grep Udp
Udp6InDatagrams                 615091             0.0
Udp6InErrors                    3904277            0.0
Udp6RcvbufErrors                3904277            0.0

After:

nstat -n ; sleep 1 ; nstat | grep Udp
Udp6InDatagrams                 816281             0.0
Udp6InErrors                    7497093            0.0
Udp6RcvbufErrors                7497093            0.0

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20250826125031.1578842-5-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/include/linux/udp.h b/include/linux/udp.h
index 4e1a672af4c57f01d10dde906b2114327387ca73..981506be1e15ad3aa831c1d4884372b2a477f988 100644 (file)
--- a/include/linux/udp.h
+++ b/include/linux/udp.h
@@ -108,6 +108,7 @@ struct udp_sock {
         * the last UDP socket cacheline.
         */
        struct hlist_node       tunnel_list;
+       struct socket_drop_counters drop_counters;
 };
 
 #define udp_test_bit(nr, sk)                   \

diff --git a/include/net/udp.h b/include/net/udp.h
index 7b26d4c50f33b94507933c407531c14b8edd306a..93b159f30e884ce7d30e2d2240b846441c5e135b 100644 (file)
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -288,6 +288,7 @@ static inline void udp_lib_init_sock(struct sock *sk)
 {
        struct udp_sock *up = udp_sk(sk);
 
+       sk->sk_drop_counters = &up->drop_counters;
        skb_queue_head_init(&up->reader_queue);
        INIT_HLIST_NODE(&up->tunnel_list);
        up->forward_threshold = sk->sk_rcvbuf >> 2;

diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c
index ffbd4b116d17ffbb9f14440c788e50490fb0f4e0..23b2aa2604de2fd0b8075c9b446230125961ae8c 100644 (file)
--- a/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp4.c
@@ -64,7 +64,8 @@ int dump_udp4(struct bpf_iter__udp *ctx)
                       0, 0L, 0, ctx->uid, 0,
                       sock_i_ino(&inet->sk),
                       inet->sk.sk_refcnt.refs.counter, udp_sk,
-                      inet->sk.sk_drops.counter);
+                      udp_sk->drop_counters.drops0.counter +
+                      udp_sk->drop_counters.drops1.counter);
 
        return 0;
 }

diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c
index 47ff7754f4fda4c9db92fbf1dc2e6a68f044174e..c48b05aa2a4b2a008b0e2dcc1d97dc84d67aff68 100644 (file)
--- a/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_udp6.c
@@ -72,7 +72,7 @@ int dump_udp6(struct bpf_iter__udp *ctx)
                       0, 0L, 0, ctx->uid, 0,
                       sock_i_ino(&inet->sk),
                       inet->sk.sk_refcnt.refs.counter, udp_sk,
-                      inet->sk.sk_drops.counter);
-
+                      udp_sk->drop_counters.drops0.counter +
+                      udp_sk->drop_counters.drops1.counter);
        return 0;
 }