tcp: Update bind bucket state on port release

[ Upstream commit d57f4b874946e997be52f5ebb5e0e1dad368c16f ]

Today, once an inet_bind_bucket enters a state where fastreuse >= 0 or
fastreuseport >= 0 after a socket is explicitly bound to a port, it remains
in that state until all sockets are removed and the bucket is destroyed.

In this state, the bucket is skipped during ephemeral port selection in
connect(). For applications using a reduced ephemeral port
range (IP_LOCAL_PORT_RANGE socket option), this can cause faster port
exhaustion since blocked buckets are excluded from reuse.

The reason the bucket state isn't updated on port release is unclear.
Possibly a performance trade-off to avoid scanning bucket owners, or just
an oversight.

Fix it by recalculating the bucket state when a socket releases a port. To
limit overhead, each inet_bind2_bucket stores its own (fastreuse,
fastreuseport) state. On port release, only the relevant port-addr bucket
is scanned, and the overall state is derived from these.

Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20250917-update-bind-bucket-state-on-unhash-v5-1-57168b661b47@cloudflare.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index 1735db332aab56173e45e8754c8d17b21e7e77c5..072347f164830be75dbd2aef94abb5a55752f9b3 100644 (file)
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -322,8 +322,9 @@ int inet_csk_listen_start(struct sock *sk);
 void inet_csk_listen_stop(struct sock *sk);
 
 /* update the fast reuse flag when adding a socket */
-void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
-                              struct sock *sk);
+void inet_csk_update_fastreuse(const struct sock *sk,
+                              struct inet_bind_bucket *tb,
+                              struct inet_bind2_bucket *tb2);
 
 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu);
 

diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index 19dbd9081d5a5e75946c1306e7a63b6fd35434c7..d6676746dabfecd3871993017080dcc8f2ee667f 100644 (file)
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -108,6 +108,8 @@ struct inet_bind2_bucket {
        struct hlist_node       bhash_node;
        /* List of sockets hashed to this bucket */
        struct hlist_head       owners;
+       signed char             fastreuse;
+       signed char             fastreuseport;
 };
 
 static inline struct net *ib_net(const struct inet_bind_bucket *ib)

diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index 67a313575780992a1b55aa26aaa2055111eb7e8d..baafef24318e03a1d48a4a176ffef45cfde4542f 100644 (file)
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -70,7 +70,8 @@ struct inet_timewait_sock {
        unsigned int            tw_transparent  : 1,
                                tw_flowlabel    : 20,
                                tw_usec_ts      : 1,
-                               tw_pad          : 2,    /* 2 bits hole */
+                               tw_connect_bind : 1,
+                               tw_pad          : 1,    /* 1 bit hole */
                                tw_tos          : 8;
        u32                     tw_txhash;
        u32                     tw_priority;

diff --git a/include/net/sock.h b/include/net/sock.h
index 2e14283c5be1adeddc024527bd10dbd8a97a54bf..57c0df29ee964f689ec996a22f01f9fd14ed51da 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1488,6 +1488,10 @@ static inline int __sk_prot_rehash(struct sock *sk)
 
 #define SOCK_BINDADDR_LOCK     4
 #define SOCK_BINDPORT_LOCK     8
+/**
+ * define SOCK_CONNECT_BIND - &sock->sk_userlocks flag for auto-bind at connect() time
+ */
+#define SOCK_CONNECT_BIND      16
 
 struct socket_alloc {
        struct socket socket;

diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index 1e2df51427fed88d8a18a61b030f5e6234dadd8f..0076c67d9bd41114a7fdc478c90b06edf6ff8c24 100644 (file)
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -423,7 +423,7 @@ success:
 }
 
 static inline int sk_reuseport_match(struct inet_bind_bucket *tb,
-                                    struct sock *sk)
+                                    const struct sock *sk)
 {
        if (tb->fastreuseport <= 0)
                return 0;
@@ -453,8 +453,9 @@ static inline int sk_reuseport_match(struct inet_bind_bucket *tb,
                                    ipv6_only_sock(sk), true, false);
 }
 
-void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
-                              struct sock *sk)
+void inet_csk_update_fastreuse(const struct sock *sk,
+                              struct inet_bind_bucket *tb,
+                              struct inet_bind2_bucket *tb2)
 {
        bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
 
@@ -501,6 +502,9 @@ void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
                        tb->fastreuseport = 0;
                }
        }
+
+       tb2->fastreuse = tb->fastreuse;
+       tb2->fastreuseport = tb->fastreuseport;
 }
 
 /* Obtain a reference to a local port for the given sock,
@@ -582,7 +586,7 @@ int inet_csk_get_port(struct sock *sk, unsigned short snum)
        }
 
 success:
-       inet_csk_update_fastreuse(tb, sk);
+       inet_csk_update_fastreuse(sk, tb, tb2);
 
        if (!inet_csk(sk)->icsk_bind_hash)
                inet_bind_hash(sk, tb, tb2, port);

diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index ceeeec9b7290aabdab8c400cd202312b0f0be70b..4316c127f789615fdb9fe1bcb6d89d5974bc7e76 100644 (file)
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -58,6 +58,14 @@ static u32 sk_ehashfn(const struct sock *sk)
                            sk->sk_daddr, sk->sk_dport);
 }
 
+static bool sk_is_connect_bind(const struct sock *sk)
+{
+       if (sk->sk_state == TCP_TIME_WAIT)
+               return inet_twsk(sk)->tw_connect_bind;
+       else
+               return sk->sk_userlocks & SOCK_CONNECT_BIND;
+}
+
 /*
  * Allocate and initialize a new local port bind bucket.
  * The bindhash mutex for snum's hash chain must be held here.
@@ -87,10 +95,22 @@ struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
  */
 void inet_bind_bucket_destroy(struct inet_bind_bucket *tb)
 {
+       const struct inet_bind2_bucket *tb2;
+
        if (hlist_empty(&tb->bhash2)) {
                hlist_del_rcu(&tb->node);
                kfree_rcu(tb, rcu);
+               return;
+       }
+
+       if (tb->fastreuse == -1 && tb->fastreuseport == -1)
+               return;
+       hlist_for_each_entry(tb2, &tb->bhash2, bhash_node) {
+               if (tb2->fastreuse != -1 || tb2->fastreuseport != -1)
+                       return;
        }
+       tb->fastreuse = -1;
+       tb->fastreuseport = -1;
 }
 
 bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, const struct net *net,
@@ -121,6 +141,8 @@ static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb2,
 #else
        tb2->rcv_saddr = sk->sk_rcv_saddr;
 #endif
+       tb2->fastreuse = 0;
+       tb2->fastreuseport = 0;
        INIT_HLIST_HEAD(&tb2->owners);
        hlist_add_head(&tb2->node, &head->chain);
        hlist_add_head(&tb2->bhash_node, &tb->bhash2);
@@ -143,11 +165,23 @@ struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep,
 /* Caller must hold hashbucket lock for this tb with local BH disabled */
 void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb)
 {
+       const struct sock *sk;
+
        if (hlist_empty(&tb->owners)) {
                __hlist_del(&tb->node);
                __hlist_del(&tb->bhash_node);
                kmem_cache_free(cachep, tb);
+               return;
        }
+
+       if (tb->fastreuse == -1 && tb->fastreuseport == -1)
+               return;
+       sk_for_each_bound(sk, &tb->owners) {
+               if (!sk_is_connect_bind(sk))
+                       return;
+       }
+       tb->fastreuse = -1;
+       tb->fastreuseport = -1;
 }
 
 static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2,
@@ -191,6 +225,7 @@ static void __inet_put_port(struct sock *sk)
        tb = inet_csk(sk)->icsk_bind_hash;
        inet_csk(sk)->icsk_bind_hash = NULL;
        inet_sk(sk)->inet_num = 0;
+       sk->sk_userlocks &= ~SOCK_CONNECT_BIND;
 
        spin_lock(&head2->lock);
        if (inet_csk(sk)->icsk_bind2_hash) {
@@ -277,7 +312,7 @@ bhash2_find:
                }
        }
        if (update_fastreuse)
-               inet_csk_update_fastreuse(tb, child);
+               inet_csk_update_fastreuse(child, tb, tb2);
        inet_bind_hash(child, tb, tb2, port);
        spin_unlock(&head2->lock);
        spin_unlock(&head->lock);
@@ -966,6 +1001,10 @@ static int __inet_bhash2_update_saddr(struct sock *sk, void *saddr, int family,
        if (!tb2) {
                tb2 = new_tb2;
                inet_bind2_bucket_init(tb2, net, head2, inet_csk(sk)->icsk_bind_hash, sk);
+               if (sk_is_connect_bind(sk)) {
+                       tb2->fastreuse = -1;
+                       tb2->fastreuseport = -1;
+               }
        }
        inet_csk(sk)->icsk_bind2_hash = tb2;
        sk_add_bind_node(sk, &tb2->owners);
@@ -1136,6 +1175,8 @@ ok:
                                               head2, tb, sk);
                if (!tb2)
                        goto error;
+               tb2->fastreuse = -1;
+               tb2->fastreuseport = -1;
        }
 
        /* Here we want to add a little bit of randomness to the next source
@@ -1148,6 +1189,7 @@ ok:
 
        /* Head lock still held and bh's disabled */
        inet_bind_hash(sk, tb, tb2, port);
+       sk->sk_userlocks |= SOCK_CONNECT_BIND;
 
        if (sk_unhashed(sk)) {
                inet_sk(sk)->inet_sport = htons(port);

diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index 875ff923a8ed05411628f55f788f33cbe3dee5b3..6fb9efdbee27a30283fee05bb52a3ac4f98d4098 100644 (file)
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -206,6 +206,7 @@ struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
                tw->tw_hash         = sk->sk_hash;
                tw->tw_ipv6only     = 0;
                tw->tw_transparent  = inet_test_bit(TRANSPARENT, sk);
+               tw->tw_connect_bind = !!(sk->sk_userlocks & SOCK_CONNECT_BIND);
                tw->tw_prot         = sk->sk_prot_creator;
                atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
                twsk_net_set(tw, sock_net(sk));