net: remove duplicate reuseport_lookup functions

[ Upstream commit 0f495f7617229772403e683033abc473f0f0553c ]

There are currently four copies of reuseport_lookup: one each for
(TCP, UDP)x(IPv4, IPv6). This forces us to duplicate all callers of
those functions as well. This is already the case for sk_lookup
helpers (inet,inet6,udp4,udp6)_lookup_run_bpf.

There are two differences between the reuseport_lookup helpers:

1. They call different hash functions depending on protocol
2. UDP reuseport_lookup checks that sk_state != TCP_ESTABLISHED

Move the check for sk_state into the caller and use the INDIRECT_CALL
infrastructure to cut down the helpers to one per IP version.

Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Lorenz Bauer <lmb@isovalent.com>
Link: https://lore.kernel.org/r/20230720-so-reuseport-v6-4-7021b683cdae@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Stable-dep-of: 50aee97d1511 ("udp: Avoid call to compute_score on multiple sites")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h
index 032ddab48f8f875fc819dae1e0edb4d2e963a489..f89320b6fee315081abf18f61b22c4a78b3b32e1 100644 (file)
--- a/include/net/inet6_hashtables.h
+++ b/include/net/inet6_hashtables.h
@@ -48,12 +48,21 @@ struct sock *__inet6_lookup_established(struct net *net,
                                        const u16 hnum, const int dif,
                                        const int sdif);
 
+typedef u32 (inet6_ehashfn_t)(const struct net *net,
+                              const struct in6_addr *laddr, const u16 lport,
+                              const struct in6_addr *faddr, const __be16 fport);
+
+inet6_ehashfn_t inet6_ehashfn;
+
+INDIRECT_CALLABLE_DECLARE(inet6_ehashfn_t udp6_ehashfn);
+
 struct sock *inet6_lookup_reuseport(struct net *net, struct sock *sk,
                                    struct sk_buff *skb, int doff,
                                    const struct in6_addr *saddr,
                                    __be16 sport,
                                    const struct in6_addr *daddr,
-                                   unsigned short hnum);
+                                   unsigned short hnum,
+                                   inet6_ehashfn_t *ehashfn);
 
 struct sock *inet6_lookup_listener(struct net *net,
                                   struct inet_hashinfo *hashinfo,

diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index 94c418d2e345effd3264fe29ac4a7ef5d30fc3b8..7778422cfac55fa8d70c3cc4e85510e5caaf7de2 100644 (file)
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -313,10 +313,19 @@ struct sock *__inet_lookup_established(struct net *net,
                                       const __be32 daddr, const u16 hnum,
                                       const int dif, const int sdif);
 
+typedef u32 (inet_ehashfn_t)(const struct net *net,
+                             const __be32 laddr, const __u16 lport,
+                             const __be32 faddr, const __be16 fport);
+
+inet_ehashfn_t inet_ehashfn;
+
+INDIRECT_CALLABLE_DECLARE(inet_ehashfn_t udp_ehashfn);
+
 struct sock *inet_lookup_reuseport(struct net *net, struct sock *sk,
                                   struct sk_buff *skb, int doff,
                                   __be32 saddr, __be16 sport,
-                                  __be32 daddr, unsigned short hnum);
+                                  __be32 daddr, unsigned short hnum,
+                                  inet_ehashfn_t *ehashfn);
 
 static inline struct sock *
        inet_lookup_established(struct net *net, struct inet_hashinfo *hashinfo,
@@ -387,10 +396,6 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo,
                             refcounted);
 }
 
-u32 inet6_ehashfn(const struct net *net,
-                 const struct in6_addr *laddr, const u16 lport,
-                 const struct in6_addr *faddr, const __be16 fport);
-
 static inline void sk_daddr_set(struct sock *sk, __be32 addr)
 {
        sk->sk_daddr = addr; /* alias of inet_daddr */

diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index bea88219d531321258099ea5c1aa798f034ce590..56deddeac1b0ee27b53126e2f1419512672c4e00 100644 (file)
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -28,9 +28,9 @@
 #include <net/tcp.h>
 #include <net/sock_reuseport.h>
 
-static u32 inet_ehashfn(const struct net *net, const __be32 laddr,
-                       const __u16 lport, const __be32 faddr,
-                       const __be16 fport)
+u32 inet_ehashfn(const struct net *net, const __be32 laddr,
+                const __u16 lport, const __be32 faddr,
+                const __be16 fport)
 {
        static u32 inet_ehash_secret __read_mostly;
 
@@ -39,6 +39,7 @@ static u32 inet_ehashfn(const struct net *net, const __be32 laddr,
        return __inet_ehashfn(laddr, lport, faddr, fport,
                              inet_ehash_secret + net_hash_mix(net));
 }
+EXPORT_SYMBOL_GPL(inet_ehashfn);
 
 /* This function handles inet_sock, but also timewait and request sockets
  * for IPv4/IPv6.
@@ -252,16 +253,20 @@ static inline int compute_score(struct sock *sk, struct net *net,
        return score;
 }
 
+INDIRECT_CALLABLE_DECLARE(inet_ehashfn_t udp_ehashfn);
+
 struct sock *inet_lookup_reuseport(struct net *net, struct sock *sk,
                                   struct sk_buff *skb, int doff,
                                   __be32 saddr, __be16 sport,
-                                  __be32 daddr, unsigned short hnum)
+                                  __be32 daddr, unsigned short hnum,
+                                  inet_ehashfn_t *ehashfn)
 {
        struct sock *reuse_sk = NULL;
        u32 phash;
 
        if (sk->sk_reuseport) {
-               phash = inet_ehashfn(net, daddr, hnum, saddr, sport);
+               phash = INDIRECT_CALL_2(ehashfn, udp_ehashfn, inet_ehashfn,
+                                       net, daddr, hnum, saddr, sport);
                reuse_sk = reuseport_select_sock(sk, phash, skb, doff);
        }
        return reuse_sk;
@@ -292,7 +297,7 @@ static struct sock *inet_lhash2_lookup(struct net *net,
                score = compute_score(sk, net, hnum, daddr, dif, sdif);
                if (score > hiscore) {
                        result = inet_lookup_reuseport(net, sk, skb, doff,
-                                                      saddr, sport, daddr, hnum);
+                                                      saddr, sport, daddr, hnum, inet_ehashfn);
                        if (result)
                                return result;
 
@@ -321,7 +326,8 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net,
        if (no_reuseport || IS_ERR_OR_NULL(sk))
                return sk;
 
-       reuse_sk = inet_lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum);
+       reuse_sk = inet_lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum,
+                                        inet_ehashfn);
        if (reuse_sk)
                sk = reuse_sk;
        return sk;

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 16ff3962b24dbb2b8536eda7d410280944a500ae..127aa7b5ce97724a7ee9213536cbcda830820598 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -398,9 +398,9 @@ static int compute_score(struct sock *sk, struct net *net,
        return score;
 }
 
-static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
-                      const __u16 lport, const __be32 faddr,
-                      const __be16 fport)
+INDIRECT_CALLABLE_SCOPE
+u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
+               const __be32 faddr, const __be16 fport)
 {
        static u32 udp_ehash_secret __read_mostly;
 
@@ -410,22 +410,6 @@ static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
                              udp_ehash_secret + net_hash_mix(net));
 }
 
-static struct sock *lookup_reuseport(struct net *net, struct sock *sk,
-                                    struct sk_buff *skb,
-                                    __be32 saddr, __be16 sport,
-                                    __be32 daddr, unsigned short hnum)
-{
-       struct sock *reuse_sk = NULL;
-       u32 hash;
-
-       if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) {
-               hash = udp_ehashfn(net, daddr, hnum, saddr, sport);
-               reuse_sk = reuseport_select_sock(sk, hash, skb,
-                                                sizeof(struct udphdr));
-       }
-       return reuse_sk;
-}
-
 /* called with rcu_read_lock() */
 static struct sock *udp4_lib_lookup2(struct net *net,
                                     __be32 saddr, __be16 sport,
@@ -444,7 +428,14 @@ static struct sock *udp4_lib_lookup2(struct net *net,
                                      daddr, hnum, dif, sdif);
                if (score > badness) {
                        badness = score;
-                       result = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
+
+                       if (sk->sk_state == TCP_ESTABLISHED) {
+                               result = sk;
+                               continue;
+                       }
+
+                       result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
+                                                      saddr, sport, daddr, hnum, udp_ehashfn);
                        if (!result) {
                                result = sk;
                                continue;
@@ -483,7 +474,8 @@ static struct sock *udp4_lookup_run_bpf(struct net *net,
        if (no_reuseport || IS_ERR_OR_NULL(sk))
                return sk;
 
-       reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
+       reuse_sk = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
+                                        saddr, sport, daddr, hnum, udp_ehashfn);
        if (reuse_sk)
                sk = reuse_sk;
        return sk;

diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 2267f973e2df3717353ea217fee5bb5134f6ccf0..21b4fc835487bf43e8b7359fe6a5158caf31161d 100644 (file)
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -41,6 +41,7 @@ u32 inet6_ehashfn(const struct net *net,
        return __inet6_ehashfn(lhash, lport, fhash, fport,
                               inet6_ehash_secret + net_hash_mix(net));
 }
+EXPORT_SYMBOL_GPL(inet6_ehashfn);
 
 /*
  * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
@@ -113,18 +114,22 @@ static inline int compute_score(struct sock *sk, struct net *net,
        return score;
 }
 
+INDIRECT_CALLABLE_DECLARE(inet6_ehashfn_t udp6_ehashfn);
+
 struct sock *inet6_lookup_reuseport(struct net *net, struct sock *sk,
                                    struct sk_buff *skb, int doff,
                                    const struct in6_addr *saddr,
                                    __be16 sport,
                                    const struct in6_addr *daddr,
-                                   unsigned short hnum)
+                                   unsigned short hnum,
+                                   inet6_ehashfn_t *ehashfn)
 {
        struct sock *reuse_sk = NULL;
        u32 phash;
 
        if (sk->sk_reuseport) {
-               phash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
+               phash = INDIRECT_CALL_INET(ehashfn, udp6_ehashfn, inet6_ehashfn,
+                                          net, daddr, hnum, saddr, sport);
                reuse_sk = reuseport_select_sock(sk, phash, skb, doff);
        }
        return reuse_sk;
@@ -148,7 +153,7 @@ static struct sock *inet6_lhash2_lookup(struct net *net,
                score = compute_score(sk, net, hnum, daddr, dif, sdif);
                if (score > hiscore) {
                        result = inet6_lookup_reuseport(net, sk, skb, doff,
-                                                       saddr, sport, daddr, hnum);
+                                                       saddr, sport, daddr, hnum, inet6_ehashfn);
                        if (result)
                                return result;
 
@@ -179,7 +184,8 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net,
        if (no_reuseport || IS_ERR_OR_NULL(sk))
                return sk;
 
-       reuse_sk = inet6_lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum);
+       reuse_sk = inet6_lookup_reuseport(net, sk, skb, doff,
+                                         saddr, sport, daddr, hnum, inet6_ehashfn);
        if (reuse_sk)
                sk = reuse_sk;
        return sk;

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 8c9672e7a7dd6919e46681fa2625bdd54cde4aa6..1632b940347c5caf75ca5c900010f921fc789ea8 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -67,11 +67,12 @@ int udpv6_init_sock(struct sock *sk)
        return 0;
 }
 
-static u32 udp6_ehashfn(const struct net *net,
-                       const struct in6_addr *laddr,
-                       const u16 lport,
-                       const struct in6_addr *faddr,
-                       const __be16 fport)
+INDIRECT_CALLABLE_SCOPE
+u32 udp6_ehashfn(const struct net *net,
+                const struct in6_addr *laddr,
+                const u16 lport,
+                const struct in6_addr *faddr,
+                const __be16 fport)
 {
        static u32 udp6_ehash_secret __read_mostly;
        static u32 udp_ipv6_hash_secret __read_mostly;
@@ -155,24 +156,6 @@ static int compute_score(struct sock *sk, struct net *net,
        return score;
 }
 
-static struct sock *lookup_reuseport(struct net *net, struct sock *sk,
-                                    struct sk_buff *skb,
-                                    const struct in6_addr *saddr,
-                                    __be16 sport,
-                                    const struct in6_addr *daddr,
-                                    unsigned int hnum)
-{
-       struct sock *reuse_sk = NULL;
-       u32 hash;
-
-       if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) {
-               hash = udp6_ehashfn(net, daddr, hnum, saddr, sport);
-               reuse_sk = reuseport_select_sock(sk, hash, skb,
-                                                sizeof(struct udphdr));
-       }
-       return reuse_sk;
-}
-
 /* called with rcu_read_lock() */
 static struct sock *udp6_lib_lookup2(struct net *net,
                const struct in6_addr *saddr, __be16 sport,
@@ -190,7 +173,14 @@ static struct sock *udp6_lib_lookup2(struct net *net,
                                      daddr, hnum, dif, sdif);
                if (score > badness) {
                        badness = score;
-                       result = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
+
+                       if (sk->sk_state == TCP_ESTABLISHED) {
+                               result = sk;
+                               continue;
+                       }
+
+                       result = inet6_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
+                                                       saddr, sport, daddr, hnum, udp6_ehashfn);
                        if (!result) {
                                result = sk;
                                continue;
@@ -230,7 +220,8 @@ static inline struct sock *udp6_lookup_run_bpf(struct net *net,
        if (no_reuseport || IS_ERR_OR_NULL(sk))
                return sk;
 
-       reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
+       reuse_sk = inet6_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
+                                         saddr, sport, daddr, hnum, udp6_ehashfn);
        if (reuse_sk)
                sk = reuse_sk;
        return sk;