tcp: md5: add tcp_md5_needed jump label

Most linux hosts never setup TCP MD5 keys. We can avoid a
cache line miss (accessing tp->md5ig_info) on RX and TX
using a jump label.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 3b522259da7d5a54d7d3730ddd8d8c9ef24313e1..0681afc62354c6693e316ae66cb7e72a1cb8bd0a 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1555,9 +1555,21 @@ struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
                                         const struct sock *addr_sk);
 
 #ifdef CONFIG_TCP_MD5SIG
-struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
-                                        const union tcp_md5_addr *addr,
-                                        int family);
+#include <linux/jump_label.h>
+extern struct static_key tcp_md5_needed;
+struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
+                                          const union tcp_md5_addr *addr,
+                                          int family);
+static inline struct tcp_md5sig_key *
+tcp_md5_do_lookup(const struct sock *sk,
+                 const union tcp_md5_addr *addr,
+                 int family)
+{
+       if (!static_key_false(&tcp_md5_needed))
+               return NULL;
+       return __tcp_md5_do_lookup(sk, addr, family);
+}
+
 #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key)
 #else
 static inline struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 252048776dbb0727737eb60bde6d606dbf057488..215e4d3b361657c434ff019ebe638437b65dd351 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -3656,8 +3656,11 @@ bool tcp_alloc_md5sig_pool(void)
        if (unlikely(!tcp_md5sig_pool_populated)) {
                mutex_lock(&tcp_md5sig_mutex);
 
-               if (!tcp_md5sig_pool_populated)
+               if (!tcp_md5sig_pool_populated) {
                        __tcp_alloc_md5sig_pool();
+                       if (tcp_md5sig_pool_populated)
+                               static_key_slow_inc(&tcp_md5_needed);
+               }
 
                mutex_unlock(&tcp_md5sig_mutex);
        }

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 4904250a9aac5001410f9454258cbb8978bb8202..efc6fef692ffdca4dcdd3f4b87a837656dd66c8c 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -970,10 +970,13 @@ static void tcp_v4_reqsk_destructor(struct request_sock *req)
  * We need to maintain these in the sk structure.
  */
 
+struct static_key tcp_md5_needed __read_mostly;
+EXPORT_SYMBOL(tcp_md5_needed);
+
 /* Find the Key structure for an address.  */
-struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
-                                        const union tcp_md5_addr *addr,
-                                        int family)
+struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk,
+                                          const union tcp_md5_addr *addr,
+                                          int family)
 {
        const struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_key *key;
@@ -1011,7 +1014,7 @@ struct tcp_md5sig_key *tcp_md5_do_lookup(const struct sock *sk,
        }
        return best_match;
 }
-EXPORT_SYMBOL(tcp_md5_do_lookup);
+EXPORT_SYMBOL(__tcp_md5_do_lookup);
 
 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
                                                      const union tcp_md5_addr *addr,

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index e4c1e51b18c1f645286eb837e8aa69011d8de406..d3b691f3a9e896ce340ae081a0d24092024361c8 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -594,7 +594,8 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
 
        *md5 = NULL;
 #ifdef CONFIG_TCP_MD5SIG
-       if (unlikely(rcu_access_pointer(tp->md5sig_info))) {
+       if (static_key_false(&tcp_md5_needed) &&
+           rcu_access_pointer(tp->md5sig_info)) {
                *md5 = tp->af_specific->md5_lookup(sk, sk);
                if (*md5) {
                        opts->options |= OPTION_MD5;
@@ -730,7 +731,8 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
 
        *md5 = NULL;
 #ifdef CONFIG_TCP_MD5SIG
-       if (unlikely(rcu_access_pointer(tp->md5sig_info))) {
+       if (static_key_false(&tcp_md5_needed) &&
+           rcu_access_pointer(tp->md5sig_info)) {
                *md5 = tp->af_specific->md5_lookup(sk, sk);
                if (*md5) {
                        opts->options |= OPTION_MD5;