#include <net/udp.h>
#include <net/protocol.h>
#include <net/inet_common.h>
-#include <net/udp_tunnel.h>
-
-#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
-
-/*
- * Dummy GRO tunnel callback, exists mainly to avoid dangling/NULL
- * values for the udp tunnel static call.
- */
-static struct sk_buff *dummy_gro_rcv(struct sock *sk,
- struct list_head *head,
- struct sk_buff *skb)
-{
- NAPI_GRO_CB(skb)->flush = 1;
- return NULL;
-}
-
-typedef struct sk_buff *(*udp_tunnel_gro_rcv_t)(struct sock *sk,
- struct list_head *head,
- struct sk_buff *skb);
-
-struct udp_tunnel_type_entry {
- udp_tunnel_gro_rcv_t gro_receive;
- refcount_t count;
-};
-
-#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
- IS_ENABLED(CONFIG_VXLAN) * 2 + \
- IS_ENABLED(CONFIG_NET_FOU) * 2)
-
-DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
-static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
-static struct mutex udp_tunnel_gro_type_lock;
-static struct udp_tunnel_type_entry udp_tunnel_gro_types[UDP_MAX_TUNNEL_TYPES];
-static unsigned int udp_tunnel_gro_type_nr;
-static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
-
-void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
-{
- bool is_ipv6 = sk->sk_family == AF_INET6;
- struct udp_sock *tup, *up = udp_sk(sk);
- struct udp_tunnel_gro *udp_tunnel_gro;
-
- spin_lock(&udp_tunnel_gro_lock);
- udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
- if (add)
- hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
- else
- hlist_del_init(&up->tunnel_list);
-
- if (udp_tunnel_gro->list.first &&
- !udp_tunnel_gro->list.first->next) {
- tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
- tunnel_list);
-
- rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
- } else {
- RCU_INIT_POINTER(udp_tunnel_gro->sk, NULL);
- }
-
- spin_unlock(&udp_tunnel_gro_lock);
-}
-EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
-
-void udp_tunnel_update_gro_rcv(struct sock *sk, bool add)
-{
- struct udp_tunnel_type_entry *cur = NULL;
- struct udp_sock *up = udp_sk(sk);
- int i, old_gro_type_nr;
-
- if (!up->gro_receive)
- return;
-
- mutex_lock(&udp_tunnel_gro_type_lock);
- for (i = 0; i < udp_tunnel_gro_type_nr; i++)
- if (udp_tunnel_gro_types[i].gro_receive == up->gro_receive)
- cur = &udp_tunnel_gro_types[i];
-
- old_gro_type_nr = udp_tunnel_gro_type_nr;
- if (add) {
- /*
- * Update the matching entry, if found, or add a new one
- * if needed
- */
- if (cur) {
- refcount_inc(&cur->count);
- goto out;
- }
-
- if (unlikely(udp_tunnel_gro_type_nr == UDP_MAX_TUNNEL_TYPES)) {
- pr_err_once("Too many UDP tunnel types, please increase UDP_MAX_TUNNEL_TYPES\n");
- /* Ensure static call will never be enabled */
- udp_tunnel_gro_type_nr = UDP_MAX_TUNNEL_TYPES + 2;
- goto out;
- }
-
- cur = &udp_tunnel_gro_types[udp_tunnel_gro_type_nr++];
- refcount_set(&cur->count, 1);
- cur->gro_receive = up->gro_receive;
- } else {
- /*
- * The stack cleanups only successfully added tunnel, the
- * lookup on removal should never fail.
- */
- if (WARN_ON_ONCE(!cur))
- goto out;
-
- if (!refcount_dec_and_test(&cur->count))
- goto out;
-
- /* avoid gaps, so that the enable tunnel has always id 0 */
- *cur = udp_tunnel_gro_types[--udp_tunnel_gro_type_nr];
- }
-
- if (udp_tunnel_gro_type_nr == 1) {
- static_call_update(udp_tunnel_gro_rcv,
- udp_tunnel_gro_types[0].gro_receive);
- static_branch_enable(&udp_tunnel_static_call);
- } else if (old_gro_type_nr == 1) {
- static_branch_disable(&udp_tunnel_static_call);
- static_call_update(udp_tunnel_gro_rcv, dummy_gro_rcv);
- }
-
-out:
- mutex_unlock(&udp_tunnel_gro_type_lock);
-}
-EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv);
-
-static void udp_tunnel_gro_init(void)
-{
- mutex_init(&udp_tunnel_gro_type_lock);
-}
-
-static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
- struct list_head *head,
- struct sk_buff *skb)
-{
- if (static_branch_likely(&udp_tunnel_static_call)) {
- if (unlikely(gro_recursion_inc_test(skb))) {
- NAPI_GRO_CB(skb)->flush |= 1;
- return NULL;
- }
- return static_call(udp_tunnel_gro_rcv)(sk, head, skb);
- }
- return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
-}
-
-#else
-
-static void udp_tunnel_gro_init(void) {}
-
-static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
- struct list_head *head,
- struct sk_buff *skb)
-{
- return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
-}
-
-#endif
static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
netdev_features_t features,
skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
- pp = udp_tunnel_gro_rcv(sk, head, skb);
+ pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
out:
skb_gro_flush_final(skb, pp, flush);
{
const struct iphdr *iph = skb_gro_network_header(skb);
struct net *net = dev_net_rcu(skb->dev);
- struct sock *sk;
int iif, sdif;
- sk = udp_tunnel_sk(net, false);
- if (sk && dport == htons(sk->sk_num))
- return sk;
-
inet_get_iif_sdif(skb, &iif, &sdif);
return __udp4_lib_lookup(net, iph->saddr, sport,
.gro_complete = udp4_gro_complete,
},
};
-
- udp_tunnel_gro_init();
return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
}
projects / thirdparty / kernel / linux.git / commitdiff
