geneve: Allow users to specify source port range

Recently, in case of Cilium, we run into users on Azure who require to use
tunneling for east/west traffic due to hitting IPAM API limits for Kubernetes
Pods if they would have gone with publicly routable IPs for Pods. In case
of tunneling, Cilium supports the option of vxlan or geneve. In order to
RSS spread flows among remote CPUs both derive a source port hash via
udp_flow_src_port() which takes the inner packet's skb->hash into account.
For clusters with many nodes, this can then hit a new limitation [0]: Today,
the Azure networking stack supports 1M total flows (500k inbound and 500k
outbound) for a VM. [...] Once this limit is hit, other connections are
dropped. [...] Each flow is distinguished by a 5-tuple (protocol, local IP
address, remote IP address, local port, and remote port) information. [...]

For vxlan and geneve, this can create a massive amount of UDP flows which
then run into the limits if stale flows are not evicted fast enough. One
option to mitigate this for vxlan is to narrow the source port range via
IFLA_VXLAN_PORT_RANGE while still being able to benefit from RSS. However,
geneve currently does not have this option and it spreads traffic across
the full source port range of [1, USHRT_MAX]. To overcome this limitation
also for geneve, add an equivalent IFLA_GENEVE_PORT_RANGE setting for users.

Note that struct geneve_config before/after still remains at 2 cachelines
on x86-64. The low/high members of struct ifla_geneve_port_range (which is
uapi exposed) are of type __be16. While they would be perfectly fine to be
of __u16 type, the consensus was that it would be good to be consistent
with the existing struct ifla_vxlan_port_range from a uapi consumer PoV.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://learn.microsoft.com/en-us/azure/virtual-network/virtual-machine-network-throughput
Link: https://patch.msgid.link/20250226182030.89440-1-daniel@iogearbox.net
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index fc62b25e0362a49d4d017193f73c8fb7213535e9..2c65f867fd31d04e2316335a368cb8f2c4e34f05 100644 (file)
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -57,6 +57,8 @@ struct geneve_config {
        bool                    ttl_inherit;
        enum ifla_geneve_df     df;
        bool                    inner_proto_inherit;
+       u16                     port_min;
+       u16                     port_max;
 };
 
 /* Pseudo network device */
@@ -835,7 +837,9 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 
        use_cache = ip_tunnel_dst_cache_usable(skb, info);
        tos = geneve_get_dsfield(skb, dev, info, &use_cache);
-       sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
+       sport = udp_flow_src_port(geneve->net, skb,
+                                 geneve->cfg.port_min,
+                                 geneve->cfg.port_max, true);
 
        rt = udp_tunnel_dst_lookup(skb, dev, geneve->net, 0, &saddr,
                                   &info->key,
@@ -945,7 +949,9 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 
        use_cache = ip_tunnel_dst_cache_usable(skb, info);
        prio = geneve_get_dsfield(skb, dev, info, &use_cache);
-       sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
+       sport = udp_flow_src_port(geneve->net, skb,
+                                 geneve->cfg.port_min,
+                                 geneve->cfg.port_max, true);
 
        dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
                                     &saddr, key, sport,
@@ -1084,7 +1090,8 @@ static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
                use_cache = ip_tunnel_dst_cache_usable(skb, info);
                tos = geneve_get_dsfield(skb, dev, info, &use_cache);
                sport = udp_flow_src_port(geneve->net, skb,
-                                         1, USHRT_MAX, true);
+                                         geneve->cfg.port_min,
+                                         geneve->cfg.port_max, true);
 
                rt = udp_tunnel_dst_lookup(skb, dev, geneve->net, 0, &saddr,
                                           &info->key,
@@ -1110,7 +1117,8 @@ static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
                use_cache = ip_tunnel_dst_cache_usable(skb, info);
                prio = geneve_get_dsfield(skb, dev, info, &use_cache);
                sport = udp_flow_src_port(geneve->net, skb,
-                                         1, USHRT_MAX, true);
+                                         geneve->cfg.port_min,
+                                         geneve->cfg.port_max, true);
 
                dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
                                             &saddr, &info->key, sport,
@@ -1234,6 +1242,7 @@ static const struct nla_policy geneve_policy[IFLA_GENEVE_MAX + 1] = {
        [IFLA_GENEVE_TTL_INHERIT]       = { .type = NLA_U8 },
        [IFLA_GENEVE_DF]                = { .type = NLA_U8 },
        [IFLA_GENEVE_INNER_PROTO_INHERIT]       = { .type = NLA_FLAG },
+       [IFLA_GENEVE_PORT_RANGE]        = NLA_POLICY_EXACT_LEN(sizeof(struct ifla_geneve_port_range)),
 };
 
 static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
@@ -1279,6 +1288,17 @@ static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
                }
        }
 
+       if (data[IFLA_GENEVE_PORT_RANGE]) {
+               const struct ifla_geneve_port_range *p;
+
+               p = nla_data(data[IFLA_GENEVE_PORT_RANGE]);
+               if (ntohs(p->high) < ntohs(p->low)) {
+                       NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_PORT_RANGE],
+                                           "Invalid source port range");
+                       return -EINVAL;
+               }
+       }
+
        return 0;
 }
 
@@ -1506,6 +1526,18 @@ static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
                info->key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
        }
 
+       if (data[IFLA_GENEVE_PORT_RANGE]) {
+               const struct ifla_geneve_port_range *p;
+
+               if (changelink) {
+                       attrtype = IFLA_GENEVE_PORT_RANGE;
+                       goto change_notsup;
+               }
+               p = nla_data(data[IFLA_GENEVE_PORT_RANGE]);
+               cfg->port_min = ntohs(p->low);
+               cfg->port_max = ntohs(p->high);
+       }
+
        if (data[IFLA_GENEVE_COLLECT_METADATA]) {
                if (changelink) {
                        attrtype = IFLA_GENEVE_COLLECT_METADATA;
@@ -1626,6 +1658,8 @@ static int geneve_newlink(struct net_device *dev,
                .use_udp6_rx_checksums = false,
                .ttl_inherit = false,
                .collect_md = false,
+               .port_min = 1,
+               .port_max = USHRT_MAX,
        };
        int err;
 
@@ -1744,6 +1778,7 @@ static size_t geneve_get_size(const struct net_device *dev)
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_RX */
                nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TTL_INHERIT */
                nla_total_size(0) +      /* IFLA_GENEVE_INNER_PROTO_INHERIT */
+               nla_total_size(sizeof(struct ifla_geneve_port_range)) + /* IFLA_GENEVE_PORT_RANGE */
                0;
 }
 
@@ -1753,6 +1788,10 @@ static int geneve_fill_info(struct sk_buff *skb, const struct net_device *dev)
        struct ip_tunnel_info *info = &geneve->cfg.info;
        bool ttl_inherit = geneve->cfg.ttl_inherit;
        bool metadata = geneve->cfg.collect_md;
+       struct ifla_geneve_port_range ports = {
+               .low    = htons(geneve->cfg.port_min),
+               .high   = htons(geneve->cfg.port_max),
+       };
        __u8 tmp_vni[3];
        __u32 vni;
 
@@ -1809,6 +1848,9 @@ static int geneve_fill_info(struct sk_buff *skb, const struct net_device *dev)
            nla_put_flag(skb, IFLA_GENEVE_INNER_PROTO_INHERIT))
                goto nla_put_failure;
 
+       if (nla_put(skb, IFLA_GENEVE_PORT_RANGE, sizeof(ports), &ports))
+               goto nla_put_failure;
+
        return 0;
 
 nla_put_failure:
@@ -1841,6 +1883,8 @@ struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
                .use_udp6_rx_checksums = true,
                .ttl_inherit = false,
                .collect_md = true,
+               .port_min = 1,
+               .port_max = USHRT_MAX,
        };
 
        memset(tb, 0, sizeof(tb));

diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index bfe880fbbb24bc765bee73212f2c83d53db168e2..3b586fb0bc4cddf6655d48ba7b31ab1c200532d6 100644 (file)
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -1438,6 +1438,7 @@ enum {
        IFLA_GENEVE_TTL_INHERIT,
        IFLA_GENEVE_DF,
        IFLA_GENEVE_INNER_PROTO_INHERIT,
+       IFLA_GENEVE_PORT_RANGE,
        __IFLA_GENEVE_MAX
 };
 #define IFLA_GENEVE_MAX        (__IFLA_GENEVE_MAX - 1)
@@ -1450,6 +1451,11 @@ enum ifla_geneve_df {
        GENEVE_DF_MAX = __GENEVE_DF_END - 1,
 };
 
+struct ifla_geneve_port_range {
+       __be16 low;
+       __be16 high;
+};
+
 /* Bareudp section  */
 enum {
        IFLA_BAREUDP_UNSPEC,