From be538b8b279bec726dd0d4db1f9c9ceb7d737197 Mon Sep 17 00:00:00 2001 From: Cole Dishington Date: Tue, 19 Oct 2021 08:53:25 +1300 Subject: [PATCH] Make number of rx and tx queues configurable for veths Distribute traffic over cpu cores of container by configuring more than 1 tx/rx queue. Signed-off-by: Cole Dishington --- src/lxc/cmd/lxc_user_nic.c | 2 +- src/lxc/confile.c | 135 +++++++++++++++++++++++++++++++++++++ src/lxc/network.c | 18 ++++- src/lxc/network.h | 4 +- 4 files changed, 155 insertions(+), 4 deletions(-) diff --git a/src/lxc/cmd/lxc_user_nic.c b/src/lxc/cmd/lxc_user_nic.c index 83e79cd1e..a91e2259d 100644 --- a/src/lxc/cmd/lxc_user_nic.c +++ b/src/lxc/cmd/lxc_user_nic.c @@ -469,7 +469,7 @@ static int instantiate_veth(char *veth1, char *veth2, pid_t pid, unsigned int mt { int ret; - ret = lxc_veth_create(veth1, veth2, pid, mtu); + ret = lxc_veth_create(veth1, veth2, pid, mtu, -1, -1); if (ret < 0) { CMD_SYSERROR("Failed to create %s-%s\n", veth1, veth2); return ret_errno(-ret); diff --git a/src/lxc/confile.c b/src/lxc/confile.c index fafcc7ba9..e9ea23eb0 100644 --- a/src/lxc/confile.c +++ b/src/lxc/confile.c @@ -125,6 +125,8 @@ lxc_config_define(net_script_down); lxc_config_define(net_script_up); lxc_config_define(net_type); lxc_config_define(net_veth_mode); +lxc_config_define(net_veth_n_rxqueues); +lxc_config_define(net_veth_n_txqueues); lxc_config_define(net_veth_pair); lxc_config_define(net_veth_ipv4_route); lxc_config_define(net_veth_ipv6_route); @@ -307,6 +309,8 @@ static struct lxc_config_net_t config_jump_table_net[] = { { "type", true, set_config_net_type, get_config_net_type, clr_config_net_type, }, { "vlan.id", true, set_config_net_vlan_id, get_config_net_vlan_id, clr_config_net_vlan_id, }, { "veth.mode", true, set_config_net_veth_mode, get_config_net_veth_mode, clr_config_net_veth_mode, }, + { "veth.n_rxqueues", true, set_config_net_veth_n_rxqueues, get_config_net_veth_n_rxqueues, clr_config_net_veth_n_rxqueues, }, + { "veth.n_txqueues", true, set_config_net_veth_n_txqueues, get_config_net_veth_n_txqueues, clr_config_net_veth_n_txqueues, }, { "veth.pair", true, set_config_net_veth_pair, get_config_net_veth_pair, clr_config_net_veth_pair, }, { "veth.ipv4.route", true, set_config_net_veth_ipv4_route, get_config_net_veth_ipv4_route, clr_config_net_veth_ipv4_route, }, { "veth.ipv6.route", true, set_config_net_veth_ipv6_route, get_config_net_veth_ipv6_route, clr_config_net_veth_ipv6_route, }, @@ -587,6 +591,56 @@ static int set_config_net_veth_mode(const char *key, const char *value, return lxc_veth_mode_to_flag(&netdev->priv.veth_attr.mode, value); } +static int set_config_net_veth_n_rxqueues(const char *key, const char *value, + struct lxc_conf *lxc_conf, void *data) +{ + int n_rxqueues; + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return ret_errno(EINVAL); + + if (lxc_config_value_empty(value)) + return clr_config_net_veth_n_rxqueues(key, lxc_conf, data); + + if (lxc_safe_int(value, &n_rxqueues)) + return ret_errno(EINVAL); + + if (n_rxqueues <= 0) + return ret_errno(EINVAL); + + netdev->priv.veth_attr.n_rxqueues = n_rxqueues; + return 0; +} + +static int set_config_net_veth_n_txqueues(const char *key, const char *value, + struct lxc_conf *lxc_conf, void *data) +{ + int n_txqueues; + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return ret_errno(EINVAL); + + if (lxc_config_value_empty(value)) + return clr_config_net_veth_n_txqueues(key, lxc_conf, data); + + if (lxc_safe_int(value, &n_txqueues)) + return ret_errno(EINVAL); + + if (n_txqueues <= 0) + return ret_errno(EINVAL); + + netdev->priv.veth_attr.n_txqueues = n_txqueues; + return 0; +} + static int set_config_net_veth_pair(const char *key, const char *value, struct lxc_conf *lxc_conf, void *data) { @@ -5572,6 +5626,39 @@ static int clr_config_net_veth_mode(const char *key, return 0; } +static int clr_config_net_veth_n_rxqueues(const char *key, struct lxc_conf *lxc_conf, + void *data) +{ + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return 0; + + netdev->priv.veth_attr.n_rxqueues = -1; + + return 0; +} + +static int clr_config_net_veth_n_txqueues(const char *key, struct lxc_conf *lxc_conf, + void *data) +{ + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return 0; + + netdev->priv.veth_attr.n_txqueues = -1; + + return 0; +} + + static int clr_config_net_veth_pair(const char *key, struct lxc_conf *lxc_conf, void *data) { @@ -6062,6 +6149,54 @@ static int get_config_net_veth_mode(const char *key, char *retv, int inlen, return fulllen; } +static int get_config_net_veth_n_rxqueues(const char *key, char *retv, int inlen, + struct lxc_conf *c, void *data) +{ + int len; + int fulllen = 0; + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return ret_errno(EINVAL); + + if (!retv) + inlen = 0; + else + memset(retv, 0, inlen); + + if (netdev->priv.veth_attr.n_rxqueues > 0) + strprint(retv, inlen, "%d", netdev->priv.veth_attr.n_rxqueues); + + return fulllen; +} + +static int get_config_net_veth_n_txqueues(const char *key, char *retv, int inlen, + struct lxc_conf *c, void *data) +{ + int len; + int fulllen = 0; + struct lxc_netdev *netdev = data; + + if (!netdev) + return ret_errno(EINVAL); + + if (netdev->type != LXC_NET_VETH) + return ret_errno(EINVAL); + + if (!retv) + inlen = 0; + else + memset(retv, 0, inlen); + + if (netdev->priv.veth_attr.n_txqueues > 0) + strprint(retv, inlen, "%d", netdev->priv.veth_attr.n_txqueues); + + return fulllen; +} + static int get_config_net_veth_pair(const char *key, char *retv, int inlen, struct lxc_conf *c, void *data) { diff --git a/src/lxc/network.c b/src/lxc/network.c index ad43694f2..81efc8a7a 100644 --- a/src/lxc/network.c +++ b/src/lxc/network.c @@ -656,7 +656,8 @@ static int netdev_configure_server_veth(struct lxc_handler *handler, struct lxc_ } } - err = lxc_veth_create(veth1, veth2, handler->pid, mtu); + err = lxc_veth_create(veth1, veth2, handler->pid, mtu, + netdev->priv.veth_attr.n_rxqueues, netdev->priv.veth_attr.n_txqueues); if (err) return log_error_errno(-1, -err, "Failed to create veth pair \"%s\" and \"%s\"", veth1, veth2); @@ -2070,7 +2071,8 @@ int lxc_netdev_down(const char *name) return netdev_set_flag(name, 0); } -int lxc_veth_create(const char *name1, const char *name2, pid_t pid, unsigned int mtu) +int lxc_veth_create(const char *name1, const char *name2, pid_t pid, unsigned int mtu, + int n_rxqueues, int n_txqueues) { call_cleaner(nlmsg_free) struct nlmsg *answer = NULL, *nlmsg = NULL; struct nl_handler nlh; @@ -2130,6 +2132,12 @@ int lxc_veth_create(const char *name1, const char *name2, pid_t pid, unsigned in if (nla_put_string(nlmsg, IFLA_IFNAME, name2)) return ret_errno(ENOMEM); + if (n_rxqueues > 0 && nla_put_u32(nlmsg, IFLA_NUM_RX_QUEUES, (unsigned int)n_rxqueues)) + return ret_errno(ENOMEM); + + if (n_txqueues > 0 && nla_put_u32(nlmsg, IFLA_NUM_TX_QUEUES, (unsigned int)n_txqueues)) + return ret_errno(ENOMEM); + if (mtu > 0 && nla_put_u32(nlmsg, IFLA_MTU, mtu)) return ret_errno(ENOMEM); @@ -2143,6 +2151,12 @@ int lxc_veth_create(const char *name1, const char *name2, pid_t pid, unsigned in if (nla_put_string(nlmsg, IFLA_IFNAME, name1)) return ret_errno(ENOMEM); + if (n_txqueues > 0 && nla_put_u32(nlmsg, IFLA_NUM_RX_QUEUES, (unsigned int)n_txqueues)) + return ret_errno(ENOMEM); + + if (n_rxqueues > 0 && nla_put_u32(nlmsg, IFLA_NUM_TX_QUEUES, (unsigned int)n_rxqueues)) + return ret_errno(ENOMEM); + return netlink_transaction(nlh_ptr, nlmsg, answer); } diff --git a/src/lxc/network.h b/src/lxc/network.h index d5d3d54b6..fd93242be 100644 --- a/src/lxc/network.h +++ b/src/lxc/network.h @@ -77,6 +77,8 @@ struct ifla_veth { struct list_head ipv4_routes; struct list_head ipv6_routes; int mode; /* bridge, router */ + int n_rxqueues; + int n_txqueues; short vlan_id; bool vlan_id_set; struct lxc_list vlan_tagged_ids; @@ -207,7 +209,7 @@ __hidden extern int lxc_netdev_set_mtu(const char *name, int mtu); /* Create a virtual network devices. */ __hidden extern int lxc_veth_create(const char *name1, const char *name2, pid_t pid, - unsigned int mtu); + unsigned int mtu, int n_rxqueues, int n_txqueues); __hidden extern int lxc_macvlan_create(const char *parent, const char *name, int mode); __hidden extern int lxc_vlan_create(const char *parent, const char *name, unsigned short vid); -- 2.47.2