From: David Arinzon Date: Tue, 17 Jun 2025 11:05:39 +0000 (+0300) Subject: net: ena: Add device reload capability through devlink X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=15115b1a255404795158fa92a1cba87a9acff15d;p=thirdparty%2Flinux.git net: ena: Add device reload capability through devlink Adding basic devlink capability support of reloading the driver. This capability is required to support driver init type devlink params (DEVLINK_PARAM_CMODE_DRIVERINIT). Such params require reloading of the driver (destroy/restore sequence). The reloading is done by the devlink framework using the hooks provided by the driver. Signed-off-by: David Arinzon Link: https://patch.msgid.link/20250617110545.5659-4-darinzon@amazon.com Signed-off-by: Jakub Kicinski --- diff --git a/Documentation/networking/device_drivers/ethernet/amazon/ena.rst b/Documentation/networking/device_drivers/ethernet/amazon/ena.rst index 98dc62174a406..112a3994bc854 100644 --- a/Documentation/networking/device_drivers/ethernet/amazon/ena.rst +++ b/Documentation/networking/device_drivers/ethernet/amazon/ena.rst @@ -57,6 +57,7 @@ ena_ethtool.c ethtool callbacks. ena_xdp.[ch] XDP files ena_pci_id_tbl.h Supported device IDs. ena_phc.[ch] PTP hardware clock infrastructure (see `PHC`_ for more info) +ena_devlink.[ch] devlink files. ================= ====================================================== Management Interface: @@ -269,6 +270,18 @@ RSS - The user can provide a hash key, hash function, and configure the indirection table through `ethtool(8)`. +DEVLINK SUPPORT +=============== +.. _`devlink`: https://www.kernel.org/doc/html/latest/networking/devlink/index.html + +`devlink`_ supports reloading the driver and initiating re-negotiation with the ENA device + +.. code-block:: shell + + sudo devlink dev reload pci/ + # for example: + sudo devlink dev reload pci/0000:00:06.0 + DATA PATH ========= diff --git a/drivers/net/ethernet/amazon/Kconfig b/drivers/net/ethernet/amazon/Kconfig index 8d61bc62e295a..95dcc3969f0c1 100644 --- a/drivers/net/ethernet/amazon/Kconfig +++ b/drivers/net/ethernet/amazon/Kconfig @@ -21,6 +21,7 @@ config ENA_ETHERNET depends on PCI_MSI && !CPU_BIG_ENDIAN depends on PTP_1588_CLOCK_OPTIONAL select DIMLIB + select NET_DEVLINK help This driver supports Elastic Network Adapter (ENA)" diff --git a/drivers/net/ethernet/amazon/ena/Makefile b/drivers/net/ethernet/amazon/ena/Makefile index 8c874177468fe..4b6511db873e0 100644 --- a/drivers/net/ethernet/amazon/ena/Makefile +++ b/drivers/net/ethernet/amazon/ena/Makefile @@ -5,4 +5,4 @@ obj-$(CONFIG_ENA_ETHERNET) += ena.o -ena-y := ena_netdev.o ena_com.o ena_eth_com.o ena_ethtool.o ena_xdp.o ena_phc.o +ena-y := ena_netdev.o ena_com.o ena_eth_com.o ena_ethtool.o ena_xdp.o ena_phc.o ena_devlink.o diff --git a/drivers/net/ethernet/amazon/ena/ena_devlink.c b/drivers/net/ethernet/amazon/ena/ena_devlink.c new file mode 100644 index 0000000000000..db56916c7f126 --- /dev/null +++ b/drivers/net/ethernet/amazon/ena/ena_devlink.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ + +#include "linux/pci.h" +#include "ena_devlink.h" + +static int ena_devlink_reload_down(struct devlink *devlink, + bool netns_change, + enum devlink_reload_action action, + enum devlink_reload_limit limit, + struct netlink_ext_ack *extack) +{ + struct ena_adapter *adapter = ENA_DEVLINK_PRIV(devlink); + + if (netns_change) { + NL_SET_ERR_MSG_MOD(extack, + "Namespace change is not supported"); + return -EOPNOTSUPP; + } + + rtnl_lock(); + ena_destroy_device(adapter, false); + rtnl_unlock(); + + return 0; +} + +static int ena_devlink_reload_up(struct devlink *devlink, + enum devlink_reload_action action, + enum devlink_reload_limit limit, + u32 *actions_performed, + struct netlink_ext_ack *extack) +{ + struct ena_adapter *adapter = ENA_DEVLINK_PRIV(devlink); + int err = 0; + + rtnl_lock(); + /* Check that no other routine initialized the device (e.g. + * ena_fw_reset_device()). Also we're under devlink_mutex here, + * so devlink isn't freed under our feet. + */ + if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) + err = ena_restore_device(adapter); + + rtnl_unlock(); + + if (!err) + *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); + + return err; +} + +static const struct devlink_ops ena_devlink_ops = { + .reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT), + .reload_down = ena_devlink_reload_down, + .reload_up = ena_devlink_reload_up, +}; + +struct devlink *ena_devlink_alloc(struct ena_adapter *adapter) +{ + struct device *dev = &adapter->pdev->dev; + struct devlink *devlink; + + devlink = devlink_alloc(&ena_devlink_ops, + sizeof(struct ena_adapter *), + dev); + if (!devlink) { + netdev_err(adapter->netdev, + "Failed to allocate devlink struct\n"); + return NULL; + } + + ENA_DEVLINK_PRIV(devlink) = adapter; + adapter->devlink = devlink; + + return devlink; +} + +void ena_devlink_free(struct devlink *devlink) +{ + devlink_free(devlink); +} + +void ena_devlink_register(struct devlink *devlink, struct device *dev) +{ + devlink_register(devlink); +} + +void ena_devlink_unregister(struct devlink *devlink) +{ + devlink_unregister(devlink); +} diff --git a/drivers/net/ethernet/amazon/ena/ena_devlink.h b/drivers/net/ethernet/amazon/ena/ena_devlink.h new file mode 100644 index 0000000000000..cb1a5f21a72e3 --- /dev/null +++ b/drivers/net/ethernet/amazon/ena/ena_devlink.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ +#ifndef DEVLINK_H +#define DEVLINK_H + +#include "ena_netdev.h" +#include + +#define ENA_DEVLINK_PRIV(devlink) \ + (*(struct ena_adapter **)devlink_priv(devlink)) + +struct devlink *ena_devlink_alloc(struct ena_adapter *adapter); +void ena_devlink_free(struct devlink *devlink); +void ena_devlink_register(struct devlink *devlink, struct device *dev); +void ena_devlink_unregister(struct devlink *devlink); + +#endif /* DEVLINK_H */ diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 14a85916464b6..ab88895b874e6 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -21,6 +21,8 @@ #include "ena_phc.h" +#include "ena_devlink.h" + MODULE_AUTHOR("Amazon.com, Inc. or its affiliates"); MODULE_DESCRIPTION(DEVICE_NAME); MODULE_LICENSE("GPL"); @@ -41,8 +43,6 @@ MODULE_DEVICE_TABLE(pci, ena_pci_tbl); static int ena_rss_init_default(struct ena_adapter *adapter); static void check_for_admin_com_state(struct ena_adapter *adapter); -static int ena_destroy_device(struct ena_adapter *adapter, bool graceful); -static int ena_restore_device(struct ena_adapter *adapter); static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue) { @@ -3240,7 +3240,7 @@ err_disable_msix: return rc; } -static int ena_destroy_device(struct ena_adapter *adapter, bool graceful) +int ena_destroy_device(struct ena_adapter *adapter, bool graceful) { struct net_device *netdev = adapter->netdev; struct ena_com_dev *ena_dev = adapter->ena_dev; @@ -3291,7 +3291,7 @@ static int ena_destroy_device(struct ena_adapter *adapter, bool graceful) return rc; } -static int ena_restore_device(struct ena_adapter *adapter) +int ena_restore_device(struct ena_adapter *adapter) { struct ena_com_dev_get_features_ctx get_feat_ctx; struct ena_com_dev *ena_dev = adapter->ena_dev; @@ -3877,6 +3877,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct ena_adapter *adapter; struct net_device *netdev; static int adapters_found; + struct devlink *devlink; u32 max_num_io_queues; bool wd_state; int bars, rc; @@ -3960,12 +3961,20 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_metrics_destroy; } + /* Need to do this before ena_device_init */ + devlink = ena_devlink_alloc(adapter); + if (!devlink) { + netdev_err(netdev, "ena_devlink_alloc failed\n"); + rc = -ENOMEM; + goto err_metrics_destroy; + } + rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state); if (rc) { dev_err(&pdev->dev, "ENA device init failed\n"); if (rc == -ETIME) rc = -EPROBE_DEFER; - goto err_metrics_destroy; + goto ena_devlink_destroy; } /* Initial TX and RX interrupt delay. Assumes 1 usec granularity. @@ -4070,6 +4079,12 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapters_found++; + /* From this point, the devlink device is visible to users. + * Perform the registration last to ensure that all the resources + * are available and that the netdevice is registered. + */ + ena_devlink_register(devlink, &pdev->dev); + return 0; err_rss: @@ -4086,6 +4101,8 @@ err_worker_destroy: err_device_destroy: ena_com_delete_host_info(ena_dev); ena_com_admin_destroy(ena_dev); +ena_devlink_destroy: + ena_devlink_free(devlink); err_metrics_destroy: ena_com_delete_customer_metrics_buffer(ena_dev); err_free_phc: @@ -4132,6 +4149,9 @@ static void __ena_shutoff(struct pci_dev *pdev, bool shutdown) ena_phc_free(adapter); + ena_devlink_unregister(adapter->devlink); + ena_devlink_free(adapter->devlink); + if (shutdown) { netif_device_detach(netdev); dev_close(netdev); diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h index 7867cd7f25bc1..a732a19ea97ba 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.h +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h @@ -16,6 +16,7 @@ #include #include #include +#include #include "ena_com.h" #include "ena_eth_com.h" @@ -387,6 +388,8 @@ struct ena_adapter { struct bpf_prog *xdp_bpf_prog; u32 xdp_first_ring; u32 xdp_num_queues; + + struct devlink *devlink; }; void ena_set_ethtool_ops(struct net_device *netdev); @@ -416,6 +419,8 @@ static inline void ena_reset_device(struct ena_adapter *adapter, set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); } +int ena_destroy_device(struct ena_adapter *adapter, bool graceful); +int ena_restore_device(struct ena_adapter *adapter); int handle_invalid_req_id(struct ena_ring *ring, u16 req_id, struct ena_tx_buffer *tx_info, bool is_xdp);