From: Konstantin Taranov Date: Tue, 22 Jul 2025 08:55:15 +0000 (-0700) Subject: RDMA/mana_ib: add support of multiple ports X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=60c9a34df2d83dee2e6ec7e47c2310293c98f7e4;p=thirdparty%2Fkernel%2Flinux.git RDMA/mana_ib: add support of multiple ports If the HW indicates support of multiple ports for rdma, create an IB device with a port per netdev in the ethernet mana driver. CM is only available on port 1, but RC QPs are supported on all ports. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index 65d0af740e5a8..fa60872f169f4 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -77,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this, struct gdma_context *gc = dev->gdma_dev->gdma_context; struct mana_context *mc = gc->mana.driver_data; struct net_device *ndev; + int i; /* Only process events from our parent device */ - if (event_dev != mc->ports[0]) - return NOTIFY_DONE; - - switch (event) { - case NETDEV_CHANGEUPPER: - ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker); - /* - * RDMA core will setup GID based on updated netdev. - * It's not possible to race with the core as rtnl lock is being - * held. - */ - ib_device_set_netdev(&dev->ib_dev, ndev, 1); - - /* mana_get_primary_netdev() returns ndev with refcount held */ - netdev_put(ndev, &dev->dev_tracker); - - return NOTIFY_OK; - default: - return NOTIFY_DONE; - } + for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) + if (event_dev == mc->ports[i]) { + switch (event) { + case NETDEV_CHANGEUPPER: + ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker); + /* + * RDMA core will setup GID based on updated netdev. + * It's not possible to race with the core as rtnl lock is being + * held. + */ + ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); + + /* mana_get_primary_netdev() returns ndev with refcount held */ + if (ndev) + netdev_put(ndev, &dev->dev_tracker); + + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } + } + return NOTIFY_DONE; } static int mana_ib_probe(struct auxiliary_device *adev, @@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, struct net_device *ndev; struct mana_ib_dev *dev; u8 mac_addr[ETH_ALEN]; - int ret; + int ret, i; dev = ib_alloc_device(mana_ib_dev, ib_dev); if (!dev) @@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev, if (mana_ib_is_rnic(dev)) { dev->ib_dev.phys_port_cnt = 1; - ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker); - if (!ndev) { - ret = -ENODEV; - ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1"); - goto free_ib_device; - } - ether_addr_copy(mac_addr, ndev->dev_addr); - addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); - ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1); - /* mana_get_primary_netdev() returns ndev with refcount held */ - netdev_put(ndev, &dev->dev_tracker); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); - goto free_ib_device; - } - - dev->nb.notifier_call = mana_ib_netdev_event; - ret = register_netdevice_notifier(&dev->nb); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", - ret); - goto free_ib_device; - } - + addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr); ret = mana_ib_gd_query_adapter_caps(dev); if (ret) { ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret); - goto deregister_net_notifier; + goto free_ib_device; } ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); @@ -163,16 +143,42 @@ static int mana_ib_probe(struct auxiliary_device *adev, ret = mana_ib_create_eqs(dev); if (ret) { ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); - goto deregister_net_notifier; + goto free_ib_device; } ret = mana_ib_gd_create_rnic_adapter(dev); if (ret) goto destroy_eqs; - ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); + if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT) + dev->ib_dev.phys_port_cnt = mc->num_ports; + + for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) { + ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker); + if (!ndev) { + ret = -ENODEV; + ibdev_err(&dev->ib_dev, + "Failed to get netdev for IB port %d", i + 1); + goto destroy_rnic; + } + ether_addr_copy(mac_addr, ndev->dev_addr); + ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); + /* mana_get_primary_netdev() returns ndev with refcount held */ + netdev_put(ndev, &dev->dev_tracker); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); + goto destroy_rnic; + } + ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret); + goto destroy_rnic; + } + } + dev->nb.notifier_call = mana_ib_netdev_event; + ret = register_netdevice_notifier(&dev->nb); if (ret) { - ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret); + ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret); goto destroy_rnic; } } else { @@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev, MANA_AV_BUFFER_SIZE, 0); if (!dev->av_pool) { ret = -ENOMEM; - goto destroy_rnic; + goto deregister_net_notifier; } ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev, @@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev, deallocate_pool: dma_pool_destroy(dev->av_pool); +deregister_net_notifier: + if (mana_ib_is_rnic(dev)) + unregister_netdevice_notifier(&dev->nb); destroy_rnic: if (mana_ib_is_rnic(dev)) mana_ib_gd_destroy_rnic_adapter(dev); destroy_eqs: if (mana_ib_is_rnic(dev)) mana_ib_destroy_eqs(dev); -deregister_net_notifier: - if (mana_ib_is_rnic(dev)) - unregister_netdevice_notifier(&dev->nb); free_ib_device: xa_destroy(&dev->qp_table_wq); ib_dealloc_device(&dev->ib_dev); @@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev) ib_unregister_device(&dev->ib_dev); dma_pool_destroy(dev->av_pool); if (mana_ib_is_rnic(dev)) { + unregister_netdevice_notifier(&dev->nb); mana_ib_gd_destroy_rnic_adapter(dev); mana_ib_destroy_eqs(dev); - unregister_netdevice_notifier(&dev->nb); } xa_destroy(&dev->qp_table_wq); ib_dealloc_device(&dev->ib_dev); diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 41a24a186f9d7..6a2471f2e8047 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, immutable->gid_tbl_len = attr.gid_tbl_len; if (mana_ib_is_rnic(dev)) { - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; - immutable->max_mad_size = IB_MGMT_MAD_SIZE; + if (port_num == 1) { + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + } else { + immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP + | RDMA_CORE_CAP_ETH_AH; + immutable->max_mad_size = 0; + } } else { immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; } @@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { props->gid_tbl_len = 16; - props->port_cap_flags = IB_PORT_CM_SUP; props->ip_gids = true; + if (port == 1) + props->port_cap_flags = IB_PORT_CM_SUP; } return 0; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 60a53f1958cfa..5d31034ac7fb3 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req { enum mana_ib_adapter_features { MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4), MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5), + MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6), }; struct mana_ib_query_adapter_caps_resp {