RDMA/mana_ib: add support of multiple ports

If the HW indicates support of multiple ports for rdma, create an IB device
with a port per netdev in the ethernet mana driver.

CM is only available on port 1, but RC QPs are supported on all
ports.

Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Link: https://patch.msgid.link/1753174515-23634-1-git-send-email-kotaranov@linux.microsoft.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index 65d0af740e5a898404bd471c5bf2a3769965c0fb..fa60872f169f4cddc01bd9e8bdb91b3526f9599f 100644 (file)
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -77,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
        struct gdma_context *gc = dev->gdma_dev->gdma_context;
        struct mana_context *mc = gc->mana.driver_data;
        struct net_device *ndev;
+       int i;
 
        /* Only process events from our parent device */
-       if (event_dev != mc->ports[0])
-               return NOTIFY_DONE;
-
-       switch (event) {
-       case NETDEV_CHANGEUPPER:
-               ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
-               /*
-                * RDMA core will setup GID based on updated netdev.
-                * It's not possible to race with the core as rtnl lock is being
-                * held.
-                */
-               ib_device_set_netdev(&dev->ib_dev, ndev, 1);
-
-               /* mana_get_primary_netdev() returns ndev with refcount held */
-               netdev_put(ndev, &dev->dev_tracker);
-
-               return NOTIFY_OK;
-       default:
-               return NOTIFY_DONE;
-       }
+       for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
+               if (event_dev == mc->ports[i]) {
+                       switch (event) {
+                       case NETDEV_CHANGEUPPER:
+                               ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
+                               /*
+                                * RDMA core will setup GID based on updated netdev.
+                                * It's not possible to race with the core as rtnl lock is being
+                                * held.
+                                */
+                               ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
+
+                               /* mana_get_primary_netdev() returns ndev with refcount held */
+                               if (ndev)
+                                       netdev_put(ndev, &dev->dev_tracker);
+
+                               return NOTIFY_OK;
+                       default:
+                               return NOTIFY_DONE;
+                       }
+               }
+       return NOTIFY_DONE;
 }
 
 static int mana_ib_probe(struct auxiliary_device *adev,
@@ -111,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
        struct net_device *ndev;
        struct mana_ib_dev *dev;
        u8 mac_addr[ETH_ALEN];
-       int ret;
+       int ret, i;
 
        dev = ib_alloc_device(mana_ib_dev, ib_dev);
        if (!dev)
@@ -126,34 +129,11 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 
        if (mana_ib_is_rnic(dev)) {
                dev->ib_dev.phys_port_cnt = 1;
-               ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
-               if (!ndev) {
-                       ret = -ENODEV;
-                       ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
-                       goto free_ib_device;
-               }
-               ether_addr_copy(mac_addr, ndev->dev_addr);
-               addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
-               ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
-               /* mana_get_primary_netdev() returns ndev with refcount held */
-               netdev_put(ndev, &dev->dev_tracker);
-               if (ret) {
-                       ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
-                       goto free_ib_device;
-               }
-
-               dev->nb.notifier_call = mana_ib_netdev_event;
-               ret = register_netdevice_notifier(&dev->nb);
-               if (ret) {
-                       ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
-                                 ret);
-                       goto free_ib_device;
-               }
-
+               addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
                ret = mana_ib_gd_query_adapter_caps(dev);
                if (ret) {
                        ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
-                       goto deregister_net_notifier;
+                       goto free_ib_device;
                }
 
                ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
@@ -163,16 +143,42 @@ static int mana_ib_probe(struct auxiliary_device *adev,
                ret = mana_ib_create_eqs(dev);
                if (ret) {
                        ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
-                       goto deregister_net_notifier;
+                       goto free_ib_device;
                }
 
                ret = mana_ib_gd_create_rnic_adapter(dev);
                if (ret)
                        goto destroy_eqs;
 
-               ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
+               if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
+                       dev->ib_dev.phys_port_cnt = mc->num_ports;
+
+               for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
+                       ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
+                       if (!ndev) {
+                               ret = -ENODEV;
+                               ibdev_err(&dev->ib_dev,
+                                         "Failed to get netdev for IB port %d", i + 1);
+                               goto destroy_rnic;
+                       }
+                       ether_addr_copy(mac_addr, ndev->dev_addr);
+                       ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
+                       /* mana_get_primary_netdev() returns ndev with refcount held */
+                       netdev_put(ndev, &dev->dev_tracker);
+                       if (ret) {
+                               ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
+                               goto destroy_rnic;
+                       }
+                       ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
+                       if (ret) {
+                               ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
+                               goto destroy_rnic;
+                       }
+               }
+               dev->nb.notifier_call = mana_ib_netdev_event;
+               ret = register_netdevice_notifier(&dev->nb);
                if (ret) {
-                       ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
+                       ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
                        goto destroy_rnic;
                }
        } else {
@@ -188,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
                                       MANA_AV_BUFFER_SIZE, 0);
        if (!dev->av_pool) {
                ret = -ENOMEM;
-               goto destroy_rnic;
+               goto deregister_net_notifier;
        }
 
        ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@@ -205,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
 
 deallocate_pool:
        dma_pool_destroy(dev->av_pool);
+deregister_net_notifier:
+       if (mana_ib_is_rnic(dev))
+               unregister_netdevice_notifier(&dev->nb);
 destroy_rnic:
        if (mana_ib_is_rnic(dev))
                mana_ib_gd_destroy_rnic_adapter(dev);
 destroy_eqs:
        if (mana_ib_is_rnic(dev))
                mana_ib_destroy_eqs(dev);
-deregister_net_notifier:
-       if (mana_ib_is_rnic(dev))
-               unregister_netdevice_notifier(&dev->nb);
 free_ib_device:
        xa_destroy(&dev->qp_table_wq);
        ib_dealloc_device(&dev->ib_dev);
@@ -227,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
        ib_unregister_device(&dev->ib_dev);
        dma_pool_destroy(dev->av_pool);
        if (mana_ib_is_rnic(dev)) {
+               unregister_netdevice_notifier(&dev->nb);
                mana_ib_gd_destroy_rnic_adapter(dev);
                mana_ib_destroy_eqs(dev);
-               unregister_netdevice_notifier(&dev->nb);
        }
        xa_destroy(&dev->qp_table_wq);
        ib_dealloc_device(&dev->ib_dev);

diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index 41a24a186f9d7d5f58e19f00472da4b4e272d603..6a2471f2e804765e5ab0ceabe353b87f62d31892 100644 (file)
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
        immutable->gid_tbl_len = attr.gid_tbl_len;
 
        if (mana_ib_is_rnic(dev)) {
-               immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
-               immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+               if (port_num == 1) {
+                       immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+                       immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+               } else {
+                       immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
+                                                   | RDMA_CORE_CAP_ETH_AH;
+                       immutable->max_mad_size = 0;
+               }
        } else {
                immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
        }
@@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
        props->pkey_tbl_len = 1;
        if (mana_ib_is_rnic(dev)) {
                props->gid_tbl_len = 16;
-               props->port_cap_flags = IB_PORT_CM_SUP;
                props->ip_gids = true;
+               if (port == 1)
+                       props->port_cap_flags = IB_PORT_CM_SUP;
        }
 
        return 0;

diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 60a53f1958cfab15cacc92df2f931108320148d5..5d31034ac7fb31c38c579fe26350219bb4a5ee4e 100644 (file)
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -220,6 +220,7 @@ struct mana_ib_query_adapter_caps_req {
 enum mana_ib_adapter_features {
        MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
        MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
+       MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
 };
 
 struct mana_ib_query_adapter_caps_resp {