[infiniband] Parse MLID, rate, and SL from multicast membership record

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 8165f96e0c7c9439c60b31ebd4e36350d08dafb1..8bad2b2a2020f43933181361b906727b5a93221a 100644 (file)
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -73,6 +73,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** Number of IPoIB completion entries */
 #define IPOIB_NUM_CQES 8
 
+/** An IPoIB broadcast address */
+struct ipoib_broadcast {
+       /** MAC address */
+       struct ipoib_mac mac;
+       /** Address vector */
+       struct ib_address_vector av;
+       /** Multicast group membership */
+       struct ib_mc_membership membership;
+};
+
 /** An IPoIB device */
 struct ipoib_device {
        /** Network device */
@@ -87,10 +97,8 @@ struct ipoib_device {
        struct ib_queue_pair *qp;
        /** Local MAC */
        struct ipoib_mac mac;
-       /** Broadcast MAC */
-       struct ipoib_mac broadcast;
-       /** IPv4 broadcast multicast group membership */
-       struct ib_mc_membership membership;
+       /** Broadcast address */
+       struct ipoib_broadcast broadcast;
        /** REMAC cache */
        struct list_head peers;
 };
@@ -149,7 +157,7 @@ static struct ipoib_mac * ipoib_find_remac ( struct ipoib_device *ipoib,
         * multicasts as broadcasts for simplicity.
         */
        if ( is_multicast_ether_addr ( remac ) )
-               return &ipoib->broadcast;
+               return &ipoib->broadcast.mac;
 
        /* Try to find via REMAC cache */
        list_for_each_entry ( peer, &ipoib->peers, list ) {
@@ -559,6 +567,7 @@ static int ipoib_transmit ( struct net_device *netdev,
        /* Construct address vector */
        memset ( &dest, 0, sizeof ( dest ) );
        dest.qpn = ( ntohl ( mac->flags__qpn ) & IB_QPN_MASK );
+       dest.qkey = ipoib->broadcast.av.qkey;
        dest.gid_present = 1;
        memcpy ( &dest.gid, &mac->gid, sizeof ( dest.gid ) );
        if ( ( rc = ib_resolve_path ( ibdev, &dest ) ) != 0 ) {
@@ -650,8 +659,9 @@ static void ipoib_complete_recv ( struct ib_device *ibdev __unused,
        ethhdr->h_protocol = net_proto;
 
        /* Construct destination address */
-       if ( dest->gid_present && ( memcmp ( &dest->gid, &ipoib->broadcast.gid,
-                                            sizeof ( dest->gid ) ) == 0 ) ) {
+       if ( dest->gid_present &&
+            ( memcmp ( &dest->gid, &ipoib->broadcast.mac.gid,
+                       sizeof ( dest->gid ) ) == 0 ) ) {
                /* Broadcast GID; use the Ethernet broadcast address */
                memcpy ( &ethhdr->h_dest, eth_broadcast,
                         sizeof ( ethhdr->h_dest ) );
@@ -726,18 +736,13 @@ static void ipoib_poll ( struct net_device *netdev ) {
 /**
  * Handle IPv4 broadcast multicast group join completion
  *
- * @v ibdev            Infiniband device
- * @v qp               Queue pair
  * @v membership       Multicast group membership
  * @v rc               Status code
- * @v mad              Response MAD (or NULL on error)
  */
-void ipoib_join_complete ( struct ib_device *ibdev __unused,
-                          struct ib_queue_pair *qp __unused,
-                          struct ib_mc_membership *membership, int rc,
-                          union ib_mad *mad __unused ) {
-       struct ipoib_device *ipoib =
-               container_of ( membership, struct ipoib_device, membership );
+void ipoib_join_complete ( struct ib_mc_membership *membership, int rc ) {
+       struct ipoib_device *ipoib = container_of ( membership,
+                                                   struct ipoib_device,
+                                                   broadcast.membership );
 
        /* Record join status as link status */
        netdev_link_err ( ipoib->netdev, rc );
@@ -752,8 +757,10 @@ void ipoib_join_complete ( struct ib_device *ibdev __unused,
 static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
        int rc;
 
+       /* Join multicast group */
        if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
-                                   &ipoib->membership, &ipoib->broadcast.gid,
+                                   &ipoib->broadcast.membership,
+                                   &ipoib->broadcast.av,
                                    ipoib_join_complete ) ) != 0 ) {
                DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
                       ipoib, strerror ( rc ) );
@@ -770,7 +777,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
  */
 static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
 
-       ib_mcast_leave ( ipoib->ibdev, ipoib->qp, &ipoib->membership );
+       /* Leave multicast group */
+       ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
+                        &ipoib->broadcast.membership );
 }
 
 /**
@@ -791,10 +800,17 @@ static void ipoib_link_state_changed ( struct ipoib_device *ipoib ) {
        memcpy ( &ipoib->mac.gid.s.prefix, &ibdev->gid.s.prefix,
                 sizeof ( ipoib->mac.gid.s.prefix ) );
 
-       /* Update broadcast GID based on potentially-new partition key */
-       ipoib->broadcast.gid.words[2] =
+       /* Update broadcast MAC GID based on potentially-new partition key */
+       ipoib->broadcast.mac.gid.words[2] =
                htons ( ibdev->pkey | IB_PKEY_FULL );
 
+       /* Construct broadcast address vector from broadcast MAC address */
+       memset ( &ipoib->broadcast.av, 0, sizeof ( ipoib->broadcast.av ) );
+       ipoib->broadcast.av.qpn = IB_QPN_BROADCAST;
+       ipoib->broadcast.av.gid_present = 1;
+       memcpy ( &ipoib->broadcast.av.gid, &ipoib->broadcast.mac.gid,
+                sizeof ( ipoib->broadcast.av.gid ) );
+
        /* Set net device link state to reflect Infiniband link state */
        rc = ib_link_rc ( ibdev );
        netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
@@ -936,8 +952,8 @@ static int ipoib_probe ( struct ib_device *ibdev ) {
                 sizeof ( ipoib->mac.gid.s.guid ) );
 
        /* Set default broadcast MAC address */
-       memcpy ( &ipoib->broadcast, &ipoib_broadcast,
-                sizeof ( ipoib->broadcast ) );
+       memcpy ( &ipoib->broadcast.mac, &ipoib_broadcast,
+                sizeof ( ipoib->broadcast.mac ) );
 
        /* Add to list of IPoIB devices */
        list_add_tail ( &ipoib->list, &ipoib_devices );

diff --git a/src/include/ipxe/ib_mcast.h b/src/include/ipxe/ib_mcast.h
index 202f8e297271322915e578b4f6cd00aa9a810c10..789a2f3bfa7fcd608e9033c0b6a4b29b0f16efae 100644 (file)
--- a/src/include/ipxe/ib_mcast.h
+++ b/src/include/ipxe/ib_mcast.h
@@ -17,32 +17,25 @@ struct ib_mad_transaction;
 struct ib_mc_membership {
        /** Queue pair */
        struct ib_queue_pair *qp;
-       /** Multicast GID */
-       union ib_gid gid;
+       /** Address vector */
+       struct ib_address_vector *av;
        /** Attached to multicast GID */
        int attached;
        /** Multicast group join transaction */
        struct ib_mad_transaction *madx;
        /** Handle join success/failure
         *
-        * @v ibdev             Infiniband device
-        * @v qp                Queue pair
         * @v membership        Multicast group membership
         * @v rc                Status code
-        * @v mad               Response MAD (or NULL on error)
         */
-       void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
-                             struct ib_mc_membership *membership, int rc,
-                             union ib_mad *mad );
+       void ( * complete ) ( struct ib_mc_membership *membership, int rc );
 };
 
 extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
                           struct ib_mc_membership *membership,
-                          union ib_gid *gid,
-                          void ( * joined ) ( struct ib_device *ibdev,
-                                              struct ib_queue_pair *qp,
-                                              struct ib_mc_membership *memb,
-                                              int rc, union ib_mad *mad ) );
+                          struct ib_address_vector *av,
+                          void ( * joined ) ( struct ib_mc_membership *memb,
+                                              int rc ) );
 
 extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
                             struct ib_mc_membership *membership );

diff --git a/src/net/infiniband/ib_mcast.c b/src/net/infiniband/ib_mcast.c
index 3a055908119d7a3505387a78ca4ba0aee5e11a1a..6f904331c506022d7a7265fd751a0286fbdf47d6 100644 (file)
--- a/src/net/infiniband/ib_mcast.c
+++ b/src/net/infiniband/ib_mcast.c
@@ -42,11 +42,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Generate multicast membership MAD
  *
  * @v ibdev            Infiniband device
- * @v gid              Multicast GID
+ * @v av               Address vector
  * @v join             Join (rather than leave) group
  * @v mad              MAD to fill in
  */
-static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
+static void ib_mcast_mad ( struct ib_device *ibdev,
+                          struct ib_address_vector *av,
                           int join, union ib_mad *mad ) {
        struct ib_mad_sa *sa = &mad->sa;
 
@@ -61,7 +62,7 @@ static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
                htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
                        IB_SA_MCMEMBER_REC_JOIN_STATE );
        sa->sa_data.mc_member_record.scope__join_state = 1;
-       memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
+       memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
                 sizeof ( sa->sa_data.mc_member_record.mgid ) );
        memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
                 sizeof ( sa->sa_data.mc_member_record.port_gid ) );
@@ -75,20 +76,19 @@ static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
  * @v madx             Management transaction
  * @v rc               Status code
  * @v mad              Received MAD (or NULL on error)
- * @v av               Source address vector (or NULL on error)
+ * @v src              Source address vector (or NULL on error)
  */
 static void ib_mcast_complete ( struct ib_device *ibdev,
                                struct ib_mad_interface *mi __unused,
                                struct ib_mad_transaction *madx,
                                int rc, union ib_mad *mad,
-                               struct ib_address_vector *av __unused ) {
+                               struct ib_address_vector *src __unused ) {
        struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
        struct ib_queue_pair *qp = membership->qp;
-       union ib_gid *gid = &membership->gid;
+       struct ib_address_vector *av = membership->av;
        struct ib_mc_member_record *mc_member_record =
                &mad->sa.sa_data.mc_member_record;
        int joined;
-       unsigned long qkey;
 
        /* Report failures */
        if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
@@ -101,13 +101,17 @@ static void ib_mcast_complete ( struct ib_device *ibdev,
 
        /* Extract values from MAD */
        joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
-       qkey = ntohl ( mc_member_record->qkey );
-       DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %lx\n",
+       av->qkey = ntohl ( mc_member_record->qkey );
+       av->lid = ntohs ( mc_member_record->mlid );
+       av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
+       av->sl = ( ( ntohl ( mc_member_record->sl__flow_label__hop_limit )
+                    >> 28 ) & 0x0f );
+       DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
               ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
-              IB_GID_ARGS ( gid ), qkey );
+              IB_GID_ARGS ( &av->gid ), av->qkey );
 
        /* Set queue key */
-       qp->qkey = qkey;
+       qp->qkey = av->qkey;
        if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
                DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
                       ibdev->name, qp->qpn, strerror ( rc ) );
@@ -120,7 +124,7 @@ static void ib_mcast_complete ( struct ib_device *ibdev,
        membership->madx = NULL;
 
        /* Hand off to upper completion handler */
-       membership->complete ( ibdev, qp, membership, rc, mad );
+       membership->complete ( membership, rc );
 }
 
 /** Multicast membership management transaction completion operations */
@@ -134,21 +138,20 @@ static struct ib_mad_transaction_operations ib_mcast_op = {
  * @v ibdev            Infiniband device
  * @v qp               Queue pair
  * @v membership       Multicast group membership
- * @v gid              Multicast GID to join
+ * @v av               Address vector to fill in
  * @v joined           Join completion handler
  * @ret rc             Return status code
  */
 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
-                   struct ib_mc_membership *membership, union ib_gid *gid,
-                   void ( * complete ) ( struct ib_device *ibdev,
-                                         struct ib_queue_pair *qp,
-                                         struct ib_mc_membership *membership,
-                                         int rc, union ib_mad *mad ) ) {
+                   struct ib_mc_membership *membership,
+                   struct ib_address_vector *av,
+                   void ( * complete ) ( struct ib_mc_membership *membership,
+                                         int rc ) ) {
        union ib_mad mad;
        int rc;
 
        DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
-              ibdev->name, qp->qpn, IB_GID_ARGS ( gid ) );
+              ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
 
        /* Sanity checks */
        assert ( qp != NULL );
@@ -156,11 +159,11 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 
        /* Initialise structure */
        membership->qp = qp;
-       memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
+       membership->av = av;
        membership->complete = complete;
 
        /* Attach queue pair to multicast GID */
-       if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
+       if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
                DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
                       ibdev->name, qp->qpn, strerror ( rc ) );
                goto err_mcast_attach;
@@ -168,7 +171,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
        membership->attached = 1;
 
        /* Initiate multicast membership join */
-       ib_mcast_mad ( ibdev, gid, 1, &mad );
+       ib_mcast_mad ( ibdev, av, 1, &mad );
        membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
                                            &ib_mcast_op );
        if ( ! membership->madx ) {
@@ -183,7 +186,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 
        ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
  err_create_madx:
-       ib_mcast_detach ( ibdev, qp, gid );
+       ib_mcast_detach ( ibdev, qp, &av->gid );
        membership->attached = 0;
  err_mcast_attach:
        return rc;
@@ -198,7 +201,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
  */
 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
                      struct ib_mc_membership *membership ) {
-       union ib_gid *gid = &membership->gid;
+       struct ib_address_vector *av = membership->av;
        union ib_mad mad;
        int rc;
 
@@ -207,13 +210,13 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
                return;
 
        DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
-              ibdev->name, qp->qpn, IB_GID_ARGS ( gid ) );
+              ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
 
        /* Sanity check */
        assert ( qp != NULL );
 
        /* Detach from multicast GID */
-       ib_mcast_detach ( ibdev, qp, &membership->gid );
+       ib_mcast_detach ( ibdev, qp, &av->gid );
        membership->attached = 0;
 
        /* Cancel multicast membership join, if applicable */
@@ -223,7 +226,7 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
        }
 
        /* Send a single group leave MAD */
-       ib_mcast_mad ( ibdev, &membership->gid, 0, &mad );
+       ib_mcast_mad ( ibdev, av, 0, &mad );
        if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
                DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
                       "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );