Broadcast GID is now calculated by IPoIB layer.

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 81011405c2ea7ff629d72cb12a269282cad0e548..c4eea5a73f179d69128ecb157b76cfbe6b1754a2 100644 (file)
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -84,6 +84,8 @@ struct ipoib_device {
        struct ipoib_queue_set data;
        /** Data queue set */
        struct ipoib_queue_set meta;
+       /** Broadcast GID */
+       struct ib_gid broadcast_gid;
 };
 
 /**
@@ -112,6 +114,15 @@ static struct ipoib_cached_path ipoib_path_cache[IPOIB_NUM_CACHED_PATHS];
 /** Oldest IPoIB path cache entry index */
 static unsigned int ipoib_path_cache_idx = 0;
 
+/** IPoIB metadata TID */
+static uint32_t ipoib_meta_tid = 0;
+
+/** IPv4 broadcast GID */
+static const struct ib_gid ipv4_broadcast_gid = {
+       { { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
+           0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } }
+};
+
 /****************************************************************************
  *
  * IPoIB link layer
@@ -319,7 +330,6 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib,
        struct io_buffer *iobuf;
        struct ib_mad_path_record *path_record;
        struct ib_address_vector av;
-       static uint32_t tid = 0;
        int rc;
 
        /* Allocate I/O buffer */
@@ -336,7 +346,7 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib,
        path_record->mad_hdr.class_version = 2;
        path_record->mad_hdr.method = IB_MGMT_METHOD_GET;
        path_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
-       path_record->mad_hdr.tid = tid++;
+       path_record->mad_hdr.tid = ipoib_meta_tid++;
        path_record->sa_hdr.comp_mask[1] =
                htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
        memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) );
@@ -361,6 +371,65 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib,
        return 0;
 }
 
+/**
+ * Transmit multicast group membership request
+ *
+ * @v ipoib            IPoIB device
+ * @v gid              Multicast GID
+ * @v join             Join (rather than leave) group
+ * @ret rc             Return status code
+ */
+static int ipoib_mc_member_record ( struct ipoib_device *ipoib,
+                                   struct ib_gid *gid, int join ) {
+       struct ib_device *ibdev = ipoib->ibdev;
+       struct io_buffer *iobuf;
+       struct ib_mad_mc_member_record *mc_member_record;
+       struct ib_address_vector av;
+       int rc;
+
+       /* Allocate I/O buffer */
+       iobuf = alloc_iob ( sizeof ( *mc_member_record ) );
+       if ( ! iobuf )
+               return -ENOMEM;
+       iob_put ( iobuf, sizeof ( *mc_member_record ) );
+       mc_member_record = iobuf->data;
+       memset ( mc_member_record, 0, sizeof ( *mc_member_record ) );
+
+       /* Construct path record request */
+       mc_member_record->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
+       mc_member_record->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
+       mc_member_record->mad_hdr.class_version = 2;
+       mc_member_record->mad_hdr.method = 
+               ( join ? IB_MGMT_METHOD_SET : IB_MGMT_METHOD_DELETE );
+       mc_member_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
+       mc_member_record->mad_hdr.tid = ipoib_meta_tid++;
+       mc_member_record->sa_hdr.comp_mask[1] =
+               htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
+                       IB_SA_MCMEMBER_REC_JOIN_STATE );
+       mc_member_record->scope__join_state = 1;
+       memcpy ( &mc_member_record->mgid, gid,
+                sizeof ( mc_member_record->mgid ) );
+       memcpy ( &mc_member_record->port_gid, &ibdev->port_gid,
+                sizeof ( mc_member_record->port_gid ) );
+
+       /* Construct address vector */
+       memset ( &av, 0, sizeof ( av ) );
+       av.dlid = ibdev->sm_lid;
+       av.dest_qp = IB_SA_QPN;
+       av.qkey = IB_GLOBAL_QKEY;
+
+       /* Post send request */
+       if ( ( rc = ib_post_send ( ibdev, ipoib->meta.qp, &av,
+                                  iobuf ) ) != 0 ) {
+               DBGC ( ipoib, "IPoIB %p could not send get path record: %s\n",
+                      ipoib, strerror ( rc ) );
+               free_iob ( iobuf );
+               return rc;
+       }
+
+       return 0;
+}
+
 /**
  * Transmit packet via IPoIB network device
  *
@@ -591,7 +660,7 @@ static int ipoib_open ( struct net_device *netdev ) {
 
        /* Attach to broadcast multicast GID */
        if ( ( rc = ib_mcast_attach ( ibdev, ipoib->data.qp,
-                                     &ibdev->broadcast_gid ) ) != 0 ) {
+                                     &ipoib->broadcast_gid ) ) != 0 ) {
                DBG ( "Could not attach to broadcast GID: %s\n",
                      strerror ( rc ) );
                return rc;
@@ -628,6 +697,27 @@ static struct net_device_operations ipoib_operations = {
        .irq            = ipoib_irq,
 };
 
+/**
+ * Join IPoIB broadcast group
+ *
+ * @v ipoib            IPoIB device
+ * @ret rc             Return status code
+ */
+int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
+       int rc;
+
+       /* Send join request */
+       if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid,
+                                            1 ) ) != 0 ) {
+               DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n",
+                      ipoib, strerror ( rc ) );
+               return rc;
+       }
+
+
+       return 0;
+}
+
 /**
  * Probe IPoIB device
  *
@@ -652,6 +742,11 @@ int ipoib_probe ( struct ib_device *ibdev ) {
        ipoib->netdev = netdev;
        ipoib->ibdev = ibdev;
 
+       /* Calculate broadcast GID */
+       memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
+                sizeof ( ipoib->broadcast_gid ) );
+       ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
+
        /* Allocate metadata queue set */
        if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
                                        IPOIB_META_NUM_CQES,
@@ -663,6 +758,8 @@ int ipoib_probe ( struct ib_device *ibdev ) {
                goto err_create_meta_qset;
        }
 
+
+
        /* Allocate data queue set */
        if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
                                        IPOIB_DATA_NUM_CQES,

diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c
index 5f3c3c4da3ecc52e9a323379827f2ff9f3703c86..920874cc3c56c60c7e1ec69a1cf9da9120c8f79e 100644 (file)
--- a/src/drivers/net/mlx_ipoib/mt25218.c
+++ b/src/drivers/net/mlx_ipoib/mt25218.c
@@ -1251,25 +1251,13 @@ static int arbel_get_sm_lid ( struct arbel *arbel,
        return 0;
 }
 
-static int arbel_get_broadcast_gid ( struct arbel *arbel,
-                                    struct ib_gid *broadcast_gid ) {
-       static const struct ib_gid ipv4_broadcast_gid = {
-               { { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
-                   0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } }
-       };
+static int arbel_get_pkey ( struct arbel *arbel, unsigned long *pkey ) {
        struct ib_mad_pkey_table pkey_table;
        int rc;
 
-       /* Start with the IPv4 broadcast GID */
-       memcpy ( broadcast_gid, &ipv4_broadcast_gid,
-                sizeof ( *broadcast_gid ) );
-
-       /* Add partition key */
        if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 )
                return rc;
-       memcpy ( &broadcast_gid->u.bytes[4], &pkey_table.pkey[0][0],
-                sizeof ( pkey_table.pkey[0][0] ) );
-
+       *pkey = ntohs ( pkey_table.pkey[0][0] );
        return 0;
 }
 
@@ -1340,12 +1328,11 @@ static int arbel_probe ( struct pci_device *pci,
                goto err_get_port_gid;
        }
 
-       /* Get broadcast GID */
-       if ( ( rc = arbel_get_broadcast_gid ( arbel,
-                                             &ibdev->broadcast_gid ) ) != 0 ){
-               DBGC ( arbel, "Arbel %p could not determine broadcast GID: "
+       /* Get partition key */
+       if ( ( rc = arbel_get_pkey ( arbel, &ibdev->pkey ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p could not determine partition key: "
                       "%s\n", arbel, strerror ( rc ) );
-               goto err_get_broadcast_gid;
+               goto err_get_pkey;
        }
 
        struct ud_av_st *bcast_av = ib_data.bcast_av;
@@ -1370,7 +1357,7 @@ static int arbel_probe ( struct pci_device *pci,
        return 0;
 
  err_ipoib_probe:
- err_get_broadcast_gid:
+ err_get_pkey:
  err_get_port_gid:
  err_get_sm_lid:
  err_query_dev_lim:

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index 845c4c228197aa2becbd704d57cb88c8c83ceb6f..8e358bce9afe351a762af342989829a8d87d98b6 100644 (file)
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -20,6 +20,7 @@
 struct ib_gid {
        union {
                uint8_t bytes[16];
+               uint16_t words[8];
                uint32_t dwords[4];
        } u;
 };
@@ -258,10 +259,10 @@ struct ib_device_operations {
 struct ib_device {
        /** Port GID */
        struct ib_gid port_gid;
-       /** Broadcast GID */
-       struct ib_gid broadcast_gid;
        /** Subnet manager LID */
        unsigned long sm_lid;
+       /** Partition key */
+       unsigned int pkey;
        /** Underlying device */
        struct device *dev;
        /** Infiniband operations */
@@ -541,6 +542,26 @@ struct ib_mad_path_record {
        uint32_t reserved2[35];
 } __attribute__ (( packed ));
 
+struct ib_mad_mc_member_record {
+       struct ib_mad_hdr mad_hdr;
+       struct ib_rmpp_hdr rmpp_hdr;
+       struct ib_sa_hdr sa_hdr;
+       struct ib_gid mgid;
+       struct ib_gid port_gid;
+       uint32_t qkey;
+       uint16_t mlid;
+       uint8_t mtu_selector__mtu;
+       uint8_t tclass;
+       uint16_t pkey;
+       uint8_t rate_selector__rate;
+       uint8_t packet_lifetime_selector__packet_lifetime;
+       uint32_t sl__flow_label__hop_limit;
+       uint8_t scope__join_state;
+       uint8_t proxy_join__reserved;
+       uint16_t reserved0;
+       uint32_t reserved1[37];
+} __attribute__ (( packed ));
+
 union ib_mad {
        struct ib_mad_hdr mad_hdr;
        struct ib_mad_data data;