[infiniband] Add the concept of an Infiniband upper-layer driver

Replace the explicit calls from the Infiniband core to the IPoIB layer
with the general concept of an Infiniband upper-layer driver
(analogous to a PCI driver) which can create arbitrary devices on top
of Infiniband devices.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index baa1c644aee4effc425ec8b7c71f6825a35a0336..82d7ca9994823c36e010529b42d31df30def76f3 100644 (file)
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -597,6 +597,42 @@ static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
        }
 }
 
+/**
+ * Handle link status change
+ *
+ * @v ibdev            Infiniband device
+ */
+static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
+       struct net_device *netdev = ib_get_ownerdata ( ibdev );
+       struct ipoib_device *ipoib = netdev->priv;
+       struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
+       int rc;
+
+       /* Leave existing broadcast group */
+       ipoib_leave_broadcast_group ( ipoib );
+
+       /* Update MAC address based on potentially-new GID prefix */
+       memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
+                sizeof ( mac->gid.u.half[0] ) );
+
+       /* Update broadcast GID based on potentially-new partition key */
+       ipoib->broadcast.gid.u.words[2] =
+               htons ( ibdev->pkey | IB_PKEY_FULL );
+
+       /* Set net device link state to reflect Infiniband link state */
+       rc = ib_link_rc ( ibdev );
+       netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
+
+       /* Join new broadcast group */
+       if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
+            ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
+               DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
+                      "%s\n", ipoib, strerror ( rc ) );
+               netdev_link_err ( netdev, rc );
+               return;
+       }
+}
+
 /**
  * Open IPoIB network device
  *
@@ -690,49 +726,13 @@ static struct net_device_operations ipoib_operations = {
        .irq            = ipoib_irq,
 };
 
-/**
- * Handle link status change
- *
- * @v ibdev            Infiniband device
- */
-void ipoib_link_state_changed ( struct ib_device *ibdev ) {
-       struct net_device *netdev = ib_get_ownerdata ( ibdev );
-       struct ipoib_device *ipoib = netdev->priv;
-       struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
-       int rc;
-
-       /* Leave existing broadcast group */
-       ipoib_leave_broadcast_group ( ipoib );
-
-       /* Update MAC address based on potentially-new GID prefix */
-       memcpy ( &mac->gid.u.half[0], &ibdev->gid.u.half[0],
-                sizeof ( mac->gid.u.half[0] ) );
-
-       /* Update broadcast GID based on potentially-new partition key */
-       ipoib->broadcast.gid.u.words[2] =
-               htons ( ibdev->pkey | IB_PKEY_FULL );
-
-       /* Set net device link state to reflect Infiniband link state */
-       rc = ib_link_rc ( ibdev );
-       netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
-
-       /* Join new broadcast group */
-       if ( ib_link_ok ( ibdev ) &&
-            ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
-               DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
-                      "%s\n", ipoib, strerror ( rc ) );
-               netdev_link_err ( netdev, rc );
-               return;
-       }
-}
-
 /**
  * Probe IPoIB device
  *
  * @v ibdev            Infiniband device
  * @ret rc             Return status code
  */
-int ipoib_probe ( struct ib_device *ibdev ) {
+static int ipoib_probe ( struct ib_device *ibdev ) {
        struct net_device *netdev;
        struct ipoib_device *ipoib;
        int rc;
@@ -775,10 +775,18 @@ int ipoib_probe ( struct ib_device *ibdev ) {
  *
  * @v ibdev            Infiniband device
  */
-void ipoib_remove ( struct ib_device *ibdev ) {
+static void ipoib_remove ( struct ib_device *ibdev ) {
        struct net_device *netdev = ib_get_ownerdata ( ibdev );
 
        unregister_netdev ( netdev );
        netdev_nullify ( netdev );
        netdev_put ( netdev );
 }
+
+/** IPoIB driver */
+struct ib_driver ipoib_driver __ib_driver = {
+       .name = "IPoIB",
+       .probe = ipoib_probe,
+       .notify = ipoib_link_state_changed,
+       .remove = ipoib_remove,
+};

diff --git a/src/include/ipxe/infiniband.h b/src/include/ipxe/infiniband.h
index f2eb57cd5d82ae811f47cc5fa30cbf8d32243032..edcce371ccec6d2218421cfd94bdd1128a875cfc 100644 (file)
--- a/src/include/ipxe/infiniband.h
+++ b/src/include/ipxe/infiniband.h
@@ -12,6 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <stdint.h>
 #include <ipxe/refcnt.h>
 #include <ipxe/device.h>
+#include <ipxe/tables.h>
 #include <ipxe/ib_packet.h>
 #include <ipxe/ib_mad.h>
 
@@ -432,6 +433,34 @@ struct ib_device {
        void *owner_priv;
 };
 
+/** An Infiniband upper-layer driver */
+struct ib_driver {
+       /** Name */
+       const char *name;
+       /** Probe device
+        *
+        * @v ibdev             Infiniband device
+        * @ret rc              Return status code
+        */
+       int ( * probe ) ( struct ib_device *ibdev );
+       /** Notify of device or link state change
+        *
+        * @v ibdev             Infiniband device
+        */
+       void ( * notify ) ( struct ib_device *ibdev );
+       /** Remove device
+        *
+        * @v ibdev             Infiniband device
+        */
+       void ( * remove ) ( struct ib_device *ibdev );
+};
+
+/** Infiniband driver table */
+#define IB_DRIVERS __table ( struct ib_driver, "ib_drivers" )
+
+/** Declare an Infiniband driver */
+#define __ib_driver __table_entry ( IB_DRIVERS, 01 )
+
 extern struct ib_completion_queue *
 ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
               struct ib_completion_queue_operations *op );
@@ -492,7 +521,7 @@ extern struct list_head ib_devices;
        list_for_each_entry ( (ibdev), &ib_devices, list )
 
 /**
- * Check link state
+ * Check link state of Infiniband device
  *
  * @v ibdev            Infiniband device
  * @ret link_up                Link is up
@@ -502,6 +531,17 @@ ib_link_ok ( struct ib_device *ibdev ) {
        return ( ibdev->port_state == IB_PORT_STATE_ACTIVE );
 }
 
+/**
+ * Check whether or not Infiniband device is open
+ *
+ * @v ibdev            Infiniband device
+ * @v is_open          Infiniband device is open
+ */
+static inline __attribute__ (( always_inline )) int
+ib_is_open ( struct ib_device *ibdev ) {
+       return ( ibdev->open_count > 0 );
+}
+
 /**
  * Get reference to Infiniband device
  *

diff --git a/src/include/ipxe/ipoib.h b/src/include/ipxe/ipoib.h
index 31b0c1b6ddbb66e877db5fc71779224b0ea2ad30..6a3fd607a0d79926c4f9f56e0fdb2df9fefb3838 100644 (file)
--- a/src/include/ipxe/ipoib.h
+++ b/src/include/ipxe/ipoib.h
@@ -53,9 +53,6 @@ struct ipoib_hdr {
 } __attribute__ (( packed ));
 
 extern const char * ipoib_ntoa ( const void *ll_addr );
-extern void ipoib_link_state_changed ( struct ib_device *ibdev );
-extern int ipoib_probe ( struct ib_device *ibdev );
-extern void ipoib_remove ( struct ib_device *ibdev );
 extern struct net_device * alloc_ipoibdev ( size_t priv_size );
 
 #endif /* _IPXE_IPOIB_H */

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index 5daecd52edff492a889d7eedd44bbdf0a478affa..76fce3bd9e25f17327643060bc196200d4b94761 100644 (file)
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/if_arp.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/iobuf.h>
-#include <ipxe/ipoib.h>
 #include <ipxe/process.h>
 #include <ipxe/infiniband.h>
 #include <ipxe/ib_mi.h>
@@ -538,6 +537,64 @@ void ib_refill_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
  ***************************************************************************
  */
 
+/**
+ * Get link state
+ *
+ * @v ibdev            Infiniband device
+ * @ret rc             Link status code
+ */
+int ib_link_rc ( struct ib_device *ibdev ) {
+       switch ( ibdev->port_state ) {
+       case IB_PORT_STATE_DOWN:        return -ENOTCONN;
+       case IB_PORT_STATE_INIT:        return -EINPROGRESS_INIT;
+       case IB_PORT_STATE_ARMED:       return -EINPROGRESS_ARMED;
+       case IB_PORT_STATE_ACTIVE:      return 0;
+       default:                        return -EINVAL;
+       }
+}
+
+/**
+ * Textual representation of Infiniband link state
+ *
+ * @v ibdev            Infiniband device
+ * @ret link_text      Link state text
+ */
+static const char * ib_link_state_text ( struct ib_device *ibdev ) {
+       switch ( ibdev->port_state ) {
+       case IB_PORT_STATE_DOWN:        return "DOWN";
+       case IB_PORT_STATE_INIT:        return "INIT";
+       case IB_PORT_STATE_ARMED:       return "ARMED";
+       case IB_PORT_STATE_ACTIVE:      return "ACTIVE";
+       default:                        return "UNKNOWN";
+       }
+}
+
+/**
+ * Notify drivers of Infiniband device or link state change
+ *
+ * @v ibdev            Infiniband device
+ */
+static void ib_notify ( struct ib_device *ibdev ) {
+       struct ib_driver *driver;
+
+       for_each_table_entry ( driver, IB_DRIVERS )
+               driver->notify ( ibdev );
+}
+
+/**
+ * Notify of Infiniband link state change
+ *
+ * @v ibdev            Infiniband device
+ */
+void ib_link_state_changed ( struct ib_device *ibdev ) {
+
+       DBGC ( ibdev, "IBDEV %p link state is %s\n",
+              ibdev, ib_link_state_text ( ibdev ) );
+
+       /* Notify drivers of link state change */
+       ib_notify ( ibdev );
+}
+
 /**
  * Open port
  *
@@ -586,6 +643,9 @@ int ib_open ( struct ib_device *ibdev ) {
        /* Add to head of open devices list */
        list_add ( &ibdev->open_list, &open_ib_devices );
 
+       /* Notify drivers of device state change */
+       ib_notify ( ibdev );
+
        assert ( ibdev->open_count == 1 );
        return 0;
 
@@ -614,6 +674,7 @@ void ib_close ( struct ib_device *ibdev ) {
 
        /* Close device if this was the last remaining requested opening */
        if ( ibdev->open_count == 0 ) {
+               ib_notify ( ibdev );
                list_del ( &ibdev->open_list );
                ib_destroy_mi ( ibdev, ibdev->gsi );
                ib_destroy_sma ( ibdev, ibdev->smi );
@@ -622,22 +683,6 @@ void ib_close ( struct ib_device *ibdev ) {
        }
 }
 
-/**
- * Get link state
- *
- * @v ibdev            Infiniband device
- * @ret rc             Link status code
- */
-int ib_link_rc ( struct ib_device *ibdev ) {
-       switch ( ibdev->port_state ) {
-       case IB_PORT_STATE_DOWN:        return -ENOTCONN;
-       case IB_PORT_STATE_INIT:        return -EINPROGRESS_INIT;
-       case IB_PORT_STATE_ARMED:       return -EINPROGRESS_ARMED;
-       case IB_PORT_STATE_ACTIVE:      return 0;
-       default:                        return -EINVAL;
-       }
-}
-
 /***************************************************************************
  *
  * Multicast
@@ -799,17 +844,6 @@ int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad ) {
  ***************************************************************************
  */
 
-/**
- * Handle Infiniband link state change
- *
- * @v ibdev            Infiniband device
- */
-void ib_link_state_changed ( struct ib_device *ibdev ) {
-
-       /* Notify IPoIB of link state change */
-       ipoib_link_state_changed ( ibdev );
-}
-
 /**
  * Poll event queue
  *
@@ -883,24 +917,29 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
  * @ret rc             Return status code
  */
 int register_ibdev ( struct ib_device *ibdev ) {
+       struct ib_driver *driver;
        int rc;
 
        /* Add to device list */
        ibdev_get ( ibdev );
        list_add_tail ( &ibdev->list, &ib_devices );
+       DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
+              ibdev->dev->name );
 
-       /* Add IPoIB device */
-       if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
-               DBGC ( ibdev, "IBDEV %p could not add IPoIB device: %s\n",
-                      ibdev, strerror ( rc ) );
-               goto err_ipoib_probe;
+       /* Probe device */
+       for_each_table_entry ( driver, IB_DRIVERS ) {
+               if ( ( rc = driver->probe ( ibdev ) ) != 0 ) {
+                       DBGC ( ibdev, "IBDEV %p could not add %s device: %s\n",
+                              ibdev, driver->name, strerror ( rc ) );
+                       goto err_probe;
+               }
        }
 
-       DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
-              ibdev->dev->name );
        return 0;
 
- err_ipoib_probe:
+ err_probe:
+       for_each_table_entry_continue_reverse ( driver, IB_DRIVERS )
+               driver->remove ( ibdev );
        list_del ( &ibdev->list );
        ibdev_put ( ibdev );
        return rc;
@@ -912,9 +951,11 @@ int register_ibdev ( struct ib_device *ibdev ) {
  * @v ibdev            Infiniband device
  */
 void unregister_ibdev ( struct ib_device *ibdev ) {
+       struct ib_driver *driver;
 
-       /* Close device */
-       ipoib_remove ( ibdev );
+       /* Remove device */
+       for_each_table_entry_reverse ( driver, IB_DRIVERS )
+               driver->remove ( ibdev );
 
        /* Remove from device list */
        list_del ( &ibdev->list );
@@ -953,3 +994,6 @@ struct ib_device * last_opened_ibdev ( void ) {
 
        return NULL;
 }
+
+/* Drag in IPoIB */
+REQUIRE_OBJECT ( ipoib );