[infiniband] Flush uncompleted work queue entries at QP teardown

Avoid leaking I/O buffers in ib_destroy_qp() by completing any
outstanding work queue entries with a generic error code.  This
requires the completion handlers to be available to ib_destroy_qp(),
which is done by making them static configuration parameters of the CQ
(set by ib_create_cq()) rather than being provided on each call to
ib_poll_cq().

This mimics the functionality of netdev_{tx,rx}_flush().  The netdev
flush functions would previously have been catching any I/O buffers
leaked by the IPoIB data queue (though not by the IPoIB metadata
queue).

diff --git a/src/drivers/infiniband/arbel.c b/src/drivers/infiniband/arbel.c
index 1b55131b86507a028378d3d2cea1f5985a70cf92..2649a4aba4d8cdd2e818719342ae793c44c12407 100644 (file)
--- a/src/drivers/infiniband/arbel.c
+++ b/src/drivers/infiniband/arbel.c
@@ -1106,15 +1106,11 @@ static int arbel_post_recv ( struct ib_device *ibdev,
  * @v ibdev            Infiniband device
  * @v cq               Completion queue
  * @v cqe              Hardware completion queue entry
- * @v complete_send    Send completion handler
- * @v complete_recv    Receive completion handler
  * @ret rc             Return status code
  */
 static int arbel_complete ( struct ib_device *ibdev,
                            struct ib_completion_queue *cq,
-                           union arbelprm_completion_entry *cqe,
-                           ib_completer_t complete_send,
-                           ib_completer_t complete_recv ) {
+                           union arbelprm_completion_entry *cqe ) {
        struct arbel *arbel = ib_get_drvdata ( ibdev );
        struct ib_completion completion;
        struct ib_work_queue *wq;
@@ -1124,7 +1120,6 @@ static int arbel_complete ( struct ib_device *ibdev,
        struct arbel_recv_work_queue *arbel_recv_wq;
        struct arbelprm_recv_wqe *recv_wqe;
        struct io_buffer *iobuf;
-       ib_completer_t complete;
        unsigned int opcode;
        unsigned long qpn;
        int is_send;
@@ -1201,8 +1196,11 @@ static int arbel_complete ( struct ib_device *ibdev,
        }
 
        /* Pass off to caller's completion handler */
-       complete = ( is_send ? complete_send : complete_recv );
-       complete ( ibdev, qp, &completion, iobuf );
+       if ( is_send ) {
+               ib_complete_send ( ibdev, qp, &completion, iobuf );
+       } else {
+               ib_complete_recv ( ibdev, qp, &completion, iobuf );
+       }
 
        return rc;
 }                           
@@ -1212,13 +1210,9 @@ static int arbel_complete ( struct ib_device *ibdev,
  *
  * @v ibdev            Infiniband device
  * @v cq               Completion queue
- * @v complete_send    Send completion handler
- * @v complete_recv    Receive completion handler
  */
 static void arbel_poll_cq ( struct ib_device *ibdev,
-                           struct ib_completion_queue *cq,
-                           ib_completer_t complete_send,
-                           ib_completer_t complete_recv ) {
+                           struct ib_completion_queue *cq ) {
        struct arbel *arbel = ib_get_drvdata ( ibdev );
        struct arbel_completion_queue *arbel_cq = ib_cq_get_drvdata ( cq );
        struct arbelprm_cq_ci_db_record *ci_db_rec;
@@ -1236,8 +1230,7 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
                }
 
                /* Handle completion */
-               if ( ( rc = arbel_complete ( ibdev, cq, cqe, complete_send,
-                                            complete_recv ) ) != 0 ) {
+               if ( ( rc = arbel_complete ( ibdev, cq, cqe ) ) != 0 ) {
                        DBGC ( arbel, "Arbel %p failed to complete: %s\n",
                               arbel, strerror ( rc ) );
                        DBGC_HD ( arbel, cqe, sizeof ( *cqe ) );

diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c
index 3ca6003365aa10ed9c31363c8bf408d62f594f51..a8907634f21645a91ee7ff29608f38ed4b6c8eb5 100644 (file)
--- a/src/drivers/infiniband/hermon.c
+++ b/src/drivers/infiniband/hermon.c
@@ -1101,22 +1101,17 @@ static int hermon_post_recv ( struct ib_device *ibdev,
  * @v ibdev            Infiniband device
  * @v cq               Completion queue
  * @v cqe              Hardware completion queue entry
- * @v complete_send    Send completion handler
- * @v complete_recv    Receive completion handler
  * @ret rc             Return status code
  */
 static int hermon_complete ( struct ib_device *ibdev,
                             struct ib_completion_queue *cq,
-                            union hermonprm_completion_entry *cqe,
-                            ib_completer_t complete_send,
-                            ib_completer_t complete_recv ) {
+                            union hermonprm_completion_entry *cqe ) {
        struct hermon *hermon = ib_get_drvdata ( ibdev );
        struct ib_completion completion;
        struct ib_work_queue *wq;
        struct ib_queue_pair *qp;
        struct hermon_queue_pair *hermon_qp;
        struct io_buffer *iobuf;
-       ib_completer_t complete;
        unsigned int opcode;
        unsigned long qpn;
        int is_send;
@@ -1172,8 +1167,11 @@ static int hermon_complete ( struct ib_device *ibdev,
        }
 
        /* Pass off to caller's completion handler */
-       complete = ( is_send ? complete_send : complete_recv );
-       complete ( ibdev, qp, &completion, iobuf );
+       if ( is_send ) {
+               ib_complete_send ( ibdev, qp, &completion, iobuf );
+       } else {
+               ib_complete_recv ( ibdev, qp, &completion, iobuf );
+       }
 
        return rc;
 }
@@ -1183,13 +1181,9 @@ static int hermon_complete ( struct ib_device *ibdev,
  *
  * @v ibdev            Infiniband device
  * @v cq               Completion queue
- * @v complete_send    Send completion handler
- * @v complete_recv    Receive completion handler
  */
 static void hermon_poll_cq ( struct ib_device *ibdev,
-                            struct ib_completion_queue *cq,
-                            ib_completer_t complete_send,
-                            ib_completer_t complete_recv ) {
+                            struct ib_completion_queue *cq ) {
        struct hermon *hermon = ib_get_drvdata ( ibdev );
        struct hermon_completion_queue *hermon_cq = ib_cq_get_drvdata ( cq );
        union hermonprm_completion_entry *cqe;
@@ -1209,8 +1203,7 @@ static void hermon_poll_cq ( struct ib_device *ibdev,
                DBGCP_HD ( hermon, cqe, sizeof ( *cqe ) );
 
                /* Handle completion */
-               if ( ( rc = hermon_complete ( ibdev, cq, cqe, complete_send,
-                                             complete_recv ) ) != 0 ) {
+               if ( ( rc = hermon_complete ( ibdev, cq, cqe ) ) != 0 ) {
                        DBGC ( hermon, "Hermon %p failed to complete: %s\n",
                               hermon, strerror ( rc ) );
                        DBGC_HD ( hermon, cqe, sizeof ( *cqe ) );

diff --git a/src/drivers/net/ipoib.c b/src/drivers/net/ipoib.c
index 812c11c953ace7a776919096199dc7786cf4feca..a56ceb89e57161088dbee8902b94374f919e2e81 100644 (file)
--- a/src/drivers/net/ipoib.c
+++ b/src/drivers/net/ipoib.c
@@ -278,13 +278,21 @@ static void ipoib_destroy_qset ( struct ipoib_device *ipoib,
  *
  * @v ipoib            IPoIB device
  * @v qset             Queue set
+ * @v num_cqes         Number of completion queue entries
+ * @v num_send_wqes    Number of send work queue entries
+ * @v complete_send    Send completion handler
+ * @v num_recv_wqes    Number of receive work queue entries
+ * @v complete_recv    Receive completion handler
+ * @v qkey             Queue key
  * @ret rc             Return status code
  */
 static int ipoib_create_qset ( struct ipoib_device *ipoib,
                               struct ipoib_queue_set *qset,
                               unsigned int num_cqes,
                               unsigned int num_send_wqes,
+                              ib_completer_t complete_send,
                               unsigned int num_recv_wqes,
+                              ib_completer_t complete_recv,
                               unsigned long qkey ) {
        struct ib_device *ibdev = ipoib->ibdev;
        int rc;
@@ -297,7 +305,8 @@ static int ipoib_create_qset ( struct ipoib_device *ipoib,
        qset->recv_max_fill = num_recv_wqes;
 
        /* Allocate completion queue */
-       qset->cq = ib_create_cq ( ibdev, num_cqes );
+       qset->cq = ib_create_cq ( ibdev, num_cqes, complete_send,
+                                 complete_recv );
        if ( ! qset->cq ) {
                DBGC ( ipoib, "IPoIB %p could not allocate completion queue\n",
                       ipoib );
@@ -759,10 +768,8 @@ static void ipoib_poll ( struct net_device *netdev ) {
        struct ipoib_device *ipoib = netdev->priv;
        struct ib_device *ibdev = ipoib->ibdev;
 
-       ib_poll_cq ( ibdev, ipoib->meta.cq, ipoib_meta_complete_send,
-                    ipoib_meta_complete_recv );
-       ib_poll_cq ( ibdev, ipoib->data.cq, ipoib_data_complete_send,
-                    ipoib_data_complete_recv );
+       ib_poll_cq ( ibdev, ipoib->meta.cq );
+       ib_poll_cq ( ibdev, ipoib->data.cq );
        ipoib_refill_recv ( ipoib, &ipoib->meta );
        ipoib_refill_recv ( ipoib, &ipoib->data );
 }
@@ -847,7 +854,9 @@ static int ipoib_open ( struct net_device *netdev ) {
        if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
                                        IPOIB_META_NUM_CQES,
                                        IPOIB_META_NUM_SEND_WQES,
+                                       ipoib_meta_complete_send,
                                        IPOIB_META_NUM_RECV_WQES,
+                                       ipoib_meta_complete_recv,
                                        IB_GLOBAL_QKEY ) ) != 0 ) {
                DBGC ( ipoib, "IPoIB %p could not allocate metadata QP: %s\n",
                       ipoib, strerror ( rc ) );
@@ -858,7 +867,9 @@ static int ipoib_open ( struct net_device *netdev ) {
        if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
                                        IPOIB_DATA_NUM_CQES,
                                        IPOIB_DATA_NUM_SEND_WQES,
+                                       ipoib_data_complete_send,
                                        IPOIB_DATA_NUM_RECV_WQES,
+                                       ipoib_data_complete_recv,
                                        IB_GLOBAL_QKEY ) ) != 0 ) {
                DBGC ( ipoib, "IPoIB %p could not allocate data QP: %s\n",
                       ipoib, strerror ( rc ) );

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index f9e65348ee9f2460ca65069cf435ef94e48fff4a..82b7a0a8b29164fac0c37350e1a828021e24a696 100644 (file)
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -101,26 +101,6 @@ enum ib_queue_pair_mods {
        IB_MODIFY_QKEY = 0x0001,
 };
 
-/** An Infiniband Completion Queue */
-struct ib_completion_queue {
-       /** Completion queue number */
-       unsigned long cqn;
-       /** Number of completion queue entries */
-       unsigned int num_cqes;
-       /** Next completion queue entry index
-        *
-        * This is the index of the next entry to be filled (i.e. the
-        * first empty entry).  This value is not bounded by num_wqes;
-        * users must logical-AND with (num_wqes-1) to generate an
-        * array index.
-        */
-       unsigned long next_idx;
-       /** List of work queues completing to this queue */
-       struct list_head work_queues;
-       /** Driver private data */
-       void *drv_priv;
-};
-
 /** An Infiniband completion */
 struct ib_completion {
        /** Syndrome
@@ -132,6 +112,14 @@ struct ib_completion {
        size_t len;
 };
 
+/** Infiniband completion syndromes */
+enum ib_syndrome {
+       IB_SYN_NONE = 0,
+       IB_SYN_LOCAL_LENGTH = 1,
+       IB_SYN_LOCAL_QP = 2,
+       IB_SYN_LOCAL_PROT = 4,
+};
+
 /** An Infiniband completion handler
  *
  * @v ibdev            Infiniband device
@@ -144,6 +132,30 @@ typedef void ( * ib_completer_t ) ( struct ib_device *ibdev,
                                    struct ib_completion *completion,
                                    struct io_buffer *iobuf );
 
+/** An Infiniband Completion Queue */
+struct ib_completion_queue {
+       /** Completion queue number */
+       unsigned long cqn;
+       /** Number of completion queue entries */
+       unsigned int num_cqes;
+       /** Next completion queue entry index
+        *
+        * This is the index of the next entry to be filled (i.e. the
+        * first empty entry).  This value is not bounded by num_wqes;
+        * users must logical-AND with (num_wqes-1) to generate an
+        * array index.
+        */
+       unsigned long next_idx;
+       /** List of work queues completing to this queue */
+       struct list_head work_queues;
+       /** Send completion handler */
+       ib_completer_t complete_send;
+       /** Receive completion handler */
+       ib_completer_t complete_recv;
+       /** Driver private data */
+       void *drv_priv;
+};
+
 /** An Infiniband Address Vector */
 struct ib_address_vector {
        /** Destination Queue Pair */
@@ -246,15 +258,12 @@ struct ib_device_operations {
         *
         * @v ibdev             Infiniband device
         * @v cq                Completion queue
-        * @v complete_send     Send completion handler
-        * @v complete_recv     Receive completion handler
         *
-        * The completion handler takes ownership of the I/O buffer.
+        * The relevant completion handler (specified at completion
+        * queue creation time) takes ownership of the I/O buffer.
         */
        void ( * poll_cq ) ( struct ib_device *ibdev,
-                            struct ib_completion_queue *cq,
-                            ib_completer_t complete_send,
-                            ib_completer_t complete_recv );
+                            struct ib_completion_queue *cq );
        /**
         * Poll event queue
         *
@@ -331,8 +340,9 @@ struct ib_device {
        void *owner_priv;
 };
 
-extern struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
-                                                  unsigned int num_cqes );
+extern struct ib_completion_queue *
+ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
+              ib_completer_t complete_send, ib_completer_t complete_recv );
 extern void ib_destroy_cq ( struct ib_device *ibdev,
                            struct ib_completion_queue *cq );
 extern struct ib_queue_pair *
@@ -379,18 +389,45 @@ ib_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
        return ibdev->op->post_recv ( ibdev, qp, iobuf );
 }
 
+/**
+ * Complete send work queue entry
+ *
+ * @v ibdev            Infiniband device
+ * @v qp               Queue pair
+ * @v completion       Completion
+ * @v iobuf            I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+ib_complete_send ( struct ib_device *ibdev, struct ib_queue_pair *qp,
+                  struct ib_completion *completion,
+                  struct io_buffer *iobuf ) {
+       return qp->send.cq->complete_send ( ibdev, qp, completion, iobuf );
+}
+
+/**
+ * Complete receive work queue entry
+ *
+ * @v ibdev            Infiniband device
+ * @v qp               Queue pair
+ * @v completion       Completion
+ * @v iobuf            I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+ib_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp,
+                  struct ib_completion *completion,
+                  struct io_buffer *iobuf ) {
+       return qp->recv.cq->complete_recv ( ibdev, qp, completion, iobuf );
+}
+
 /**
  * Poll completion queue
  *
  * @v ibdev            Infiniband device
  * @v cq               Completion queue
- * @v complete_send    Send completion handler
- * @v complete_recv    Receive completion handler
  */
 static inline __attribute__ (( always_inline )) void
-ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq,
-            ib_completer_t complete_send, ib_completer_t complete_recv ) {
-       ibdev->op->poll_cq ( ibdev, cq, complete_send, complete_recv );
+ib_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) {
+       ibdev->op->poll_cq ( ibdev, cq );
 }
 
 /**

diff --git a/src/net/infiniband.c b/src/net/infiniband.c
index ab76742e75fa35db909618a304d9090d8b7580be..8437b4ceb0362a4ed5f9643249dca187b1675147 100644 (file)
--- a/src/net/infiniband.c
+++ b/src/net/infiniband.c
@@ -46,10 +46,13 @@ struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices );
  *
  * @v ibdev            Infiniband device
  * @v num_cqes         Number of completion queue entries
+ * @v complete_send    Send completion handler
+ * @v complete_recv    Receive completion handler
  * @ret cq             New completion queue
  */
-struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
-                                           unsigned int num_cqes ) {
+struct ib_completion_queue *
+ib_create_cq ( struct ib_device *ibdev, unsigned int num_cqes,
+              ib_completer_t complete_send, ib_completer_t complete_recv ) {
        struct ib_completion_queue *cq;
        int rc;
 
@@ -61,6 +64,8 @@ struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
                return NULL;
        cq->num_cqes = num_cqes;
        INIT_LIST_HEAD ( &cq->work_queues );
+       cq->complete_send = complete_send;
+       cq->complete_recv = complete_recv;
 
        /* Perform device-specific initialisation and get CQN */
        if ( ( rc = ibdev->op->create_cq ( ibdev, cq ) ) != 0 ) {
@@ -190,11 +195,33 @@ int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
  * @v qp               Queue pair
  */
 void ib_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
+       struct ib_completion completion = {
+               .syndrome = IB_SYN_LOCAL_QP,
+       };
+       struct io_buffer *iobuf;
+       unsigned int i;
+
        DBGC ( ibdev, "IBDEV %p destroying QPN %#lx\n",
               ibdev, qp->qpn );
+
+       /* Perform device-specific destruction */
        ibdev->op->destroy_qp ( ibdev, qp );
+
+       /* Complete any remaining I/O buffers with errors */
+       for ( i = 0 ; i < qp->send.num_wqes ; i++ ) {
+               if ( ( iobuf = qp->send.iobufs[i] ) != NULL )
+                       ib_complete_send ( ibdev, qp, &completion, iobuf );
+       }
+       for ( i = 0 ; i < qp->recv.num_wqes ; i++ ) {
+               if ( ( iobuf = qp->recv.iobufs[i] ) != NULL )
+                       ib_complete_recv ( ibdev, qp, &completion, iobuf );
+       }
+
+       /* Remove work queues from completion queue */
        list_del ( &qp->send.list );
        list_del ( &qp->recv.list );
+
+       /* Free QP */
        free ( qp );
 }