destroy_cq() now implemented (not tested).

diff --git a/src/drivers/net/mlx_ipoib/arbel.h b/src/drivers/net/mlx_ipoib/arbel.h
index 37fe3693b3e53f999dafcfa4e03b703b1c9034c1..d38424676e069bf362bff193269db32b58ed4e2d 100644 (file)
--- a/src/drivers/net/mlx_ipoib/arbel.h
+++ b/src/drivers/net/mlx_ipoib/arbel.h
@@ -28,6 +28,7 @@
 /* HCA command register opcodes */
 #define ARBEL_HCR_QUERY_DEV_LIM                0x0003
 #define ARBEL_HCR_SW2HW_CQ             0x0016
+#define ARBEL_HCR_HW2SW_CQ             0x0017
 
 /*
  * Wrapper structures for hardware datatypes
@@ -247,6 +248,9 @@ struct arbel {
 #define ARBEL_HCR_OUT_CMD( _opcode, _out_mbox, _out_len )                   \
        ARBEL_HCR_CMD ( _opcode, 0, 0, _out_mbox, _out_len )
 
+#define ARBEL_HCR_VOID_CMD( _opcode )                                       \
+       ARBEL_HCR_CMD ( _opcode, 0, 0, 0, 0 )
+
 /*
  * Doorbell record allocation
  *

diff --git a/src/drivers/net/mlx_ipoib/mt25218.c b/src/drivers/net/mlx_ipoib/mt25218.c
index dfdba4b5648d0d47b592fc3d3568b1a39f222d75..64ae992bad16407fbb1467079b94adeaaceae34c 100644 (file)
--- a/src/drivers/net/mlx_ipoib/mt25218.c
+++ b/src/drivers/net/mlx_ipoib/mt25218.c
@@ -270,6 +270,13 @@ static struct net_device_operations mlx_operations = {
 
 
 
+/***************************************************************************
+ *
+ * Queue number allocation
+ *
+ ***************************************************************************
+ */
+
 /**
  * Allocate queue number
  *
@@ -444,6 +451,13 @@ arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
                           0, cqctx, cqn, NULL );
 }
 
+static inline int
+arbel_cmd_hw2sw_cq ( struct arbel *arbel, unsigned long cqn ) {
+       return arbel_cmd ( arbel,
+                          ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_CQ ),
+                          1, NULL, cqn, NULL );
+}
+
 /***************************************************************************
  *
  * Completion queue operations
@@ -548,8 +562,9 @@ static int arbel_create_cq ( struct ib_device *ibdev,
        return 0;
 
  err_sw2hw:
-       memset ( ci_db_rec, 0, sizeof ( *ci_db_rec ) );
-       memset ( arm_db_rec, 0, sizeof ( *arm_db_rec ) );
+       MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
+       MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
+       free_dma ( arbel_cq->cqe, cqe_size );
  err_cqe:
        free ( arbel_cq );
  err_arbel_cq:
@@ -558,6 +573,50 @@ static int arbel_create_cq ( struct ib_device *ibdev,
        return rc;
 }
 
+/**
+ * Destroy completion queue
+ *
+ * @v ibdev            Infiniband device
+ * @v cq               Completion queue
+ */
+static void arbel_destroy_cq ( struct ib_device *ibdev,
+                              struct ib_completion_queue *cq ) {
+       struct arbel *arbel = ibdev->priv;
+       struct arbel_completion_queue *arbel_cq =
+               container_of ( cq, struct arbel_completion_queue, cq );
+       struct arbelprm_cq_ci_db_record *ci_db_rec;
+       struct arbelprm_cq_arm_db_record *arm_db_rec;
+       int cqn_offset;
+       size_t cqe_size;
+       unsigned int ci_doorbell_idx;
+       unsigned int arm_doorbell_idx;
+       int rc;
+
+       assert ( list_empty ( &cq->work_queues ) );
+
+       /* Take ownership back from hardware */
+       if ( ( rc = arbel_cmd_hw2sw_cq ( arbel, cq->cqn ) ) != 0 ) {
+               DBGC ( arbel, "Arbel %p FATAL HW2SW_CQ failed: %s\n",
+                      arbel, strerror ( rc ) );
+               /* Leak memory and return; at least we avoid corruption */
+               return;
+       }
+
+       /* Clear doorbell records */
+       cqn_offset = ( cq->cqn - arbel->limits.reserved_cqs );
+       ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( cqn_offset );
+       arm_doorbell_idx = arbel_cq_arm_doorbell_idx ( cqn_offset );
+       ci_db_rec = &arbel->db_rec[ci_doorbell_idx].cq_ci;
+       arm_db_rec = &arbel->db_rec[arm_doorbell_idx].cq_arm;
+       MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
+       MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
+
+       /* Free memory */
+       cqe_size = ( cq->num_cqes * sizeof ( arbel_cq->cqe[0] ) );
+       free_dma ( arbel_cq->cqe, cqe_size );
+       free ( arbel_cq );
+       arbel_free_qn_offset ( arbel->cq_inuse, cqn_offset );
+}
 
 /***************************************************************************
  *
@@ -863,6 +922,8 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
 
 /** Arbel Infiniband operations */
 static struct ib_device_operations arbel_ib_operations = {
+       .create_cq      = arbel_create_cq,
+       .destroy_cq     = arbel_destroy_cq,
        .post_send      = arbel_post_send,
        .post_recv      = arbel_post_recv,
        .poll_cq        = arbel_poll_cq,

diff --git a/src/include/gpxe/infiniband.h b/src/include/gpxe/infiniband.h
index dd8022fb6d0fd2fa035f6f1397f063e027561ab4..973c582387a90e83c00290dd60e5a3a4460c1bcb 100644 (file)
--- a/src/include/gpxe/infiniband.h
+++ b/src/include/gpxe/infiniband.h
@@ -168,6 +168,25 @@ struct ib_address_vector {
  * These represent a subset of the Infiniband Verbs.
  */
 struct ib_device_operations {
+       /**
+        * Create completion queue
+        *
+        * @v ibdev             Infiniband device
+        * @v log2_num_cqes     Log2 of the number of completion queue entries
+        * @ret new_cq          New completion queue
+        * @ret rc              Return status code
+        */
+       int ( * create_cq ) ( struct ib_device *ibdev,
+                             unsigned int log2_num_cqes,
+                             struct ib_completion_queue **new_cq );
+       /**
+        * Destroy completion queue
+        *
+        * @v ibdev             Infiniband device
+        * @v cq                Completion queue
+        */
+       void ( * destroy_cq ) ( struct ib_device *ibdev,
+                               struct ib_completion_queue *cq );
        /** Post send work queue entry
         *
         * @v ibdev             Infiniband device