[PATCH] IB/mthca: Use mmiowb after doorbell ring

We discovered a problem when running IPoIB applications on multiple
CPUs on an Altix system. Many messages such as:

ib_mthca 0002:01:00.0: SQ 000014 full (19941644 head, 19941707 tail, 64 max, 0 nreq)

appear in syslog, and the driver wedges up.

Apparently this is because writes to the doorbells from different CPUs
reach the device out of order. The following patch adds mmiowb() calls
after doorbell rings to ensure the doorbell writes are ordered.

Signed-off-by: Arthur Kepner <akepner@sgi.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
Signed-off-by: Chris Wright <chrisw@sous-sol.org>

diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 3e27a084257e9f23b7a4fba49561e15cfddf5b91..8030d6adee08dd25f775b98b328dbde507b1b146 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -39,6 +39,8 @@
 #include <linux/init.h>
 #include <linux/hardirq.h>
 
+#include <asm/io.h>
+
 #include <rdma/ib_pack.h>
 
 #include "mthca_dev.h"
@@ -210,6 +212,11 @@ static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
                mthca_write64(doorbell,
                              dev->kar + MTHCA_CQ_DOORBELL,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+               /*
+                * Make sure doorbells don't leak out of CQ spinlock
+                * and reach the HCA out of order:
+                */
+               mmiowb();
        }
 }
 

diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 2e8f6f36e0a5b862e87a3de76e0f0478ad62e25f..b66aa49c8fe3761b68b0b901875a78531edc9741 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -39,6 +39,8 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
+#include <asm/io.h>
+
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_cache.h>
 #include <rdma/ib_pack.h>
@@ -1730,6 +1732,11 @@ out:
                mthca_write64(doorbell,
                              dev->kar + MTHCA_SEND_DOORBELL,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+               /*
+                * Make sure doorbells don't leak out of SQ spinlock
+                * and reach the HCA out of order:
+                */
+               mmiowb();
        }
 
        qp->sq.next_ind = ind;
@@ -1849,6 +1856,12 @@ out:
        qp->rq.next_ind = ind;
        qp->rq.head    += nreq;
 
+       /*
+        * Make sure doorbells don't leak out of RQ spinlock and reach
+        * the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&qp->rq.lock, flags);
        return err;
 }
@@ -2110,6 +2123,12 @@ out:
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
        }
 
+       /*
+        * Make sure doorbells don't leak out of SQ spinlock and reach
+        * the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&qp->sq.lock, flags);
        return err;
 }

diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c
index b60a9d79ae54ae569e7e7c765389c0ee05fd8260..a01f1a827f723ba62111bed76ff7ff6210bf36c1 100644 (file)
--- a/drivers/infiniband/hw/mthca/mthca_srq.c
+++ b/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -35,6 +35,8 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 
+#include <asm/io.h>
+
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
 #include "mthca_memfree.h"
@@ -593,6 +595,12 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
                              MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
        }
 
+       /*
+        * Make sure doorbells don't leak out of SRQ spinlock and
+        * reach the HCA out of order:
+        */
+       mmiowb();
+
        spin_unlock_irqrestore(&srq->lock, flags);
        return err;
 }