smb: client: allocate enough space for MR WRs and ib_drain_qp()

The IB_WR_REG_MR and IB_WR_LOCAL_INV operations for smbdirect_mr_io
structures should never fail because the submission or completion queues
are too small. So we allocate more send_wr depending on the (local) max
number of MRs.

While there also add additional space for ib_drain_qp().

This should make sure ib_post_send() will never fail
because the submission queue is full.

Fixes: f198186aa9bb ("CIFS: SMBD: Establish SMB Direct connection")
Fixes: cc55f65dd352 ("smb: client: make use of common smbdirect_socket_parameters")
Cc: stable@vger.kernel.org
Cc: Steve French <smfrench@gmail.com>
Cc: Tom Talpey <tom@talpey.com>
Cc: Long Li <longli@microsoft.com>
Cc: Namjae Jeon <linkinjeon@kernel.org>
Cc: linux-cifs@vger.kernel.org
Cc: samba-technical@lists.samba.org
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Signed-off-by: Steve French <stfrench@microsoft.com>

diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c
index 49e2df3ad1f0a4ea5217aaed6ebec091a2c51b04..b1218ea4aa8b9663d5a231956d12396e736e98bd 100644 (file)
--- a/fs/smb/client/smbdirect.c
+++ b/fs/smb/client/smbdirect.c
@@ -1767,6 +1767,7 @@ static struct smbd_connection *_smbd_get_connection(
        struct smbdirect_socket *sc;
        struct smbdirect_socket_parameters *sp;
        struct rdma_conn_param conn_param;
+       struct ib_qp_cap qp_cap;
        struct ib_qp_init_attr qp_attr;
        struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr;
        struct ib_port_immutable port_immutable;
@@ -1838,6 +1839,25 @@ static struct smbd_connection *_smbd_get_connection(
                goto config_failed;
        }
 
+       sp->responder_resources =
+               min_t(u8, sp->responder_resources,
+                     sc->ib.dev->attrs.max_qp_rd_atom);
+       log_rdma_mr(INFO, "responder_resources=%d\n",
+               sp->responder_resources);
+
+       /*
+        * We use allocate sp->responder_resources * 2 MRs
+        * and each MR needs WRs for REG and INV, so
+        * we use '* 4'.
+        *
+        * +1 for ib_drain_qp()
+        */
+       memset(&qp_cap, 0, sizeof(qp_cap));
+       qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1;
+       qp_cap.max_recv_wr = sp->recv_credit_max + 1;
+       qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
+       qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
+
        sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0);
        if (IS_ERR(sc->ib.pd)) {
                rc = PTR_ERR(sc->ib.pd);
@@ -1848,7 +1868,7 @@ static struct smbd_connection *_smbd_get_connection(
 
        sc->ib.send_cq =
                ib_alloc_cq_any(sc->ib.dev, sc,
-                               sp->send_credit_target, IB_POLL_SOFTIRQ);
+                               qp_cap.max_send_wr, IB_POLL_SOFTIRQ);
        if (IS_ERR(sc->ib.send_cq)) {
                sc->ib.send_cq = NULL;
                goto alloc_cq_failed;
@@ -1856,7 +1876,7 @@ static struct smbd_connection *_smbd_get_connection(
 
        sc->ib.recv_cq =
                ib_alloc_cq_any(sc->ib.dev, sc,
-                               sp->recv_credit_max, IB_POLL_SOFTIRQ);
+                               qp_cap.max_recv_wr, IB_POLL_SOFTIRQ);
        if (IS_ERR(sc->ib.recv_cq)) {
                sc->ib.recv_cq = NULL;
                goto alloc_cq_failed;
@@ -1865,11 +1885,7 @@ static struct smbd_connection *_smbd_get_connection(
        memset(&qp_attr, 0, sizeof(qp_attr));
        qp_attr.event_handler = smbd_qp_async_error_upcall;
        qp_attr.qp_context = sc;
-       qp_attr.cap.max_send_wr = sp->send_credit_target;
-       qp_attr.cap.max_recv_wr = sp->recv_credit_max;
-       qp_attr.cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE;
-       qp_attr.cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE;
-       qp_attr.cap.max_inline_data = 0;
+       qp_attr.cap = qp_cap;
        qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
        qp_attr.qp_type = IB_QPT_RC;
        qp_attr.send_cq = sc->ib.send_cq;
@@ -1883,12 +1899,6 @@ static struct smbd_connection *_smbd_get_connection(
        }
        sc->ib.qp = sc->rdma.cm_id->qp;
 
-       sp->responder_resources =
-               min_t(u8, sp->responder_resources,
-                     sc->ib.dev->attrs.max_qp_rd_atom);
-       log_rdma_mr(INFO, "responder_resources=%d\n",
-               sp->responder_resources);
-
        memset(&conn_param, 0, sizeof(conn_param));
        conn_param.initiator_depth = sp->initiator_depth;
        conn_param.responder_resources = sp->responder_resources;