From efb9713152d9a28e025641c79838f8523f1c3aa9 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Tue, 28 Aug 2018 07:21:01 +0200 Subject: [PATCH] 4.18-stable patches added patches: xprtrdma-fix-disconnect-regression.patch --- queue-4.18/series | 1 + .../xprtrdma-fix-disconnect-regression.patch | 99 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 queue-4.18/xprtrdma-fix-disconnect-regression.patch diff --git a/queue-4.18/series b/queue-4.18/series index db8424a3406..1dfb401bc62 100644 --- a/queue-4.18/series +++ b/queue-4.18/series @@ -1,3 +1,4 @@ patch-scripts-kernel-doc.patch scripts-kernel-doc-escape-all-literal-braces-in-regexes.patch scsi-libsas-dynamically-allocate-and-free-ata-host.patch +xprtrdma-fix-disconnect-regression.patch diff --git a/queue-4.18/xprtrdma-fix-disconnect-regression.patch b/queue-4.18/xprtrdma-fix-disconnect-regression.patch new file mode 100644 index 00000000000..41a19e81076 --- /dev/null +++ b/queue-4.18/xprtrdma-fix-disconnect-regression.patch @@ -0,0 +1,99 @@ +From 8d4fb8ff427a23e573c9373b2bb3d1d6e8ea4399 Mon Sep 17 00:00:00 2001 +From: Chuck Lever +Date: Sat, 28 Jul 2018 10:46:47 -0400 +Subject: xprtrdma: Fix disconnect regression + +From: Chuck Lever + +commit 8d4fb8ff427a23e573c9373b2bb3d1d6e8ea4399 upstream. + +I found that injecting disconnects with v4.18-rc resulted in +random failures of the multi-threaded git regression test. + +The root cause appears to be that, after a reconnect, the +RPC/RDMA transport is waking pending RPCs before the transport has +posted enough Receive buffers to receive the Replies. If a Reply +arrives before enough Receive buffers are posted, the connection +is dropped. A few connection drops happen in quick succession as +the client and server struggle to regain credit synchronization. + +This regression was introduced with commit 7c8d9e7c8863 ("xprtrdma: +Move Receive posting to Receive handler"). The client is supposed to +post a single Receive when a connection is established because +it's not supposed to send more than one RPC Call before it gets +a fresh credit grant in the first RPC Reply [RFC 8166, Section +3.3.3]. + +Unfortunately there appears to be a longstanding bug in the Linux +client's credit accounting mechanism. On connect, it simply dumps +all pending RPC Calls onto the new connection. It's possible it has +done this ever since the RPC/RDMA transport was added to the kernel +ten years ago. + +Servers have so far been tolerant of this bad behavior. Currently no +server implementation ever changes its credit grant over reconnects, +and servers always repost enough Receives before connections are +fully established. + +The Linux client implementation used to post a Receive before each +of these Calls. This has covered up the flooding send behavior. + +I could try to correct this old bug so that the client sends exactly +one RPC Call and waits for a Reply. Since we are so close to the +next merge window, I'm going to instead provide a simple patch to +post enough Receives before a reconnect completes (based on the +number of credits granted to the previous connection). + +The spurious disconnects will be gone, but the client will still +send multiple RPC Calls immediately after a reconnect. + +Addressing the latter problem will wait for a merge window because +a) I expect it to be a large change requiring lots of testing, and +b) obviously the Linux client has interoperated successfully since +day zero while still being broken. + +Fixes: 7c8d9e7c8863 ("xprtrdma: Move Receive posting to ... ") +Cc: stable@vger.kernel.org # v4.18+ +Signed-off-by: Chuck Lever +Signed-off-by: Anna Schumaker +Signed-off-by: Greg Kroah-Hartman + +--- + net/sunrpc/xprtrdma/verbs.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +--- a/net/sunrpc/xprtrdma/verbs.c ++++ b/net/sunrpc/xprtrdma/verbs.c +@@ -280,7 +280,6 @@ rpcrdma_conn_upcall(struct rdma_cm_id *i + ++xprt->rx_xprt.connect_cookie; + connstate = -ECONNABORTED; + connected: +- xprt->rx_buf.rb_credits = 1; + ep->rep_connected = connstate; + rpcrdma_conn_func(ep); + wake_up_all(&ep->rep_connect_wait); +@@ -755,6 +754,7 @@ retry: + } + + ep->rep_connected = 0; ++ rpcrdma_post_recvs(r_xprt, true); + + rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); + if (rc) { +@@ -773,8 +773,6 @@ retry: + + dprintk("RPC: %s: connected\n", __func__); + +- rpcrdma_post_recvs(r_xprt, true); +- + out: + if (rc) + ep->rep_connected = rc; +@@ -1171,6 +1169,7 @@ rpcrdma_buffer_create(struct rpcrdma_xpr + list_add(&req->rl_list, &buf->rb_send_bufs); + } + ++ buf->rb_credits = 1; + buf->rb_posted_receives = 0; + INIT_LIST_HEAD(&buf->rb_recv_bufs); + -- 2.47.3