]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5d252f90 | 2 | /* |
4201c746 | 3 | * Copyright (c) 2015-2018 Oracle. All rights reserved. |
5d252f90 CL |
4 | * |
5 | * Support for backward direction RPCs on RPC/RDMA (server-side). | |
6 | */ | |
7 | ||
8 | #include <linux/sunrpc/svc_rdma.h> | |
bd2abef3 | 9 | |
5d252f90 | 10 | #include "xprt_rdma.h" |
bd2abef3 | 11 | #include <trace/events/rpcrdma.h> |
5d252f90 CL |
12 | |
13 | #define RPCDBG_FACILITY RPCDBG_SVCXPRT | |
14 | ||
15 | #undef SVCRDMA_BACKCHANNEL_DEBUG | |
16 | ||
f5821c76 CL |
17 | /** |
18 | * svc_rdma_handle_bc_reply - Process incoming backchannel reply | |
19 | * @xprt: controlling backchannel transport | |
20 | * @rdma_resp: pointer to incoming transport header | |
21 | * @rcvbuf: XDR buffer into which to decode the reply | |
22 | * | |
23 | * Returns: | |
24 | * %0 if @rcvbuf is filled in, xprt_complete_rqst called, | |
25 | * %-EAGAIN if server should call ->recvfrom again. | |
26 | */ | |
27 | int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp, | |
5d252f90 CL |
28 | struct xdr_buf *rcvbuf) |
29 | { | |
30 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
31 | struct kvec *dst, *src = &rcvbuf->head[0]; | |
32 | struct rpc_rqst *req; | |
5d252f90 CL |
33 | u32 credits; |
34 | size_t len; | |
35 | __be32 xid; | |
36 | __be32 *p; | |
37 | int ret; | |
38 | ||
39 | p = (__be32 *)src->iov_base; | |
40 | len = src->iov_len; | |
f5821c76 | 41 | xid = *rdma_resp; |
5d252f90 CL |
42 | |
43 | #ifdef SVCRDMA_BACKCHANNEL_DEBUG | |
44 | pr_info("%s: xid=%08x, length=%zu\n", | |
45 | __func__, be32_to_cpu(xid), len); | |
46 | pr_info("%s: RPC/RDMA: %*ph\n", | |
f5821c76 | 47 | __func__, (int)RPCRDMA_HDRLEN_MIN, rdma_resp); |
5d252f90 CL |
48 | pr_info("%s: RPC: %*ph\n", |
49 | __func__, (int)len, p); | |
50 | #endif | |
51 | ||
52 | ret = -EAGAIN; | |
53 | if (src->iov_len < 24) | |
54 | goto out_shortreply; | |
55 | ||
75c84151 | 56 | spin_lock(&xprt->queue_lock); |
5d252f90 CL |
57 | req = xprt_lookup_rqst(xprt, xid); |
58 | if (!req) | |
59 | goto out_notfound; | |
60 | ||
61 | dst = &req->rq_private_buf.head[0]; | |
62 | memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); | |
63 | if (dst->iov_len < len) | |
64 | goto out_unlock; | |
65 | memcpy(dst->iov_base, p, len); | |
596f2a19 | 66 | xprt_pin_rqst(req); |
310c7585 | 67 | spin_unlock(&xprt->queue_lock); |
5d252f90 | 68 | |
f5821c76 | 69 | credits = be32_to_cpup(rdma_resp + 2); |
5d252f90 CL |
70 | if (credits == 0) |
71 | credits = 1; /* don't deadlock */ | |
72 | else if (credits > r_xprt->rx_buf.rb_bc_max_requests) | |
73 | credits = r_xprt->rx_buf.rb_bc_max_requests; | |
74 | ||
b5e92419 | 75 | spin_lock(&xprt->transport_lock); |
5d252f90 | 76 | xprt->cwnd = credits << RPC_CWNDSHIFT; |
b5e92419 | 77 | spin_unlock(&xprt->transport_lock); |
ce7c252a | 78 | |
310c7585 | 79 | spin_lock(&xprt->queue_lock); |
5d252f90 CL |
80 | ret = 0; |
81 | xprt_complete_rqst(req->rq_task, rcvbuf->len); | |
596f2a19 | 82 | xprt_unpin_rqst(req); |
5d252f90 CL |
83 | rcvbuf->len = 0; |
84 | ||
85 | out_unlock: | |
75c84151 | 86 | spin_unlock(&xprt->queue_lock); |
5d252f90 CL |
87 | out: |
88 | return ret; | |
89 | ||
90 | out_shortreply: | |
91 | dprintk("svcrdma: short bc reply: xprt=%p, len=%zu\n", | |
92 | xprt, src->iov_len); | |
93 | goto out; | |
94 | ||
95 | out_notfound: | |
96 | dprintk("svcrdma: unrecognized bc reply: xprt=%p, xid=%08x\n", | |
97 | xprt, be32_to_cpu(xid)); | |
5d252f90 CL |
98 | goto out_unlock; |
99 | } | |
100 | ||
101 | /* Send a backwards direction RPC call. | |
102 | * | |
103 | * Caller holds the connection's mutex and has already marshaled | |
104 | * the RPC/RDMA request. | |
105 | * | |
9a6a180b CL |
106 | * This is similar to svc_rdma_send_reply_msg, but takes a struct |
107 | * rpc_rqst instead, does not support chunks, and avoids blocking | |
108 | * memory allocation. | |
5d252f90 CL |
109 | * |
110 | * XXX: There is still an opportunity to block in svc_rdma_send() | |
111 | * if there are no SQ entries to post the Send. This may occur if | |
112 | * the adapter has a small maximum SQ depth. | |
113 | */ | |
114 | static int svc_rdma_bc_sendto(struct svcxprt_rdma *rdma, | |
99722fe4 CL |
115 | struct rpc_rqst *rqst, |
116 | struct svc_rdma_send_ctxt *ctxt) | |
5d252f90 | 117 | { |
5d252f90 CL |
118 | int ret; |
119 | ||
4554755e | 120 | ret = svc_rdma_map_reply_msg(rdma, ctxt, NULL, &rqst->rq_snd_buf); |
6e6092ca | 121 | if (ret < 0) |
99722fe4 | 122 | return -EIO; |
5d252f90 | 123 | |
0bad47ca CL |
124 | /* Bump page refcnt so Send completion doesn't release |
125 | * the rq_buffer before all retransmits are complete. | |
126 | */ | |
127 | get_page(virt_to_page(rqst->rq_buffer)); | |
986b7889 | 128 | ctxt->sc_send_wr.opcode = IB_WR_SEND; |
99722fe4 | 129 | return svc_rdma_send(rdma, &ctxt->sc_send_wr); |
5d252f90 CL |
130 | } |
131 | ||
132 | /* Server-side transport endpoint wants a whole page for its send | |
133 | * buffer. The client RPC code constructs the RPC header in this | |
134 | * buffer before it invokes ->send_request. | |
5d252f90 | 135 | */ |
5fe6eaa1 CL |
136 | static int |
137 | xprt_rdma_bc_allocate(struct rpc_task *task) | |
5d252f90 CL |
138 | { |
139 | struct rpc_rqst *rqst = task->tk_rqstp; | |
5fe6eaa1 | 140 | size_t size = rqst->rq_callsize; |
5d252f90 CL |
141 | struct page *page; |
142 | ||
5fe6eaa1 | 143 | if (size > PAGE_SIZE) { |
5d252f90 CL |
144 | WARN_ONCE(1, "svcrdma: large bc buffer request (size %zu)\n", |
145 | size); | |
5fe6eaa1 CL |
146 | return -EINVAL; |
147 | } | |
5d252f90 CL |
148 | |
149 | page = alloc_page(RPCRDMA_DEF_GFP); | |
150 | if (!page) | |
5fe6eaa1 | 151 | return -ENOMEM; |
5fe6eaa1 | 152 | rqst->rq_buffer = page_address(page); |
8d42629b CL |
153 | |
154 | rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, RPCRDMA_DEF_GFP); | |
155 | if (!rqst->rq_rbuffer) { | |
156 | put_page(page); | |
157 | return -ENOMEM; | |
158 | } | |
5fe6eaa1 | 159 | return 0; |
5d252f90 CL |
160 | } |
161 | ||
162 | static void | |
3435c74a | 163 | xprt_rdma_bc_free(struct rpc_task *task) |
5d252f90 | 164 | { |
8d42629b CL |
165 | struct rpc_rqst *rqst = task->tk_rqstp; |
166 | ||
0bad47ca | 167 | put_page(virt_to_page(rqst->rq_buffer)); |
8d42629b | 168 | kfree(rqst->rq_rbuffer); |
5d252f90 CL |
169 | } |
170 | ||
171 | static int | |
172 | rpcrdma_bc_send_request(struct svcxprt_rdma *rdma, struct rpc_rqst *rqst) | |
173 | { | |
174 | struct rpc_xprt *xprt = rqst->rq_xprt; | |
175 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
99722fe4 | 176 | struct svc_rdma_send_ctxt *ctxt; |
c2ccf64a | 177 | __be32 *p; |
5d252f90 CL |
178 | int rc; |
179 | ||
99722fe4 CL |
180 | ctxt = svc_rdma_send_ctxt_get(rdma); |
181 | if (!ctxt) | |
182 | goto drop_connection; | |
183 | ||
6fd5034d CL |
184 | p = xdr_reserve_space(&ctxt->sc_stream, RPCRDMA_HDRLEN_MIN); |
185 | if (!p) | |
186 | goto put_ctxt; | |
c2ccf64a CL |
187 | *p++ = rqst->rq_xid; |
188 | *p++ = rpcrdma_version; | |
189 | *p++ = cpu_to_be32(r_xprt->rx_buf.rb_bc_max_requests); | |
190 | *p++ = rdma_msg; | |
191 | *p++ = xdr_zero; | |
192 | *p++ = xdr_zero; | |
193 | *p = xdr_zero; | |
5d252f90 CL |
194 | |
195 | #ifdef SVCRDMA_BACKCHANNEL_DEBUG | |
196 | pr_info("%s: %*ph\n", __func__, 64, rqst->rq_buffer); | |
197 | #endif | |
198 | ||
8729aaba | 199 | rqst->rq_xtime = ktime_get(); |
99722fe4 | 200 | rc = svc_rdma_bc_sendto(rdma, rqst, ctxt); |
6fd5034d CL |
201 | if (rc) |
202 | goto put_ctxt; | |
0c0829bc | 203 | return 0; |
5d252f90 | 204 | |
6fd5034d CL |
205 | put_ctxt: |
206 | svc_rdma_send_ctxt_put(rdma, ctxt); | |
207 | ||
5d252f90 CL |
208 | drop_connection: |
209 | dprintk("svcrdma: failed to send bc call\n"); | |
5d252f90 CL |
210 | return -ENOTCONN; |
211 | } | |
212 | ||
213 | /* Send an RPC call on the passive end of a transport | |
214 | * connection. | |
215 | */ | |
216 | static int | |
adfa7144 | 217 | xprt_rdma_bc_send_request(struct rpc_rqst *rqst) |
5d252f90 | 218 | { |
5d252f90 CL |
219 | struct svc_xprt *sxprt = rqst->rq_xprt->bc_xprt; |
220 | struct svcxprt_rdma *rdma; | |
221 | int ret; | |
222 | ||
223 | dprintk("svcrdma: sending bc call with xid: %08x\n", | |
224 | be32_to_cpu(rqst->rq_xid)); | |
225 | ||
c544577d | 226 | mutex_lock(&sxprt->xpt_mutex); |
5d252f90 CL |
227 | |
228 | ret = -ENOTCONN; | |
229 | rdma = container_of(sxprt, struct svcxprt_rdma, sc_xprt); | |
0c0829bc | 230 | if (!test_bit(XPT_DEAD, &sxprt->xpt_flags)) { |
5d252f90 | 231 | ret = rpcrdma_bc_send_request(rdma, rqst); |
0c0829bc CL |
232 | if (ret == -ENOTCONN) |
233 | svc_close_xprt(sxprt); | |
234 | } | |
5d252f90 CL |
235 | |
236 | mutex_unlock(&sxprt->xpt_mutex); | |
237 | ||
238 | if (ret < 0) | |
239 | return ret; | |
240 | return 0; | |
241 | } | |
242 | ||
243 | static void | |
244 | xprt_rdma_bc_close(struct rpc_xprt *xprt) | |
245 | { | |
246 | dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); | |
6221f1d9 CL |
247 | |
248 | xprt_disconnect_done(xprt); | |
ef739b21 | 249 | xprt->cwnd = RPC_CWNDSHIFT; |
5d252f90 CL |
250 | } |
251 | ||
252 | static void | |
253 | xprt_rdma_bc_put(struct rpc_xprt *xprt) | |
254 | { | |
255 | dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); | |
256 | ||
1a33d8a2 | 257 | xprt_rdma_free_addresses(xprt); |
5d252f90 | 258 | xprt_free(xprt); |
5d252f90 CL |
259 | } |
260 | ||
d31ae254 | 261 | static const struct rpc_xprt_ops xprt_rdma_bc_procs = { |
5d252f90 CL |
262 | .reserve_xprt = xprt_reserve_xprt_cong, |
263 | .release_xprt = xprt_release_xprt_cong, | |
264 | .alloc_slot = xprt_alloc_slot, | |
a9cde23a | 265 | .free_slot = xprt_free_slot, |
5d252f90 CL |
266 | .release_request = xprt_release_rqst_cong, |
267 | .buf_alloc = xprt_rdma_bc_allocate, | |
268 | .buf_free = xprt_rdma_bc_free, | |
269 | .send_request = xprt_rdma_bc_send_request, | |
8ba6a92d | 270 | .wait_for_reply_request = xprt_wait_for_reply_request_def, |
5d252f90 CL |
271 | .close = xprt_rdma_bc_close, |
272 | .destroy = xprt_rdma_bc_put, | |
273 | .print_stats = xprt_rdma_print_stats | |
274 | }; | |
275 | ||
276 | static const struct rpc_timeout xprt_rdma_bc_timeout = { | |
277 | .to_initval = 60 * HZ, | |
278 | .to_maxval = 60 * HZ, | |
279 | }; | |
280 | ||
281 | /* It shouldn't matter if the number of backchannel session slots | |
282 | * doesn't match the number of RPC/RDMA credits. That just means | |
283 | * one or the other will have extra slots that aren't used. | |
284 | */ | |
285 | static struct rpc_xprt * | |
286 | xprt_setup_rdma_bc(struct xprt_create *args) | |
287 | { | |
288 | struct rpc_xprt *xprt; | |
289 | struct rpcrdma_xprt *new_xprt; | |
290 | ||
291 | if (args->addrlen > sizeof(xprt->addr)) { | |
292 | dprintk("RPC: %s: address too large\n", __func__); | |
293 | return ERR_PTR(-EBADF); | |
294 | } | |
295 | ||
296 | xprt = xprt_alloc(args->net, sizeof(*new_xprt), | |
297 | RPCRDMA_MAX_BC_REQUESTS, | |
298 | RPCRDMA_MAX_BC_REQUESTS); | |
299 | if (!xprt) { | |
300 | dprintk("RPC: %s: couldn't allocate rpc_xprt\n", | |
301 | __func__); | |
302 | return ERR_PTR(-ENOMEM); | |
303 | } | |
304 | ||
305 | xprt->timeout = &xprt_rdma_bc_timeout; | |
306 | xprt_set_bound(xprt); | |
307 | xprt_set_connected(xprt); | |
308 | xprt->bind_timeout = RPCRDMA_BIND_TO; | |
309 | xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; | |
310 | xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO; | |
311 | ||
312 | xprt->prot = XPRT_TRANSPORT_BC_RDMA; | |
5d252f90 CL |
313 | xprt->ops = &xprt_rdma_bc_procs; |
314 | ||
315 | memcpy(&xprt->addr, args->dstaddr, args->addrlen); | |
316 | xprt->addrlen = args->addrlen; | |
317 | xprt_rdma_format_addresses(xprt, (struct sockaddr *)&xprt->addr); | |
318 | xprt->resvport = 0; | |
319 | ||
320 | xprt->max_payload = xprt_rdma_max_inline_read; | |
321 | ||
322 | new_xprt = rpcx_to_rdmax(xprt); | |
323 | new_xprt->rx_buf.rb_bc_max_requests = xprt->max_reqs; | |
324 | ||
325 | xprt_get(xprt); | |
326 | args->bc_xprt->xpt_bc_xprt = xprt; | |
327 | xprt->bc_xprt = args->bc_xprt; | |
328 | ||
5d252f90 CL |
329 | /* Final put for backchannel xprt is in __svc_rdma_free */ |
330 | xprt_get(xprt); | |
331 | return xprt; | |
5d252f90 CL |
332 | } |
333 | ||
334 | struct xprt_class xprt_rdma_bc = { | |
335 | .list = LIST_HEAD_INIT(xprt_rdma_bc.list), | |
336 | .name = "rdma backchannel", | |
337 | .owner = THIS_MODULE, | |
338 | .ident = XPRT_TRANSPORT_BC_RDMA, | |
339 | .setup = xprt_setup_rdma_bc, | |
340 | }; |