]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5f08318f UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Connection Data Control (CDC) | |
6 | * handles flow control | |
7 | * | |
8 | * Copyright IBM Corp. 2016 | |
9 | * | |
10 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <linux/spinlock.h> | |
14 | ||
15 | #include "smc.h" | |
16 | #include "smc_wr.h" | |
17 | #include "smc_cdc.h" | |
e6727f39 | 18 | #include "smc_tx.h" |
952310cc | 19 | #include "smc_rx.h" |
b38d7324 | 20 | #include "smc_close.h" |
5f08318f UB |
21 | |
22 | /********************************** send *************************************/ | |
23 | ||
5f08318f UB |
24 | /* handler for send/transmission completion of a CDC msg */ |
25 | static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd, | |
26 | struct smc_link *link, | |
27 | enum ib_wc_status wc_status) | |
28 | { | |
29 | struct smc_cdc_tx_pend *cdcpend = (struct smc_cdc_tx_pend *)pnd_snd; | |
bac6de7b | 30 | struct smc_connection *conn = cdcpend->conn; |
5f08318f UB |
31 | struct smc_sock *smc; |
32 | int diff; | |
33 | ||
bac6de7b | 34 | smc = container_of(conn, struct smc_sock, conn); |
5f08318f UB |
35 | bh_lock_sock(&smc->sk); |
36 | if (!wc_status) { | |
69cb7dc0 | 37 | diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len, |
5f08318f UB |
38 | &cdcpend->conn->tx_curs_fin, |
39 | &cdcpend->cursor); | |
40 | /* sndbuf_space is decreased in smc_sendmsg */ | |
41 | smp_mb__before_atomic(); | |
42 | atomic_add(diff, &cdcpend->conn->sndbuf_space); | |
69cb7dc0 | 43 | /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ |
5f08318f | 44 | smp_mb__after_atomic(); |
bac6de7b | 45 | smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn); |
f0ec4f1d KG |
46 | smc_curs_copy(&conn->local_tx_ctrl_fin, &cdcpend->p_cursor, |
47 | conn); | |
48 | conn->tx_cdc_seq_fin = cdcpend->ctrl_seq; | |
5f08318f | 49 | } |
349d4312 | 50 | |
dcd2cf5f | 51 | if (atomic_dec_and_test(&conn->cdc_pend_tx_wr)) { |
6b88af83 DL |
52 | /* If user owns the sock_lock, mark the connection need sending. |
53 | * User context will later try to send when it release sock_lock | |
54 | * in smc_release_cb() | |
dcd2cf5f | 55 | */ |
6b88af83 DL |
56 | if (sock_owned_by_user(&smc->sk)) |
57 | conn->tx_in_release_sock = true; | |
58 | else | |
59 | smc_tx_pending(conn); | |
60 | ||
dcd2cf5f DL |
61 | if (unlikely(wq_has_sleeper(&conn->cdc_pend_tx_wq))) |
62 | wake_up(&conn->cdc_pend_tx_wq); | |
63 | } | |
349d4312 DL |
64 | WARN_ON(atomic_read(&conn->cdc_pend_tx_wr) < 0); |
65 | ||
e6727f39 | 66 | smc_tx_sndbuf_nonfull(smc); |
5f08318f UB |
67 | bh_unlock_sock(&smc->sk); |
68 | } | |
69 | ||
51957bc5 | 70 | int smc_cdc_get_free_slot(struct smc_connection *conn, |
c6f02ebe | 71 | struct smc_link *link, |
5f08318f | 72 | struct smc_wr_buf **wr_buf, |
ad6f317f | 73 | struct smc_rdma_wr **wr_rdma_buf, |
5f08318f UB |
74 | struct smc_cdc_tx_pend **pend) |
75 | { | |
1a0a04c7 | 76 | int rc; |
51957bc5 | 77 | |
1a0a04c7 | 78 | rc = smc_wr_tx_get_free_slot(link, smc_cdc_tx_handler, wr_buf, |
ad6f317f | 79 | wr_rdma_buf, |
1a0a04c7 | 80 | (struct smc_wr_tx_pend_priv **)pend); |
2bced6ae | 81 | if (conn->killed) { |
1a0a04c7 | 82 | /* abnormal termination */ |
2bced6ae KG |
83 | if (!rc) |
84 | smc_wr_tx_put_slot(link, | |
e225c9a5 | 85 | (struct smc_wr_tx_pend_priv *)(*pend)); |
1a0a04c7 | 86 | rc = -EPIPE; |
2bced6ae | 87 | } |
1a0a04c7 | 88 | return rc; |
5f08318f UB |
89 | } |
90 | ||
91 | static inline void smc_cdc_add_pending_send(struct smc_connection *conn, | |
92 | struct smc_cdc_tx_pend *pend) | |
93 | { | |
94 | BUILD_BUG_ON_MSG( | |
95 | sizeof(struct smc_cdc_msg) > SMC_WR_BUF_SIZE, | |
96 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_cdc_msg)"); | |
97 | BUILD_BUG_ON_MSG( | |
b9a22dd9 | 98 | offsetofend(struct smc_cdc_msg, reserved) > SMC_WR_TX_SIZE, |
5f08318f UB |
99 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_cdc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); |
100 | BUILD_BUG_ON_MSG( | |
101 | sizeof(struct smc_cdc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
102 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_cdc_tx_pend)"); | |
103 | pend->conn = conn; | |
104 | pend->cursor = conn->tx_curs_sent; | |
105 | pend->p_cursor = conn->local_tx_ctrl.prod; | |
106 | pend->ctrl_seq = conn->tx_cdc_seq; | |
107 | } | |
108 | ||
109 | int smc_cdc_msg_send(struct smc_connection *conn, | |
110 | struct smc_wr_buf *wr_buf, | |
111 | struct smc_cdc_tx_pend *pend) | |
112 | { | |
387707fd | 113 | struct smc_link *link = conn->lnk; |
b8649efa | 114 | union smc_host_cursor cfed; |
5f08318f UB |
115 | int rc; |
116 | ||
5f08318f UB |
117 | smc_cdc_add_pending_send(conn, pend); |
118 | ||
119 | conn->tx_cdc_seq++; | |
120 | conn->local_tx_ctrl.seqno = conn->tx_cdc_seq; | |
ccc8ca9b | 121 | smc_host_msg_to_cdc((struct smc_cdc_msg *)wr_buf, conn, &cfed); |
349d4312 DL |
122 | |
123 | atomic_inc(&conn->cdc_pend_tx_wr); | |
124 | smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */ | |
125 | ||
5f08318f | 126 | rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); |
4dff63c2 | 127 | if (!rc) { |
b8649efa | 128 | smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn); |
4dff63c2 | 129 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; |
f0ec4f1d KG |
130 | } else { |
131 | conn->tx_cdc_seq--; | |
132 | conn->local_tx_ctrl.seqno = conn->tx_cdc_seq; | |
349d4312 | 133 | atomic_dec(&conn->cdc_pend_tx_wr); |
4dff63c2 | 134 | } |
5f08318f UB |
135 | |
136 | return rc; | |
137 | } | |
138 | ||
29bd73db | 139 | /* send a validation msg indicating the move of a conn to an other QP link */ |
b8ded9de KG |
140 | int smcr_cdc_msg_send_validation(struct smc_connection *conn, |
141 | struct smc_cdc_tx_pend *pend, | |
142 | struct smc_wr_buf *wr_buf) | |
29bd73db KG |
143 | { |
144 | struct smc_host_cdc_msg *local = &conn->local_tx_ctrl; | |
145 | struct smc_link *link = conn->lnk; | |
29bd73db KG |
146 | struct smc_cdc_msg *peer; |
147 | int rc; | |
148 | ||
29bd73db KG |
149 | peer = (struct smc_cdc_msg *)wr_buf; |
150 | peer->common.type = local->common.type; | |
151 | peer->len = local->len; | |
152 | peer->seqno = htons(conn->tx_cdc_seq_fin); /* seqno last compl. tx */ | |
153 | peer->token = htonl(local->token); | |
154 | peer->prod_flags.failover_validation = 1; | |
155 | ||
349d4312 DL |
156 | /* We need to set pend->conn here to make sure smc_cdc_tx_handler() |
157 | * can handle properly | |
158 | */ | |
159 | smc_cdc_add_pending_send(conn, pend); | |
160 | ||
161 | atomic_inc(&conn->cdc_pend_tx_wr); | |
162 | smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */ | |
163 | ||
29bd73db | 164 | rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); |
349d4312 DL |
165 | if (unlikely(rc)) |
166 | atomic_dec(&conn->cdc_pend_tx_wr); | |
167 | ||
29bd73db KG |
168 | return rc; |
169 | } | |
170 | ||
be244f28 | 171 | static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn) |
5f08318f UB |
172 | { |
173 | struct smc_cdc_tx_pend *pend; | |
174 | struct smc_wr_buf *wr_buf; | |
c6f02ebe KG |
175 | struct smc_link *link; |
176 | bool again = false; | |
5f08318f UB |
177 | int rc; |
178 | ||
c6f02ebe KG |
179 | again: |
180 | link = conn->lnk; | |
95f7f3e7 KG |
181 | if (!smc_wr_tx_link_hold(link)) |
182 | return -ENOLINK; | |
c6f02ebe | 183 | rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend); |
5f08318f | 184 | if (rc) |
95f7f3e7 | 185 | goto put_out; |
5f08318f | 186 | |
2dee25af | 187 | spin_lock_bh(&conn->send_lock); |
c6f02ebe KG |
188 | if (link != conn->lnk) { |
189 | /* link of connection changed, try again one time*/ | |
190 | spin_unlock_bh(&conn->send_lock); | |
191 | smc_wr_tx_put_slot(link, | |
192 | (struct smc_wr_tx_pend_priv *)pend); | |
95f7f3e7 | 193 | smc_wr_tx_link_put(link); |
c6f02ebe KG |
194 | if (again) |
195 | return -ENOLINK; | |
196 | again = true; | |
197 | goto again; | |
198 | } | |
2dee25af KG |
199 | rc = smc_cdc_msg_send(conn, wr_buf, pend); |
200 | spin_unlock_bh(&conn->send_lock); | |
95f7f3e7 KG |
201 | put_out: |
202 | smc_wr_tx_link_put(link); | |
2dee25af | 203 | return rc; |
5f08318f UB |
204 | } |
205 | ||
be244f28 HW |
206 | int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn) |
207 | { | |
208 | int rc; | |
209 | ||
ea89c6c0 WG |
210 | if (!smc_conn_lgr_valid(conn) || |
211 | (conn->lgr->is_smcd && conn->lgr->peer_shutdown)) | |
50c6b20e UB |
212 | return -EPIPE; |
213 | ||
be244f28 HW |
214 | if (conn->lgr->is_smcd) { |
215 | spin_lock_bh(&conn->send_lock); | |
216 | rc = smcd_cdc_msg_send(conn); | |
217 | spin_unlock_bh(&conn->send_lock); | |
218 | } else { | |
219 | rc = smcr_cdc_get_slot_and_msg_send(conn); | |
220 | } | |
221 | ||
222 | return rc; | |
223 | } | |
224 | ||
349d4312 | 225 | void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn) |
5f08318f | 226 | { |
349d4312 | 227 | wait_event(conn->cdc_pend_tx_wq, !atomic_read(&conn->cdc_pend_tx_wr)); |
5f08318f UB |
228 | } |
229 | ||
be244f28 HW |
230 | /* Send a SMC-D CDC header. |
231 | * This increments the free space available in our send buffer. | |
232 | * Also update the confirmed receive buffer with what was sent to the peer. | |
233 | */ | |
234 | int smcd_cdc_msg_send(struct smc_connection *conn) | |
235 | { | |
236 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
b9a22dd9 | 237 | union smc_host_cursor curs; |
be244f28 HW |
238 | struct smcd_cdc_msg cdc; |
239 | int rc, diff; | |
240 | ||
241 | memset(&cdc, 0, sizeof(cdc)); | |
242 | cdc.common.type = SMC_CDC_MSG_TYPE; | |
b9a22dd9 UB |
243 | curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.prod.acurs); |
244 | cdc.prod.wrap = curs.wrap; | |
245 | cdc.prod.count = curs.count; | |
246 | curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.cons.acurs); | |
247 | cdc.cons.wrap = curs.wrap; | |
248 | cdc.cons.count = curs.count; | |
249 | cdc.cons.prod_flags = conn->local_tx_ctrl.prod_flags; | |
250 | cdc.cons.conn_state_flags = conn->local_tx_ctrl.conn_state_flags; | |
be244f28 HW |
251 | rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1); |
252 | if (rc) | |
253 | return rc; | |
b9a22dd9 | 254 | smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn); |
4dff63c2 | 255 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; |
be244f28 HW |
256 | /* Calculate transmitted data and increment free send buffer space */ |
257 | diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin, | |
258 | &conn->tx_curs_sent); | |
259 | /* increased by confirmed number of bytes */ | |
260 | smp_mb__before_atomic(); | |
261 | atomic_add(diff, &conn->sndbuf_space); | |
262 | /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ | |
263 | smp_mb__after_atomic(); | |
bac6de7b | 264 | smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn); |
be244f28 HW |
265 | |
266 | smc_tx_sndbuf_nonfull(smc); | |
267 | return rc; | |
268 | } | |
269 | ||
5f08318f UB |
270 | /********************************* receive ***********************************/ |
271 | ||
272 | static inline bool smc_cdc_before(u16 seq1, u16 seq2) | |
273 | { | |
274 | return (s16)(seq1 - seq2) < 0; | |
275 | } | |
276 | ||
de8474eb SR |
277 | static void smc_cdc_handle_urg_data_arrival(struct smc_sock *smc, |
278 | int *diff_prod) | |
279 | { | |
280 | struct smc_connection *conn = &smc->conn; | |
281 | char *base; | |
282 | ||
283 | /* new data included urgent business */ | |
bac6de7b | 284 | smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn); |
de8474eb SR |
285 | conn->urg_state = SMC_URG_VALID; |
286 | if (!sock_flag(&smc->sk, SOCK_URGINLINE)) | |
287 | /* we'll skip the urgent byte, so don't account for it */ | |
288 | (*diff_prod)--; | |
be244f28 | 289 | base = (char *)conn->rmb_desc->cpu_addr + conn->rx_off; |
de8474eb SR |
290 | if (conn->urg_curs.count) |
291 | conn->urg_rx_byte = *(base + conn->urg_curs.count - 1); | |
292 | else | |
293 | conn->urg_rx_byte = *(base + conn->rmb_desc->len - 1); | |
294 | sk_send_sigurg(&smc->sk); | |
295 | } | |
296 | ||
b286a065 KG |
297 | static void smc_cdc_msg_validate(struct smc_sock *smc, struct smc_cdc_msg *cdc, |
298 | struct smc_link *link) | |
299 | { | |
300 | struct smc_connection *conn = &smc->conn; | |
301 | u16 recv_seq = ntohs(cdc->seqno); | |
302 | s16 diff; | |
303 | ||
304 | /* check that seqnum was seen before */ | |
305 | diff = conn->local_rx_ctrl.seqno - recv_seq; | |
306 | if (diff < 0) { /* diff larger than 0x7fff */ | |
307 | /* drop connection */ | |
308 | conn->out_of_sync = 1; /* prevent any further receives */ | |
309 | spin_lock_bh(&conn->send_lock); | |
310 | conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; | |
311 | conn->lnk = link; | |
312 | spin_unlock_bh(&conn->send_lock); | |
313 | sock_hold(&smc->sk); /* sock_put in abort_work */ | |
22ef473d | 314 | if (!queue_work(smc_close_wq, &conn->abort_work)) |
b286a065 KG |
315 | sock_put(&smc->sk); |
316 | } | |
317 | } | |
318 | ||
5f08318f | 319 | static void smc_cdc_msg_recv_action(struct smc_sock *smc, |
5f08318f UB |
320 | struct smc_cdc_msg *cdc) |
321 | { | |
322 | union smc_host_cursor cons_old, prod_old; | |
323 | struct smc_connection *conn = &smc->conn; | |
324 | int diff_cons, diff_prod; | |
325 | ||
bac6de7b SR |
326 | smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn); |
327 | smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn); | |
5f08318f UB |
328 | smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn); |
329 | ||
330 | diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old, | |
331 | &conn->local_rx_ctrl.cons); | |
332 | if (diff_cons) { | |
333 | /* peer_rmbe_space is decreased during data transfer with RDMA | |
334 | * write | |
335 | */ | |
336 | smp_mb__before_atomic(); | |
337 | atomic_add(diff_cons, &conn->peer_rmbe_space); | |
338 | /* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */ | |
339 | smp_mb__after_atomic(); | |
340 | } | |
341 | ||
69cb7dc0 | 342 | diff_prod = smc_curs_diff(conn->rmb_desc->len, &prod_old, |
5f08318f UB |
343 | &conn->local_rx_ctrl.prod); |
344 | if (diff_prod) { | |
de8474eb SR |
345 | if (conn->local_rx_ctrl.prod_flags.urg_data_present) |
346 | smc_cdc_handle_urg_data_arrival(smc, &diff_prod); | |
5f08318f UB |
347 | /* bytes_to_rcv is decreased in smc_recvmsg */ |
348 | smp_mb__before_atomic(); | |
349 | atomic_add(diff_prod, &conn->bytes_to_rcv); | |
69cb7dc0 | 350 | /* guarantee 0 <= bytes_to_rcv <= rmb_desc->len */ |
5f08318f | 351 | smp_mb__after_atomic(); |
952310cc | 352 | smc->sk.sk_data_ready(&smc->sk); |
de8474eb | 353 | } else { |
cf0cfe53 KG |
354 | if (conn->local_rx_ctrl.prod_flags.write_blocked) |
355 | smc->sk.sk_data_ready(&smc->sk); | |
356 | if (conn->local_rx_ctrl.prod_flags.urg_data_pending) | |
357 | conn->urg_state = SMC_URG_NOTYET; | |
5f08318f UB |
358 | } |
359 | ||
51f1de79 | 360 | /* trigger sndbuf consumer: RDMA write into peer RMBE and CDC */ |
cf0cfe53 KG |
361 | if ((diff_cons && smc_tx_prepared_sends(conn)) || |
362 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req || | |
6b88af83 DL |
363 | conn->local_rx_ctrl.prod_flags.urg_data_pending) { |
364 | if (!sock_owned_by_user(&smc->sk)) | |
365 | smc_tx_pending(conn); | |
366 | else | |
367 | conn->tx_in_release_sock = true; | |
368 | } | |
cf0cfe53 | 369 | |
de8474eb SR |
370 | if (diff_cons && conn->urg_tx_pend && |
371 | atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) { | |
372 | /* urg data confirmed by peer, indicate we're ready for more */ | |
373 | conn->urg_tx_pend = false; | |
374 | smc->sk.sk_write_space(&smc->sk); | |
375 | } | |
51f1de79 | 376 | |
b38d7324 | 377 | if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) { |
5f08318f | 378 | smc->sk.sk_err = ECONNRESET; |
b38d7324 UB |
379 | conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; |
380 | } | |
46c28dbd UB |
381 | if (smc_cdc_rxed_any_close_or_senddone(conn)) { |
382 | smc->sk.sk_shutdown |= RCV_SHUTDOWN; | |
383 | if (smc->clcsock && smc->clcsock->sk) | |
384 | smc->clcsock->sk->sk_shutdown |= RCV_SHUTDOWN; | |
385 | sock_set_flag(&smc->sk, SOCK_DONE); | |
51f1de79 | 386 | sock_hold(&smc->sk); /* sock_put in close_work */ |
22ef473d | 387 | if (!queue_work(smc_close_wq, &conn->close_work)) |
51f1de79 | 388 | sock_put(&smc->sk); |
b38d7324 | 389 | } |
5f08318f UB |
390 | } |
391 | ||
392 | /* called under tasklet context */ | |
d7b0e37c | 393 | static void smc_cdc_msg_recv(struct smc_sock *smc, struct smc_cdc_msg *cdc) |
5f08318f | 394 | { |
5f08318f | 395 | sock_hold(&smc->sk); |
5f08318f | 396 | bh_lock_sock(&smc->sk); |
d7b0e37c | 397 | smc_cdc_msg_recv_action(smc, cdc); |
5f08318f UB |
398 | bh_unlock_sock(&smc->sk); |
399 | sock_put(&smc->sk); /* no free sk in softirq-context */ | |
400 | } | |
401 | ||
be244f28 HW |
402 | /* Schedule a tasklet for this connection. Triggered from the ISM device IRQ |
403 | * handler to indicate update in the DMBE. | |
404 | * | |
405 | * Context: | |
406 | * - tasklet context | |
407 | */ | |
fcb8e3a3 | 408 | static void smcd_cdc_rx_tsklet(struct tasklet_struct *t) |
be244f28 | 409 | { |
fcb8e3a3 | 410 | struct smc_connection *conn = from_tasklet(conn, t, rx_tsklet); |
b9a22dd9 | 411 | struct smcd_cdc_msg *data_cdc; |
be244f28 HW |
412 | struct smcd_cdc_msg cdc; |
413 | struct smc_sock *smc; | |
414 | ||
b2900980 | 415 | if (!conn || conn->killed) |
be244f28 HW |
416 | return; |
417 | ||
b9a22dd9 UB |
418 | data_cdc = (struct smcd_cdc_msg *)conn->rmb_desc->cpu_addr; |
419 | smcd_curs_copy(&cdc.prod, &data_cdc->prod, conn); | |
420 | smcd_curs_copy(&cdc.cons, &data_cdc->cons, conn); | |
be244f28 HW |
421 | smc = container_of(conn, struct smc_sock, conn); |
422 | smc_cdc_msg_recv(smc, (struct smc_cdc_msg *)&cdc); | |
423 | } | |
424 | ||
425 | /* Initialize receive tasklet. Called from ISM device IRQ handler to start | |
426 | * receiver side. | |
427 | */ | |
428 | void smcd_cdc_rx_init(struct smc_connection *conn) | |
429 | { | |
fcb8e3a3 | 430 | tasklet_setup(&conn->rx_tsklet, smcd_cdc_rx_tsklet); |
be244f28 HW |
431 | } |
432 | ||
5f08318f UB |
433 | /***************************** init, exit, misc ******************************/ |
434 | ||
435 | static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) | |
436 | { | |
437 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
438 | struct smc_cdc_msg *cdc = buf; | |
d7b0e37c HW |
439 | struct smc_connection *conn; |
440 | struct smc_link_group *lgr; | |
441 | struct smc_sock *smc; | |
5f08318f UB |
442 | |
443 | if (wc->byte_len < offsetof(struct smc_cdc_msg, reserved)) | |
444 | return; /* short message */ | |
cbba07a7 | 445 | if (cdc->len != SMC_WR_TX_SIZE) |
5f08318f | 446 | return; /* invalid message */ |
d7b0e37c HW |
447 | |
448 | /* lookup connection */ | |
00e5fb26 | 449 | lgr = smc_get_lgr(link); |
d7b0e37c HW |
450 | read_lock_bh(&lgr->conns_lock); |
451 | conn = smc_lgr_find_conn(ntohl(cdc->token), lgr); | |
452 | read_unlock_bh(&lgr->conns_lock); | |
b286a065 | 453 | if (!conn || conn->out_of_sync) |
d7b0e37c HW |
454 | return; |
455 | smc = container_of(conn, struct smc_sock, conn); | |
456 | ||
b286a065 KG |
457 | if (cdc->prod_flags.failover_validation) { |
458 | smc_cdc_msg_validate(smc, cdc, link); | |
459 | return; | |
d7b0e37c | 460 | } |
b286a065 KG |
461 | if (smc_cdc_before(ntohs(cdc->seqno), |
462 | conn->local_rx_ctrl.seqno)) | |
463 | /* received seqno is old */ | |
464 | return; | |
465 | ||
d7b0e37c | 466 | smc_cdc_msg_recv(smc, cdc); |
5f08318f UB |
467 | } |
468 | ||
469 | static struct smc_wr_rx_handler smc_cdc_rx_handlers[] = { | |
470 | { | |
471 | .handler = smc_cdc_rx_handler, | |
472 | .type = SMC_CDC_MSG_TYPE | |
473 | }, | |
474 | { | |
475 | .handler = NULL, | |
476 | } | |
477 | }; | |
478 | ||
479 | int __init smc_cdc_init(void) | |
480 | { | |
481 | struct smc_wr_rx_handler *handler; | |
482 | int rc = 0; | |
483 | ||
484 | for (handler = smc_cdc_rx_handlers; handler->handler; handler++) { | |
485 | INIT_HLIST_NODE(&handler->list); | |
486 | rc = smc_wr_rx_register_handler(handler); | |
487 | if (rc) | |
488 | break; | |
489 | } | |
490 | return rc; | |
491 | } |