2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
9 * Initial restrictions:
10 * - non-blocking connect postponed
11 * - IPv6 support postponed
12 * - support for alternate links postponed
13 * - partial support for non-blocking sockets only
14 * - support for urgent data postponed
16 * Copyright IBM Corp. 2016
18 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
19 * based on prototype from Frank Blaschka
22 #define KMSG_COMPONENT "smc"
23 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
25 #include <linux/module.h>
26 #include <linux/socket.h>
27 #include <linux/inetdevice.h>
28 #include <linux/workqueue.h>
30 #include <linux/sched/signal.h>
45 #include "smc_close.h"
47 static DEFINE_MUTEX(smc_create_lgr_pending
); /* serialize link group
51 struct smc_lgr_list smc_lgr_list
= { /* established link groups */
52 .lock
= __SPIN_LOCK_UNLOCKED(smc_lgr_list
.lock
),
53 .list
= LIST_HEAD_INIT(smc_lgr_list
.list
),
56 static void smc_tcp_listen_work(struct work_struct
*);
58 static void smc_set_keepalive(struct sock
*sk
, int val
)
60 struct smc_sock
*smc
= smc_sk(sk
);
62 smc
->clcsock
->sk
->sk_prot
->keepalive(smc
->clcsock
->sk
, val
);
65 static struct smc_hashinfo smc_v4_hashinfo
= {
66 .lock
= __RW_LOCK_UNLOCKED(smc_v4_hashinfo
.lock
),
69 int smc_hash_sk(struct sock
*sk
)
71 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
72 struct hlist_head
*head
;
76 write_lock_bh(&h
->lock
);
77 sk_add_node(sk
, head
);
78 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
79 write_unlock_bh(&h
->lock
);
83 EXPORT_SYMBOL_GPL(smc_hash_sk
);
85 void smc_unhash_sk(struct sock
*sk
)
87 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
89 write_lock_bh(&h
->lock
);
90 if (sk_del_node_init(sk
))
91 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
92 write_unlock_bh(&h
->lock
);
94 EXPORT_SYMBOL_GPL(smc_unhash_sk
);
96 struct proto smc_proto
= {
99 .keepalive
= smc_set_keepalive
,
101 .unhash
= smc_unhash_sk
,
102 .obj_size
= sizeof(struct smc_sock
),
103 .h
.smc_hash
= &smc_v4_hashinfo
,
104 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
106 EXPORT_SYMBOL_GPL(smc_proto
);
108 static int smc_release(struct socket
*sock
)
110 struct sock
*sk
= sock
->sk
;
111 struct smc_sock
*smc
;
119 if (sk
->sk_state
== SMC_LISTEN
)
120 /* smc_close_non_accepted() is called and acquires
121 * sock lock for child sockets again
123 lock_sock_nested(sk
, SINGLE_DEPTH_NESTING
);
127 if (smc
->use_fallback
) {
128 sk
->sk_state
= SMC_CLOSED
;
129 sk
->sk_state_change(sk
);
131 rc
= smc_close_active(smc
);
132 sock_set_flag(sk
, SOCK_DEAD
);
133 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
136 if (smc
->use_fallback
&& sk
->sk_state
== SMC_LISTEN
) {
137 /* wake up clcsock accept */
138 rc
= kernel_sock_shutdown(smc
->clcsock
, SHUT_RDWR
);
140 mutex_lock(&smc
->clcsock_release_lock
);
141 sock_release(smc
->clcsock
);
143 mutex_unlock(&smc
->clcsock_release_lock
);
149 if (smc
->use_fallback
) {
150 schedule_delayed_work(&smc
->sock_put_work
, TCP_TIMEWAIT_LEN
);
151 } else if (sk
->sk_state
== SMC_CLOSED
) {
152 smc_conn_free(&smc
->conn
);
153 schedule_delayed_work(&smc
->sock_put_work
,
154 SMC_CLOSE_SOCK_PUT_DELAY
);
163 static void smc_destruct(struct sock
*sk
)
165 if (sk
->sk_state
!= SMC_CLOSED
)
167 if (!sock_flag(sk
, SOCK_DEAD
))
170 sk_refcnt_debug_dec(sk
);
173 static struct sock
*smc_sock_alloc(struct net
*net
, struct socket
*sock
)
175 struct smc_sock
*smc
;
178 sk
= sk_alloc(net
, PF_SMC
, GFP_KERNEL
, &smc_proto
, 0);
182 sock_init_data(sock
, sk
); /* sets sk_refcnt to 1 */
183 sk
->sk_state
= SMC_INIT
;
184 sk
->sk_destruct
= smc_destruct
;
185 sk
->sk_protocol
= SMCPROTO_SMC
;
187 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
188 INIT_LIST_HEAD(&smc
->accept_q
);
189 spin_lock_init(&smc
->accept_q_lock
);
190 INIT_DELAYED_WORK(&smc
->sock_put_work
, smc_close_sock_put_work
);
191 sk
->sk_prot
->hash(sk
);
192 sk_refcnt_debug_inc(sk
);
193 mutex_init(&smc
->clcsock_release_lock
);
198 static int smc_bind(struct socket
*sock
, struct sockaddr
*uaddr
,
201 struct sockaddr_in
*addr
= (struct sockaddr_in
*)uaddr
;
202 struct sock
*sk
= sock
->sk
;
203 struct smc_sock
*smc
;
208 /* replicate tests from inet_bind(), to be safe wrt. future changes */
210 if (addr_len
< sizeof(struct sockaddr_in
))
214 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
215 if ((addr
->sin_family
!= AF_INET
) &&
216 ((addr
->sin_family
!= AF_UNSPEC
) ||
217 (addr
->sin_addr
.s_addr
!= htonl(INADDR_ANY
))))
222 /* Check if socket is already active */
224 if (sk
->sk_state
!= SMC_INIT
)
227 smc
->clcsock
->sk
->sk_reuse
= sk
->sk_reuse
;
228 rc
= kernel_bind(smc
->clcsock
, uaddr
, addr_len
);
236 static void smc_copy_sock_settings(struct sock
*nsk
, struct sock
*osk
,
239 /* options we don't get control via setsockopt for */
240 nsk
->sk_type
= osk
->sk_type
;
241 nsk
->sk_sndbuf
= osk
->sk_sndbuf
;
242 nsk
->sk_rcvbuf
= osk
->sk_rcvbuf
;
243 nsk
->sk_sndtimeo
= osk
->sk_sndtimeo
;
244 nsk
->sk_rcvtimeo
= osk
->sk_rcvtimeo
;
245 nsk
->sk_mark
= osk
->sk_mark
;
246 nsk
->sk_priority
= osk
->sk_priority
;
247 nsk
->sk_rcvlowat
= osk
->sk_rcvlowat
;
248 nsk
->sk_bound_dev_if
= osk
->sk_bound_dev_if
;
249 nsk
->sk_err
= osk
->sk_err
;
251 nsk
->sk_flags
&= ~mask
;
252 nsk
->sk_flags
|= osk
->sk_flags
& mask
;
255 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
256 (1UL << SOCK_KEEPOPEN) | \
257 (1UL << SOCK_LINGER) | \
258 (1UL << SOCK_BROADCAST) | \
259 (1UL << SOCK_TIMESTAMP) | \
260 (1UL << SOCK_DBG) | \
261 (1UL << SOCK_RCVTSTAMP) | \
262 (1UL << SOCK_RCVTSTAMPNS) | \
263 (1UL << SOCK_LOCALROUTE) | \
264 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
265 (1UL << SOCK_RXQ_OVFL) | \
266 (1UL << SOCK_WIFI_STATUS) | \
267 (1UL << SOCK_NOFCS) | \
268 (1UL << SOCK_FILTER_LOCKED))
269 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
270 * clc socket (since smc is not called for these options from net/core)
272 static void smc_copy_sock_settings_to_clc(struct smc_sock
*smc
)
274 smc_copy_sock_settings(smc
->clcsock
->sk
, &smc
->sk
, SK_FLAGS_SMC_TO_CLC
);
277 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
278 (1UL << SOCK_KEEPOPEN) | \
279 (1UL << SOCK_LINGER) | \
281 /* copy only settings and flags relevant for smc from clc to smc socket */
282 static void smc_copy_sock_settings_to_smc(struct smc_sock
*smc
)
284 smc_copy_sock_settings(&smc
->sk
, smc
->clcsock
->sk
, SK_FLAGS_CLC_TO_SMC
);
287 /* determine subnet and mask of internal TCP socket */
288 int smc_netinfo_by_tcpsk(struct socket
*clcsock
,
289 __be32
*subnet
, u8
*prefix_len
)
291 struct dst_entry
*dst
= sk_dst_get(clcsock
->sk
);
292 struct in_device
*in_dev
;
293 struct sockaddr_in addr
;
306 /* get address to which the internal TCP socket is bound */
307 kernel_getsockname(clcsock
, (struct sockaddr
*)&addr
, &len
);
308 /* analyze IPv4 specific data of net_device belonging to TCP socket */
310 in_dev
= __in_dev_get_rcu(dst
->dev
);
312 if (!inet_ifa_match(addr
.sin_addr
.s_addr
, ifa
))
314 *prefix_len
= inet_mask_len(ifa
->ifa_mask
);
315 *subnet
= ifa
->ifa_address
& ifa
->ifa_mask
;
318 } endfor_ifa(in_dev
);
327 static int smc_clnt_conf_first_link(struct smc_sock
*smc
, union ib_gid
*gid
)
329 struct smc_link_group
*lgr
= smc
->conn
.lgr
;
330 struct smc_link
*link
;
334 link
= &lgr
->lnk
[SMC_SINGLE_LINK
];
335 /* receive CONFIRM LINK request from server over RoCE fabric */
336 rest
= wait_for_completion_interruptible_timeout(
338 SMC_LLC_WAIT_FIRST_TIME
);
340 struct smc_clc_msg_decline dclc
;
342 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
347 rc
= smc_ib_modify_qp_rts(link
);
349 return SMC_CLC_DECL_INTERR
;
351 smc_wr_remember_qp_attr(link
);
353 rc
= smc_wr_reg_send(link
,
354 smc
->conn
.rmb_desc
->mr_rx
[SMC_SINGLE_LINK
]);
356 return SMC_CLC_DECL_INTERR
;
358 /* send CONFIRM LINK response over RoCE fabric */
359 rc
= smc_llc_send_confirm_link(link
,
360 link
->smcibdev
->mac
[link
->ibport
- 1],
363 return SMC_CLC_DECL_TCL
;
368 static void smc_conn_save_peer_info(struct smc_sock
*smc
,
369 struct smc_clc_msg_accept_confirm
*clc
)
371 smc
->conn
.peer_conn_idx
= clc
->conn_idx
;
372 smc
->conn
.local_tx_ctrl
.token
= ntohl(clc
->rmbe_alert_token
);
373 smc
->conn
.peer_rmbe_size
= smc_uncompress_bufsize(clc
->rmbe_size
);
374 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
377 static void smc_link_save_peer_info(struct smc_link
*link
,
378 struct smc_clc_msg_accept_confirm
*clc
)
380 link
->peer_qpn
= ntoh24(clc
->qpn
);
381 memcpy(link
->peer_gid
, clc
->lcl
.gid
, SMC_GID_SIZE
);
382 memcpy(link
->peer_mac
, clc
->lcl
.mac
, sizeof(link
->peer_mac
));
383 link
->peer_psn
= ntoh24(clc
->psn
);
384 link
->peer_mtu
= clc
->qp_mtu
;
387 /* setup for RDMA connection of client */
388 static int smc_connect_rdma(struct smc_sock
*smc
)
390 struct sockaddr_in
*inaddr
= (struct sockaddr_in
*)smc
->addr
;
391 struct smc_clc_msg_accept_confirm aclc
;
392 int local_contact
= SMC_FIRST_CONTACT
;
393 struct smc_ib_device
*smcibdev
;
394 struct smc_link
*link
;
395 u8 srv_first_contact
;
400 /* IPSec connections opt out of SMC-R optimizations */
401 if (using_ipsec(smc
)) {
402 reason_code
= SMC_CLC_DECL_IPSEC
;
406 /* PNET table look up: search active ib_device and port
407 * within same PNETID that also contains the ethernet device
408 * used for the internal TCP socket
410 smc_pnet_find_roce_resource(smc
->clcsock
->sk
, &smcibdev
, &ibport
);
412 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
416 /* do inband token exchange */
417 reason_code
= smc_clc_send_proposal(smc
, smcibdev
, ibport
);
418 if (reason_code
< 0) {
422 if (reason_code
> 0) /* configuration error */
424 /* receive SMC Accept CLC message */
425 reason_code
= smc_clc_wait_msg(smc
, &aclc
, sizeof(aclc
),
427 if (reason_code
< 0) {
434 srv_first_contact
= aclc
.hdr
.flag
;
435 mutex_lock(&smc_create_lgr_pending
);
436 local_contact
= smc_conn_create(smc
, inaddr
->sin_addr
.s_addr
, smcibdev
,
437 ibport
, &aclc
.lcl
, srv_first_contact
);
438 if (local_contact
< 0) {
441 reason_code
= SMC_CLC_DECL_MEM
;/* insufficient memory*/
442 else if (rc
== -ENOLINK
)
443 reason_code
= SMC_CLC_DECL_SYNCERR
; /* synchr. error */
444 goto decline_rdma_unlock
;
446 link
= &smc
->conn
.lgr
->lnk
[SMC_SINGLE_LINK
];
448 smc_conn_save_peer_info(smc
, &aclc
);
450 /* create send buffer and rmb */
451 rc
= smc_buf_create(smc
);
453 reason_code
= SMC_CLC_DECL_MEM
;
454 goto decline_rdma_unlock
;
457 if (local_contact
== SMC_FIRST_CONTACT
)
458 smc_link_save_peer_info(link
, &aclc
);
460 rc
= smc_rmb_rtoken_handling(&smc
->conn
, &aclc
);
462 reason_code
= SMC_CLC_DECL_INTERR
;
463 goto decline_rdma_unlock
;
469 if (local_contact
== SMC_FIRST_CONTACT
) {
470 rc
= smc_ib_ready_link(link
);
472 reason_code
= SMC_CLC_DECL_INTERR
;
473 goto decline_rdma_unlock
;
476 struct smc_buf_desc
*buf_desc
= smc
->conn
.rmb_desc
;
478 if (!buf_desc
->reused
) {
479 /* register memory region for new rmb */
480 rc
= smc_wr_reg_send(link
,
481 buf_desc
->mr_rx
[SMC_SINGLE_LINK
]);
483 reason_code
= SMC_CLC_DECL_INTERR
;
484 goto decline_rdma_unlock
;
488 smc_rmb_sync_sg_for_device(&smc
->conn
);
490 rc
= smc_clc_send_confirm(smc
);
494 if (local_contact
== SMC_FIRST_CONTACT
) {
495 /* QP confirmation over RoCE fabric */
496 reason_code
= smc_clnt_conf_first_link(
497 smc
, &smcibdev
->gid
[ibport
- 1]);
498 if (reason_code
< 0) {
503 goto decline_rdma_unlock
;
506 mutex_unlock(&smc_create_lgr_pending
);
510 smc_copy_sock_settings_to_clc(smc
);
511 if (smc
->sk
.sk_state
== SMC_INIT
)
512 smc
->sk
.sk_state
= SMC_ACTIVE
;
514 return rc
? rc
: local_contact
;
517 mutex_unlock(&smc_create_lgr_pending
);
518 smc_conn_free(&smc
->conn
);
520 /* RDMA setup failed, switch back to TCP */
521 smc
->use_fallback
= true;
522 if (reason_code
&& (reason_code
!= SMC_CLC_DECL_REPLY
)) {
523 rc
= smc_clc_send_decline(smc
, reason_code
);
524 if (rc
< sizeof(struct smc_clc_msg_decline
))
530 mutex_unlock(&smc_create_lgr_pending
);
531 smc_conn_free(&smc
->conn
);
536 static int smc_connect(struct socket
*sock
, struct sockaddr
*addr
,
539 struct sock
*sk
= sock
->sk
;
540 struct smc_sock
*smc
;
545 /* separate smc parameter checking to be safe */
546 if (alen
< sizeof(addr
->sa_family
))
548 if (addr
->sa_family
!= AF_INET
)
550 smc
->addr
= addr
; /* needed for nonblocking connect */
553 switch (sk
->sk_state
) {
564 smc_copy_sock_settings_to_clc(smc
);
565 rc
= kernel_connect(smc
->clcsock
, addr
, alen
, flags
);
569 /* setup RDMA connection */
570 rc
= smc_connect_rdma(smc
);
574 rc
= 0; /* success cases including fallback */
582 static int smc_clcsock_accept(struct smc_sock
*lsmc
, struct smc_sock
**new_smc
)
584 struct sock
*sk
= &lsmc
->sk
;
585 struct socket
*new_clcsock
;
589 release_sock(&lsmc
->sk
);
590 new_sk
= smc_sock_alloc(sock_net(sk
), NULL
);
593 lsmc
->sk
.sk_err
= ENOMEM
;
595 lock_sock(&lsmc
->sk
);
598 *new_smc
= smc_sk(new_sk
);
600 mutex_lock(&lsmc
->clcsock_release_lock
);
602 rc
= kernel_accept(lsmc
->clcsock
, &new_clcsock
, 0);
603 mutex_unlock(&lsmc
->clcsock_release_lock
);
604 lock_sock(&lsmc
->sk
);
606 lsmc
->sk
.sk_err
= -rc
;
607 new_sk
->sk_state
= SMC_CLOSED
;
608 sock_set_flag(new_sk
, SOCK_DEAD
);
609 sk
->sk_prot
->unhash(new_sk
);
614 if (lsmc
->sk
.sk_state
== SMC_CLOSED
) {
616 sock_release(new_clcsock
);
617 new_sk
->sk_state
= SMC_CLOSED
;
618 sock_set_flag(new_sk
, SOCK_DEAD
);
619 sk
->sk_prot
->unhash(new_sk
);
625 (*new_smc
)->clcsock
= new_clcsock
;
630 /* add a just created sock to the accept queue of the listen sock as
631 * candidate for a following socket accept call from user space
633 static void smc_accept_enqueue(struct sock
*parent
, struct sock
*sk
)
635 struct smc_sock
*par
= smc_sk(parent
);
638 spin_lock(&par
->accept_q_lock
);
639 list_add_tail(&smc_sk(sk
)->accept_q
, &par
->accept_q
);
640 spin_unlock(&par
->accept_q_lock
);
641 sk_acceptq_added(parent
);
644 /* remove a socket from the accept queue of its parental listening socket */
645 static void smc_accept_unlink(struct sock
*sk
)
647 struct smc_sock
*par
= smc_sk(sk
)->listen_smc
;
649 spin_lock(&par
->accept_q_lock
);
650 list_del_init(&smc_sk(sk
)->accept_q
);
651 spin_unlock(&par
->accept_q_lock
);
652 sk_acceptq_removed(&smc_sk(sk
)->listen_smc
->sk
);
656 /* remove a sock from the accept queue to bind it to a new socket created
657 * for a socket accept call from user space
659 struct sock
*smc_accept_dequeue(struct sock
*parent
,
660 struct socket
*new_sock
)
662 struct smc_sock
*isk
, *n
;
665 list_for_each_entry_safe(isk
, n
, &smc_sk(parent
)->accept_q
, accept_q
) {
666 new_sk
= (struct sock
*)isk
;
668 smc_accept_unlink(new_sk
);
669 if (new_sk
->sk_state
== SMC_CLOSED
) {
670 new_sk
->sk_prot
->unhash(new_sk
);
675 sock_graft(new_sk
, new_sock
);
681 /* clean up for a created but never accepted sock */
682 void smc_close_non_accepted(struct sock
*sk
)
684 struct smc_sock
*smc
= smc_sk(sk
);
688 if (!sk
->sk_lingertime
)
689 /* wait for peer closing */
690 sk
->sk_lingertime
= SMC_MAX_STREAM_WAIT_TIMEOUT
;
691 if (smc
->use_fallback
) {
692 sk
->sk_state
= SMC_CLOSED
;
694 smc_close_active(smc
);
695 sock_set_flag(sk
, SOCK_DEAD
);
696 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
705 if (smc
->use_fallback
) {
706 schedule_delayed_work(&smc
->sock_put_work
, TCP_TIMEWAIT_LEN
);
707 } else if (sk
->sk_state
== SMC_CLOSED
) {
708 smc_conn_free(&smc
->conn
);
709 schedule_delayed_work(&smc
->sock_put_work
,
710 SMC_CLOSE_SOCK_PUT_DELAY
);
716 static int smc_serv_conf_first_link(struct smc_sock
*smc
)
718 struct smc_link_group
*lgr
= smc
->conn
.lgr
;
719 struct smc_link
*link
;
723 link
= &lgr
->lnk
[SMC_SINGLE_LINK
];
725 rc
= smc_wr_reg_send(link
,
726 smc
->conn
.rmb_desc
->mr_rx
[SMC_SINGLE_LINK
]);
728 return SMC_CLC_DECL_INTERR
;
730 /* send CONFIRM LINK request to client over the RoCE fabric */
731 rc
= smc_llc_send_confirm_link(link
,
732 link
->smcibdev
->mac
[link
->ibport
- 1],
733 &link
->smcibdev
->gid
[link
->ibport
- 1],
736 return SMC_CLC_DECL_TCL
;
738 /* receive CONFIRM LINK response from client over the RoCE fabric */
739 rest
= wait_for_completion_interruptible_timeout(
740 &link
->llc_confirm_resp
,
741 SMC_LLC_WAIT_FIRST_TIME
);
743 struct smc_clc_msg_decline dclc
;
745 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
752 /* setup for RDMA connection of server */
753 static void smc_listen_work(struct work_struct
*work
)
755 struct smc_sock
*new_smc
= container_of(work
, struct smc_sock
,
757 struct socket
*newclcsock
= new_smc
->clcsock
;
758 struct smc_sock
*lsmc
= new_smc
->listen_smc
;
759 struct smc_clc_msg_accept_confirm cclc
;
760 int local_contact
= SMC_REUSE_CONTACT
;
761 struct sock
*newsmcsk
= &new_smc
->sk
;
762 struct smc_clc_msg_proposal pclc
;
763 struct smc_ib_device
*smcibdev
;
764 struct sockaddr_in peeraddr
;
765 struct smc_link
*link
;
772 /* do inband token exchange -
773 *wait for and receive SMC Proposal CLC message
775 reason_code
= smc_clc_wait_msg(new_smc
, &pclc
, sizeof(pclc
),
782 /* IPSec connections opt out of SMC-R optimizations */
783 if (using_ipsec(new_smc
)) {
784 reason_code
= SMC_CLC_DECL_IPSEC
;
788 /* PNET table look up: search active ib_device and port
789 * within same PNETID that also contains the ethernet device
790 * used for the internal TCP socket
792 smc_pnet_find_roce_resource(newclcsock
->sk
, &smcibdev
, &ibport
);
794 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
798 /* determine subnet and mask from internal TCP socket */
799 rc
= smc_netinfo_by_tcpsk(newclcsock
, &subnet
, &prefix_len
);
801 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
804 if ((pclc
.outgoing_subnet
!= subnet
) ||
805 (pclc
.prefix_len
!= prefix_len
)) {
806 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
810 /* get address of the peer connected to the internal TCP socket */
811 kernel_getpeername(newclcsock
, (struct sockaddr
*)&peeraddr
, &len
);
813 /* allocate connection / link group */
814 mutex_lock(&smc_create_lgr_pending
);
815 local_contact
= smc_conn_create(new_smc
, peeraddr
.sin_addr
.s_addr
,
816 smcibdev
, ibport
, &pclc
.lcl
, 0);
817 if (local_contact
< 0) {
820 reason_code
= SMC_CLC_DECL_MEM
;/* insufficient memory*/
823 link
= &new_smc
->conn
.lgr
->lnk
[SMC_SINGLE_LINK
];
825 /* create send buffer and rmb */
826 rc
= smc_buf_create(new_smc
);
828 reason_code
= SMC_CLC_DECL_MEM
;
832 smc_close_init(new_smc
);
833 smc_rx_init(new_smc
);
835 if (local_contact
!= SMC_FIRST_CONTACT
) {
836 struct smc_buf_desc
*buf_desc
= new_smc
->conn
.rmb_desc
;
838 if (!buf_desc
->reused
) {
839 /* register memory region for new rmb */
840 rc
= smc_wr_reg_send(link
,
841 buf_desc
->mr_rx
[SMC_SINGLE_LINK
]);
843 reason_code
= SMC_CLC_DECL_INTERR
;
848 smc_rmb_sync_sg_for_device(&new_smc
->conn
);
850 rc
= smc_clc_send_accept(new_smc
, local_contact
);
854 /* receive SMC Confirm CLC message */
855 reason_code
= smc_clc_wait_msg(new_smc
, &cclc
, sizeof(cclc
),
861 smc_conn_save_peer_info(new_smc
, &cclc
);
862 if (local_contact
== SMC_FIRST_CONTACT
)
863 smc_link_save_peer_info(link
, &cclc
);
865 rc
= smc_rmb_rtoken_handling(&new_smc
->conn
, &cclc
);
867 reason_code
= SMC_CLC_DECL_INTERR
;
871 if (local_contact
== SMC_FIRST_CONTACT
) {
872 rc
= smc_ib_ready_link(link
);
874 reason_code
= SMC_CLC_DECL_INTERR
;
877 /* QP confirmation over RoCE fabric */
878 reason_code
= smc_serv_conf_first_link(new_smc
);
879 if (reason_code
< 0) {
880 /* peer is not aware of a problem */
888 smc_tx_init(new_smc
);
891 sk_refcnt_debug_inc(newsmcsk
);
892 if (newsmcsk
->sk_state
== SMC_INIT
)
893 newsmcsk
->sk_state
= SMC_ACTIVE
;
895 mutex_unlock(&smc_create_lgr_pending
);
896 lock_sock_nested(&lsmc
->sk
, SINGLE_DEPTH_NESTING
);
897 if (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
898 smc_accept_enqueue(&lsmc
->sk
, newsmcsk
);
899 } else { /* no longer listening */
900 smc_close_non_accepted(newsmcsk
);
902 release_sock(&lsmc
->sk
);
905 lsmc
->sk
.sk_data_ready(&lsmc
->sk
);
906 sock_put(&lsmc
->sk
); /* sock_hold in smc_tcp_listen_work */
910 /* RDMA setup failed, switch back to TCP */
911 smc_conn_free(&new_smc
->conn
);
912 new_smc
->use_fallback
= true;
913 if (reason_code
&& (reason_code
!= SMC_CLC_DECL_REPLY
)) {
914 rc
= smc_clc_send_decline(new_smc
, reason_code
);
915 if (rc
< sizeof(struct smc_clc_msg_decline
))
921 newsmcsk
->sk_state
= SMC_CLOSED
;
922 smc_conn_free(&new_smc
->conn
);
923 goto enqueue
; /* queue new sock with sk_err set */
926 static void smc_tcp_listen_work(struct work_struct
*work
)
928 struct smc_sock
*lsmc
= container_of(work
, struct smc_sock
,
930 struct smc_sock
*new_smc
;
933 lock_sock(&lsmc
->sk
);
934 while (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
935 rc
= smc_clcsock_accept(lsmc
, &new_smc
);
941 new_smc
->listen_smc
= lsmc
;
942 new_smc
->use_fallback
= false; /* assume rdma capability first*/
943 sock_hold(&lsmc
->sk
); /* sock_put in smc_listen_work */
944 INIT_WORK(&new_smc
->smc_listen_work
, smc_listen_work
);
945 smc_copy_sock_settings_to_smc(new_smc
);
946 schedule_work(&new_smc
->smc_listen_work
);
950 release_sock(&lsmc
->sk
);
951 lsmc
->sk
.sk_data_ready(&lsmc
->sk
); /* no more listening, wake accept */
954 static int smc_listen(struct socket
*sock
, int backlog
)
956 struct sock
*sk
= sock
->sk
;
957 struct smc_sock
*smc
;
964 if ((sk
->sk_state
!= SMC_INIT
) && (sk
->sk_state
!= SMC_LISTEN
))
968 if (sk
->sk_state
== SMC_LISTEN
) {
969 sk
->sk_max_ack_backlog
= backlog
;
972 /* some socket options are handled in core, so we could not apply
973 * them to the clc socket -- copy smc socket options to clc socket
975 smc_copy_sock_settings_to_clc(smc
);
977 rc
= kernel_listen(smc
->clcsock
, backlog
);
980 sk
->sk_max_ack_backlog
= backlog
;
981 sk
->sk_ack_backlog
= 0;
982 sk
->sk_state
= SMC_LISTEN
;
983 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
984 schedule_work(&smc
->tcp_listen_work
);
991 static int smc_accept(struct socket
*sock
, struct socket
*new_sock
,
992 int flags
, bool kern
)
994 struct sock
*sk
= sock
->sk
, *nsk
;
995 DECLARE_WAITQUEUE(wait
, current
);
996 struct smc_sock
*lsmc
;
1003 if (lsmc
->sk
.sk_state
!= SMC_LISTEN
) {
1008 /* Wait for an incoming connection */
1009 timeo
= sock_rcvtimeo(sk
, flags
& O_NONBLOCK
);
1010 add_wait_queue_exclusive(sk_sleep(sk
), &wait
);
1011 while (!(nsk
= smc_accept_dequeue(sk
, new_sock
))) {
1012 set_current_state(TASK_INTERRUPTIBLE
);
1018 timeo
= schedule_timeout(timeo
);
1019 /* wakeup by sk_data_ready in smc_listen_work() */
1020 sched_annotate_sleep();
1022 if (signal_pending(current
)) {
1023 rc
= sock_intr_errno(timeo
);
1027 set_current_state(TASK_RUNNING
);
1028 remove_wait_queue(sk_sleep(sk
), &wait
);
1031 rc
= sock_error(nsk
);
1038 static int smc_getname(struct socket
*sock
, struct sockaddr
*addr
,
1041 struct smc_sock
*smc
;
1043 if (peer
&& (sock
->sk
->sk_state
!= SMC_ACTIVE
) &&
1044 (sock
->sk
->sk_state
!= SMC_APPCLOSEWAIT1
))
1047 smc
= smc_sk(sock
->sk
);
1049 return smc
->clcsock
->ops
->getname(smc
->clcsock
, addr
, len
, peer
);
1052 static int smc_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
)
1054 struct sock
*sk
= sock
->sk
;
1055 struct smc_sock
*smc
;
1060 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
1061 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1062 (sk
->sk_state
!= SMC_INIT
))
1064 if (smc
->use_fallback
)
1065 rc
= smc
->clcsock
->ops
->sendmsg(smc
->clcsock
, msg
, len
);
1067 rc
= smc_tx_sendmsg(smc
, msg
, len
);
1073 static int smc_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
,
1076 struct sock
*sk
= sock
->sk
;
1077 struct smc_sock
*smc
;
1082 if ((sk
->sk_state
== SMC_INIT
) ||
1083 (sk
->sk_state
== SMC_LISTEN
) ||
1084 (sk
->sk_state
== SMC_CLOSED
))
1087 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
1092 if (smc
->use_fallback
)
1093 rc
= smc
->clcsock
->ops
->recvmsg(smc
->clcsock
, msg
, len
, flags
);
1095 rc
= smc_rx_recvmsg(smc
, msg
, len
, flags
);
1102 static unsigned int smc_accept_poll(struct sock
*parent
)
1104 struct smc_sock
*isk
;
1108 list_for_each_entry(isk
, &smc_sk(parent
)->accept_q
, accept_q
) {
1109 sk
= (struct sock
*)isk
;
1111 if (sk
->sk_state
== SMC_ACTIVE
) {
1112 release_sock(parent
);
1113 return POLLIN
| POLLRDNORM
;
1116 release_sock(parent
);
1121 static unsigned int smc_poll(struct file
*file
, struct socket
*sock
,
1124 struct sock
*sk
= sock
->sk
;
1125 unsigned int mask
= 0;
1126 struct smc_sock
*smc
;
1129 smc
= smc_sk(sock
->sk
);
1130 if ((sk
->sk_state
== SMC_INIT
) || smc
->use_fallback
) {
1131 /* delegate to CLC child sock */
1132 mask
= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
, wait
);
1133 /* if non-blocking connect finished ... */
1135 if ((sk
->sk_state
== SMC_INIT
) && (mask
& POLLOUT
)) {
1136 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
1140 rc
= smc_connect_rdma(smc
);
1144 /* success cases including fallback */
1145 mask
|= POLLOUT
| POLLWRNORM
;
1150 sock_poll_wait(file
, sk_sleep(sk
), wait
);
1151 if (sk
->sk_state
== SMC_LISTEN
)
1152 /* woken up by sk_data_ready in smc_listen_work() */
1153 mask
|= smc_accept_poll(sk
);
1156 if (atomic_read(&smc
->conn
.sndbuf_space
) ||
1157 (sk
->sk_shutdown
& SEND_SHUTDOWN
)) {
1158 mask
|= POLLOUT
| POLLWRNORM
;
1160 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
1161 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
1163 if (atomic_read(&smc
->conn
.bytes_to_rcv
))
1164 mask
|= POLLIN
| POLLRDNORM
;
1165 if ((sk
->sk_shutdown
== SHUTDOWN_MASK
) ||
1166 (sk
->sk_state
== SMC_CLOSED
))
1168 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
1169 mask
|= POLLIN
| POLLRDNORM
| POLLRDHUP
;
1170 if (sk
->sk_state
== SMC_APPCLOSEWAIT1
)
1178 static int smc_shutdown(struct socket
*sock
, int how
)
1180 struct sock
*sk
= sock
->sk
;
1181 struct smc_sock
*smc
;
1187 if ((how
< SHUT_RD
) || (how
> SHUT_RDWR
))
1193 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
1194 (sk
->sk_state
!= SMC_PEERCLOSEWAIT1
) &&
1195 (sk
->sk_state
!= SMC_PEERCLOSEWAIT2
) &&
1196 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1197 (sk
->sk_state
!= SMC_APPCLOSEWAIT2
) &&
1198 (sk
->sk_state
!= SMC_APPFINCLOSEWAIT
))
1200 if (smc
->use_fallback
) {
1201 rc
= kernel_sock_shutdown(smc
->clcsock
, how
);
1202 sk
->sk_shutdown
= smc
->clcsock
->sk
->sk_shutdown
;
1203 if (sk
->sk_shutdown
== SHUTDOWN_MASK
)
1204 sk
->sk_state
= SMC_CLOSED
;
1208 case SHUT_RDWR
: /* shutdown in both directions */
1209 rc
= smc_close_active(smc
);
1212 rc
= smc_close_shutdown_write(smc
);
1216 /* nothing more to do because peer is not involved */
1220 rc1
= kernel_sock_shutdown(smc
->clcsock
, how
);
1221 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1222 sk
->sk_shutdown
|= how
+ 1;
1226 return rc
? rc
: rc1
;
1229 static int smc_setsockopt(struct socket
*sock
, int level
, int optname
,
1230 char __user
*optval
, unsigned int optlen
)
1232 struct sock
*sk
= sock
->sk
;
1233 struct smc_sock
*smc
;
1237 /* generic setsockopts reaching us here always apply to the
1240 return smc
->clcsock
->ops
->setsockopt(smc
->clcsock
, level
, optname
,
1244 static int smc_getsockopt(struct socket
*sock
, int level
, int optname
,
1245 char __user
*optval
, int __user
*optlen
)
1247 struct smc_sock
*smc
;
1249 smc
= smc_sk(sock
->sk
);
1250 /* socket options apply to the CLC socket */
1251 return smc
->clcsock
->ops
->getsockopt(smc
->clcsock
, level
, optname
,
1255 static int smc_ioctl(struct socket
*sock
, unsigned int cmd
,
1258 struct smc_sock
*smc
;
1260 smc
= smc_sk(sock
->sk
);
1261 if (smc
->use_fallback
)
1262 return smc
->clcsock
->ops
->ioctl(smc
->clcsock
, cmd
, arg
);
1264 return sock_no_ioctl(sock
, cmd
, arg
);
1267 static ssize_t
smc_sendpage(struct socket
*sock
, struct page
*page
,
1268 int offset
, size_t size
, int flags
)
1270 struct sock
*sk
= sock
->sk
;
1271 struct smc_sock
*smc
;
1276 if (sk
->sk_state
!= SMC_ACTIVE
) {
1281 if (smc
->use_fallback
)
1282 rc
= kernel_sendpage(smc
->clcsock
, page
, offset
,
1285 rc
= sock_no_sendpage(sock
, page
, offset
, size
, flags
);
1291 static ssize_t
smc_splice_read(struct socket
*sock
, loff_t
*ppos
,
1292 struct pipe_inode_info
*pipe
, size_t len
,
1295 struct sock
*sk
= sock
->sk
;
1296 struct smc_sock
*smc
;
1301 if ((sk
->sk_state
!= SMC_ACTIVE
) && (sk
->sk_state
!= SMC_CLOSED
))
1303 if (smc
->use_fallback
) {
1304 rc
= smc
->clcsock
->ops
->splice_read(smc
->clcsock
, ppos
,
1314 /* must look like tcp */
1315 static const struct proto_ops smc_sock_ops
= {
1317 .owner
= THIS_MODULE
,
1318 .release
= smc_release
,
1320 .connect
= smc_connect
,
1321 .socketpair
= sock_no_socketpair
,
1322 .accept
= smc_accept
,
1323 .getname
= smc_getname
,
1326 .listen
= smc_listen
,
1327 .shutdown
= smc_shutdown
,
1328 .setsockopt
= smc_setsockopt
,
1329 .getsockopt
= smc_getsockopt
,
1330 .sendmsg
= smc_sendmsg
,
1331 .recvmsg
= smc_recvmsg
,
1332 .mmap
= sock_no_mmap
,
1333 .sendpage
= smc_sendpage
,
1334 .splice_read
= smc_splice_read
,
1337 static int smc_create(struct net
*net
, struct socket
*sock
, int protocol
,
1340 struct smc_sock
*smc
;
1344 rc
= -ESOCKTNOSUPPORT
;
1345 if (sock
->type
!= SOCK_STREAM
)
1348 rc
= -EPROTONOSUPPORT
;
1349 if ((protocol
!= IPPROTO_IP
) && (protocol
!= IPPROTO_TCP
))
1353 sock
->ops
= &smc_sock_ops
;
1354 sk
= smc_sock_alloc(net
, sock
);
1358 /* create internal TCP socket for CLC handshake and fallback */
1360 smc
->use_fallback
= false; /* assume rdma capability first */
1361 rc
= sock_create_kern(net
, PF_INET
, SOCK_STREAM
,
1362 IPPROTO_TCP
, &smc
->clcsock
);
1364 sk_common_release(sk
);
1367 smc
->sk
.sk_sndbuf
= max(smc
->clcsock
->sk
->sk_sndbuf
, SMC_BUF_MIN_SIZE
);
1368 smc
->sk
.sk_rcvbuf
= max(smc
->clcsock
->sk
->sk_rcvbuf
, SMC_BUF_MIN_SIZE
);
1374 static const struct net_proto_family smc_sock_family_ops
= {
1376 .owner
= THIS_MODULE
,
1377 .create
= smc_create
,
1380 static int __init
smc_init(void)
1384 rc
= smc_pnet_init();
1388 rc
= smc_llc_init();
1390 pr_err("%s: smc_llc_init fails with %d\n", __func__
, rc
);
1394 rc
= smc_cdc_init();
1396 pr_err("%s: smc_cdc_init fails with %d\n", __func__
, rc
);
1400 rc
= proto_register(&smc_proto
, 1);
1402 pr_err("%s: proto_register fails with %d\n", __func__
, rc
);
1406 rc
= sock_register(&smc_sock_family_ops
);
1408 pr_err("%s: sock_register fails with %d\n", __func__
, rc
);
1411 INIT_HLIST_HEAD(&smc_v4_hashinfo
.ht
);
1413 rc
= smc_ib_register_client();
1415 pr_err("%s: ib_register fails with %d\n", __func__
, rc
);
1422 sock_unregister(PF_SMC
);
1424 proto_unregister(&smc_proto
);
1430 static void __exit
smc_exit(void)
1432 struct smc_link_group
*lgr
, *lg
;
1433 LIST_HEAD(lgr_freeing_list
);
1435 spin_lock_bh(&smc_lgr_list
.lock
);
1436 if (!list_empty(&smc_lgr_list
.list
))
1437 list_splice_init(&smc_lgr_list
.list
, &lgr_freeing_list
);
1438 spin_unlock_bh(&smc_lgr_list
.lock
);
1439 list_for_each_entry_safe(lgr
, lg
, &lgr_freeing_list
, list
) {
1440 list_del_init(&lgr
->list
);
1441 smc_lgr_free(lgr
); /* free link group */
1443 smc_ib_unregister_client();
1444 sock_unregister(PF_SMC
);
1445 proto_unregister(&smc_proto
);
1449 module_init(smc_init
);
1450 module_exit(smc_exit
);
1452 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1453 MODULE_DESCRIPTION("smc socket address family");
1454 MODULE_LICENSE("GPL");
1455 MODULE_ALIAS_NETPROTO(PF_SMC
);