1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
5 * Definitions for SMC Connections, Link Groups and Links
7 * Copyright IBM Corp. 2016
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
15 #include <linux/atomic.h>
16 #include <linux/smc.h>
17 #include <linux/pci.h>
18 #include <rdma/ib_verbs.h>
19 #include <net/genetlink.h>
24 #define SMC_RMBS_PER_LGR_MAX 255 /* max. # of RMBs per link group */
26 struct smc_lgr_list
{ /* list of link group definition */
27 struct list_head list
;
28 spinlock_t lock
; /* protects list of link groups */
29 u32 num
; /* unique link group number */
32 enum smc_lgr_role
{ /* possible roles of a link group */
33 SMC_CLNT
, /* client */
37 enum smc_link_state
{ /* possible states of a link */
38 SMC_LNK_UNUSED
, /* link is unused */
39 SMC_LNK_INACTIVE
, /* link is inactive */
40 SMC_LNK_ACTIVATING
, /* link is being activated */
41 SMC_LNK_ACTIVE
, /* link is active */
44 #define SMC_WR_BUF_SIZE 48 /* size of work request buffer */
45 #define SMC_WR_BUF_V2_SIZE 8192 /* size of v2 work request buffer */
48 u8 raw
[SMC_WR_BUF_SIZE
];
51 struct smc_wr_v2_buf
{
52 u8 raw
[SMC_WR_BUF_V2_SIZE
];
55 #define SMC_WR_REG_MR_WAIT_TIME (5 * HZ)/* wait time for ib_wr_reg_mr result */
57 enum smc_wr_reg_state
{
58 POSTED
, /* ib_wr_reg_mr request posted */
59 CONFIRMED
, /* ib_wr_reg_mr response: successful */
60 FAILED
/* ib_wr_reg_mr response: failure */
63 struct smc_rdma_sge
{ /* sges for RDMA writes */
64 struct ib_sge wr_tx_rdma_sge
[SMC_IB_MAX_SEND_SGE
];
67 #define SMC_MAX_RDMA_WRITES 2 /* max. # of RDMA writes per
71 struct smc_rdma_sges
{ /* sges per message send */
72 struct smc_rdma_sge tx_rdma_sge
[SMC_MAX_RDMA_WRITES
];
75 struct smc_rdma_wr
{ /* work requests per message
78 struct ib_rdma_wr wr_tx_rdma
[SMC_MAX_RDMA_WRITES
];
81 #define SMC_LGR_ID_SIZE 4
84 struct smc_ib_device
*smcibdev
; /* ib-device */
85 u8 ibport
; /* port - values 1 | 2 */
86 struct ib_pd
*roce_pd
; /* IB protection domain,
87 * unique for every RoCE QP
89 struct ib_qp
*roce_qp
; /* IB queue pair */
90 struct ib_qp_attr qp_attr
; /* IB queue pair attributes */
92 struct smc_wr_buf
*wr_tx_bufs
; /* WR send payload buffers */
93 struct ib_send_wr
*wr_tx_ibs
; /* WR send meta data */
94 struct ib_sge
*wr_tx_sges
; /* WR send gather meta data */
95 struct smc_rdma_sges
*wr_tx_rdma_sges
;/*RDMA WRITE gather meta data*/
96 struct smc_rdma_wr
*wr_tx_rdmas
; /* WR RDMA WRITE */
97 struct smc_wr_tx_pend
*wr_tx_pends
; /* WR send waiting for CQE */
98 struct completion
*wr_tx_compl
; /* WR send CQE completion */
99 /* above four vectors have wr_tx_cnt elements and use the same index */
100 struct ib_send_wr
*wr_tx_v2_ib
; /* WR send v2 meta data */
101 struct ib_sge
*wr_tx_v2_sge
; /* WR send v2 gather meta data*/
102 struct smc_wr_tx_pend
*wr_tx_v2_pend
; /* WR send v2 waiting for CQE */
103 dma_addr_t wr_tx_dma_addr
; /* DMA address of wr_tx_bufs */
104 dma_addr_t wr_tx_v2_dma_addr
; /* DMA address of v2 tx buf*/
105 atomic_long_t wr_tx_id
; /* seq # of last sent WR */
106 unsigned long *wr_tx_mask
; /* bit mask of used indexes */
107 u32 wr_tx_cnt
; /* number of WR send buffers */
108 wait_queue_head_t wr_tx_wait
; /* wait for free WR send buf */
109 atomic_t wr_tx_refcnt
; /* tx refs to link */
111 struct smc_wr_buf
*wr_rx_bufs
; /* WR recv payload buffers */
112 struct ib_recv_wr
*wr_rx_ibs
; /* WR recv meta data */
113 struct ib_sge
*wr_rx_sges
; /* WR recv scatter meta data */
114 /* above three vectors have wr_rx_cnt elements and use the same index */
115 dma_addr_t wr_rx_dma_addr
; /* DMA address of wr_rx_bufs */
116 dma_addr_t wr_rx_v2_dma_addr
; /* DMA address of v2 rx buf*/
117 u64 wr_rx_id
; /* seq # of last recv WR */
118 u32 wr_rx_cnt
; /* number of WR recv buffers */
119 unsigned long wr_rx_tstamp
; /* jiffies when last buf rx */
121 struct ib_reg_wr wr_reg
; /* WR register memory region */
122 wait_queue_head_t wr_reg_wait
; /* wait for wr_reg result */
123 atomic_t wr_reg_refcnt
; /* reg refs to link */
124 enum smc_wr_reg_state wr_reg_state
; /* state of wr_reg request */
126 u8 gid
[SMC_GID_SIZE
];/* gid matching used vlan id*/
127 u8 sgid_index
; /* gid index for vlan id */
128 u32 peer_qpn
; /* QP number of peer */
129 enum ib_mtu path_mtu
; /* used mtu */
130 enum ib_mtu peer_mtu
; /* mtu size of peer */
131 u32 psn_initial
; /* QP tx initial packet seqno */
132 u32 peer_psn
; /* QP rx initial packet seqno */
133 u8 peer_mac
[ETH_ALEN
]; /* = gid[8:10||13:15] */
134 u8 peer_gid
[SMC_GID_SIZE
]; /* gid of peer*/
135 u8 link_id
; /* unique # within link group */
136 u8 link_uid
[SMC_LGR_ID_SIZE
]; /* unique lnk id */
137 u8 peer_link_uid
[SMC_LGR_ID_SIZE
]; /* peer uid */
138 u8 link_idx
; /* index in lgr link array */
139 u8 link_is_asym
; /* is link asymmetric? */
140 u8 clearing
: 1; /* link is being cleared */
141 refcount_t refcnt
; /* link reference count */
142 struct smc_link_group
*lgr
; /* parent link group */
143 struct work_struct link_down_wrk
; /* wrk to bring link down */
144 char ibname
[IB_DEVICE_NAME_MAX
]; /* ib device name */
145 int ndev_ifidx
; /* network device ifindex */
147 enum smc_link_state state
; /* state of link */
148 struct delayed_work llc_testlink_wrk
; /* testlink worker */
149 struct completion llc_testlink_resp
; /* wait for rx of testlink */
150 int llc_testlink_time
; /* testlink interval */
151 atomic_t conn_cnt
; /* connections on this link */
154 /* For now we just allow one parallel link per link group. The SMC protocol
155 * allows more (up to 8).
157 #define SMC_LINKS_PER_LGR_MAX 3
158 #define SMC_SINGLE_LINK 0
160 /* tx/rx buffer list element for sndbufs list and rmbs list of a lgr */
161 struct smc_buf_desc
{
162 struct list_head list
;
163 void *cpu_addr
; /* virtual address of buffer */
165 int len
; /* length of buffer */
166 u32 used
; /* currently used / unused */
169 struct sg_table sgt
[SMC_LINKS_PER_LGR_MAX
];
171 struct ib_mr
*mr_rx
[SMC_LINKS_PER_LGR_MAX
];
172 /* for rmb only: memory region
173 * incl. rkey provided to peer
175 u32 order
; /* allocation order */
178 /* confirm_rkey done */
179 u8 is_reg_mr
[SMC_LINKS_PER_LGR_MAX
];
180 /* mem region registered */
181 u8 is_map_ib
[SMC_LINKS_PER_LGR_MAX
];
182 /* mem region mapped to lnk */
184 /* buffer registration err */
187 unsigned short sba_idx
;
188 /* SBA index number */
190 /* DMB token number */
197 struct smc_rtoken
{ /* address/key of remote RMB */
202 #define SMC_BUF_MIN_SIZE 16384 /* minimum size of an RMB */
203 #define SMC_RMBE_SIZES 16 /* number of distinct RMBE sizes */
204 /* theoretically, the RFC states that largest size would be 512K,
205 * i.e. compressed 5 and thus 6 sizes (0..5), despite
206 * struct smc_clc_msg_accept_confirm.rmbe_size being a 4 bit value (0..15)
211 enum smc_lgr_type
{ /* redundancy state of lgr */
212 SMC_LGR_NONE
, /* no active links, lgr to be deleted */
213 SMC_LGR_SINGLE
, /* 1 active RNIC on each peer */
214 SMC_LGR_SYMMETRIC
, /* 2 active RNICs on each peer */
215 SMC_LGR_ASYMMETRIC_PEER
, /* local has 2, peer 1 active RNICs */
216 SMC_LGR_ASYMMETRIC_LOCAL
, /* local has 1, peer 2 active RNICs */
219 enum smc_llc_flowtype
{
220 SMC_LLC_FLOW_NONE
= 0,
221 SMC_LLC_FLOW_ADD_LINK
= 2,
222 SMC_LLC_FLOW_DEL_LINK
= 4,
223 SMC_LLC_FLOW_REQ_ADD_LINK
= 5,
224 SMC_LLC_FLOW_RKEY
= 6,
227 struct smc_llc_qentry
;
229 struct smc_llc_flow
{
230 enum smc_llc_flowtype type
;
231 struct smc_llc_qentry
*qentry
;
234 struct smc_link_group
{
235 struct list_head list
;
236 struct rb_root conns_all
; /* connection tree */
237 rwlock_t conns_lock
; /* protects conns_all */
238 unsigned int conns_num
; /* current # of connections */
239 unsigned short vlan_id
; /* vlan id of link group */
241 struct list_head sndbufs
[SMC_RMBE_SIZES
];/* tx buffers */
242 struct mutex sndbufs_lock
; /* protects tx buffers */
243 struct list_head rmbs
[SMC_RMBE_SIZES
]; /* rx buffers */
244 struct mutex rmbs_lock
; /* protects rx buffers */
246 u8 id
[SMC_LGR_ID_SIZE
]; /* unique lgr id */
247 struct delayed_work free_work
; /* delayed freeing of an lgr */
248 struct work_struct terminate_work
; /* abnormal lgr termination */
249 struct workqueue_struct
*tx_wq
; /* wq for conn. tx workers */
250 u8 sync_err
: 1; /* lgr no longer fits to peer */
251 u8 terminating
: 1;/* lgr is terminating */
252 u8 freeing
: 1; /* lgr is being freed */
254 refcount_t refcnt
; /* lgr reference count */
255 bool is_smcd
; /* SMC-R or SMC-D */
257 u8 negotiated_eid
[SMC_MAX_EID_LEN
];
258 u8 peer_os
; /* peer operating system */
260 u8 peer_hostname
[SMC_MAX_HOSTNAME_LEN
];
263 enum smc_lgr_role role
;
264 /* client or server */
265 struct smc_link lnk
[SMC_LINKS_PER_LGR_MAX
];
267 struct smc_wr_v2_buf
*wr_rx_buf_v2
;
268 /* WR v2 recv payload buffer */
269 struct smc_wr_v2_buf
*wr_tx_buf_v2
;
270 /* WR v2 send payload buffer */
271 char peer_systemid
[SMC_SYSTEMID_LEN
];
272 /* unique system_id of peer */
273 struct smc_rtoken rtokens
[SMC_RMBS_PER_LGR_MAX
]
274 [SMC_LINKS_PER_LGR_MAX
];
275 /* remote addr/key pairs */
276 DECLARE_BITMAP(rtokens_used_mask
, SMC_RMBS_PER_LGR_MAX
);
277 /* used rtoken elements */
279 enum smc_lgr_type type
;
280 /* redundancy state */
281 u8 pnet_id
[SMC_MAX_PNETID_LEN
+ 1];
282 /* pnet id of this lgr */
283 struct list_head llc_event_q
;
284 /* queue for llc events */
285 spinlock_t llc_event_q_lock
;
286 /* protects llc_event_q */
287 struct mutex llc_conf_mutex
;
288 /* protects lgr reconfig. */
289 struct work_struct llc_add_link_work
;
290 struct work_struct llc_del_link_work
;
291 struct work_struct llc_event_work
;
292 /* llc event worker */
293 wait_queue_head_t llc_flow_waiter
;
294 /* w4 next llc event */
295 wait_queue_head_t llc_msg_waiter
;
296 /* w4 next llc msg */
297 struct smc_llc_flow llc_flow_lcl
;
298 /* llc local control field */
299 struct smc_llc_flow llc_flow_rmt
;
300 /* llc remote control field */
301 struct smc_llc_qentry
*delayed_event
;
302 /* arrived when flow active */
303 spinlock_t llc_flow_lock
;
304 /* protects llc flow */
305 int llc_testlink_time
;
306 /* link keep alive time */
307 u32 llc_termination_rsn
;
308 /* rsn code for termination */
309 u8 nexthop_mac
[ETH_ALEN
];
317 /* Peer GID (remote) */
318 struct smcd_dev
*smcd
;
319 /* ISM device for VLAN reg. */
320 u8 peer_shutdown
: 1;
321 /* peer triggered shutdownn */
326 struct smc_clc_msg_local
;
328 #define GID_LIST_SIZE 2
332 u8 list
[GID_LIST_SIZE
][SMC_GID_SIZE
];
335 struct smc_init_info_smcrv2
{
341 /* Output fields when saddr is set */
342 struct smc_ib_device
*ib_dev_v2
;
344 u8 ib_gid_v2
[SMC_GID_SIZE
];
346 /* Additional output fields when clc_sk and daddr is set as well */
348 u8 nexthop_mac
[ETH_ALEN
];
350 struct smc_gidlist gidlist
;
353 struct smc_init_info
{
357 u8 first_contact_peer
;
358 u8 first_contact_local
;
359 unsigned short vlan_id
;
361 u8 negotiated_eid
[SMC_MAX_EID_LEN
];
365 u8 peer_gid
[SMC_GID_SIZE
];
366 u8 peer_mac
[ETH_ALEN
];
367 u8 peer_systemid
[SMC_SYSTEMID_LEN
];
368 struct smc_ib_device
*ib_dev
;
369 u8 ib_gid
[SMC_GID_SIZE
];
372 struct smc_init_info_smcrv2 smcrv2
;
374 u64 ism_peer_gid
[SMC_MAX_ISM_DEVS
+ 1];
375 struct smcd_dev
*ism_dev
[SMC_MAX_ISM_DEVS
+ 1];
376 u16 ism_chid
[SMC_MAX_ISM_DEVS
+ 1];
377 u8 ism_offered_cnt
; /* # of ISM devices offered */
378 u8 ism_selected
; /* index of selected ISM dev*/
382 /* Find the connection associated with the given alert token in the link group.
383 * To use rbtrees we have to implement our own search core.
384 * Requires @conns_lock
385 * @token alert token to search for
386 * @lgr link group to search in
387 * Returns connection associated with token if found, NULL otherwise.
389 static inline struct smc_connection
*smc_lgr_find_conn(
390 u32 token
, struct smc_link_group
*lgr
)
392 struct smc_connection
*res
= NULL
;
393 struct rb_node
*node
;
395 node
= lgr
->conns_all
.rb_node
;
397 struct smc_connection
*cur
= rb_entry(node
,
398 struct smc_connection
, alert_node
);
400 if (cur
->alert_token_local
> token
) {
401 node
= node
->rb_left
;
403 if (cur
->alert_token_local
< token
) {
404 node
= node
->rb_right
;
415 static inline bool smc_conn_lgr_valid(struct smc_connection
*conn
)
417 return conn
->lgr
&& conn
->alert_token_local
;
421 * Returns true if the specified link is usable.
423 * usable means the link is ready to receive RDMA messages, map memory
424 * on the link, etc. This doesn't ensure we are able to send RDMA messages
425 * on this link, if sending RDMA messages is needed, use smc_link_sendable()
427 static inline bool smc_link_usable(struct smc_link
*lnk
)
429 if (lnk
->state
== SMC_LNK_UNUSED
|| lnk
->state
== SMC_LNK_INACTIVE
)
435 * Returns true if the specified link is ready to receive AND send RDMA
438 * For the client side in first contact, the underlying QP may still in
439 * RESET or RTR when the link state is ACTIVATING, checks in smc_link_usable()
440 * is not strong enough. For those places that need to send any CDC or LLC
441 * messages, use smc_link_sendable(), otherwise, use smc_link_usable() instead
443 static inline bool smc_link_sendable(struct smc_link
*lnk
)
445 return smc_link_usable(lnk
) &&
446 lnk
->qp_attr
.cur_qp_state
== IB_QPS_RTS
;
449 static inline bool smc_link_active(struct smc_link
*lnk
)
451 return lnk
->state
== SMC_LNK_ACTIVE
;
454 static inline void smc_gid_be16_convert(__u8
*buf
, u8
*gid_raw
)
456 sprintf(buf
, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x",
457 be16_to_cpu(((__be16
*)gid_raw
)[0]),
458 be16_to_cpu(((__be16
*)gid_raw
)[1]),
459 be16_to_cpu(((__be16
*)gid_raw
)[2]),
460 be16_to_cpu(((__be16
*)gid_raw
)[3]),
461 be16_to_cpu(((__be16
*)gid_raw
)[4]),
462 be16_to_cpu(((__be16
*)gid_raw
)[5]),
463 be16_to_cpu(((__be16
*)gid_raw
)[6]),
464 be16_to_cpu(((__be16
*)gid_raw
)[7]));
472 __u8 pci_id
[SMC_PCI_ID_STR_LEN
];
475 static inline void smc_set_pci_values(struct pci_dev
*pci_dev
,
476 struct smc_pci_dev
*smc_dev
)
478 smc_dev
->pci_vendor
= pci_dev
->vendor
;
479 smc_dev
->pci_device
= pci_dev
->device
;
480 snprintf(smc_dev
->pci_id
, sizeof(smc_dev
->pci_id
), "%s",
482 #if IS_ENABLED(CONFIG_S390)
483 { /* Set s390 specific PCI information */
484 struct zpci_dev
*zdev
;
486 zdev
= to_zpci(pci_dev
);
487 smc_dev
->pci_fid
= zdev
->fid
;
488 smc_dev
->pci_pchid
= zdev
->pchid
;
494 struct smc_clc_msg_accept_confirm
;
496 void smc_lgr_cleanup_early(struct smc_link_group
*lgr
);
497 void smc_lgr_terminate_sched(struct smc_link_group
*lgr
);
498 void smc_lgr_hold(struct smc_link_group
*lgr
);
499 void smc_lgr_put(struct smc_link_group
*lgr
);
500 void smcr_port_add(struct smc_ib_device
*smcibdev
, u8 ibport
);
501 void smcr_port_err(struct smc_ib_device
*smcibdev
, u8 ibport
);
502 void smc_smcd_terminate(struct smcd_dev
*dev
, u64 peer_gid
,
503 unsigned short vlan
);
504 void smc_smcd_terminate_all(struct smcd_dev
*dev
);
505 void smc_smcr_terminate_all(struct smc_ib_device
*smcibdev
);
506 int smc_buf_create(struct smc_sock
*smc
, bool is_smcd
);
507 int smc_uncompress_bufsize(u8 compressed
);
508 int smc_rmb_rtoken_handling(struct smc_connection
*conn
, struct smc_link
*link
,
509 struct smc_clc_msg_accept_confirm
*clc
);
510 int smc_rtoken_add(struct smc_link
*lnk
, __be64 nw_vaddr
, __be32 nw_rkey
);
511 int smc_rtoken_delete(struct smc_link
*lnk
, __be32 nw_rkey
);
512 void smc_rtoken_set(struct smc_link_group
*lgr
, int link_idx
, int link_idx_new
,
513 __be32 nw_rkey_known
, __be64 nw_vaddr
, __be32 nw_rkey
);
514 void smc_rtoken_set2(struct smc_link_group
*lgr
, int rtok_idx
, int link_id
,
515 __be64 nw_vaddr
, __be32 nw_rkey
);
516 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection
*conn
);
517 void smc_sndbuf_sync_sg_for_device(struct smc_connection
*conn
);
518 void smc_rmb_sync_sg_for_cpu(struct smc_connection
*conn
);
519 void smc_rmb_sync_sg_for_device(struct smc_connection
*conn
);
520 int smc_vlan_by_tcpsk(struct socket
*clcsock
, struct smc_init_info
*ini
);
522 void smc_conn_free(struct smc_connection
*conn
);
523 int smc_conn_create(struct smc_sock
*smc
, struct smc_init_info
*ini
);
524 void smc_lgr_schedule_free_work_fast(struct smc_link_group
*lgr
);
525 int smc_core_init(void);
526 void smc_core_exit(void);
528 int smcr_link_init(struct smc_link_group
*lgr
, struct smc_link
*lnk
,
529 u8 link_idx
, struct smc_init_info
*ini
);
530 void smcr_link_clear(struct smc_link
*lnk
, bool log
);
531 void smcr_link_hold(struct smc_link
*lnk
);
532 void smcr_link_put(struct smc_link
*lnk
);
533 void smc_switch_link_and_count(struct smc_connection
*conn
,
534 struct smc_link
*to_lnk
);
535 int smcr_buf_map_lgr(struct smc_link
*lnk
);
536 int smcr_buf_reg_lgr(struct smc_link
*lnk
);
537 void smcr_lgr_set_type(struct smc_link_group
*lgr
, enum smc_lgr_type new_type
);
538 void smcr_lgr_set_type_asym(struct smc_link_group
*lgr
,
539 enum smc_lgr_type new_type
, int asym_lnk_idx
);
540 int smcr_link_reg_rmb(struct smc_link
*link
, struct smc_buf_desc
*rmb_desc
);
541 struct smc_link
*smc_switch_conns(struct smc_link_group
*lgr
,
542 struct smc_link
*from_lnk
, bool is_dev_err
);
543 void smcr_link_down_cond(struct smc_link
*lnk
);
544 void smcr_link_down_cond_sched(struct smc_link
*lnk
);
545 int smc_nl_get_sys_info(struct sk_buff
*skb
, struct netlink_callback
*cb
);
546 int smcr_nl_get_lgr(struct sk_buff
*skb
, struct netlink_callback
*cb
);
547 int smcr_nl_get_link(struct sk_buff
*skb
, struct netlink_callback
*cb
);
548 int smcd_nl_get_lgr(struct sk_buff
*skb
, struct netlink_callback
*cb
);
550 static inline struct smc_link_group
*smc_get_lgr(struct smc_link
*link
)