1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
6 #include <linux/module.h>
7 #include <linux/mlx5/qp.h>
8 #include <linux/slab.h>
9 #include <rdma/ib_umem.h>
10 #include <rdma/ib_user_verbs.h>
14 static void *get_wqe(struct mlx5_ib_srq
*srq
, int n
)
16 return mlx5_frag_buf_get_wqe(&srq
->fbc
, n
);
19 static void mlx5_ib_srq_event(struct mlx5_core_srq
*srq
, enum mlx5_event type
)
21 struct ib_event event
;
22 struct ib_srq
*ibsrq
= &to_mibsrq(srq
)->ibsrq
;
24 if (ibsrq
->event_handler
) {
25 event
.device
= ibsrq
->device
;
26 event
.element
.srq
= ibsrq
;
28 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT
:
29 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
31 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR
:
32 event
.event
= IB_EVENT_SRQ_ERR
;
35 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n",
40 ibsrq
->event_handler(&event
, ibsrq
->srq_context
);
44 static int create_srq_user(struct ib_pd
*pd
, struct mlx5_ib_srq
*srq
,
45 struct mlx5_srq_attr
*in
,
46 struct ib_udata
*udata
, int buf_size
)
48 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
49 struct mlx5_ib_create_srq ucmd
= {};
56 u32 uidx
= MLX5_IB_DEFAULT_UIDX
;
58 ucmdlen
= min(udata
->inlen
, sizeof(ucmd
));
60 if (ib_copy_from_udata(&ucmd
, udata
, ucmdlen
)) {
61 mlx5_ib_dbg(dev
, "failed copy udata\n");
65 if (ucmd
.reserved0
|| ucmd
.reserved1
)
68 if (udata
->inlen
> sizeof(ucmd
) &&
69 !ib_is_udata_cleared(udata
, sizeof(ucmd
),
70 udata
->inlen
- sizeof(ucmd
)))
73 if (in
->type
!= IB_SRQT_BASIC
) {
74 err
= get_srq_user_index(to_mucontext(pd
->uobject
->context
),
75 &ucmd
, udata
->inlen
, &uidx
);
80 srq
->wq_sig
= !!(ucmd
.flags
& MLX5_SRQ_FLAG_SIGNATURE
);
82 srq
->umem
= ib_umem_get(pd
->uobject
->context
, ucmd
.buf_addr
, buf_size
,
84 if (IS_ERR(srq
->umem
)) {
85 mlx5_ib_dbg(dev
, "failed umem get, size %d\n", buf_size
);
86 err
= PTR_ERR(srq
->umem
);
90 mlx5_ib_cont_pages(srq
->umem
, ucmd
.buf_addr
, 0, &npages
,
91 &page_shift
, &ncont
, NULL
);
92 err
= mlx5_ib_get_buf_offset(ucmd
.buf_addr
, page_shift
,
95 mlx5_ib_warn(dev
, "bad offset\n");
99 in
->pas
= kvcalloc(ncont
, sizeof(*in
->pas
), GFP_KERNEL
);
105 mlx5_ib_populate_pas(dev
, srq
->umem
, page_shift
, in
->pas
, 0);
107 err
= mlx5_ib_db_map_user(to_mucontext(pd
->uobject
->context
),
108 ucmd
.db_addr
, &srq
->db
);
110 mlx5_ib_dbg(dev
, "map doorbell failed\n");
114 in
->log_page_size
= page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
115 in
->page_offset
= offset
;
116 in
->uid
= (in
->type
!= IB_SRQT_XRC
) ? to_mpd(pd
)->uid
: 0;
117 if (MLX5_CAP_GEN(dev
->mdev
, cqe_version
) == MLX5_CQE_VERSION_V1
&&
118 in
->type
!= IB_SRQT_BASIC
)
119 in
->user_index
= uidx
;
127 ib_umem_release(srq
->umem
);
132 static int create_srq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_srq
*srq
,
133 struct mlx5_srq_attr
*in
, int buf_size
)
137 struct mlx5_wqe_srq_next_seg
*next
;
139 err
= mlx5_db_alloc(dev
->mdev
, &srq
->db
);
141 mlx5_ib_warn(dev
, "alloc dbell rec failed\n");
145 if (mlx5_frag_buf_alloc_node(dev
->mdev
, buf_size
, &srq
->buf
,
146 dev
->mdev
->priv
.numa_node
)) {
147 mlx5_ib_dbg(dev
, "buf alloc failed\n");
152 mlx5_init_fbc(srq
->buf
.frags
, srq
->msrq
.wqe_shift
, ilog2(srq
->msrq
.max
),
156 srq
->tail
= srq
->msrq
.max
- 1;
159 for (i
= 0; i
< srq
->msrq
.max
; i
++) {
160 next
= get_wqe(srq
, i
);
161 next
->next_wqe_index
=
162 cpu_to_be16((i
+ 1) & (srq
->msrq
.max
- 1));
165 mlx5_ib_dbg(dev
, "srq->buf.page_shift = %d\n", srq
->buf
.page_shift
);
166 in
->pas
= kvcalloc(srq
->buf
.npages
, sizeof(*in
->pas
), GFP_KERNEL
);
171 mlx5_fill_page_frag_array(&srq
->buf
, in
->pas
);
173 srq
->wrid
= kvmalloc_array(srq
->msrq
.max
, sizeof(u64
), GFP_KERNEL
);
180 in
->log_page_size
= srq
->buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
181 if (MLX5_CAP_GEN(dev
->mdev
, cqe_version
) == MLX5_CQE_VERSION_V1
&&
182 in
->type
!= IB_SRQT_BASIC
)
183 in
->user_index
= MLX5_IB_DEFAULT_UIDX
;
191 mlx5_frag_buf_free(dev
->mdev
, &srq
->buf
);
194 mlx5_db_free(dev
->mdev
, &srq
->db
);
198 static void destroy_srq_user(struct ib_pd
*pd
, struct mlx5_ib_srq
*srq
)
200 mlx5_ib_db_unmap_user(to_mucontext(pd
->uobject
->context
), &srq
->db
);
201 ib_umem_release(srq
->umem
);
205 static void destroy_srq_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_srq
*srq
)
208 mlx5_frag_buf_free(dev
->mdev
, &srq
->buf
);
209 mlx5_db_free(dev
->mdev
, &srq
->db
);
212 struct ib_srq
*mlx5_ib_create_srq(struct ib_pd
*pd
,
213 struct ib_srq_init_attr
*init_attr
,
214 struct ib_udata
*udata
)
216 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
217 struct mlx5_ib_srq
*srq
;
221 struct mlx5_srq_attr in
= {0};
222 __u32 max_srq_wqes
= 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_srq_sz
);
224 /* Sanity check SRQ size before proceeding */
225 if (init_attr
->attr
.max_wr
>= max_srq_wqes
) {
226 mlx5_ib_dbg(dev
, "max_wr %d, cap %d\n",
227 init_attr
->attr
.max_wr
,
229 return ERR_PTR(-EINVAL
);
232 srq
= kmalloc(sizeof(*srq
), GFP_KERNEL
);
234 return ERR_PTR(-ENOMEM
);
236 mutex_init(&srq
->mutex
);
237 spin_lock_init(&srq
->lock
);
238 srq
->msrq
.max
= roundup_pow_of_two(init_attr
->attr
.max_wr
+ 1);
239 srq
->msrq
.max_gs
= init_attr
->attr
.max_sge
;
241 desc_size
= sizeof(struct mlx5_wqe_srq_next_seg
) +
242 srq
->msrq
.max_gs
* sizeof(struct mlx5_wqe_data_seg
);
243 if (desc_size
== 0 || srq
->msrq
.max_gs
> desc_size
) {
247 desc_size
= roundup_pow_of_two(desc_size
);
248 desc_size
= max_t(size_t, 32, desc_size
);
249 if (desc_size
< sizeof(struct mlx5_wqe_srq_next_seg
)) {
253 srq
->msrq
.max_avail_gather
= (desc_size
- sizeof(struct mlx5_wqe_srq_next_seg
)) /
254 sizeof(struct mlx5_wqe_data_seg
);
255 srq
->msrq
.wqe_shift
= ilog2(desc_size
);
256 buf_size
= srq
->msrq
.max
* desc_size
;
257 if (buf_size
< desc_size
) {
261 in
.type
= init_attr
->srq_type
;
264 err
= create_srq_user(pd
, srq
, &in
, udata
, buf_size
);
266 err
= create_srq_kernel(dev
, srq
, &in
, buf_size
);
269 mlx5_ib_warn(dev
, "create srq %s failed, err %d\n",
270 pd
->uobject
? "user" : "kernel", err
);
274 in
.log_size
= ilog2(srq
->msrq
.max
);
275 in
.wqe_shift
= srq
->msrq
.wqe_shift
- 4;
277 in
.flags
|= MLX5_SRQ_FLAG_WQ_SIG
;
279 if (init_attr
->srq_type
== IB_SRQT_XRC
)
280 in
.xrcd
= to_mxrcd(init_attr
->ext
.xrc
.xrcd
)->xrcdn
;
282 in
.xrcd
= to_mxrcd(dev
->devr
.x0
)->xrcdn
;
284 if (init_attr
->srq_type
== IB_SRQT_TM
) {
285 in
.tm_log_list_size
=
286 ilog2(init_attr
->ext
.tag_matching
.max_num_tags
) + 1;
287 if (in
.tm_log_list_size
>
288 MLX5_CAP_GEN(dev
->mdev
, log_tag_matching_list_sz
)) {
289 mlx5_ib_dbg(dev
, "TM SRQ max_num_tags exceeding limit\n");
291 goto err_usr_kern_srq
;
293 in
.flags
|= MLX5_SRQ_FLAG_RNDV
;
296 if (ib_srq_has_cq(init_attr
->srq_type
))
297 in
.cqn
= to_mcq(init_attr
->ext
.cq
)->mcq
.cqn
;
299 in
.cqn
= to_mcq(dev
->devr
.c0
)->mcq
.cqn
;
301 in
.pd
= to_mpd(pd
)->pdn
;
302 in
.db_record
= srq
->db
.dma
;
303 err
= mlx5_cmd_create_srq(dev
, &srq
->msrq
, &in
);
306 mlx5_ib_dbg(dev
, "create SRQ failed, err %d\n", err
);
307 goto err_usr_kern_srq
;
310 mlx5_ib_dbg(dev
, "create SRQ with srqn 0x%x\n", srq
->msrq
.srqn
);
312 srq
->msrq
.event
= mlx5_ib_srq_event
;
313 srq
->ibsrq
.ext
.xrc
.srq_num
= srq
->msrq
.srqn
;
316 if (ib_copy_to_udata(udata
, &srq
->msrq
.srqn
, sizeof(__u32
))) {
317 mlx5_ib_dbg(dev
, "copy to user failed\n");
322 init_attr
->attr
.max_wr
= srq
->msrq
.max
- 1;
327 mlx5_cmd_destroy_srq(dev
, &srq
->msrq
);
331 destroy_srq_user(pd
, srq
);
333 destroy_srq_kernel(dev
, srq
);
341 int mlx5_ib_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
342 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
)
344 struct mlx5_ib_dev
*dev
= to_mdev(ibsrq
->device
);
345 struct mlx5_ib_srq
*srq
= to_msrq(ibsrq
);
348 /* We don't support resizing SRQs yet */
349 if (attr_mask
& IB_SRQ_MAX_WR
)
352 if (attr_mask
& IB_SRQ_LIMIT
) {
353 if (attr
->srq_limit
>= srq
->msrq
.max
)
356 mutex_lock(&srq
->mutex
);
357 ret
= mlx5_cmd_arm_srq(dev
, &srq
->msrq
, attr
->srq_limit
, 1);
358 mutex_unlock(&srq
->mutex
);
367 int mlx5_ib_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
369 struct mlx5_ib_dev
*dev
= to_mdev(ibsrq
->device
);
370 struct mlx5_ib_srq
*srq
= to_msrq(ibsrq
);
372 struct mlx5_srq_attr
*out
;
374 out
= kzalloc(sizeof(*out
), GFP_KERNEL
);
378 ret
= mlx5_cmd_query_srq(dev
, &srq
->msrq
, out
);
382 srq_attr
->srq_limit
= out
->lwm
;
383 srq_attr
->max_wr
= srq
->msrq
.max
- 1;
384 srq_attr
->max_sge
= srq
->msrq
.max_gs
;
391 int mlx5_ib_destroy_srq(struct ib_srq
*srq
)
393 struct mlx5_ib_dev
*dev
= to_mdev(srq
->device
);
394 struct mlx5_ib_srq
*msrq
= to_msrq(srq
);
396 mlx5_cmd_destroy_srq(dev
, &msrq
->msrq
);
399 mlx5_ib_db_unmap_user(to_mucontext(srq
->uobject
->context
), &msrq
->db
);
400 ib_umem_release(msrq
->umem
);
402 destroy_srq_kernel(dev
, msrq
);
409 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq
*srq
, int wqe_index
)
411 struct mlx5_wqe_srq_next_seg
*next
;
413 /* always called with interrupts disabled. */
414 spin_lock(&srq
->lock
);
416 next
= get_wqe(srq
, srq
->tail
);
417 next
->next_wqe_index
= cpu_to_be16(wqe_index
);
418 srq
->tail
= wqe_index
;
420 spin_unlock(&srq
->lock
);
423 int mlx5_ib_post_srq_recv(struct ib_srq
*ibsrq
, const struct ib_recv_wr
*wr
,
424 const struct ib_recv_wr
**bad_wr
)
426 struct mlx5_ib_srq
*srq
= to_msrq(ibsrq
);
427 struct mlx5_wqe_srq_next_seg
*next
;
428 struct mlx5_wqe_data_seg
*scat
;
429 struct mlx5_ib_dev
*dev
= to_mdev(ibsrq
->device
);
430 struct mlx5_core_dev
*mdev
= dev
->mdev
;
436 spin_lock_irqsave(&srq
->lock
, flags
);
438 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
444 for (nreq
= 0; wr
; nreq
++, wr
= wr
->next
) {
445 if (unlikely(wr
->num_sge
> srq
->msrq
.max_gs
)) {
451 if (unlikely(srq
->head
== srq
->tail
)) {
457 srq
->wrid
[srq
->head
] = wr
->wr_id
;
459 next
= get_wqe(srq
, srq
->head
);
460 srq
->head
= be16_to_cpu(next
->next_wqe_index
);
461 scat
= (struct mlx5_wqe_data_seg
*)(next
+ 1);
463 for (i
= 0; i
< wr
->num_sge
; i
++) {
464 scat
[i
].byte_count
= cpu_to_be32(wr
->sg_list
[i
].length
);
465 scat
[i
].lkey
= cpu_to_be32(wr
->sg_list
[i
].lkey
);
466 scat
[i
].addr
= cpu_to_be64(wr
->sg_list
[i
].addr
);
469 if (i
< srq
->msrq
.max_avail_gather
) {
470 scat
[i
].byte_count
= 0;
471 scat
[i
].lkey
= cpu_to_be32(MLX5_INVALID_LKEY
);
477 srq
->wqe_ctr
+= nreq
;
479 /* Make sure that descriptors are written before
484 *srq
->db
.db
= cpu_to_be32(srq
->wqe_ctr
);
487 spin_unlock_irqrestore(&srq
->lock
, flags
);