2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_cache.h>
36 #include <rdma/ib_user_verbs.h>
39 /* not supported currently */
40 static int wq_signature
;
43 MLX5_IB_ACK_REQ_FREQ
= 8,
47 MLX5_IB_DEFAULT_SCHED_QUEUE
= 0x83,
48 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE
= 0x3f,
49 MLX5_IB_LINK_TYPE_IB
= 0,
50 MLX5_IB_LINK_TYPE_ETH
= 1
54 MLX5_IB_SQ_STRIDE
= 6,
57 static const u32 mlx5_ib_opcode
[] = {
58 [IB_WR_SEND
] = MLX5_OPCODE_SEND
,
59 [IB_WR_LSO
] = MLX5_OPCODE_LSO
,
60 [IB_WR_SEND_WITH_IMM
] = MLX5_OPCODE_SEND_IMM
,
61 [IB_WR_RDMA_WRITE
] = MLX5_OPCODE_RDMA_WRITE
,
62 [IB_WR_RDMA_WRITE_WITH_IMM
] = MLX5_OPCODE_RDMA_WRITE_IMM
,
63 [IB_WR_RDMA_READ
] = MLX5_OPCODE_RDMA_READ
,
64 [IB_WR_ATOMIC_CMP_AND_SWP
] = MLX5_OPCODE_ATOMIC_CS
,
65 [IB_WR_ATOMIC_FETCH_AND_ADD
] = MLX5_OPCODE_ATOMIC_FA
,
66 [IB_WR_SEND_WITH_INV
] = MLX5_OPCODE_SEND_INVAL
,
67 [IB_WR_LOCAL_INV
] = MLX5_OPCODE_UMR
,
68 [IB_WR_REG_MR
] = MLX5_OPCODE_UMR
,
69 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP
] = MLX5_OPCODE_ATOMIC_MASKED_CS
,
70 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD
] = MLX5_OPCODE_ATOMIC_MASKED_FA
,
71 [MLX5_IB_WR_UMR
] = MLX5_OPCODE_UMR
,
74 struct mlx5_wqe_eth_pad
{
78 enum raw_qp_set_mask_map
{
79 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID
= 1UL << 0,
80 MLX5_RAW_QP_RATE_LIMIT
= 1UL << 1,
83 struct mlx5_modify_raw_qp_param
{
86 u32 set_mask
; /* raw_qp_set_mask_map */
91 static void get_cqs(enum ib_qp_type qp_type
,
92 struct ib_cq
*ib_send_cq
, struct ib_cq
*ib_recv_cq
,
93 struct mlx5_ib_cq
**send_cq
, struct mlx5_ib_cq
**recv_cq
);
95 static int is_qp0(enum ib_qp_type qp_type
)
97 return qp_type
== IB_QPT_SMI
;
100 static int is_sqp(enum ib_qp_type qp_type
)
102 return is_qp0(qp_type
) || is_qp1(qp_type
);
105 static void *get_wqe(struct mlx5_ib_qp
*qp
, int offset
)
107 return mlx5_buf_offset(&qp
->buf
, offset
);
110 static void *get_recv_wqe(struct mlx5_ib_qp
*qp
, int n
)
112 return get_wqe(qp
, qp
->rq
.offset
+ (n
<< qp
->rq
.wqe_shift
));
115 void *mlx5_get_send_wqe(struct mlx5_ib_qp
*qp
, int n
)
117 return get_wqe(qp
, qp
->sq
.offset
+ (n
<< MLX5_IB_SQ_STRIDE
));
121 * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space.
123 * @qp: QP to copy from.
124 * @send: copy from the send queue when non-zero, use the receive queue
126 * @wqe_index: index to start copying from. For send work queues, the
127 * wqe_index is in units of MLX5_SEND_WQE_BB.
128 * For receive work queue, it is the number of work queue
129 * element in the queue.
130 * @buffer: destination buffer.
131 * @length: maximum number of bytes to copy.
133 * Copies at least a single WQE, but may copy more data.
135 * Return: the number of bytes copied, or an error code.
137 int mlx5_ib_read_user_wqe(struct mlx5_ib_qp
*qp
, int send
, int wqe_index
,
138 void *buffer
, u32 length
,
139 struct mlx5_ib_qp_base
*base
)
141 struct ib_device
*ibdev
= qp
->ibqp
.device
;
142 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
143 struct mlx5_ib_wq
*wq
= send
? &qp
->sq
: &qp
->rq
;
146 struct ib_umem
*umem
= base
->ubuffer
.umem
;
147 u32 first_copy_length
;
151 if (wq
->wqe_cnt
== 0) {
152 mlx5_ib_dbg(dev
, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n",
157 offset
= wq
->offset
+ ((wqe_index
% wq
->wqe_cnt
) << wq
->wqe_shift
);
158 wq_end
= wq
->offset
+ (wq
->wqe_cnt
<< wq
->wqe_shift
);
160 if (send
&& length
< sizeof(struct mlx5_wqe_ctrl_seg
))
163 if (offset
> umem
->length
||
164 (send
&& offset
+ sizeof(struct mlx5_wqe_ctrl_seg
) > umem
->length
))
167 first_copy_length
= min_t(u32
, offset
+ length
, wq_end
) - offset
;
168 ret
= ib_umem_copy_from(buffer
, umem
, offset
, first_copy_length
);
173 struct mlx5_wqe_ctrl_seg
*ctrl
= buffer
;
174 int ds
= be32_to_cpu(ctrl
->qpn_ds
) & MLX5_WQE_CTRL_DS_MASK
;
176 wqe_length
= ds
* MLX5_WQE_DS_UNITS
;
178 wqe_length
= 1 << wq
->wqe_shift
;
181 if (wqe_length
<= first_copy_length
)
182 return first_copy_length
;
184 ret
= ib_umem_copy_from(buffer
+ first_copy_length
, umem
, wq
->offset
,
185 wqe_length
- first_copy_length
);
192 static void mlx5_ib_qp_event(struct mlx5_core_qp
*qp
, int type
)
194 struct ib_qp
*ibqp
= &to_mibqp(qp
)->ibqp
;
195 struct ib_event event
;
197 if (type
== MLX5_EVENT_TYPE_PATH_MIG
) {
198 /* This event is only valid for trans_qps */
199 to_mibqp(qp
)->port
= to_mibqp(qp
)->trans_qp
.alt_port
;
202 if (ibqp
->event_handler
) {
203 event
.device
= ibqp
->device
;
204 event
.element
.qp
= ibqp
;
206 case MLX5_EVENT_TYPE_PATH_MIG
:
207 event
.event
= IB_EVENT_PATH_MIG
;
209 case MLX5_EVENT_TYPE_COMM_EST
:
210 event
.event
= IB_EVENT_COMM_EST
;
212 case MLX5_EVENT_TYPE_SQ_DRAINED
:
213 event
.event
= IB_EVENT_SQ_DRAINED
;
215 case MLX5_EVENT_TYPE_SRQ_LAST_WQE
:
216 event
.event
= IB_EVENT_QP_LAST_WQE_REACHED
;
218 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR
:
219 event
.event
= IB_EVENT_QP_FATAL
;
221 case MLX5_EVENT_TYPE_PATH_MIG_FAILED
:
222 event
.event
= IB_EVENT_PATH_MIG_ERR
;
224 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR
:
225 event
.event
= IB_EVENT_QP_REQ_ERR
;
227 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR
:
228 event
.event
= IB_EVENT_QP_ACCESS_ERR
;
231 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type
, qp
->qpn
);
235 ibqp
->event_handler(&event
, ibqp
->qp_context
);
239 static int set_rq_size(struct mlx5_ib_dev
*dev
, struct ib_qp_cap
*cap
,
240 int has_rq
, struct mlx5_ib_qp
*qp
, struct mlx5_ib_create_qp
*ucmd
)
245 /* Sanity check RQ size before proceeding */
246 if (cap
->max_recv_wr
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_qp_sz
)))
252 qp
->rq
.wqe_shift
= 0;
253 cap
->max_recv_wr
= 0;
254 cap
->max_recv_sge
= 0;
257 qp
->rq
.wqe_cnt
= ucmd
->rq_wqe_count
;
258 qp
->rq
.wqe_shift
= ucmd
->rq_wqe_shift
;
259 qp
->rq
.max_gs
= (1 << qp
->rq
.wqe_shift
) / sizeof(struct mlx5_wqe_data_seg
) - qp
->wq_sig
;
260 qp
->rq
.max_post
= qp
->rq
.wqe_cnt
;
262 wqe_size
= qp
->wq_sig
? sizeof(struct mlx5_wqe_signature_seg
) : 0;
263 wqe_size
+= cap
->max_recv_sge
* sizeof(struct mlx5_wqe_data_seg
);
264 wqe_size
= roundup_pow_of_two(wqe_size
);
265 wq_size
= roundup_pow_of_two(cap
->max_recv_wr
) * wqe_size
;
266 wq_size
= max_t(int, wq_size
, MLX5_SEND_WQE_BB
);
267 qp
->rq
.wqe_cnt
= wq_size
/ wqe_size
;
268 if (wqe_size
> MLX5_CAP_GEN(dev
->mdev
, max_wqe_sz_rq
)) {
269 mlx5_ib_dbg(dev
, "wqe_size %d, max %d\n",
271 MLX5_CAP_GEN(dev
->mdev
,
275 qp
->rq
.wqe_shift
= ilog2(wqe_size
);
276 qp
->rq
.max_gs
= (1 << qp
->rq
.wqe_shift
) / sizeof(struct mlx5_wqe_data_seg
) - qp
->wq_sig
;
277 qp
->rq
.max_post
= qp
->rq
.wqe_cnt
;
284 static int sq_overhead(struct ib_qp_init_attr
*attr
)
288 switch (attr
->qp_type
) {
290 size
+= sizeof(struct mlx5_wqe_xrc_seg
);
293 size
+= sizeof(struct mlx5_wqe_ctrl_seg
) +
294 max(sizeof(struct mlx5_wqe_atomic_seg
) +
295 sizeof(struct mlx5_wqe_raddr_seg
),
296 sizeof(struct mlx5_wqe_umr_ctrl_seg
) +
297 sizeof(struct mlx5_mkey_seg
));
304 size
+= sizeof(struct mlx5_wqe_ctrl_seg
) +
305 max(sizeof(struct mlx5_wqe_raddr_seg
),
306 sizeof(struct mlx5_wqe_umr_ctrl_seg
) +
307 sizeof(struct mlx5_mkey_seg
));
311 if (attr
->create_flags
& IB_QP_CREATE_IPOIB_UD_LSO
)
312 size
+= sizeof(struct mlx5_wqe_eth_pad
) +
313 sizeof(struct mlx5_wqe_eth_seg
);
316 case MLX5_IB_QPT_HW_GSI
:
317 size
+= sizeof(struct mlx5_wqe_ctrl_seg
) +
318 sizeof(struct mlx5_wqe_datagram_seg
);
321 case MLX5_IB_QPT_REG_UMR
:
322 size
+= sizeof(struct mlx5_wqe_ctrl_seg
) +
323 sizeof(struct mlx5_wqe_umr_ctrl_seg
) +
324 sizeof(struct mlx5_mkey_seg
);
334 static int calc_send_wqe(struct ib_qp_init_attr
*attr
)
339 size
= sq_overhead(attr
);
343 if (attr
->cap
.max_inline_data
) {
344 inl_size
= size
+ sizeof(struct mlx5_wqe_inline_seg
) +
345 attr
->cap
.max_inline_data
;
348 size
+= attr
->cap
.max_send_sge
* sizeof(struct mlx5_wqe_data_seg
);
349 if (attr
->create_flags
& IB_QP_CREATE_SIGNATURE_EN
&&
350 ALIGN(max_t(int, inl_size
, size
), MLX5_SEND_WQE_BB
) < MLX5_SIG_WQE_SIZE
)
351 return MLX5_SIG_WQE_SIZE
;
353 return ALIGN(max_t(int, inl_size
, size
), MLX5_SEND_WQE_BB
);
356 static int get_send_sge(struct ib_qp_init_attr
*attr
, int wqe_size
)
360 if (attr
->qp_type
== IB_QPT_RC
)
361 max_sge
= (min_t(int, wqe_size
, 512) -
362 sizeof(struct mlx5_wqe_ctrl_seg
) -
363 sizeof(struct mlx5_wqe_raddr_seg
)) /
364 sizeof(struct mlx5_wqe_data_seg
);
365 else if (attr
->qp_type
== IB_QPT_XRC_INI
)
366 max_sge
= (min_t(int, wqe_size
, 512) -
367 sizeof(struct mlx5_wqe_ctrl_seg
) -
368 sizeof(struct mlx5_wqe_xrc_seg
) -
369 sizeof(struct mlx5_wqe_raddr_seg
)) /
370 sizeof(struct mlx5_wqe_data_seg
);
372 max_sge
= (wqe_size
- sq_overhead(attr
)) /
373 sizeof(struct mlx5_wqe_data_seg
);
375 return min_t(int, max_sge
, wqe_size
- sq_overhead(attr
) /
376 sizeof(struct mlx5_wqe_data_seg
));
379 static int calc_sq_size(struct mlx5_ib_dev
*dev
, struct ib_qp_init_attr
*attr
,
380 struct mlx5_ib_qp
*qp
)
385 if (!attr
->cap
.max_send_wr
)
388 wqe_size
= calc_send_wqe(attr
);
389 mlx5_ib_dbg(dev
, "wqe_size %d\n", wqe_size
);
393 if (wqe_size
> MLX5_CAP_GEN(dev
->mdev
, max_wqe_sz_sq
)) {
394 mlx5_ib_dbg(dev
, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
395 wqe_size
, MLX5_CAP_GEN(dev
->mdev
, max_wqe_sz_sq
));
399 qp
->max_inline_data
= wqe_size
- sq_overhead(attr
) -
400 sizeof(struct mlx5_wqe_inline_seg
);
401 attr
->cap
.max_inline_data
= qp
->max_inline_data
;
403 if (attr
->create_flags
& IB_QP_CREATE_SIGNATURE_EN
)
404 qp
->signature_en
= true;
406 wq_size
= roundup_pow_of_two(attr
->cap
.max_send_wr
* wqe_size
);
407 qp
->sq
.wqe_cnt
= wq_size
/ MLX5_SEND_WQE_BB
;
408 if (qp
->sq
.wqe_cnt
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_qp_sz
))) {
409 mlx5_ib_dbg(dev
, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
410 attr
->cap
.max_send_wr
, wqe_size
, MLX5_SEND_WQE_BB
,
412 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_qp_sz
));
415 qp
->sq
.wqe_shift
= ilog2(MLX5_SEND_WQE_BB
);
416 qp
->sq
.max_gs
= get_send_sge(attr
, wqe_size
);
417 if (qp
->sq
.max_gs
< attr
->cap
.max_send_sge
)
420 attr
->cap
.max_send_sge
= qp
->sq
.max_gs
;
421 qp
->sq
.max_post
= wq_size
/ wqe_size
;
422 attr
->cap
.max_send_wr
= qp
->sq
.max_post
;
427 static int set_user_buf_size(struct mlx5_ib_dev
*dev
,
428 struct mlx5_ib_qp
*qp
,
429 struct mlx5_ib_create_qp
*ucmd
,
430 struct mlx5_ib_qp_base
*base
,
431 struct ib_qp_init_attr
*attr
)
433 int desc_sz
= 1 << qp
->sq
.wqe_shift
;
435 if (desc_sz
> MLX5_CAP_GEN(dev
->mdev
, max_wqe_sz_sq
)) {
436 mlx5_ib_warn(dev
, "desc_sz %d, max_sq_desc_sz %d\n",
437 desc_sz
, MLX5_CAP_GEN(dev
->mdev
, max_wqe_sz_sq
));
441 if (ucmd
->sq_wqe_count
&& ((1 << ilog2(ucmd
->sq_wqe_count
)) != ucmd
->sq_wqe_count
)) {
442 mlx5_ib_warn(dev
, "sq_wqe_count %d, sq_wqe_count %d\n",
443 ucmd
->sq_wqe_count
, ucmd
->sq_wqe_count
);
447 qp
->sq
.wqe_cnt
= ucmd
->sq_wqe_count
;
449 if (qp
->sq
.wqe_cnt
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_qp_sz
))) {
450 mlx5_ib_warn(dev
, "wqe_cnt %d, max_wqes %d\n",
452 1 << MLX5_CAP_GEN(dev
->mdev
, log_max_qp_sz
));
456 if (attr
->qp_type
== IB_QPT_RAW_PACKET
) {
457 base
->ubuffer
.buf_size
= qp
->rq
.wqe_cnt
<< qp
->rq
.wqe_shift
;
458 qp
->raw_packet_qp
.sq
.ubuffer
.buf_size
= qp
->sq
.wqe_cnt
<< 6;
460 base
->ubuffer
.buf_size
= (qp
->rq
.wqe_cnt
<< qp
->rq
.wqe_shift
) +
461 (qp
->sq
.wqe_cnt
<< 6);
467 static int qp_has_rq(struct ib_qp_init_attr
*attr
)
469 if (attr
->qp_type
== IB_QPT_XRC_INI
||
470 attr
->qp_type
== IB_QPT_XRC_TGT
|| attr
->srq
||
471 attr
->qp_type
== MLX5_IB_QPT_REG_UMR
||
472 !attr
->cap
.max_recv_wr
)
478 static int first_med_bfreg(void)
484 /* this is the first blue flame register in the array of bfregs assigned
485 * to a processes. Since we do not use it for blue flame but rather
486 * regular 64 bit doorbells, we do not need a lock for maintaiing
489 NUM_NON_BLUE_FLAME_BFREGS
= 1,
492 static int max_bfregs(struct mlx5_ib_dev
*dev
, struct mlx5_bfreg_info
*bfregi
)
494 return get_num_uars(dev
, bfregi
) * MLX5_NON_FP_BFREGS_PER_UAR
;
497 static int num_med_bfreg(struct mlx5_ib_dev
*dev
,
498 struct mlx5_bfreg_info
*bfregi
)
502 n
= max_bfregs(dev
, bfregi
) - bfregi
->num_low_latency_bfregs
-
503 NUM_NON_BLUE_FLAME_BFREGS
;
505 return n
>= 0 ? n
: 0;
508 static int first_hi_bfreg(struct mlx5_ib_dev
*dev
,
509 struct mlx5_bfreg_info
*bfregi
)
513 med
= num_med_bfreg(dev
, bfregi
);
517 static int alloc_high_class_bfreg(struct mlx5_ib_dev
*dev
,
518 struct mlx5_bfreg_info
*bfregi
)
522 for (i
= first_hi_bfreg(dev
, bfregi
); i
< max_bfregs(dev
, bfregi
); i
++) {
523 if (!bfregi
->count
[i
]) {
532 static int alloc_med_class_bfreg(struct mlx5_ib_dev
*dev
,
533 struct mlx5_bfreg_info
*bfregi
)
535 int minidx
= first_med_bfreg();
538 for (i
= first_med_bfreg(); i
< first_hi_bfreg(dev
, bfregi
); i
++) {
539 if (bfregi
->count
[i
] < bfregi
->count
[minidx
])
541 if (!bfregi
->count
[minidx
])
545 bfregi
->count
[minidx
]++;
549 static int alloc_bfreg(struct mlx5_ib_dev
*dev
,
550 struct mlx5_bfreg_info
*bfregi
,
551 enum mlx5_ib_latency_class lat
)
553 int bfregn
= -EINVAL
;
555 mutex_lock(&bfregi
->lock
);
557 case MLX5_IB_LATENCY_CLASS_LOW
:
558 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS
!= 1);
560 bfregi
->count
[bfregn
]++;
563 case MLX5_IB_LATENCY_CLASS_MEDIUM
:
567 bfregn
= alloc_med_class_bfreg(dev
, bfregi
);
570 case MLX5_IB_LATENCY_CLASS_HIGH
:
574 bfregn
= alloc_high_class_bfreg(dev
, bfregi
);
577 mutex_unlock(&bfregi
->lock
);
582 static void free_bfreg(struct mlx5_ib_dev
*dev
, struct mlx5_bfreg_info
*bfregi
, int bfregn
)
584 mutex_lock(&bfregi
->lock
);
585 bfregi
->count
[bfregn
]--;
586 mutex_unlock(&bfregi
->lock
);
589 static enum mlx5_qp_state
to_mlx5_state(enum ib_qp_state state
)
592 case IB_QPS_RESET
: return MLX5_QP_STATE_RST
;
593 case IB_QPS_INIT
: return MLX5_QP_STATE_INIT
;
594 case IB_QPS_RTR
: return MLX5_QP_STATE_RTR
;
595 case IB_QPS_RTS
: return MLX5_QP_STATE_RTS
;
596 case IB_QPS_SQD
: return MLX5_QP_STATE_SQD
;
597 case IB_QPS_SQE
: return MLX5_QP_STATE_SQER
;
598 case IB_QPS_ERR
: return MLX5_QP_STATE_ERR
;
603 static int to_mlx5_st(enum ib_qp_type type
)
606 case IB_QPT_RC
: return MLX5_QP_ST_RC
;
607 case IB_QPT_UC
: return MLX5_QP_ST_UC
;
608 case IB_QPT_UD
: return MLX5_QP_ST_UD
;
609 case MLX5_IB_QPT_REG_UMR
: return MLX5_QP_ST_REG_UMR
;
611 case IB_QPT_XRC_TGT
: return MLX5_QP_ST_XRC
;
612 case IB_QPT_SMI
: return MLX5_QP_ST_QP0
;
613 case MLX5_IB_QPT_HW_GSI
: return MLX5_QP_ST_QP1
;
614 case IB_QPT_RAW_IPV6
: return MLX5_QP_ST_RAW_IPV6
;
615 case IB_QPT_RAW_PACKET
:
616 case IB_QPT_RAW_ETHERTYPE
: return MLX5_QP_ST_RAW_ETHERTYPE
;
618 default: return -EINVAL
;
622 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq
*send_cq
,
623 struct mlx5_ib_cq
*recv_cq
);
624 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq
*send_cq
,
625 struct mlx5_ib_cq
*recv_cq
);
627 static int bfregn_to_uar_index(struct mlx5_ib_dev
*dev
,
628 struct mlx5_bfreg_info
*bfregi
, int bfregn
)
630 int bfregs_per_sys_page
;
631 int index_of_sys_page
;
634 bfregs_per_sys_page
= get_uars_per_sys_page(dev
, bfregi
->lib_uar_4k
) *
635 MLX5_NON_FP_BFREGS_PER_UAR
;
636 index_of_sys_page
= bfregn
/ bfregs_per_sys_page
;
638 offset
= bfregn
% bfregs_per_sys_page
/ MLX5_NON_FP_BFREGS_PER_UAR
;
640 return bfregi
->sys_pages
[index_of_sys_page
] + offset
;
643 static int mlx5_ib_umem_get(struct mlx5_ib_dev
*dev
,
645 unsigned long addr
, size_t size
,
646 struct ib_umem
**umem
,
647 int *npages
, int *page_shift
, int *ncont
,
652 *umem
= ib_umem_get(pd
->uobject
->context
, addr
, size
, 0, 0);
654 mlx5_ib_dbg(dev
, "umem_get failed\n");
655 return PTR_ERR(*umem
);
658 mlx5_ib_cont_pages(*umem
, addr
, 0, npages
, page_shift
, ncont
, NULL
);
660 err
= mlx5_ib_get_buf_offset(addr
, *page_shift
, offset
);
662 mlx5_ib_warn(dev
, "bad offset\n");
666 mlx5_ib_dbg(dev
, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
667 addr
, size
, *npages
, *page_shift
, *ncont
, *offset
);
672 ib_umem_release(*umem
);
678 static void destroy_user_rq(struct ib_pd
*pd
, struct mlx5_ib_rwq
*rwq
)
680 struct mlx5_ib_ucontext
*context
;
682 context
= to_mucontext(pd
->uobject
->context
);
683 mlx5_ib_db_unmap_user(context
, &rwq
->db
);
685 ib_umem_release(rwq
->umem
);
688 static int create_user_rq(struct mlx5_ib_dev
*dev
, struct ib_pd
*pd
,
689 struct mlx5_ib_rwq
*rwq
,
690 struct mlx5_ib_create_wq
*ucmd
)
692 struct mlx5_ib_ucontext
*context
;
702 context
= to_mucontext(pd
->uobject
->context
);
703 rwq
->umem
= ib_umem_get(pd
->uobject
->context
, ucmd
->buf_addr
,
704 rwq
->buf_size
, 0, 0);
705 if (IS_ERR(rwq
->umem
)) {
706 mlx5_ib_dbg(dev
, "umem_get failed\n");
707 err
= PTR_ERR(rwq
->umem
);
711 mlx5_ib_cont_pages(rwq
->umem
, ucmd
->buf_addr
, 0, &npages
, &page_shift
,
713 err
= mlx5_ib_get_buf_offset(ucmd
->buf_addr
, page_shift
,
714 &rwq
->rq_page_offset
);
716 mlx5_ib_warn(dev
, "bad offset\n");
720 rwq
->rq_num_pas
= ncont
;
721 rwq
->page_shift
= page_shift
;
722 rwq
->log_page_size
= page_shift
- MLX5_ADAPTER_PAGE_SHIFT
;
723 rwq
->wq_sig
= !!(ucmd
->flags
& MLX5_WQ_FLAG_SIGNATURE
);
725 mlx5_ib_dbg(dev
, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
726 (unsigned long long)ucmd
->buf_addr
, rwq
->buf_size
,
727 npages
, page_shift
, ncont
, offset
);
729 err
= mlx5_ib_db_map_user(context
, ucmd
->db_addr
, &rwq
->db
);
731 mlx5_ib_dbg(dev
, "map failed\n");
735 rwq
->create_type
= MLX5_WQ_USER
;
739 ib_umem_release(rwq
->umem
);
743 static int adjust_bfregn(struct mlx5_ib_dev
*dev
,
744 struct mlx5_bfreg_info
*bfregi
, int bfregn
)
746 return bfregn
/ MLX5_NON_FP_BFREGS_PER_UAR
* MLX5_BFREGS_PER_UAR
+
747 bfregn
% MLX5_NON_FP_BFREGS_PER_UAR
;
750 static int create_user_qp(struct mlx5_ib_dev
*dev
, struct ib_pd
*pd
,
751 struct mlx5_ib_qp
*qp
, struct ib_udata
*udata
,
752 struct ib_qp_init_attr
*attr
,
754 struct mlx5_ib_create_qp_resp
*resp
, int *inlen
,
755 struct mlx5_ib_qp_base
*base
)
757 struct mlx5_ib_ucontext
*context
;
758 struct mlx5_ib_create_qp ucmd
;
759 struct mlx5_ib_ubuffer
*ubuffer
= &base
->ubuffer
;
770 err
= ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
));
772 mlx5_ib_dbg(dev
, "copy failed\n");
776 context
= to_mucontext(pd
->uobject
->context
);
778 * TBD: should come from the verbs when we have the API
780 if (qp
->flags
& MLX5_IB_QP_CROSS_CHANNEL
)
781 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
782 bfregn
= MLX5_CROSS_CHANNEL_BFREG
;
784 bfregn
= alloc_bfreg(dev
, &context
->bfregi
, MLX5_IB_LATENCY_CLASS_HIGH
);
786 mlx5_ib_dbg(dev
, "failed to allocate low latency BFREG\n");
787 mlx5_ib_dbg(dev
, "reverting to medium latency\n");
788 bfregn
= alloc_bfreg(dev
, &context
->bfregi
, MLX5_IB_LATENCY_CLASS_MEDIUM
);
790 mlx5_ib_dbg(dev
, "failed to allocate medium latency BFREG\n");
791 mlx5_ib_dbg(dev
, "reverting to high latency\n");
792 bfregn
= alloc_bfreg(dev
, &context
->bfregi
, MLX5_IB_LATENCY_CLASS_LOW
);
794 mlx5_ib_warn(dev
, "bfreg allocation failed\n");
801 uar_index
= bfregn_to_uar_index(dev
, &context
->bfregi
, bfregn
);
802 mlx5_ib_dbg(dev
, "bfregn 0x%x, uar_index 0x%x\n", bfregn
, uar_index
);
805 qp
->sq
.wqe_shift
= ilog2(MLX5_SEND_WQE_BB
);
806 qp
->sq
.offset
= qp
->rq
.wqe_cnt
<< qp
->rq
.wqe_shift
;
808 err
= set_user_buf_size(dev
, qp
, &ucmd
, base
, attr
);
812 if (ucmd
.buf_addr
&& ubuffer
->buf_size
) {
813 ubuffer
->buf_addr
= ucmd
.buf_addr
;
814 err
= mlx5_ib_umem_get(dev
, pd
, ubuffer
->buf_addr
,
816 &ubuffer
->umem
, &npages
, &page_shift
,
821 ubuffer
->umem
= NULL
;
824 *inlen
= MLX5_ST_SZ_BYTES(create_qp_in
) +
825 MLX5_FLD_SZ_BYTES(create_qp_in
, pas
[0]) * ncont
;
826 *in
= mlx5_vzalloc(*inlen
);
832 pas
= (__be64
*)MLX5_ADDR_OF(create_qp_in
, *in
, pas
);
834 mlx5_ib_populate_pas(dev
, ubuffer
->umem
, page_shift
, pas
, 0);
836 qpc
= MLX5_ADDR_OF(create_qp_in
, *in
, qpc
);
838 MLX5_SET(qpc
, qpc
, log_page_size
, page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
839 MLX5_SET(qpc
, qpc
, page_offset
, offset
);
841 MLX5_SET(qpc
, qpc
, uar_page
, uar_index
);
842 resp
->bfreg_index
= adjust_bfregn(dev
, &context
->bfregi
, bfregn
);
845 err
= mlx5_ib_db_map_user(context
, ucmd
.db_addr
, &qp
->db
);
847 mlx5_ib_dbg(dev
, "map failed\n");
851 err
= ib_copy_to_udata(udata
, resp
, sizeof(*resp
));
853 mlx5_ib_dbg(dev
, "copy failed\n");
856 qp
->create_type
= MLX5_QP_USER
;
861 mlx5_ib_db_unmap_user(context
, &qp
->db
);
868 ib_umem_release(ubuffer
->umem
);
871 free_bfreg(dev
, &context
->bfregi
, bfregn
);
875 static void destroy_qp_user(struct mlx5_ib_dev
*dev
, struct ib_pd
*pd
,
876 struct mlx5_ib_qp
*qp
, struct mlx5_ib_qp_base
*base
)
878 struct mlx5_ib_ucontext
*context
;
880 context
= to_mucontext(pd
->uobject
->context
);
881 mlx5_ib_db_unmap_user(context
, &qp
->db
);
882 if (base
->ubuffer
.umem
)
883 ib_umem_release(base
->ubuffer
.umem
);
884 free_bfreg(dev
, &context
->bfregi
, qp
->bfregn
);
887 static int create_kernel_qp(struct mlx5_ib_dev
*dev
,
888 struct ib_qp_init_attr
*init_attr
,
889 struct mlx5_ib_qp
*qp
,
890 u32
**in
, int *inlen
,
891 struct mlx5_ib_qp_base
*base
)
897 if (init_attr
->create_flags
& ~(IB_QP_CREATE_SIGNATURE_EN
|
898 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
|
899 IB_QP_CREATE_IPOIB_UD_LSO
|
900 IB_QP_CREATE_NETIF_QP
|
901 mlx5_ib_create_qp_sqpn_qp1()))
904 if (init_attr
->qp_type
== MLX5_IB_QPT_REG_UMR
)
905 qp
->bf
.bfreg
= &dev
->fp_bfreg
;
907 qp
->bf
.bfreg
= &dev
->bfreg
;
909 /* We need to divide by two since each register is comprised of
910 * two buffers of identical size, namely odd and even
912 qp
->bf
.buf_size
= (1 << MLX5_CAP_GEN(dev
->mdev
, log_bf_reg_size
)) / 2;
913 uar_index
= qp
->bf
.bfreg
->index
;
915 err
= calc_sq_size(dev
, init_attr
, qp
);
917 mlx5_ib_dbg(dev
, "err %d\n", err
);
922 qp
->sq
.offset
= qp
->rq
.wqe_cnt
<< qp
->rq
.wqe_shift
;
923 base
->ubuffer
.buf_size
= err
+ (qp
->rq
.wqe_cnt
<< qp
->rq
.wqe_shift
);
925 err
= mlx5_buf_alloc(dev
->mdev
, base
->ubuffer
.buf_size
, &qp
->buf
);
927 mlx5_ib_dbg(dev
, "err %d\n", err
);
931 qp
->sq
.qend
= mlx5_get_send_wqe(qp
, qp
->sq
.wqe_cnt
);
932 *inlen
= MLX5_ST_SZ_BYTES(create_qp_in
) +
933 MLX5_FLD_SZ_BYTES(create_qp_in
, pas
[0]) * qp
->buf
.npages
;
934 *in
= mlx5_vzalloc(*inlen
);
940 qpc
= MLX5_ADDR_OF(create_qp_in
, *in
, qpc
);
941 MLX5_SET(qpc
, qpc
, uar_page
, uar_index
);
942 MLX5_SET(qpc
, qpc
, log_page_size
, qp
->buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
944 /* Set "fast registration enabled" for all kernel QPs */
945 MLX5_SET(qpc
, qpc
, fre
, 1);
946 MLX5_SET(qpc
, qpc
, rlky
, 1);
948 if (init_attr
->create_flags
& mlx5_ib_create_qp_sqpn_qp1()) {
949 MLX5_SET(qpc
, qpc
, deth_sqpn
, 1);
950 qp
->flags
|= MLX5_IB_QP_SQPN_QP1
;
953 mlx5_fill_page_array(&qp
->buf
,
954 (__be64
*)MLX5_ADDR_OF(create_qp_in
, *in
, pas
));
956 err
= mlx5_db_alloc(dev
->mdev
, &qp
->db
);
958 mlx5_ib_dbg(dev
, "err %d\n", err
);
962 qp
->sq
.wrid
= kmalloc(qp
->sq
.wqe_cnt
* sizeof(*qp
->sq
.wrid
), GFP_KERNEL
);
963 qp
->sq
.wr_data
= kmalloc(qp
->sq
.wqe_cnt
* sizeof(*qp
->sq
.wr_data
), GFP_KERNEL
);
964 qp
->rq
.wrid
= kmalloc(qp
->rq
.wqe_cnt
* sizeof(*qp
->rq
.wrid
), GFP_KERNEL
);
965 qp
->sq
.w_list
= kmalloc(qp
->sq
.wqe_cnt
* sizeof(*qp
->sq
.w_list
), GFP_KERNEL
);
966 qp
->sq
.wqe_head
= kmalloc(qp
->sq
.wqe_cnt
* sizeof(*qp
->sq
.wqe_head
), GFP_KERNEL
);
968 if (!qp
->sq
.wrid
|| !qp
->sq
.wr_data
|| !qp
->rq
.wrid
||
969 !qp
->sq
.w_list
|| !qp
->sq
.wqe_head
) {
973 qp
->create_type
= MLX5_QP_KERNEL
;
978 kfree(qp
->sq
.wqe_head
);
979 kfree(qp
->sq
.w_list
);
981 kfree(qp
->sq
.wr_data
);
983 mlx5_db_free(dev
->mdev
, &qp
->db
);
989 mlx5_buf_free(dev
->mdev
, &qp
->buf
);
993 static void destroy_qp_kernel(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
)
995 kfree(qp
->sq
.wqe_head
);
996 kfree(qp
->sq
.w_list
);
998 kfree(qp
->sq
.wr_data
);
1000 mlx5_db_free(dev
->mdev
, &qp
->db
);
1001 mlx5_buf_free(dev
->mdev
, &qp
->buf
);
1004 static u32
get_rx_type(struct mlx5_ib_qp
*qp
, struct ib_qp_init_attr
*attr
)
1006 if (attr
->srq
|| (attr
->qp_type
== IB_QPT_XRC_TGT
) ||
1007 (attr
->qp_type
== IB_QPT_XRC_INI
))
1009 else if (!qp
->has_rq
)
1010 return MLX5_ZERO_LEN_RQ
;
1012 return MLX5_NON_ZERO_RQ
;
1015 static int is_connected(enum ib_qp_type qp_type
)
1017 if (qp_type
== IB_QPT_RC
|| qp_type
== IB_QPT_UC
)
1023 static int create_raw_packet_qp_tis(struct mlx5_ib_dev
*dev
,
1024 struct mlx5_ib_sq
*sq
, u32 tdn
)
1026 u32 in
[MLX5_ST_SZ_DW(create_tis_in
)] = {0};
1027 void *tisc
= MLX5_ADDR_OF(create_tis_in
, in
, ctx
);
1029 MLX5_SET(tisc
, tisc
, transport_domain
, tdn
);
1030 return mlx5_core_create_tis(dev
->mdev
, in
, sizeof(in
), &sq
->tisn
);
1033 static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev
*dev
,
1034 struct mlx5_ib_sq
*sq
)
1036 mlx5_core_destroy_tis(dev
->mdev
, sq
->tisn
);
1039 static int create_raw_packet_qp_sq(struct mlx5_ib_dev
*dev
,
1040 struct mlx5_ib_sq
*sq
, void *qpin
,
1043 struct mlx5_ib_ubuffer
*ubuffer
= &sq
->ubuffer
;
1047 void *qpc
= MLX5_ADDR_OF(create_qp_in
, qpin
, qpc
);
1056 err
= mlx5_ib_umem_get(dev
, pd
, ubuffer
->buf_addr
, ubuffer
->buf_size
,
1057 &sq
->ubuffer
.umem
, &npages
, &page_shift
,
1062 inlen
= MLX5_ST_SZ_BYTES(create_sq_in
) + sizeof(u64
) * ncont
;
1063 in
= mlx5_vzalloc(inlen
);
1069 sqc
= MLX5_ADDR_OF(create_sq_in
, in
, ctx
);
1070 MLX5_SET(sqc
, sqc
, flush_in_error_en
, 1);
1071 MLX5_SET(sqc
, sqc
, state
, MLX5_SQC_STATE_RST
);
1072 MLX5_SET(sqc
, sqc
, user_index
, MLX5_GET(qpc
, qpc
, user_index
));
1073 MLX5_SET(sqc
, sqc
, cqn
, MLX5_GET(qpc
, qpc
, cqn_snd
));
1074 MLX5_SET(sqc
, sqc
, tis_lst_sz
, 1);
1075 MLX5_SET(sqc
, sqc
, tis_num_0
, sq
->tisn
);
1077 wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1078 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_CYCLIC
);
1079 MLX5_SET(wq
, wq
, pd
, MLX5_GET(qpc
, qpc
, pd
));
1080 MLX5_SET(wq
, wq
, uar_page
, MLX5_GET(qpc
, qpc
, uar_page
));
1081 MLX5_SET64(wq
, wq
, dbr_addr
, MLX5_GET64(qpc
, qpc
, dbr_addr
));
1082 MLX5_SET(wq
, wq
, log_wq_stride
, ilog2(MLX5_SEND_WQE_BB
));
1083 MLX5_SET(wq
, wq
, log_wq_sz
, MLX5_GET(qpc
, qpc
, log_sq_size
));
1084 MLX5_SET(wq
, wq
, log_wq_pg_sz
, page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
1085 MLX5_SET(wq
, wq
, page_offset
, offset
);
1087 pas
= (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
);
1088 mlx5_ib_populate_pas(dev
, sq
->ubuffer
.umem
, page_shift
, pas
, 0);
1090 err
= mlx5_core_create_sq_tracked(dev
->mdev
, in
, inlen
, &sq
->base
.mqp
);
1100 ib_umem_release(sq
->ubuffer
.umem
);
1101 sq
->ubuffer
.umem
= NULL
;
1106 static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev
*dev
,
1107 struct mlx5_ib_sq
*sq
)
1109 mlx5_core_destroy_sq_tracked(dev
->mdev
, &sq
->base
.mqp
);
1110 ib_umem_release(sq
->ubuffer
.umem
);
1113 static int get_rq_pas_size(void *qpc
)
1115 u32 log_page_size
= MLX5_GET(qpc
, qpc
, log_page_size
) + 12;
1116 u32 log_rq_stride
= MLX5_GET(qpc
, qpc
, log_rq_stride
);
1117 u32 log_rq_size
= MLX5_GET(qpc
, qpc
, log_rq_size
);
1118 u32 page_offset
= MLX5_GET(qpc
, qpc
, page_offset
);
1119 u32 po_quanta
= 1 << (log_page_size
- 6);
1120 u32 rq_sz
= 1 << (log_rq_size
+ 4 + log_rq_stride
);
1121 u32 page_size
= 1 << log_page_size
;
1122 u32 rq_sz_po
= rq_sz
+ (page_offset
* po_quanta
);
1123 u32 rq_num_pas
= (rq_sz_po
+ page_size
- 1) / page_size
;
1125 return rq_num_pas
* sizeof(u64
);
1128 static int create_raw_packet_qp_rq(struct mlx5_ib_dev
*dev
,
1129 struct mlx5_ib_rq
*rq
, void *qpin
)
1131 struct mlx5_ib_qp
*mqp
= rq
->base
.container_mibqp
;
1137 void *qpc
= MLX5_ADDR_OF(create_qp_in
, qpin
, qpc
);
1140 u32 rq_pas_size
= get_rq_pas_size(qpc
);
1142 inlen
= MLX5_ST_SZ_BYTES(create_rq_in
) + rq_pas_size
;
1143 in
= mlx5_vzalloc(inlen
);
1147 rqc
= MLX5_ADDR_OF(create_rq_in
, in
, ctx
);
1148 if (!(rq
->flags
& MLX5_IB_RQ_CVLAN_STRIPPING
))
1149 MLX5_SET(rqc
, rqc
, vsd
, 1);
1150 MLX5_SET(rqc
, rqc
, mem_rq_type
, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE
);
1151 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RST
);
1152 MLX5_SET(rqc
, rqc
, flush_in_error_en
, 1);
1153 MLX5_SET(rqc
, rqc
, user_index
, MLX5_GET(qpc
, qpc
, user_index
));
1154 MLX5_SET(rqc
, rqc
, cqn
, MLX5_GET(qpc
, qpc
, cqn_rcv
));
1156 if (mqp
->flags
& MLX5_IB_QP_CAP_SCATTER_FCS
)
1157 MLX5_SET(rqc
, rqc
, scatter_fcs
, 1);
1159 wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
1160 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_CYCLIC
);
1161 MLX5_SET(wq
, wq
, end_padding_mode
,
1162 MLX5_GET(qpc
, qpc
, end_padding_mode
));
1163 MLX5_SET(wq
, wq
, page_offset
, MLX5_GET(qpc
, qpc
, page_offset
));
1164 MLX5_SET(wq
, wq
, pd
, MLX5_GET(qpc
, qpc
, pd
));
1165 MLX5_SET64(wq
, wq
, dbr_addr
, MLX5_GET64(qpc
, qpc
, dbr_addr
));
1166 MLX5_SET(wq
, wq
, log_wq_stride
, MLX5_GET(qpc
, qpc
, log_rq_stride
) + 4);
1167 MLX5_SET(wq
, wq
, log_wq_pg_sz
, MLX5_GET(qpc
, qpc
, log_page_size
));
1168 MLX5_SET(wq
, wq
, log_wq_sz
, MLX5_GET(qpc
, qpc
, log_rq_size
));
1170 pas
= (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
);
1171 qp_pas
= (__be64
*)MLX5_ADDR_OF(create_qp_in
, qpin
, pas
);
1172 memcpy(pas
, qp_pas
, rq_pas_size
);
1174 err
= mlx5_core_create_rq_tracked(dev
->mdev
, in
, inlen
, &rq
->base
.mqp
);
1181 static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev
*dev
,
1182 struct mlx5_ib_rq
*rq
)
1184 mlx5_core_destroy_rq_tracked(dev
->mdev
, &rq
->base
.mqp
);
1187 static int create_raw_packet_qp_tir(struct mlx5_ib_dev
*dev
,
1188 struct mlx5_ib_rq
*rq
, u32 tdn
)
1195 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
1196 in
= mlx5_vzalloc(inlen
);
1200 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
1201 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_DIRECT
);
1202 MLX5_SET(tirc
, tirc
, inline_rqn
, rq
->base
.mqp
.qpn
);
1203 MLX5_SET(tirc
, tirc
, transport_domain
, tdn
);
1205 err
= mlx5_core_create_tir(dev
->mdev
, in
, inlen
, &rq
->tirn
);
1212 static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev
*dev
,
1213 struct mlx5_ib_rq
*rq
)
1215 mlx5_core_destroy_tir(dev
->mdev
, rq
->tirn
);
1218 static int create_raw_packet_qp(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
1222 struct mlx5_ib_raw_packet_qp
*raw_packet_qp
= &qp
->raw_packet_qp
;
1223 struct mlx5_ib_sq
*sq
= &raw_packet_qp
->sq
;
1224 struct mlx5_ib_rq
*rq
= &raw_packet_qp
->rq
;
1225 struct ib_uobject
*uobj
= pd
->uobject
;
1226 struct ib_ucontext
*ucontext
= uobj
->context
;
1227 struct mlx5_ib_ucontext
*mucontext
= to_mucontext(ucontext
);
1229 u32 tdn
= mucontext
->tdn
;
1231 if (qp
->sq
.wqe_cnt
) {
1232 err
= create_raw_packet_qp_tis(dev
, sq
, tdn
);
1236 err
= create_raw_packet_qp_sq(dev
, sq
, in
, pd
);
1238 goto err_destroy_tis
;
1240 sq
->base
.container_mibqp
= qp
;
1243 if (qp
->rq
.wqe_cnt
) {
1244 rq
->base
.container_mibqp
= qp
;
1246 if (qp
->flags
& MLX5_IB_QP_CVLAN_STRIPPING
)
1247 rq
->flags
|= MLX5_IB_RQ_CVLAN_STRIPPING
;
1248 err
= create_raw_packet_qp_rq(dev
, rq
, in
);
1250 goto err_destroy_sq
;
1253 err
= create_raw_packet_qp_tir(dev
, rq
, tdn
);
1255 goto err_destroy_rq
;
1258 qp
->trans_qp
.base
.mqp
.qpn
= qp
->sq
.wqe_cnt
? sq
->base
.mqp
.qpn
:
1264 destroy_raw_packet_qp_rq(dev
, rq
);
1266 if (!qp
->sq
.wqe_cnt
)
1268 destroy_raw_packet_qp_sq(dev
, sq
);
1270 destroy_raw_packet_qp_tis(dev
, sq
);
1275 static void destroy_raw_packet_qp(struct mlx5_ib_dev
*dev
,
1276 struct mlx5_ib_qp
*qp
)
1278 struct mlx5_ib_raw_packet_qp
*raw_packet_qp
= &qp
->raw_packet_qp
;
1279 struct mlx5_ib_sq
*sq
= &raw_packet_qp
->sq
;
1280 struct mlx5_ib_rq
*rq
= &raw_packet_qp
->rq
;
1282 if (qp
->rq
.wqe_cnt
) {
1283 destroy_raw_packet_qp_tir(dev
, rq
);
1284 destroy_raw_packet_qp_rq(dev
, rq
);
1287 if (qp
->sq
.wqe_cnt
) {
1288 destroy_raw_packet_qp_sq(dev
, sq
);
1289 destroy_raw_packet_qp_tis(dev
, sq
);
1293 static void raw_packet_qp_copy_info(struct mlx5_ib_qp
*qp
,
1294 struct mlx5_ib_raw_packet_qp
*raw_packet_qp
)
1296 struct mlx5_ib_sq
*sq
= &raw_packet_qp
->sq
;
1297 struct mlx5_ib_rq
*rq
= &raw_packet_qp
->rq
;
1301 sq
->doorbell
= &qp
->db
;
1302 rq
->doorbell
= &qp
->db
;
1305 static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
)
1307 mlx5_core_destroy_tir(dev
->mdev
, qp
->rss_qp
.tirn
);
1310 static int create_rss_raw_qp_tir(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
1312 struct ib_qp_init_attr
*init_attr
,
1313 struct ib_udata
*udata
)
1315 struct ib_uobject
*uobj
= pd
->uobject
;
1316 struct ib_ucontext
*ucontext
= uobj
->context
;
1317 struct mlx5_ib_ucontext
*mucontext
= to_mucontext(ucontext
);
1318 struct mlx5_ib_create_qp_resp resp
= {};
1324 u32 selected_fields
= 0;
1325 size_t min_resp_len
;
1326 u32 tdn
= mucontext
->tdn
;
1327 struct mlx5_ib_create_qp_rss ucmd
= {};
1328 size_t required_cmd_sz
;
1330 if (init_attr
->qp_type
!= IB_QPT_RAW_PACKET
)
1333 if (init_attr
->create_flags
|| init_attr
->send_cq
)
1336 min_resp_len
= offsetof(typeof(resp
), bfreg_index
) + sizeof(resp
.bfreg_index
);
1337 if (udata
->outlen
< min_resp_len
)
1340 required_cmd_sz
= offsetof(typeof(ucmd
), reserved1
) + sizeof(ucmd
.reserved1
);
1341 if (udata
->inlen
< required_cmd_sz
) {
1342 mlx5_ib_dbg(dev
, "invalid inlen\n");
1346 if (udata
->inlen
> sizeof(ucmd
) &&
1347 !ib_is_udata_cleared(udata
, sizeof(ucmd
),
1348 udata
->inlen
- sizeof(ucmd
))) {
1349 mlx5_ib_dbg(dev
, "inlen is not supported\n");
1353 if (ib_copy_from_udata(&ucmd
, udata
, min(sizeof(ucmd
), udata
->inlen
))) {
1354 mlx5_ib_dbg(dev
, "copy failed\n");
1358 if (ucmd
.comp_mask
) {
1359 mlx5_ib_dbg(dev
, "invalid comp mask\n");
1363 if (memchr_inv(ucmd
.reserved
, 0, sizeof(ucmd
.reserved
)) || ucmd
.reserved1
) {
1364 mlx5_ib_dbg(dev
, "invalid reserved\n");
1368 err
= ib_copy_to_udata(udata
, &resp
, min_resp_len
);
1370 mlx5_ib_dbg(dev
, "copy failed\n");
1374 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
1375 in
= mlx5_vzalloc(inlen
);
1379 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
1380 MLX5_SET(tirc
, tirc
, disp_type
,
1381 MLX5_TIRC_DISP_TYPE_INDIRECT
);
1382 MLX5_SET(tirc
, tirc
, indirect_table
,
1383 init_attr
->rwq_ind_tbl
->ind_tbl_num
);
1384 MLX5_SET(tirc
, tirc
, transport_domain
, tdn
);
1386 hfso
= MLX5_ADDR_OF(tirc
, tirc
, rx_hash_field_selector_outer
);
1387 switch (ucmd
.rx_hash_function
) {
1388 case MLX5_RX_HASH_FUNC_TOEPLITZ
:
1390 void *rss_key
= MLX5_ADDR_OF(tirc
, tirc
, rx_hash_toeplitz_key
);
1391 size_t len
= MLX5_FLD_SZ_BYTES(tirc
, rx_hash_toeplitz_key
);
1393 if (len
!= ucmd
.rx_key_len
) {
1398 MLX5_SET(tirc
, tirc
, rx_hash_fn
, MLX5_RX_HASH_FN_TOEPLITZ
);
1399 MLX5_SET(tirc
, tirc
, rx_hash_symmetric
, 1);
1400 memcpy(rss_key
, ucmd
.rx_hash_key
, len
);
1408 if (!ucmd
.rx_hash_fields_mask
) {
1409 /* special case when this TIR serves as steering entry without hashing */
1410 if (!init_attr
->rwq_ind_tbl
->log_ind_tbl_size
)
1416 if (((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV4
) ||
1417 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV4
)) &&
1418 ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV6
) ||
1419 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV6
))) {
1424 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1425 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV4
) ||
1426 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV4
))
1427 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
1428 MLX5_L3_PROT_TYPE_IPV4
);
1429 else if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV6
) ||
1430 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV6
))
1431 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
1432 MLX5_L3_PROT_TYPE_IPV6
);
1434 if (((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_TCP
) ||
1435 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_TCP
)) &&
1436 ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_UDP
) ||
1437 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_UDP
))) {
1442 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1443 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_TCP
) ||
1444 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_TCP
))
1445 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
1446 MLX5_L4_PROT_TYPE_TCP
);
1447 else if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_UDP
) ||
1448 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_UDP
))
1449 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
1450 MLX5_L4_PROT_TYPE_UDP
);
1452 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV4
) ||
1453 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_IPV6
))
1454 selected_fields
|= MLX5_HASH_FIELD_SEL_SRC_IP
;
1456 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV4
) ||
1457 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_IPV6
))
1458 selected_fields
|= MLX5_HASH_FIELD_SEL_DST_IP
;
1460 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_TCP
) ||
1461 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_SRC_PORT_UDP
))
1462 selected_fields
|= MLX5_HASH_FIELD_SEL_L4_SPORT
;
1464 if ((ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_TCP
) ||
1465 (ucmd
.rx_hash_fields_mask
& MLX5_RX_HASH_DST_PORT_UDP
))
1466 selected_fields
|= MLX5_HASH_FIELD_SEL_L4_DPORT
;
1468 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
, selected_fields
);
1471 err
= mlx5_core_create_tir(dev
->mdev
, in
, inlen
, &qp
->rss_qp
.tirn
);
1477 /* qpn is reserved for that QP */
1478 qp
->trans_qp
.base
.mqp
.qpn
= 0;
1479 qp
->flags
|= MLX5_IB_QP_RSS
;
1487 static int create_qp_common(struct mlx5_ib_dev
*dev
, struct ib_pd
*pd
,
1488 struct ib_qp_init_attr
*init_attr
,
1489 struct ib_udata
*udata
, struct mlx5_ib_qp
*qp
)
1491 struct mlx5_ib_resources
*devr
= &dev
->devr
;
1492 int inlen
= MLX5_ST_SZ_BYTES(create_qp_in
);
1493 struct mlx5_core_dev
*mdev
= dev
->mdev
;
1494 struct mlx5_ib_create_qp_resp resp
;
1495 struct mlx5_ib_cq
*send_cq
;
1496 struct mlx5_ib_cq
*recv_cq
;
1497 unsigned long flags
;
1498 u32 uidx
= MLX5_IB_DEFAULT_UIDX
;
1499 struct mlx5_ib_create_qp ucmd
;
1500 struct mlx5_ib_qp_base
*base
;
1505 base
= init_attr
->qp_type
== IB_QPT_RAW_PACKET
?
1506 &qp
->raw_packet_qp
.rq
.base
:
1509 mutex_init(&qp
->mutex
);
1510 spin_lock_init(&qp
->sq
.lock
);
1511 spin_lock_init(&qp
->rq
.lock
);
1513 if (init_attr
->rwq_ind_tbl
) {
1517 err
= create_rss_raw_qp_tir(dev
, qp
, pd
, init_attr
, udata
);
1521 if (init_attr
->create_flags
& IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
) {
1522 if (!MLX5_CAP_GEN(mdev
, block_lb_mc
)) {
1523 mlx5_ib_dbg(dev
, "block multicast loopback isn't supported\n");
1526 qp
->flags
|= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK
;
1530 if (init_attr
->create_flags
&
1531 (IB_QP_CREATE_CROSS_CHANNEL
|
1532 IB_QP_CREATE_MANAGED_SEND
|
1533 IB_QP_CREATE_MANAGED_RECV
)) {
1534 if (!MLX5_CAP_GEN(mdev
, cd
)) {
1535 mlx5_ib_dbg(dev
, "cross-channel isn't supported\n");
1538 if (init_attr
->create_flags
& IB_QP_CREATE_CROSS_CHANNEL
)
1539 qp
->flags
|= MLX5_IB_QP_CROSS_CHANNEL
;
1540 if (init_attr
->create_flags
& IB_QP_CREATE_MANAGED_SEND
)
1541 qp
->flags
|= MLX5_IB_QP_MANAGED_SEND
;
1542 if (init_attr
->create_flags
& IB_QP_CREATE_MANAGED_RECV
)
1543 qp
->flags
|= MLX5_IB_QP_MANAGED_RECV
;
1546 if (init_attr
->qp_type
== IB_QPT_UD
&&
1547 (init_attr
->create_flags
& IB_QP_CREATE_IPOIB_UD_LSO
))
1548 if (!MLX5_CAP_GEN(mdev
, ipoib_basic_offloads
)) {
1549 mlx5_ib_dbg(dev
, "ipoib UD lso qp isn't supported\n");
1553 if (init_attr
->create_flags
& IB_QP_CREATE_SCATTER_FCS
) {
1554 if (init_attr
->qp_type
!= IB_QPT_RAW_PACKET
) {
1555 mlx5_ib_dbg(dev
, "Scatter FCS is supported only for Raw Packet QPs");
1558 if (!MLX5_CAP_GEN(dev
->mdev
, eth_net_offloads
) ||
1559 !MLX5_CAP_ETH(dev
->mdev
, scatter_fcs
)) {
1560 mlx5_ib_dbg(dev
, "Scatter FCS isn't supported\n");
1563 qp
->flags
|= MLX5_IB_QP_CAP_SCATTER_FCS
;
1566 if (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
)
1567 qp
->sq_signal_bits
= MLX5_WQE_CTRL_CQ_UPDATE
;
1569 if (init_attr
->create_flags
& IB_QP_CREATE_CVLAN_STRIPPING
) {
1570 if (!(MLX5_CAP_GEN(dev
->mdev
, eth_net_offloads
) &&
1571 MLX5_CAP_ETH(dev
->mdev
, vlan_cap
)) ||
1572 (init_attr
->qp_type
!= IB_QPT_RAW_PACKET
))
1574 qp
->flags
|= MLX5_IB_QP_CVLAN_STRIPPING
;
1577 if (pd
&& pd
->uobject
) {
1578 if (ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
))) {
1579 mlx5_ib_dbg(dev
, "copy failed\n");
1583 err
= get_qp_user_index(to_mucontext(pd
->uobject
->context
),
1584 &ucmd
, udata
->inlen
, &uidx
);
1588 qp
->wq_sig
= !!(ucmd
.flags
& MLX5_QP_FLAG_SIGNATURE
);
1589 qp
->scat_cqe
= !!(ucmd
.flags
& MLX5_QP_FLAG_SCATTER_CQE
);
1591 qp
->wq_sig
= !!wq_signature
;
1594 qp
->has_rq
= qp_has_rq(init_attr
);
1595 err
= set_rq_size(dev
, &init_attr
->cap
, qp
->has_rq
,
1596 qp
, (pd
&& pd
->uobject
) ? &ucmd
: NULL
);
1598 mlx5_ib_dbg(dev
, "err %d\n", err
);
1605 1 << MLX5_CAP_GEN(mdev
, log_max_qp_sz
);
1606 mlx5_ib_dbg(dev
, "requested sq_wqe_count (%d)\n", ucmd
.sq_wqe_count
);
1607 if (ucmd
.rq_wqe_shift
!= qp
->rq
.wqe_shift
||
1608 ucmd
.rq_wqe_count
!= qp
->rq
.wqe_cnt
) {
1609 mlx5_ib_dbg(dev
, "invalid rq params\n");
1612 if (ucmd
.sq_wqe_count
> max_wqes
) {
1613 mlx5_ib_dbg(dev
, "requested sq_wqe_count (%d) > max allowed (%d)\n",
1614 ucmd
.sq_wqe_count
, max_wqes
);
1617 if (init_attr
->create_flags
&
1618 mlx5_ib_create_qp_sqpn_qp1()) {
1619 mlx5_ib_dbg(dev
, "user-space is not allowed to create UD QPs spoofing as QP1\n");
1622 err
= create_user_qp(dev
, pd
, qp
, udata
, init_attr
, &in
,
1623 &resp
, &inlen
, base
);
1625 mlx5_ib_dbg(dev
, "err %d\n", err
);
1627 err
= create_kernel_qp(dev
, init_attr
, qp
, &in
, &inlen
,
1630 mlx5_ib_dbg(dev
, "err %d\n", err
);
1636 in
= mlx5_vzalloc(inlen
);
1640 qp
->create_type
= MLX5_QP_EMPTY
;
1643 if (is_sqp(init_attr
->qp_type
))
1644 qp
->port
= init_attr
->port_num
;
1646 qpc
= MLX5_ADDR_OF(create_qp_in
, in
, qpc
);
1648 MLX5_SET(qpc
, qpc
, st
, to_mlx5_st(init_attr
->qp_type
));
1649 MLX5_SET(qpc
, qpc
, pm_state
, MLX5_QP_PM_MIGRATED
);
1651 if (init_attr
->qp_type
!= MLX5_IB_QPT_REG_UMR
)
1652 MLX5_SET(qpc
, qpc
, pd
, to_mpd(pd
? pd
: devr
->p0
)->pdn
);
1654 MLX5_SET(qpc
, qpc
, latency_sensitive
, 1);
1658 MLX5_SET(qpc
, qpc
, wq_signature
, 1);
1660 if (qp
->flags
& MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK
)
1661 MLX5_SET(qpc
, qpc
, block_lb_mc
, 1);
1663 if (qp
->flags
& MLX5_IB_QP_CROSS_CHANNEL
)
1664 MLX5_SET(qpc
, qpc
, cd_master
, 1);
1665 if (qp
->flags
& MLX5_IB_QP_MANAGED_SEND
)
1666 MLX5_SET(qpc
, qpc
, cd_slave_send
, 1);
1667 if (qp
->flags
& MLX5_IB_QP_MANAGED_RECV
)
1668 MLX5_SET(qpc
, qpc
, cd_slave_receive
, 1);
1670 if (qp
->scat_cqe
&& is_connected(init_attr
->qp_type
)) {
1674 rcqe_sz
= mlx5_ib_get_cqe_size(dev
, init_attr
->recv_cq
);
1675 scqe_sz
= mlx5_ib_get_cqe_size(dev
, init_attr
->send_cq
);
1678 MLX5_SET(qpc
, qpc
, cs_res
, MLX5_RES_SCAT_DATA64_CQE
);
1680 MLX5_SET(qpc
, qpc
, cs_res
, MLX5_RES_SCAT_DATA32_CQE
);
1682 if (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) {
1684 MLX5_SET(qpc
, qpc
, cs_req
, MLX5_REQ_SCAT_DATA64_CQE
);
1686 MLX5_SET(qpc
, qpc
, cs_req
, MLX5_REQ_SCAT_DATA32_CQE
);
1690 if (qp
->rq
.wqe_cnt
) {
1691 MLX5_SET(qpc
, qpc
, log_rq_stride
, qp
->rq
.wqe_shift
- 4);
1692 MLX5_SET(qpc
, qpc
, log_rq_size
, ilog2(qp
->rq
.wqe_cnt
));
1695 MLX5_SET(qpc
, qpc
, rq_type
, get_rx_type(qp
, init_attr
));
1698 MLX5_SET(qpc
, qpc
, log_sq_size
, ilog2(qp
->sq
.wqe_cnt
));
1700 MLX5_SET(qpc
, qpc
, no_sq
, 1);
1702 /* Set default resources */
1703 switch (init_attr
->qp_type
) {
1704 case IB_QPT_XRC_TGT
:
1705 MLX5_SET(qpc
, qpc
, cqn_rcv
, to_mcq(devr
->c0
)->mcq
.cqn
);
1706 MLX5_SET(qpc
, qpc
, cqn_snd
, to_mcq(devr
->c0
)->mcq
.cqn
);
1707 MLX5_SET(qpc
, qpc
, srqn_rmpn_xrqn
, to_msrq(devr
->s0
)->msrq
.srqn
);
1708 MLX5_SET(qpc
, qpc
, xrcd
, to_mxrcd(init_attr
->xrcd
)->xrcdn
);
1710 case IB_QPT_XRC_INI
:
1711 MLX5_SET(qpc
, qpc
, cqn_rcv
, to_mcq(devr
->c0
)->mcq
.cqn
);
1712 MLX5_SET(qpc
, qpc
, xrcd
, to_mxrcd(devr
->x1
)->xrcdn
);
1713 MLX5_SET(qpc
, qpc
, srqn_rmpn_xrqn
, to_msrq(devr
->s0
)->msrq
.srqn
);
1716 if (init_attr
->srq
) {
1717 MLX5_SET(qpc
, qpc
, xrcd
, to_mxrcd(devr
->x0
)->xrcdn
);
1718 MLX5_SET(qpc
, qpc
, srqn_rmpn_xrqn
, to_msrq(init_attr
->srq
)->msrq
.srqn
);
1720 MLX5_SET(qpc
, qpc
, xrcd
, to_mxrcd(devr
->x1
)->xrcdn
);
1721 MLX5_SET(qpc
, qpc
, srqn_rmpn_xrqn
, to_msrq(devr
->s1
)->msrq
.srqn
);
1725 if (init_attr
->send_cq
)
1726 MLX5_SET(qpc
, qpc
, cqn_snd
, to_mcq(init_attr
->send_cq
)->mcq
.cqn
);
1728 if (init_attr
->recv_cq
)
1729 MLX5_SET(qpc
, qpc
, cqn_rcv
, to_mcq(init_attr
->recv_cq
)->mcq
.cqn
);
1731 MLX5_SET64(qpc
, qpc
, dbr_addr
, qp
->db
.dma
);
1733 /* 0xffffff means we ask to work with cqe version 0 */
1734 if (MLX5_CAP_GEN(mdev
, cqe_version
) == MLX5_CQE_VERSION_V1
)
1735 MLX5_SET(qpc
, qpc
, user_index
, uidx
);
1737 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
1738 if (init_attr
->qp_type
== IB_QPT_UD
&&
1739 (init_attr
->create_flags
& IB_QP_CREATE_IPOIB_UD_LSO
)) {
1740 MLX5_SET(qpc
, qpc
, ulp_stateless_offload_mode
, 1);
1741 qp
->flags
|= MLX5_IB_QP_LSO
;
1744 if (init_attr
->qp_type
== IB_QPT_RAW_PACKET
) {
1745 qp
->raw_packet_qp
.sq
.ubuffer
.buf_addr
= ucmd
.sq_buf_addr
;
1746 raw_packet_qp_copy_info(qp
, &qp
->raw_packet_qp
);
1747 err
= create_raw_packet_qp(dev
, qp
, in
, pd
);
1749 err
= mlx5_core_create_qp(dev
->mdev
, &base
->mqp
, in
, inlen
);
1753 mlx5_ib_dbg(dev
, "create qp failed\n");
1759 base
->container_mibqp
= qp
;
1760 base
->mqp
.event
= mlx5_ib_qp_event
;
1762 get_cqs(init_attr
->qp_type
, init_attr
->send_cq
, init_attr
->recv_cq
,
1763 &send_cq
, &recv_cq
);
1764 spin_lock_irqsave(&dev
->reset_flow_resource_lock
, flags
);
1765 mlx5_ib_lock_cqs(send_cq
, recv_cq
);
1766 /* Maintain device to QPs access, needed for further handling via reset
1769 list_add_tail(&qp
->qps_list
, &dev
->qp_list
);
1770 /* Maintain CQ to QPs access, needed for further handling via reset flow
1773 list_add_tail(&qp
->cq_send_list
, &send_cq
->list_send_qp
);
1775 list_add_tail(&qp
->cq_recv_list
, &recv_cq
->list_recv_qp
);
1776 mlx5_ib_unlock_cqs(send_cq
, recv_cq
);
1777 spin_unlock_irqrestore(&dev
->reset_flow_resource_lock
, flags
);
1782 if (qp
->create_type
== MLX5_QP_USER
)
1783 destroy_qp_user(dev
, pd
, qp
, base
);
1784 else if (qp
->create_type
== MLX5_QP_KERNEL
)
1785 destroy_qp_kernel(dev
, qp
);
1791 static void mlx5_ib_lock_cqs(struct mlx5_ib_cq
*send_cq
, struct mlx5_ib_cq
*recv_cq
)
1792 __acquires(&send_cq
->lock
) __acquires(&recv_cq
->lock
)
1796 if (send_cq
->mcq
.cqn
< recv_cq
->mcq
.cqn
) {
1797 spin_lock(&send_cq
->lock
);
1798 spin_lock_nested(&recv_cq
->lock
,
1799 SINGLE_DEPTH_NESTING
);
1800 } else if (send_cq
->mcq
.cqn
== recv_cq
->mcq
.cqn
) {
1801 spin_lock(&send_cq
->lock
);
1802 __acquire(&recv_cq
->lock
);
1804 spin_lock(&recv_cq
->lock
);
1805 spin_lock_nested(&send_cq
->lock
,
1806 SINGLE_DEPTH_NESTING
);
1809 spin_lock(&send_cq
->lock
);
1810 __acquire(&recv_cq
->lock
);
1812 } else if (recv_cq
) {
1813 spin_lock(&recv_cq
->lock
);
1814 __acquire(&send_cq
->lock
);
1816 __acquire(&send_cq
->lock
);
1817 __acquire(&recv_cq
->lock
);
1821 static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq
*send_cq
, struct mlx5_ib_cq
*recv_cq
)
1822 __releases(&send_cq
->lock
) __releases(&recv_cq
->lock
)
1826 if (send_cq
->mcq
.cqn
< recv_cq
->mcq
.cqn
) {
1827 spin_unlock(&recv_cq
->lock
);
1828 spin_unlock(&send_cq
->lock
);
1829 } else if (send_cq
->mcq
.cqn
== recv_cq
->mcq
.cqn
) {
1830 __release(&recv_cq
->lock
);
1831 spin_unlock(&send_cq
->lock
);
1833 spin_unlock(&send_cq
->lock
);
1834 spin_unlock(&recv_cq
->lock
);
1837 __release(&recv_cq
->lock
);
1838 spin_unlock(&send_cq
->lock
);
1840 } else if (recv_cq
) {
1841 __release(&send_cq
->lock
);
1842 spin_unlock(&recv_cq
->lock
);
1844 __release(&recv_cq
->lock
);
1845 __release(&send_cq
->lock
);
1849 static struct mlx5_ib_pd
*get_pd(struct mlx5_ib_qp
*qp
)
1851 return to_mpd(qp
->ibqp
.pd
);
1854 static void get_cqs(enum ib_qp_type qp_type
,
1855 struct ib_cq
*ib_send_cq
, struct ib_cq
*ib_recv_cq
,
1856 struct mlx5_ib_cq
**send_cq
, struct mlx5_ib_cq
**recv_cq
)
1859 case IB_QPT_XRC_TGT
:
1863 case MLX5_IB_QPT_REG_UMR
:
1864 case IB_QPT_XRC_INI
:
1865 *send_cq
= ib_send_cq
? to_mcq(ib_send_cq
) : NULL
;
1870 case MLX5_IB_QPT_HW_GSI
:
1874 case IB_QPT_RAW_IPV6
:
1875 case IB_QPT_RAW_ETHERTYPE
:
1876 case IB_QPT_RAW_PACKET
:
1877 *send_cq
= ib_send_cq
? to_mcq(ib_send_cq
) : NULL
;
1878 *recv_cq
= ib_recv_cq
? to_mcq(ib_recv_cq
) : NULL
;
1889 static int modify_raw_packet_qp(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
1890 const struct mlx5_modify_raw_qp_param
*raw_qp_param
,
1891 u8 lag_tx_affinity
);
1893 static void destroy_qp_common(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
)
1895 struct mlx5_ib_cq
*send_cq
, *recv_cq
;
1896 struct mlx5_ib_qp_base
*base
= &qp
->trans_qp
.base
;
1897 unsigned long flags
;
1900 if (qp
->ibqp
.rwq_ind_tbl
) {
1901 destroy_rss_raw_qp_tir(dev
, qp
);
1905 base
= qp
->ibqp
.qp_type
== IB_QPT_RAW_PACKET
?
1906 &qp
->raw_packet_qp
.rq
.base
:
1909 if (qp
->state
!= IB_QPS_RESET
) {
1910 if (qp
->ibqp
.qp_type
!= IB_QPT_RAW_PACKET
) {
1911 err
= mlx5_core_qp_modify(dev
->mdev
,
1912 MLX5_CMD_OP_2RST_QP
, 0,
1915 struct mlx5_modify_raw_qp_param raw_qp_param
= {
1916 .operation
= MLX5_CMD_OP_2RST_QP
1919 err
= modify_raw_packet_qp(dev
, qp
, &raw_qp_param
, 0);
1922 mlx5_ib_warn(dev
, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
1926 get_cqs(qp
->ibqp
.qp_type
, qp
->ibqp
.send_cq
, qp
->ibqp
.recv_cq
,
1927 &send_cq
, &recv_cq
);
1929 spin_lock_irqsave(&dev
->reset_flow_resource_lock
, flags
);
1930 mlx5_ib_lock_cqs(send_cq
, recv_cq
);
1931 /* del from lists under both locks above to protect reset flow paths */
1932 list_del(&qp
->qps_list
);
1934 list_del(&qp
->cq_send_list
);
1937 list_del(&qp
->cq_recv_list
);
1939 if (qp
->create_type
== MLX5_QP_KERNEL
) {
1940 __mlx5_ib_cq_clean(recv_cq
, base
->mqp
.qpn
,
1941 qp
->ibqp
.srq
? to_msrq(qp
->ibqp
.srq
) : NULL
);
1942 if (send_cq
!= recv_cq
)
1943 __mlx5_ib_cq_clean(send_cq
, base
->mqp
.qpn
,
1946 mlx5_ib_unlock_cqs(send_cq
, recv_cq
);
1947 spin_unlock_irqrestore(&dev
->reset_flow_resource_lock
, flags
);
1949 if (qp
->ibqp
.qp_type
== IB_QPT_RAW_PACKET
) {
1950 destroy_raw_packet_qp(dev
, qp
);
1952 err
= mlx5_core_destroy_qp(dev
->mdev
, &base
->mqp
);
1954 mlx5_ib_warn(dev
, "failed to destroy QP 0x%x\n",
1958 if (qp
->create_type
== MLX5_QP_KERNEL
)
1959 destroy_qp_kernel(dev
, qp
);
1960 else if (qp
->create_type
== MLX5_QP_USER
)
1961 destroy_qp_user(dev
, &get_pd(qp
)->ibpd
, qp
, base
);
1964 static const char *ib_qp_type_str(enum ib_qp_type type
)
1968 return "IB_QPT_SMI";
1970 return "IB_QPT_GSI";
1977 case IB_QPT_RAW_IPV6
:
1978 return "IB_QPT_RAW_IPV6";
1979 case IB_QPT_RAW_ETHERTYPE
:
1980 return "IB_QPT_RAW_ETHERTYPE";
1981 case IB_QPT_XRC_INI
:
1982 return "IB_QPT_XRC_INI";
1983 case IB_QPT_XRC_TGT
:
1984 return "IB_QPT_XRC_TGT";
1985 case IB_QPT_RAW_PACKET
:
1986 return "IB_QPT_RAW_PACKET";
1987 case MLX5_IB_QPT_REG_UMR
:
1988 return "MLX5_IB_QPT_REG_UMR";
1991 return "Invalid QP type";
1995 struct ib_qp
*mlx5_ib_create_qp(struct ib_pd
*pd
,
1996 struct ib_qp_init_attr
*init_attr
,
1997 struct ib_udata
*udata
)
1999 struct mlx5_ib_dev
*dev
;
2000 struct mlx5_ib_qp
*qp
;
2005 dev
= to_mdev(pd
->device
);
2007 if (init_attr
->qp_type
== IB_QPT_RAW_PACKET
) {
2009 mlx5_ib_dbg(dev
, "Raw Packet QP is not supported for kernel consumers\n");
2010 return ERR_PTR(-EINVAL
);
2011 } else if (!to_mucontext(pd
->uobject
->context
)->cqe_version
) {
2012 mlx5_ib_dbg(dev
, "Raw Packet QP is only supported for CQE version > 0\n");
2013 return ERR_PTR(-EINVAL
);
2017 /* being cautious here */
2018 if (init_attr
->qp_type
!= IB_QPT_XRC_TGT
&&
2019 init_attr
->qp_type
!= MLX5_IB_QPT_REG_UMR
) {
2020 pr_warn("%s: no PD for transport %s\n", __func__
,
2021 ib_qp_type_str(init_attr
->qp_type
));
2022 return ERR_PTR(-EINVAL
);
2024 dev
= to_mdev(to_mxrcd(init_attr
->xrcd
)->ibxrcd
.device
);
2027 switch (init_attr
->qp_type
) {
2028 case IB_QPT_XRC_TGT
:
2029 case IB_QPT_XRC_INI
:
2030 if (!MLX5_CAP_GEN(dev
->mdev
, xrc
)) {
2031 mlx5_ib_dbg(dev
, "XRC not supported\n");
2032 return ERR_PTR(-ENOSYS
);
2034 init_attr
->recv_cq
= NULL
;
2035 if (init_attr
->qp_type
== IB_QPT_XRC_TGT
) {
2036 xrcdn
= to_mxrcd(init_attr
->xrcd
)->xrcdn
;
2037 init_attr
->send_cq
= NULL
;
2041 case IB_QPT_RAW_PACKET
:
2046 case MLX5_IB_QPT_HW_GSI
:
2047 case MLX5_IB_QPT_REG_UMR
:
2048 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
2050 return ERR_PTR(-ENOMEM
);
2052 err
= create_qp_common(dev
, pd
, init_attr
, udata
, qp
);
2054 mlx5_ib_dbg(dev
, "create_qp_common failed\n");
2056 return ERR_PTR(err
);
2059 if (is_qp0(init_attr
->qp_type
))
2060 qp
->ibqp
.qp_num
= 0;
2061 else if (is_qp1(init_attr
->qp_type
))
2062 qp
->ibqp
.qp_num
= 1;
2064 qp
->ibqp
.qp_num
= qp
->trans_qp
.base
.mqp
.qpn
;
2066 mlx5_ib_dbg(dev
, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
2067 qp
->ibqp
.qp_num
, qp
->trans_qp
.base
.mqp
.qpn
,
2068 init_attr
->recv_cq
? to_mcq(init_attr
->recv_cq
)->mcq
.cqn
: -1,
2069 init_attr
->send_cq
? to_mcq(init_attr
->send_cq
)->mcq
.cqn
: -1);
2071 qp
->trans_qp
.xrcdn
= xrcdn
;
2076 return mlx5_ib_gsi_create_qp(pd
, init_attr
);
2078 case IB_QPT_RAW_IPV6
:
2079 case IB_QPT_RAW_ETHERTYPE
:
2082 mlx5_ib_dbg(dev
, "unsupported qp type %d\n",
2083 init_attr
->qp_type
);
2084 /* Don't support raw QPs */
2085 return ERR_PTR(-EINVAL
);
2091 int mlx5_ib_destroy_qp(struct ib_qp
*qp
)
2093 struct mlx5_ib_dev
*dev
= to_mdev(qp
->device
);
2094 struct mlx5_ib_qp
*mqp
= to_mqp(qp
);
2096 if (unlikely(qp
->qp_type
== IB_QPT_GSI
))
2097 return mlx5_ib_gsi_destroy_qp(qp
);
2099 destroy_qp_common(dev
, mqp
);
2106 static __be32
to_mlx5_access_flags(struct mlx5_ib_qp
*qp
, const struct ib_qp_attr
*attr
,
2109 u32 hw_access_flags
= 0;
2113 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
2114 dest_rd_atomic
= attr
->max_dest_rd_atomic
;
2116 dest_rd_atomic
= qp
->trans_qp
.resp_depth
;
2118 if (attr_mask
& IB_QP_ACCESS_FLAGS
)
2119 access_flags
= attr
->qp_access_flags
;
2121 access_flags
= qp
->trans_qp
.atomic_rd_en
;
2123 if (!dest_rd_atomic
)
2124 access_flags
&= IB_ACCESS_REMOTE_WRITE
;
2126 if (access_flags
& IB_ACCESS_REMOTE_READ
)
2127 hw_access_flags
|= MLX5_QP_BIT_RRE
;
2128 if (access_flags
& IB_ACCESS_REMOTE_ATOMIC
)
2129 hw_access_flags
|= (MLX5_QP_BIT_RAE
| MLX5_ATOMIC_MODE_CX
);
2130 if (access_flags
& IB_ACCESS_REMOTE_WRITE
)
2131 hw_access_flags
|= MLX5_QP_BIT_RWE
;
2133 return cpu_to_be32(hw_access_flags
);
2137 MLX5_PATH_FLAG_FL
= 1 << 0,
2138 MLX5_PATH_FLAG_FREE_AR
= 1 << 1,
2139 MLX5_PATH_FLAG_COUNTER
= 1 << 2,
2142 static int ib_rate_to_mlx5(struct mlx5_ib_dev
*dev
, u8 rate
)
2144 if (rate
== IB_RATE_PORT_CURRENT
) {
2146 } else if (rate
< IB_RATE_2_5_GBPS
|| rate
> IB_RATE_300_GBPS
) {
2149 while (rate
!= IB_RATE_2_5_GBPS
&&
2150 !(1 << (rate
+ MLX5_STAT_RATE_OFFSET
) &
2151 MLX5_CAP_GEN(dev
->mdev
, stat_rate_support
)))
2155 return rate
+ MLX5_STAT_RATE_OFFSET
;
2158 static int modify_raw_packet_eth_prio(struct mlx5_core_dev
*dev
,
2159 struct mlx5_ib_sq
*sq
, u8 sl
)
2166 inlen
= MLX5_ST_SZ_BYTES(modify_tis_in
);
2167 in
= mlx5_vzalloc(inlen
);
2171 MLX5_SET(modify_tis_in
, in
, bitmask
.prio
, 1);
2173 tisc
= MLX5_ADDR_OF(modify_tis_in
, in
, ctx
);
2174 MLX5_SET(tisc
, tisc
, prio
, ((sl
& 0x7) << 1));
2176 err
= mlx5_core_modify_tis(dev
, sq
->tisn
, in
, inlen
);
2183 static int modify_raw_packet_tx_affinity(struct mlx5_core_dev
*dev
,
2184 struct mlx5_ib_sq
*sq
, u8 tx_affinity
)
2191 inlen
= MLX5_ST_SZ_BYTES(modify_tis_in
);
2192 in
= mlx5_vzalloc(inlen
);
2196 MLX5_SET(modify_tis_in
, in
, bitmask
.lag_tx_port_affinity
, 1);
2198 tisc
= MLX5_ADDR_OF(modify_tis_in
, in
, ctx
);
2199 MLX5_SET(tisc
, tisc
, lag_tx_port_affinity
, tx_affinity
);
2201 err
= mlx5_core_modify_tis(dev
, sq
->tisn
, in
, inlen
);
2208 static int mlx5_set_path(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
2209 const struct ib_ah_attr
*ah
,
2210 struct mlx5_qp_path
*path
, u8 port
, int attr_mask
,
2211 u32 path_flags
, const struct ib_qp_attr
*attr
,
2214 enum rdma_link_layer ll
= rdma_port_get_link_layer(&dev
->ib_dev
, port
);
2216 enum ib_gid_type gid_type
;
2218 if (attr_mask
& IB_QP_PKEY_INDEX
)
2219 path
->pkey_index
= cpu_to_be16(alt
? attr
->alt_pkey_index
:
2222 if (ah
->ah_flags
& IB_AH_GRH
) {
2223 if (ah
->grh
.sgid_index
>=
2224 dev
->mdev
->port_caps
[port
- 1].gid_table_len
) {
2225 pr_err("sgid_index (%u) too large. max is %d\n",
2227 dev
->mdev
->port_caps
[port
- 1].gid_table_len
);
2232 if (ll
== IB_LINK_LAYER_ETHERNET
) {
2233 if (!(ah
->ah_flags
& IB_AH_GRH
))
2235 err
= mlx5_get_roce_gid_type(dev
, port
, ah
->grh
.sgid_index
,
2239 memcpy(path
->rmac
, ah
->dmac
, sizeof(ah
->dmac
));
2240 path
->udp_sport
= mlx5_get_roce_udp_sport(dev
, port
,
2241 ah
->grh
.sgid_index
);
2242 path
->dci_cfi_prio_sl
= (ah
->sl
& 0x7) << 4;
2243 if (gid_type
== IB_GID_TYPE_ROCE_UDP_ENCAP
)
2244 path
->ecn_dscp
= (ah
->grh
.traffic_class
>> 2) & 0x3f;
2246 path
->fl_free_ar
= (path_flags
& MLX5_PATH_FLAG_FL
) ? 0x80 : 0;
2248 (path_flags
& MLX5_PATH_FLAG_FREE_AR
) ? 0x40 : 0;
2249 path
->rlid
= cpu_to_be16(ah
->dlid
);
2250 path
->grh_mlid
= ah
->src_path_bits
& 0x7f;
2251 if (ah
->ah_flags
& IB_AH_GRH
)
2252 path
->grh_mlid
|= 1 << 7;
2253 path
->dci_cfi_prio_sl
= ah
->sl
& 0xf;
2256 if (ah
->ah_flags
& IB_AH_GRH
) {
2257 path
->mgid_index
= ah
->grh
.sgid_index
;
2258 path
->hop_limit
= ah
->grh
.hop_limit
;
2259 path
->tclass_flowlabel
=
2260 cpu_to_be32((ah
->grh
.traffic_class
<< 20) |
2261 (ah
->grh
.flow_label
));
2262 memcpy(path
->rgid
, ah
->grh
.dgid
.raw
, 16);
2265 err
= ib_rate_to_mlx5(dev
, ah
->static_rate
);
2268 path
->static_rate
= err
;
2271 if (attr_mask
& IB_QP_TIMEOUT
)
2272 path
->ackto_lt
= (alt
? attr
->alt_timeout
: attr
->timeout
) << 3;
2274 if ((qp
->ibqp
.qp_type
== IB_QPT_RAW_PACKET
) && qp
->sq
.wqe_cnt
)
2275 return modify_raw_packet_eth_prio(dev
->mdev
,
2276 &qp
->raw_packet_qp
.sq
,
2282 static enum mlx5_qp_optpar opt_mask
[MLX5_QP_NUM_STATE
][MLX5_QP_NUM_STATE
][MLX5_QP_ST_MAX
] = {
2283 [MLX5_QP_STATE_INIT
] = {
2284 [MLX5_QP_STATE_INIT
] = {
2285 [MLX5_QP_ST_RC
] = MLX5_QP_OPTPAR_RRE
|
2286 MLX5_QP_OPTPAR_RAE
|
2287 MLX5_QP_OPTPAR_RWE
|
2288 MLX5_QP_OPTPAR_PKEY_INDEX
|
2289 MLX5_QP_OPTPAR_PRI_PORT
,
2290 [MLX5_QP_ST_UC
] = MLX5_QP_OPTPAR_RWE
|
2291 MLX5_QP_OPTPAR_PKEY_INDEX
|
2292 MLX5_QP_OPTPAR_PRI_PORT
,
2293 [MLX5_QP_ST_UD
] = MLX5_QP_OPTPAR_PKEY_INDEX
|
2294 MLX5_QP_OPTPAR_Q_KEY
|
2295 MLX5_QP_OPTPAR_PRI_PORT
,
2297 [MLX5_QP_STATE_RTR
] = {
2298 [MLX5_QP_ST_RC
] = MLX5_QP_OPTPAR_ALT_ADDR_PATH
|
2299 MLX5_QP_OPTPAR_RRE
|
2300 MLX5_QP_OPTPAR_RAE
|
2301 MLX5_QP_OPTPAR_RWE
|
2302 MLX5_QP_OPTPAR_PKEY_INDEX
,
2303 [MLX5_QP_ST_UC
] = MLX5_QP_OPTPAR_ALT_ADDR_PATH
|
2304 MLX5_QP_OPTPAR_RWE
|
2305 MLX5_QP_OPTPAR_PKEY_INDEX
,
2306 [MLX5_QP_ST_UD
] = MLX5_QP_OPTPAR_PKEY_INDEX
|
2307 MLX5_QP_OPTPAR_Q_KEY
,
2308 [MLX5_QP_ST_MLX
] = MLX5_QP_OPTPAR_PKEY_INDEX
|
2309 MLX5_QP_OPTPAR_Q_KEY
,
2310 [MLX5_QP_ST_XRC
] = MLX5_QP_OPTPAR_ALT_ADDR_PATH
|
2311 MLX5_QP_OPTPAR_RRE
|
2312 MLX5_QP_OPTPAR_RAE
|
2313 MLX5_QP_OPTPAR_RWE
|
2314 MLX5_QP_OPTPAR_PKEY_INDEX
,
2317 [MLX5_QP_STATE_RTR
] = {
2318 [MLX5_QP_STATE_RTS
] = {
2319 [MLX5_QP_ST_RC
] = MLX5_QP_OPTPAR_ALT_ADDR_PATH
|
2320 MLX5_QP_OPTPAR_RRE
|
2321 MLX5_QP_OPTPAR_RAE
|
2322 MLX5_QP_OPTPAR_RWE
|
2323 MLX5_QP_OPTPAR_PM_STATE
|
2324 MLX5_QP_OPTPAR_RNR_TIMEOUT
,
2325 [MLX5_QP_ST_UC
] = MLX5_QP_OPTPAR_ALT_ADDR_PATH
|
2326 MLX5_QP_OPTPAR_RWE
|
2327 MLX5_QP_OPTPAR_PM_STATE
,
2328 [MLX5_QP_ST_UD
] = MLX5_QP_OPTPAR_Q_KEY
,
2331 [MLX5_QP_STATE_RTS
] = {
2332 [MLX5_QP_STATE_RTS
] = {
2333 [MLX5_QP_ST_RC
] = MLX5_QP_OPTPAR_RRE
|
2334 MLX5_QP_OPTPAR_RAE
|
2335 MLX5_QP_OPTPAR_RWE
|
2336 MLX5_QP_OPTPAR_RNR_TIMEOUT
|
2337 MLX5_QP_OPTPAR_PM_STATE
|
2338 MLX5_QP_OPTPAR_ALT_ADDR_PATH
,
2339 [MLX5_QP_ST_UC
] = MLX5_QP_OPTPAR_RWE
|
2340 MLX5_QP_OPTPAR_PM_STATE
|
2341 MLX5_QP_OPTPAR_ALT_ADDR_PATH
,
2342 [MLX5_QP_ST_UD
] = MLX5_QP_OPTPAR_Q_KEY
|
2343 MLX5_QP_OPTPAR_SRQN
|
2344 MLX5_QP_OPTPAR_CQN_RCV
,
2347 [MLX5_QP_STATE_SQER
] = {
2348 [MLX5_QP_STATE_RTS
] = {
2349 [MLX5_QP_ST_UD
] = MLX5_QP_OPTPAR_Q_KEY
,
2350 [MLX5_QP_ST_MLX
] = MLX5_QP_OPTPAR_Q_KEY
,
2351 [MLX5_QP_ST_UC
] = MLX5_QP_OPTPAR_RWE
,
2352 [MLX5_QP_ST_RC
] = MLX5_QP_OPTPAR_RNR_TIMEOUT
|
2353 MLX5_QP_OPTPAR_RWE
|
2354 MLX5_QP_OPTPAR_RAE
|
2360 static int ib_nr_to_mlx5_nr(int ib_mask
)
2365 case IB_QP_CUR_STATE
:
2367 case IB_QP_EN_SQD_ASYNC_NOTIFY
:
2369 case IB_QP_ACCESS_FLAGS
:
2370 return MLX5_QP_OPTPAR_RWE
| MLX5_QP_OPTPAR_RRE
|
2372 case IB_QP_PKEY_INDEX
:
2373 return MLX5_QP_OPTPAR_PKEY_INDEX
;
2375 return MLX5_QP_OPTPAR_PRI_PORT
;
2377 return MLX5_QP_OPTPAR_Q_KEY
;
2379 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH
|
2380 MLX5_QP_OPTPAR_PRI_PORT
;
2381 case IB_QP_PATH_MTU
:
2384 return MLX5_QP_OPTPAR_ACK_TIMEOUT
;
2385 case IB_QP_RETRY_CNT
:
2386 return MLX5_QP_OPTPAR_RETRY_COUNT
;
2387 case IB_QP_RNR_RETRY
:
2388 return MLX5_QP_OPTPAR_RNR_RETRY
;
2391 case IB_QP_MAX_QP_RD_ATOMIC
:
2392 return MLX5_QP_OPTPAR_SRA_MAX
;
2393 case IB_QP_ALT_PATH
:
2394 return MLX5_QP_OPTPAR_ALT_ADDR_PATH
;
2395 case IB_QP_MIN_RNR_TIMER
:
2396 return MLX5_QP_OPTPAR_RNR_TIMEOUT
;
2399 case IB_QP_MAX_DEST_RD_ATOMIC
:
2400 return MLX5_QP_OPTPAR_RRA_MAX
| MLX5_QP_OPTPAR_RWE
|
2401 MLX5_QP_OPTPAR_RRE
| MLX5_QP_OPTPAR_RAE
;
2402 case IB_QP_PATH_MIG_STATE
:
2403 return MLX5_QP_OPTPAR_PM_STATE
;
2406 case IB_QP_DEST_QPN
:
2412 static int ib_mask_to_mlx5_opt(int ib_mask
)
2417 for (i
= 0; i
< 8 * sizeof(int); i
++) {
2418 if ((1 << i
) & ib_mask
)
2419 result
|= ib_nr_to_mlx5_nr(1 << i
);
2425 static int modify_raw_packet_qp_rq(struct mlx5_ib_dev
*dev
,
2426 struct mlx5_ib_rq
*rq
, int new_state
,
2427 const struct mlx5_modify_raw_qp_param
*raw_qp_param
)
2434 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
2435 in
= mlx5_vzalloc(inlen
);
2439 MLX5_SET(modify_rq_in
, in
, rq_state
, rq
->state
);
2441 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
2442 MLX5_SET(rqc
, rqc
, state
, new_state
);
2444 if (raw_qp_param
->set_mask
& MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID
) {
2445 if (MLX5_CAP_GEN(dev
->mdev
, modify_rq_counter_set_id
)) {
2446 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
2447 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID
);
2448 MLX5_SET(rqc
, rqc
, counter_set_id
, raw_qp_param
->rq_q_ctr_id
);
2450 pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
2454 err
= mlx5_core_modify_rq(dev
->mdev
, rq
->base
.mqp
.qpn
, in
, inlen
);
2458 rq
->state
= new_state
;
2465 static int modify_raw_packet_qp_sq(struct mlx5_core_dev
*dev
,
2466 struct mlx5_ib_sq
*sq
,
2468 const struct mlx5_modify_raw_qp_param
*raw_qp_param
)
2470 struct mlx5_ib_qp
*ibqp
= sq
->base
.container_mibqp
;
2471 u32 old_rate
= ibqp
->rate_limit
;
2472 u32 new_rate
= old_rate
;
2479 inlen
= MLX5_ST_SZ_BYTES(modify_sq_in
);
2480 in
= mlx5_vzalloc(inlen
);
2484 MLX5_SET(modify_sq_in
, in
, sq_state
, sq
->state
);
2486 sqc
= MLX5_ADDR_OF(modify_sq_in
, in
, ctx
);
2487 MLX5_SET(sqc
, sqc
, state
, new_state
);
2489 if (raw_qp_param
->set_mask
& MLX5_RAW_QP_RATE_LIMIT
) {
2490 if (new_state
!= MLX5_SQC_STATE_RDY
)
2491 pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
2494 new_rate
= raw_qp_param
->rate_limit
;
2497 if (old_rate
!= new_rate
) {
2499 err
= mlx5_rl_add_rate(dev
, new_rate
, &rl_index
);
2501 pr_err("Failed configuring rate %u: %d\n",
2507 MLX5_SET64(modify_sq_in
, in
, modify_bitmask
, 1);
2508 MLX5_SET(sqc
, sqc
, packet_pacing_rate_limit_index
, rl_index
);
2511 err
= mlx5_core_modify_sq(dev
, sq
->base
.mqp
.qpn
, in
, inlen
);
2513 /* Remove new rate from table if failed */
2515 old_rate
!= new_rate
)
2516 mlx5_rl_remove_rate(dev
, new_rate
);
2520 /* Only remove the old rate after new rate was set */
2522 (old_rate
!= new_rate
)) ||
2523 (new_state
!= MLX5_SQC_STATE_RDY
))
2524 mlx5_rl_remove_rate(dev
, old_rate
);
2526 ibqp
->rate_limit
= new_rate
;
2527 sq
->state
= new_state
;
2534 static int modify_raw_packet_qp(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
2535 const struct mlx5_modify_raw_qp_param
*raw_qp_param
,
2538 struct mlx5_ib_raw_packet_qp
*raw_packet_qp
= &qp
->raw_packet_qp
;
2539 struct mlx5_ib_rq
*rq
= &raw_packet_qp
->rq
;
2540 struct mlx5_ib_sq
*sq
= &raw_packet_qp
->sq
;
2541 int modify_rq
= !!qp
->rq
.wqe_cnt
;
2542 int modify_sq
= !!qp
->sq
.wqe_cnt
;
2547 switch (raw_qp_param
->operation
) {
2548 case MLX5_CMD_OP_RST2INIT_QP
:
2549 rq_state
= MLX5_RQC_STATE_RDY
;
2550 sq_state
= MLX5_SQC_STATE_RDY
;
2552 case MLX5_CMD_OP_2ERR_QP
:
2553 rq_state
= MLX5_RQC_STATE_ERR
;
2554 sq_state
= MLX5_SQC_STATE_ERR
;
2556 case MLX5_CMD_OP_2RST_QP
:
2557 rq_state
= MLX5_RQC_STATE_RST
;
2558 sq_state
= MLX5_SQC_STATE_RST
;
2560 case MLX5_CMD_OP_RTR2RTS_QP
:
2561 case MLX5_CMD_OP_RTS2RTS_QP
:
2562 if (raw_qp_param
->set_mask
==
2563 MLX5_RAW_QP_RATE_LIMIT
) {
2565 sq_state
= sq
->state
;
2567 return raw_qp_param
->set_mask
? -EINVAL
: 0;
2570 case MLX5_CMD_OP_INIT2INIT_QP
:
2571 case MLX5_CMD_OP_INIT2RTR_QP
:
2572 if (raw_qp_param
->set_mask
)
2582 err
= modify_raw_packet_qp_rq(dev
, rq
, rq_state
, raw_qp_param
);
2589 err
= modify_raw_packet_tx_affinity(dev
->mdev
, sq
,
2595 return modify_raw_packet_qp_sq(dev
->mdev
, sq
, sq_state
, raw_qp_param
);
2601 static int __mlx5_ib_modify_qp(struct ib_qp
*ibqp
,
2602 const struct ib_qp_attr
*attr
, int attr_mask
,
2603 enum ib_qp_state cur_state
, enum ib_qp_state new_state
)
2605 static const u16 optab
[MLX5_QP_NUM_STATE
][MLX5_QP_NUM_STATE
] = {
2606 [MLX5_QP_STATE_RST
] = {
2607 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2608 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2609 [MLX5_QP_STATE_INIT
] = MLX5_CMD_OP_RST2INIT_QP
,
2611 [MLX5_QP_STATE_INIT
] = {
2612 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2613 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2614 [MLX5_QP_STATE_INIT
] = MLX5_CMD_OP_INIT2INIT_QP
,
2615 [MLX5_QP_STATE_RTR
] = MLX5_CMD_OP_INIT2RTR_QP
,
2617 [MLX5_QP_STATE_RTR
] = {
2618 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2619 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2620 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_RTR2RTS_QP
,
2622 [MLX5_QP_STATE_RTS
] = {
2623 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2624 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2625 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_RTS2RTS_QP
,
2627 [MLX5_QP_STATE_SQD
] = {
2628 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2629 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2631 [MLX5_QP_STATE_SQER
] = {
2632 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2633 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2634 [MLX5_QP_STATE_RTS
] = MLX5_CMD_OP_SQERR2RTS_QP
,
2636 [MLX5_QP_STATE_ERR
] = {
2637 [MLX5_QP_STATE_RST
] = MLX5_CMD_OP_2RST_QP
,
2638 [MLX5_QP_STATE_ERR
] = MLX5_CMD_OP_2ERR_QP
,
2642 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
2643 struct mlx5_ib_qp
*qp
= to_mqp(ibqp
);
2644 struct mlx5_ib_qp_base
*base
= &qp
->trans_qp
.base
;
2645 struct mlx5_ib_cq
*send_cq
, *recv_cq
;
2646 struct mlx5_qp_context
*context
;
2647 struct mlx5_ib_pd
*pd
;
2648 struct mlx5_ib_port
*mibport
= NULL
;
2649 enum mlx5_qp_state mlx5_cur
, mlx5_new
;
2650 enum mlx5_qp_optpar optpar
;
2656 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
2660 err
= to_mlx5_st(ibqp
->qp_type
);
2662 mlx5_ib_dbg(dev
, "unsupported qp type %d\n", ibqp
->qp_type
);
2666 context
->flags
= cpu_to_be32(err
<< 16);
2668 if (!(attr_mask
& IB_QP_PATH_MIG_STATE
)) {
2669 context
->flags
|= cpu_to_be32(MLX5_QP_PM_MIGRATED
<< 11);
2671 switch (attr
->path_mig_state
) {
2672 case IB_MIG_MIGRATED
:
2673 context
->flags
|= cpu_to_be32(MLX5_QP_PM_MIGRATED
<< 11);
2676 context
->flags
|= cpu_to_be32(MLX5_QP_PM_REARM
<< 11);
2679 context
->flags
|= cpu_to_be32(MLX5_QP_PM_ARMED
<< 11);
2684 if ((cur_state
== IB_QPS_RESET
) && (new_state
== IB_QPS_INIT
)) {
2685 if ((ibqp
->qp_type
== IB_QPT_RC
) ||
2686 (ibqp
->qp_type
== IB_QPT_UD
&&
2687 !(qp
->flags
& MLX5_IB_QP_SQPN_QP1
)) ||
2688 (ibqp
->qp_type
== IB_QPT_UC
) ||
2689 (ibqp
->qp_type
== IB_QPT_RAW_PACKET
) ||
2690 (ibqp
->qp_type
== IB_QPT_XRC_INI
) ||
2691 (ibqp
->qp_type
== IB_QPT_XRC_TGT
)) {
2692 if (mlx5_lag_is_active(dev
->mdev
)) {
2693 tx_affinity
= (unsigned int)atomic_add_return(1,
2694 &dev
->roce
.next_port
) %
2696 context
->flags
|= cpu_to_be32(tx_affinity
<< 24);
2701 if (is_sqp(ibqp
->qp_type
)) {
2702 context
->mtu_msgmax
= (IB_MTU_256
<< 5) | 8;
2703 } else if (ibqp
->qp_type
== IB_QPT_UD
||
2704 ibqp
->qp_type
== MLX5_IB_QPT_REG_UMR
) {
2705 context
->mtu_msgmax
= (IB_MTU_4096
<< 5) | 12;
2706 } else if (attr_mask
& IB_QP_PATH_MTU
) {
2707 if (attr
->path_mtu
< IB_MTU_256
||
2708 attr
->path_mtu
> IB_MTU_4096
) {
2709 mlx5_ib_warn(dev
, "invalid mtu %d\n", attr
->path_mtu
);
2713 context
->mtu_msgmax
= (attr
->path_mtu
<< 5) |
2714 (u8
)MLX5_CAP_GEN(dev
->mdev
, log_max_msg
);
2717 if (attr_mask
& IB_QP_DEST_QPN
)
2718 context
->log_pg_sz_remote_qpn
= cpu_to_be32(attr
->dest_qp_num
);
2720 if (attr_mask
& IB_QP_PKEY_INDEX
)
2721 context
->pri_path
.pkey_index
= cpu_to_be16(attr
->pkey_index
);
2723 /* todo implement counter_index functionality */
2725 if (is_sqp(ibqp
->qp_type
))
2726 context
->pri_path
.port
= qp
->port
;
2728 if (attr_mask
& IB_QP_PORT
)
2729 context
->pri_path
.port
= attr
->port_num
;
2731 if (attr_mask
& IB_QP_AV
) {
2732 err
= mlx5_set_path(dev
, qp
, &attr
->ah_attr
, &context
->pri_path
,
2733 attr_mask
& IB_QP_PORT
? attr
->port_num
: qp
->port
,
2734 attr_mask
, 0, attr
, false);
2739 if (attr_mask
& IB_QP_TIMEOUT
)
2740 context
->pri_path
.ackto_lt
|= attr
->timeout
<< 3;
2742 if (attr_mask
& IB_QP_ALT_PATH
) {
2743 err
= mlx5_set_path(dev
, qp
, &attr
->alt_ah_attr
,
2746 attr_mask
| IB_QP_PKEY_INDEX
| IB_QP_TIMEOUT
,
2753 get_cqs(qp
->ibqp
.qp_type
, qp
->ibqp
.send_cq
, qp
->ibqp
.recv_cq
,
2754 &send_cq
, &recv_cq
);
2756 context
->flags_pd
= cpu_to_be32(pd
? pd
->pdn
: to_mpd(dev
->devr
.p0
)->pdn
);
2757 context
->cqn_send
= send_cq
? cpu_to_be32(send_cq
->mcq
.cqn
) : 0;
2758 context
->cqn_recv
= recv_cq
? cpu_to_be32(recv_cq
->mcq
.cqn
) : 0;
2759 context
->params1
= cpu_to_be32(MLX5_IB_ACK_REQ_FREQ
<< 28);
2761 if (attr_mask
& IB_QP_RNR_RETRY
)
2762 context
->params1
|= cpu_to_be32(attr
->rnr_retry
<< 13);
2764 if (attr_mask
& IB_QP_RETRY_CNT
)
2765 context
->params1
|= cpu_to_be32(attr
->retry_cnt
<< 16);
2767 if (attr_mask
& IB_QP_MAX_QP_RD_ATOMIC
) {
2768 if (attr
->max_rd_atomic
)
2770 cpu_to_be32(fls(attr
->max_rd_atomic
- 1) << 21);
2773 if (attr_mask
& IB_QP_SQ_PSN
)
2774 context
->next_send_psn
= cpu_to_be32(attr
->sq_psn
);
2776 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
) {
2777 if (attr
->max_dest_rd_atomic
)
2779 cpu_to_be32(fls(attr
->max_dest_rd_atomic
- 1) << 21);
2782 if (attr_mask
& (IB_QP_ACCESS_FLAGS
| IB_QP_MAX_DEST_RD_ATOMIC
))
2783 context
->params2
|= to_mlx5_access_flags(qp
, attr
, attr_mask
);
2785 if (attr_mask
& IB_QP_MIN_RNR_TIMER
)
2786 context
->rnr_nextrecvpsn
|= cpu_to_be32(attr
->min_rnr_timer
<< 24);
2788 if (attr_mask
& IB_QP_RQ_PSN
)
2789 context
->rnr_nextrecvpsn
|= cpu_to_be32(attr
->rq_psn
);
2791 if (attr_mask
& IB_QP_QKEY
)
2792 context
->qkey
= cpu_to_be32(attr
->qkey
);
2794 if (qp
->rq
.wqe_cnt
&& cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
)
2795 context
->db_rec_addr
= cpu_to_be64(qp
->db
.dma
);
2797 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2798 u8 port_num
= (attr_mask
& IB_QP_PORT
? attr
->port_num
:
2800 mibport
= &dev
->port
[port_num
];
2801 context
->qp_counter_set_usr_page
|=
2802 cpu_to_be32((u32
)(mibport
->cnts
.set_id
) << 24);
2805 if (!ibqp
->uobject
&& cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
)
2806 context
->sq_crq_size
|= cpu_to_be16(1 << 4);
2808 if (qp
->flags
& MLX5_IB_QP_SQPN_QP1
)
2809 context
->deth_sqpn
= cpu_to_be32(1);
2811 mlx5_cur
= to_mlx5_state(cur_state
);
2812 mlx5_new
= to_mlx5_state(new_state
);
2813 mlx5_st
= to_mlx5_st(ibqp
->qp_type
);
2817 if (mlx5_cur
>= MLX5_QP_NUM_STATE
|| mlx5_new
>= MLX5_QP_NUM_STATE
||
2818 !optab
[mlx5_cur
][mlx5_new
])
2821 op
= optab
[mlx5_cur
][mlx5_new
];
2822 optpar
= ib_mask_to_mlx5_opt(attr_mask
);
2823 optpar
&= opt_mask
[mlx5_cur
][mlx5_new
][mlx5_st
];
2825 if (qp
->ibqp
.qp_type
== IB_QPT_RAW_PACKET
) {
2826 struct mlx5_modify_raw_qp_param raw_qp_param
= {};
2828 raw_qp_param
.operation
= op
;
2829 if (cur_state
== IB_QPS_RESET
&& new_state
== IB_QPS_INIT
) {
2830 raw_qp_param
.rq_q_ctr_id
= mibport
->cnts
.set_id
;
2831 raw_qp_param
.set_mask
|= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID
;
2834 if (attr_mask
& IB_QP_RATE_LIMIT
) {
2835 raw_qp_param
.rate_limit
= attr
->rate_limit
;
2836 raw_qp_param
.set_mask
|= MLX5_RAW_QP_RATE_LIMIT
;
2839 err
= modify_raw_packet_qp(dev
, qp
, &raw_qp_param
, tx_affinity
);
2841 err
= mlx5_core_qp_modify(dev
->mdev
, op
, optpar
, context
,
2848 qp
->state
= new_state
;
2850 if (attr_mask
& IB_QP_ACCESS_FLAGS
)
2851 qp
->trans_qp
.atomic_rd_en
= attr
->qp_access_flags
;
2852 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
)
2853 qp
->trans_qp
.resp_depth
= attr
->max_dest_rd_atomic
;
2854 if (attr_mask
& IB_QP_PORT
)
2855 qp
->port
= attr
->port_num
;
2856 if (attr_mask
& IB_QP_ALT_PATH
)
2857 qp
->trans_qp
.alt_port
= attr
->alt_port_num
;
2860 * If we moved a kernel QP to RESET, clean up all old CQ
2861 * entries and reinitialize the QP.
2863 if (new_state
== IB_QPS_RESET
&& !ibqp
->uobject
) {
2864 mlx5_ib_cq_clean(recv_cq
, base
->mqp
.qpn
,
2865 ibqp
->srq
? to_msrq(ibqp
->srq
) : NULL
);
2866 if (send_cq
!= recv_cq
)
2867 mlx5_ib_cq_clean(send_cq
, base
->mqp
.qpn
, NULL
);
2873 qp
->sq
.cur_post
= 0;
2874 qp
->sq
.last_poll
= 0;
2875 qp
->db
.db
[MLX5_RCV_DBR
] = 0;
2876 qp
->db
.db
[MLX5_SND_DBR
] = 0;
2884 int mlx5_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2885 int attr_mask
, struct ib_udata
*udata
)
2887 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
2888 struct mlx5_ib_qp
*qp
= to_mqp(ibqp
);
2889 enum ib_qp_type qp_type
;
2890 enum ib_qp_state cur_state
, new_state
;
2893 enum rdma_link_layer ll
= IB_LINK_LAYER_UNSPECIFIED
;
2895 if (ibqp
->rwq_ind_tbl
)
2898 if (unlikely(ibqp
->qp_type
== IB_QPT_GSI
))
2899 return mlx5_ib_gsi_modify_qp(ibqp
, attr
, attr_mask
);
2901 qp_type
= (unlikely(ibqp
->qp_type
== MLX5_IB_QPT_HW_GSI
)) ?
2902 IB_QPT_GSI
: ibqp
->qp_type
;
2904 mutex_lock(&qp
->mutex
);
2906 cur_state
= attr_mask
& IB_QP_CUR_STATE
? attr
->cur_qp_state
: qp
->state
;
2907 new_state
= attr_mask
& IB_QP_STATE
? attr
->qp_state
: cur_state
;
2909 if (!(cur_state
== new_state
&& cur_state
== IB_QPS_RESET
)) {
2910 port
= attr_mask
& IB_QP_PORT
? attr
->port_num
: qp
->port
;
2911 ll
= dev
->ib_dev
.get_link_layer(&dev
->ib_dev
, port
);
2914 if (qp_type
!= MLX5_IB_QPT_REG_UMR
&&
2915 !ib_modify_qp_is_ok(cur_state
, new_state
, qp_type
, attr_mask
, ll
)) {
2916 mlx5_ib_dbg(dev
, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
2917 cur_state
, new_state
, ibqp
->qp_type
, attr_mask
);
2921 if ((attr_mask
& IB_QP_PORT
) &&
2922 (attr
->port_num
== 0 ||
2923 attr
->port_num
> MLX5_CAP_GEN(dev
->mdev
, num_ports
))) {
2924 mlx5_ib_dbg(dev
, "invalid port number %d. number of ports is %d\n",
2925 attr
->port_num
, dev
->num_ports
);
2929 if (attr_mask
& IB_QP_PKEY_INDEX
) {
2930 port
= attr_mask
& IB_QP_PORT
? attr
->port_num
: qp
->port
;
2931 if (attr
->pkey_index
>=
2932 dev
->mdev
->port_caps
[port
- 1].pkey_table_len
) {
2933 mlx5_ib_dbg(dev
, "invalid pkey index %d\n",
2939 if (attr_mask
& IB_QP_MAX_QP_RD_ATOMIC
&&
2940 attr
->max_rd_atomic
>
2941 (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_ra_res_qp
))) {
2942 mlx5_ib_dbg(dev
, "invalid max_rd_atomic value %d\n",
2943 attr
->max_rd_atomic
);
2947 if (attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
&&
2948 attr
->max_dest_rd_atomic
>
2949 (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_ra_req_qp
))) {
2950 mlx5_ib_dbg(dev
, "invalid max_dest_rd_atomic value %d\n",
2951 attr
->max_dest_rd_atomic
);
2955 if (cur_state
== new_state
&& cur_state
== IB_QPS_RESET
) {
2960 err
= __mlx5_ib_modify_qp(ibqp
, attr
, attr_mask
, cur_state
, new_state
);
2963 mutex_unlock(&qp
->mutex
);
2967 static int mlx5_wq_overflow(struct mlx5_ib_wq
*wq
, int nreq
, struct ib_cq
*ib_cq
)
2969 struct mlx5_ib_cq
*cq
;
2972 cur
= wq
->head
- wq
->tail
;
2973 if (likely(cur
+ nreq
< wq
->max_post
))
2977 spin_lock(&cq
->lock
);
2978 cur
= wq
->head
- wq
->tail
;
2979 spin_unlock(&cq
->lock
);
2981 return cur
+ nreq
>= wq
->max_post
;
2984 static __always_inline
void set_raddr_seg(struct mlx5_wqe_raddr_seg
*rseg
,
2985 u64 remote_addr
, u32 rkey
)
2987 rseg
->raddr
= cpu_to_be64(remote_addr
);
2988 rseg
->rkey
= cpu_to_be32(rkey
);
2992 static void *set_eth_seg(struct mlx5_wqe_eth_seg
*eseg
,
2993 struct ib_send_wr
*wr
, void *qend
,
2994 struct mlx5_ib_qp
*qp
, int *size
)
2998 memset(eseg
, 0, sizeof(struct mlx5_wqe_eth_seg
));
3000 if (wr
->send_flags
& IB_SEND_IP_CSUM
)
3001 eseg
->cs_flags
= MLX5_ETH_WQE_L3_CSUM
|
3002 MLX5_ETH_WQE_L4_CSUM
;
3004 seg
+= sizeof(struct mlx5_wqe_eth_seg
);
3005 *size
+= sizeof(struct mlx5_wqe_eth_seg
) / 16;
3007 if (wr
->opcode
== IB_WR_LSO
) {
3008 struct ib_ud_wr
*ud_wr
= container_of(wr
, struct ib_ud_wr
, wr
);
3009 int size_of_inl_hdr_start
= sizeof(eseg
->inline_hdr
.start
);
3010 u64 left
, leftlen
, copysz
;
3011 void *pdata
= ud_wr
->header
;
3014 eseg
->mss
= cpu_to_be16(ud_wr
->mss
);
3015 eseg
->inline_hdr
.sz
= cpu_to_be16(left
);
3018 * check if there is space till the end of queue, if yes,
3019 * copy all in one shot, otherwise copy till the end of queue,
3020 * rollback and than the copy the left
3022 leftlen
= qend
- (void *)eseg
->inline_hdr
.start
;
3023 copysz
= min_t(u64
, leftlen
, left
);
3025 memcpy(seg
- size_of_inl_hdr_start
, pdata
, copysz
);
3027 if (likely(copysz
> size_of_inl_hdr_start
)) {
3028 seg
+= ALIGN(copysz
- size_of_inl_hdr_start
, 16);
3029 *size
+= ALIGN(copysz
- size_of_inl_hdr_start
, 16) / 16;
3032 if (unlikely(copysz
< left
)) { /* the last wqe in the queue */
3033 seg
= mlx5_get_send_wqe(qp
, 0);
3036 memcpy(seg
, pdata
, left
);
3037 seg
+= ALIGN(left
, 16);
3038 *size
+= ALIGN(left
, 16) / 16;
3045 static void set_datagram_seg(struct mlx5_wqe_datagram_seg
*dseg
,
3046 struct ib_send_wr
*wr
)
3048 memcpy(&dseg
->av
, &to_mah(ud_wr(wr
)->ah
)->av
, sizeof(struct mlx5_av
));
3049 dseg
->av
.dqp_dct
= cpu_to_be32(ud_wr(wr
)->remote_qpn
| MLX5_EXTENDED_UD_AV
);
3050 dseg
->av
.key
.qkey
.qkey
= cpu_to_be32(ud_wr(wr
)->remote_qkey
);
3053 static void set_data_ptr_seg(struct mlx5_wqe_data_seg
*dseg
, struct ib_sge
*sg
)
3055 dseg
->byte_count
= cpu_to_be32(sg
->length
);
3056 dseg
->lkey
= cpu_to_be32(sg
->lkey
);
3057 dseg
->addr
= cpu_to_be64(sg
->addr
);
3060 static u64
get_xlt_octo(u64 bytes
)
3062 return ALIGN(bytes
, MLX5_IB_UMR_XLT_ALIGNMENT
) /
3063 MLX5_IB_UMR_OCTOWORD
;
3066 static __be64
frwr_mkey_mask(void)
3070 result
= MLX5_MKEY_MASK_LEN
|
3071 MLX5_MKEY_MASK_PAGE_SIZE
|
3072 MLX5_MKEY_MASK_START_ADDR
|
3073 MLX5_MKEY_MASK_EN_RINVAL
|
3074 MLX5_MKEY_MASK_KEY
|
3080 MLX5_MKEY_MASK_SMALL_FENCE
|
3081 MLX5_MKEY_MASK_FREE
;
3083 return cpu_to_be64(result
);
3086 static __be64
sig_mkey_mask(void)
3090 result
= MLX5_MKEY_MASK_LEN
|
3091 MLX5_MKEY_MASK_PAGE_SIZE
|
3092 MLX5_MKEY_MASK_START_ADDR
|
3093 MLX5_MKEY_MASK_EN_SIGERR
|
3094 MLX5_MKEY_MASK_EN_RINVAL
|
3095 MLX5_MKEY_MASK_KEY
|
3100 MLX5_MKEY_MASK_SMALL_FENCE
|
3101 MLX5_MKEY_MASK_FREE
|
3102 MLX5_MKEY_MASK_BSF_EN
;
3104 return cpu_to_be64(result
);
3107 static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg
*umr
,
3108 struct mlx5_ib_mr
*mr
)
3110 int size
= mr
->ndescs
* mr
->desc_size
;
3112 memset(umr
, 0, sizeof(*umr
));
3114 umr
->flags
= MLX5_UMR_CHECK_NOT_FREE
;
3115 umr
->xlt_octowords
= cpu_to_be16(get_xlt_octo(size
));
3116 umr
->mkey_mask
= frwr_mkey_mask();
3119 static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg
*umr
)
3121 memset(umr
, 0, sizeof(*umr
));
3122 umr
->mkey_mask
= cpu_to_be64(MLX5_MKEY_MASK_FREE
);
3123 umr
->flags
= MLX5_UMR_INLINE
;
3126 static __be64
get_umr_enable_mr_mask(void)
3130 result
= MLX5_MKEY_MASK_KEY
|
3131 MLX5_MKEY_MASK_FREE
;
3133 return cpu_to_be64(result
);
3136 static __be64
get_umr_disable_mr_mask(void)
3140 result
= MLX5_MKEY_MASK_FREE
;
3142 return cpu_to_be64(result
);
3145 static __be64
get_umr_update_translation_mask(void)
3149 result
= MLX5_MKEY_MASK_LEN
|
3150 MLX5_MKEY_MASK_PAGE_SIZE
|
3151 MLX5_MKEY_MASK_START_ADDR
;
3153 return cpu_to_be64(result
);
3156 static __be64
get_umr_update_access_mask(int atomic
)
3160 result
= MLX5_MKEY_MASK_LR
|
3166 result
|= MLX5_MKEY_MASK_A
;
3168 return cpu_to_be64(result
);
3171 static __be64
get_umr_update_pd_mask(void)
3175 result
= MLX5_MKEY_MASK_PD
;
3177 return cpu_to_be64(result
);
3180 static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg
*umr
,
3181 struct ib_send_wr
*wr
, int atomic
)
3183 struct mlx5_umr_wr
*umrwr
= umr_wr(wr
);
3185 memset(umr
, 0, sizeof(*umr
));
3187 if (wr
->send_flags
& MLX5_IB_SEND_UMR_FAIL_IF_FREE
)
3188 umr
->flags
= MLX5_UMR_CHECK_FREE
; /* fail if free */
3190 umr
->flags
= MLX5_UMR_CHECK_NOT_FREE
; /* fail if not free */
3192 umr
->xlt_octowords
= cpu_to_be16(get_xlt_octo(umrwr
->xlt_size
));
3193 if (wr
->send_flags
& MLX5_IB_SEND_UMR_UPDATE_XLT
) {
3194 u64 offset
= get_xlt_octo(umrwr
->offset
);
3196 umr
->xlt_offset
= cpu_to_be16(offset
& 0xffff);
3197 umr
->xlt_offset_47_16
= cpu_to_be32(offset
>> 16);
3198 umr
->flags
|= MLX5_UMR_TRANSLATION_OFFSET_EN
;
3200 if (wr
->send_flags
& MLX5_IB_SEND_UMR_UPDATE_TRANSLATION
)
3201 umr
->mkey_mask
|= get_umr_update_translation_mask();
3202 if (wr
->send_flags
& MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS
) {
3203 umr
->mkey_mask
|= get_umr_update_access_mask(atomic
);
3204 umr
->mkey_mask
|= get_umr_update_pd_mask();
3206 if (wr
->send_flags
& MLX5_IB_SEND_UMR_ENABLE_MR
)
3207 umr
->mkey_mask
|= get_umr_enable_mr_mask();
3208 if (wr
->send_flags
& MLX5_IB_SEND_UMR_DISABLE_MR
)
3209 umr
->mkey_mask
|= get_umr_disable_mr_mask();
3212 umr
->flags
|= MLX5_UMR_INLINE
;
3215 static u8
get_umr_flags(int acc
)
3217 return (acc
& IB_ACCESS_REMOTE_ATOMIC
? MLX5_PERM_ATOMIC
: 0) |
3218 (acc
& IB_ACCESS_REMOTE_WRITE
? MLX5_PERM_REMOTE_WRITE
: 0) |
3219 (acc
& IB_ACCESS_REMOTE_READ
? MLX5_PERM_REMOTE_READ
: 0) |
3220 (acc
& IB_ACCESS_LOCAL_WRITE
? MLX5_PERM_LOCAL_WRITE
: 0) |
3221 MLX5_PERM_LOCAL_READ
| MLX5_PERM_UMR_EN
;
3224 static void set_reg_mkey_seg(struct mlx5_mkey_seg
*seg
,
3225 struct mlx5_ib_mr
*mr
,
3226 u32 key
, int access
)
3228 int ndescs
= ALIGN(mr
->ndescs
, 8) >> 1;
3230 memset(seg
, 0, sizeof(*seg
));
3232 if (mr
->access_mode
== MLX5_MKC_ACCESS_MODE_MTT
)
3233 seg
->log2_page_size
= ilog2(mr
->ibmr
.page_size
);
3234 else if (mr
->access_mode
== MLX5_MKC_ACCESS_MODE_KLMS
)
3235 /* KLMs take twice the size of MTTs */
3238 seg
->flags
= get_umr_flags(access
) | mr
->access_mode
;
3239 seg
->qpn_mkey7_0
= cpu_to_be32((key
& 0xff) | 0xffffff00);
3240 seg
->flags_pd
= cpu_to_be32(MLX5_MKEY_REMOTE_INVAL
);
3241 seg
->start_addr
= cpu_to_be64(mr
->ibmr
.iova
);
3242 seg
->len
= cpu_to_be64(mr
->ibmr
.length
);
3243 seg
->xlt_oct_size
= cpu_to_be32(ndescs
);
3246 static void set_linv_mkey_seg(struct mlx5_mkey_seg
*seg
)
3248 memset(seg
, 0, sizeof(*seg
));
3249 seg
->status
= MLX5_MKEY_STATUS_FREE
;
3252 static void set_reg_mkey_segment(struct mlx5_mkey_seg
*seg
, struct ib_send_wr
*wr
)
3254 struct mlx5_umr_wr
*umrwr
= umr_wr(wr
);
3256 memset(seg
, 0, sizeof(*seg
));
3257 if (wr
->send_flags
& MLX5_IB_SEND_UMR_DISABLE_MR
)
3258 seg
->status
= MLX5_MKEY_STATUS_FREE
;
3260 seg
->flags
= convert_access(umrwr
->access_flags
);
3262 seg
->flags_pd
= cpu_to_be32(to_mpd(umrwr
->pd
)->pdn
);
3263 if (wr
->send_flags
& MLX5_IB_SEND_UMR_UPDATE_TRANSLATION
&&
3265 seg
->flags_pd
|= cpu_to_be32(MLX5_MKEY_LEN64
);
3267 seg
->start_addr
= cpu_to_be64(umrwr
->virt_addr
);
3268 seg
->len
= cpu_to_be64(umrwr
->length
);
3269 seg
->log2_page_size
= umrwr
->page_shift
;
3270 seg
->qpn_mkey7_0
= cpu_to_be32(0xffffff00 |
3271 mlx5_mkey_variant(umrwr
->mkey
));
3274 static void set_reg_data_seg(struct mlx5_wqe_data_seg
*dseg
,
3275 struct mlx5_ib_mr
*mr
,
3276 struct mlx5_ib_pd
*pd
)
3278 int bcount
= mr
->desc_size
* mr
->ndescs
;
3280 dseg
->addr
= cpu_to_be64(mr
->desc_map
);
3281 dseg
->byte_count
= cpu_to_be32(ALIGN(bcount
, 64));
3282 dseg
->lkey
= cpu_to_be32(pd
->ibpd
.local_dma_lkey
);
3285 static __be32
send_ieth(struct ib_send_wr
*wr
)
3287 switch (wr
->opcode
) {
3288 case IB_WR_SEND_WITH_IMM
:
3289 case IB_WR_RDMA_WRITE_WITH_IMM
:
3290 return wr
->ex
.imm_data
;
3292 case IB_WR_SEND_WITH_INV
:
3293 return cpu_to_be32(wr
->ex
.invalidate_rkey
);
3300 static u8
calc_sig(void *wqe
, int size
)
3306 for (i
= 0; i
< size
; i
++)
3312 static u8
wq_sig(void *wqe
)
3314 return calc_sig(wqe
, (*((u8
*)wqe
+ 8) & 0x3f) << 4);
3317 static int set_data_inl_seg(struct mlx5_ib_qp
*qp
, struct ib_send_wr
*wr
,
3320 struct mlx5_wqe_inline_seg
*seg
;
3321 void *qend
= qp
->sq
.qend
;
3329 wqe
+= sizeof(*seg
);
3330 for (i
= 0; i
< wr
->num_sge
; i
++) {
3331 addr
= (void *)(unsigned long)(wr
->sg_list
[i
].addr
);
3332 len
= wr
->sg_list
[i
].length
;
3335 if (unlikely(inl
> qp
->max_inline_data
))
3338 if (unlikely(wqe
+ len
> qend
)) {
3340 memcpy(wqe
, addr
, copy
);
3343 wqe
= mlx5_get_send_wqe(qp
, 0);
3345 memcpy(wqe
, addr
, len
);
3349 seg
->byte_count
= cpu_to_be32(inl
| MLX5_INLINE_SEG
);
3351 *sz
= ALIGN(inl
+ sizeof(seg
->byte_count
), 16) / 16;
3356 static u16
prot_field_size(enum ib_signature_type type
)
3359 case IB_SIG_TYPE_T10_DIF
:
3360 return MLX5_DIF_SIZE
;
3366 static u8
bs_selector(int block_size
)
3368 switch (block_size
) {
3369 case 512: return 0x1;
3370 case 520: return 0x2;
3371 case 4096: return 0x3;
3372 case 4160: return 0x4;
3373 case 1073741824: return 0x5;
3378 static void mlx5_fill_inl_bsf(struct ib_sig_domain
*domain
,
3379 struct mlx5_bsf_inl
*inl
)
3381 /* Valid inline section and allow BSF refresh */
3382 inl
->vld_refresh
= cpu_to_be16(MLX5_BSF_INL_VALID
|
3383 MLX5_BSF_REFRESH_DIF
);
3384 inl
->dif_apptag
= cpu_to_be16(domain
->sig
.dif
.app_tag
);
3385 inl
->dif_reftag
= cpu_to_be32(domain
->sig
.dif
.ref_tag
);
3386 /* repeating block */
3387 inl
->rp_inv_seed
= MLX5_BSF_REPEAT_BLOCK
;
3388 inl
->sig_type
= domain
->sig
.dif
.bg_type
== IB_T10DIF_CRC
?
3389 MLX5_DIF_CRC
: MLX5_DIF_IPCS
;
3391 if (domain
->sig
.dif
.ref_remap
)
3392 inl
->dif_inc_ref_guard_check
|= MLX5_BSF_INC_REFTAG
;
3394 if (domain
->sig
.dif
.app_escape
) {
3395 if (domain
->sig
.dif
.ref_escape
)
3396 inl
->dif_inc_ref_guard_check
|= MLX5_BSF_APPREF_ESCAPE
;
3398 inl
->dif_inc_ref_guard_check
|= MLX5_BSF_APPTAG_ESCAPE
;
3401 inl
->dif_app_bitmask_check
=
3402 cpu_to_be16(domain
->sig
.dif
.apptag_check_mask
);
3405 static int mlx5_set_bsf(struct ib_mr
*sig_mr
,
3406 struct ib_sig_attrs
*sig_attrs
,
3407 struct mlx5_bsf
*bsf
, u32 data_size
)
3409 struct mlx5_core_sig_ctx
*msig
= to_mmr(sig_mr
)->sig
;
3410 struct mlx5_bsf_basic
*basic
= &bsf
->basic
;
3411 struct ib_sig_domain
*mem
= &sig_attrs
->mem
;
3412 struct ib_sig_domain
*wire
= &sig_attrs
->wire
;
3414 memset(bsf
, 0, sizeof(*bsf
));
3416 /* Basic + Extended + Inline */
3417 basic
->bsf_size_sbs
= 1 << 7;
3418 /* Input domain check byte mask */
3419 basic
->check_byte_mask
= sig_attrs
->check_mask
;
3420 basic
->raw_data_size
= cpu_to_be32(data_size
);
3423 switch (sig_attrs
->mem
.sig_type
) {
3424 case IB_SIG_TYPE_NONE
:
3426 case IB_SIG_TYPE_T10_DIF
:
3427 basic
->mem
.bs_selector
= bs_selector(mem
->sig
.dif
.pi_interval
);
3428 basic
->m_bfs_psv
= cpu_to_be32(msig
->psv_memory
.psv_idx
);
3429 mlx5_fill_inl_bsf(mem
, &bsf
->m_inl
);
3436 switch (sig_attrs
->wire
.sig_type
) {
3437 case IB_SIG_TYPE_NONE
:
3439 case IB_SIG_TYPE_T10_DIF
:
3440 if (mem
->sig
.dif
.pi_interval
== wire
->sig
.dif
.pi_interval
&&
3441 mem
->sig_type
== wire
->sig_type
) {
3442 /* Same block structure */
3443 basic
->bsf_size_sbs
|= 1 << 4;
3444 if (mem
->sig
.dif
.bg_type
== wire
->sig
.dif
.bg_type
)
3445 basic
->wire
.copy_byte_mask
|= MLX5_CPY_GRD_MASK
;
3446 if (mem
->sig
.dif
.app_tag
== wire
->sig
.dif
.app_tag
)
3447 basic
->wire
.copy_byte_mask
|= MLX5_CPY_APP_MASK
;
3448 if (mem
->sig
.dif
.ref_tag
== wire
->sig
.dif
.ref_tag
)
3449 basic
->wire
.copy_byte_mask
|= MLX5_CPY_REF_MASK
;
3451 basic
->wire
.bs_selector
= bs_selector(wire
->sig
.dif
.pi_interval
);
3453 basic
->w_bfs_psv
= cpu_to_be32(msig
->psv_wire
.psv_idx
);
3454 mlx5_fill_inl_bsf(wire
, &bsf
->w_inl
);
3463 static int set_sig_data_segment(struct ib_sig_handover_wr
*wr
,
3464 struct mlx5_ib_qp
*qp
, void **seg
, int *size
)
3466 struct ib_sig_attrs
*sig_attrs
= wr
->sig_attrs
;
3467 struct ib_mr
*sig_mr
= wr
->sig_mr
;
3468 struct mlx5_bsf
*bsf
;
3469 u32 data_len
= wr
->wr
.sg_list
->length
;
3470 u32 data_key
= wr
->wr
.sg_list
->lkey
;
3471 u64 data_va
= wr
->wr
.sg_list
->addr
;
3476 (data_key
== wr
->prot
->lkey
&&
3477 data_va
== wr
->prot
->addr
&&
3478 data_len
== wr
->prot
->length
)) {
3480 * Source domain doesn't contain signature information
3481 * or data and protection are interleaved in memory.
3482 * So need construct:
3483 * ------------------
3485 * ------------------
3487 * ------------------
3489 struct mlx5_klm
*data_klm
= *seg
;
3491 data_klm
->bcount
= cpu_to_be32(data_len
);
3492 data_klm
->key
= cpu_to_be32(data_key
);
3493 data_klm
->va
= cpu_to_be64(data_va
);
3494 wqe_size
= ALIGN(sizeof(*data_klm
), 64);
3497 * Source domain contains signature information
3498 * So need construct a strided block format:
3499 * ---------------------------
3500 * | stride_block_ctrl |
3501 * ---------------------------
3503 * ---------------------------
3505 * ---------------------------
3507 * ---------------------------
3509 struct mlx5_stride_block_ctrl_seg
*sblock_ctrl
;
3510 struct mlx5_stride_block_entry
*data_sentry
;
3511 struct mlx5_stride_block_entry
*prot_sentry
;
3512 u32 prot_key
= wr
->prot
->lkey
;
3513 u64 prot_va
= wr
->prot
->addr
;
3514 u16 block_size
= sig_attrs
->mem
.sig
.dif
.pi_interval
;
3518 data_sentry
= (void *)sblock_ctrl
+ sizeof(*sblock_ctrl
);
3519 prot_sentry
= (void *)data_sentry
+ sizeof(*data_sentry
);
3521 prot_size
= prot_field_size(sig_attrs
->mem
.sig_type
);
3523 pr_err("Bad block size given: %u\n", block_size
);
3526 sblock_ctrl
->bcount_per_cycle
= cpu_to_be32(block_size
+
3528 sblock_ctrl
->op
= cpu_to_be32(MLX5_STRIDE_BLOCK_OP
);
3529 sblock_ctrl
->repeat_count
= cpu_to_be32(data_len
/ block_size
);
3530 sblock_ctrl
->num_entries
= cpu_to_be16(2);
3532 data_sentry
->bcount
= cpu_to_be16(block_size
);
3533 data_sentry
->key
= cpu_to_be32(data_key
);
3534 data_sentry
->va
= cpu_to_be64(data_va
);
3535 data_sentry
->stride
= cpu_to_be16(block_size
);
3537 prot_sentry
->bcount
= cpu_to_be16(prot_size
);
3538 prot_sentry
->key
= cpu_to_be32(prot_key
);
3539 prot_sentry
->va
= cpu_to_be64(prot_va
);
3540 prot_sentry
->stride
= cpu_to_be16(prot_size
);
3542 wqe_size
= ALIGN(sizeof(*sblock_ctrl
) + sizeof(*data_sentry
) +
3543 sizeof(*prot_sentry
), 64);
3547 *size
+= wqe_size
/ 16;
3548 if (unlikely((*seg
== qp
->sq
.qend
)))
3549 *seg
= mlx5_get_send_wqe(qp
, 0);
3552 ret
= mlx5_set_bsf(sig_mr
, sig_attrs
, bsf
, data_len
);
3556 *seg
+= sizeof(*bsf
);
3557 *size
+= sizeof(*bsf
) / 16;
3558 if (unlikely((*seg
== qp
->sq
.qend
)))
3559 *seg
= mlx5_get_send_wqe(qp
, 0);
3564 static void set_sig_mkey_segment(struct mlx5_mkey_seg
*seg
,
3565 struct ib_sig_handover_wr
*wr
, u32 size
,
3566 u32 length
, u32 pdn
)
3568 struct ib_mr
*sig_mr
= wr
->sig_mr
;
3569 u32 sig_key
= sig_mr
->rkey
;
3570 u8 sigerr
= to_mmr(sig_mr
)->sig
->sigerr_count
& 1;
3572 memset(seg
, 0, sizeof(*seg
));
3574 seg
->flags
= get_umr_flags(wr
->access_flags
) |
3575 MLX5_MKC_ACCESS_MODE_KLMS
;
3576 seg
->qpn_mkey7_0
= cpu_to_be32((sig_key
& 0xff) | 0xffffff00);
3577 seg
->flags_pd
= cpu_to_be32(MLX5_MKEY_REMOTE_INVAL
| sigerr
<< 26 |
3578 MLX5_MKEY_BSF_EN
| pdn
);
3579 seg
->len
= cpu_to_be64(length
);
3580 seg
->xlt_oct_size
= cpu_to_be32(get_xlt_octo(size
));
3581 seg
->bsfs_octo_size
= cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE
);
3584 static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg
*umr
,
3587 memset(umr
, 0, sizeof(*umr
));
3589 umr
->flags
= MLX5_FLAGS_INLINE
| MLX5_FLAGS_CHECK_FREE
;
3590 umr
->xlt_octowords
= cpu_to_be16(get_xlt_octo(size
));
3591 umr
->bsf_octowords
= cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE
);
3592 umr
->mkey_mask
= sig_mkey_mask();
3596 static int set_sig_umr_wr(struct ib_send_wr
*send_wr
, struct mlx5_ib_qp
*qp
,
3597 void **seg
, int *size
)
3599 struct ib_sig_handover_wr
*wr
= sig_handover_wr(send_wr
);
3600 struct mlx5_ib_mr
*sig_mr
= to_mmr(wr
->sig_mr
);
3601 u32 pdn
= get_pd(qp
)->pdn
;
3603 int region_len
, ret
;
3605 if (unlikely(wr
->wr
.num_sge
!= 1) ||
3606 unlikely(wr
->access_flags
& IB_ACCESS_REMOTE_ATOMIC
) ||
3607 unlikely(!sig_mr
->sig
) || unlikely(!qp
->signature_en
) ||
3608 unlikely(!sig_mr
->sig
->sig_status_checked
))
3611 /* length of the protected region, data + protection */
3612 region_len
= wr
->wr
.sg_list
->length
;
3614 (wr
->prot
->lkey
!= wr
->wr
.sg_list
->lkey
||
3615 wr
->prot
->addr
!= wr
->wr
.sg_list
->addr
||
3616 wr
->prot
->length
!= wr
->wr
.sg_list
->length
))
3617 region_len
+= wr
->prot
->length
;
3620 * KLM octoword size - if protection was provided
3621 * then we use strided block format (3 octowords),
3622 * else we use single KLM (1 octoword)
3624 xlt_size
= wr
->prot
? 0x30 : sizeof(struct mlx5_klm
);
3626 set_sig_umr_segment(*seg
, xlt_size
);
3627 *seg
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
);
3628 *size
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
) / 16;
3629 if (unlikely((*seg
== qp
->sq
.qend
)))
3630 *seg
= mlx5_get_send_wqe(qp
, 0);
3632 set_sig_mkey_segment(*seg
, wr
, xlt_size
, region_len
, pdn
);
3633 *seg
+= sizeof(struct mlx5_mkey_seg
);
3634 *size
+= sizeof(struct mlx5_mkey_seg
) / 16;
3635 if (unlikely((*seg
== qp
->sq
.qend
)))
3636 *seg
= mlx5_get_send_wqe(qp
, 0);
3638 ret
= set_sig_data_segment(wr
, qp
, seg
, size
);
3642 sig_mr
->sig
->sig_status_checked
= false;
3646 static int set_psv_wr(struct ib_sig_domain
*domain
,
3647 u32 psv_idx
, void **seg
, int *size
)
3649 struct mlx5_seg_set_psv
*psv_seg
= *seg
;
3651 memset(psv_seg
, 0, sizeof(*psv_seg
));
3652 psv_seg
->psv_num
= cpu_to_be32(psv_idx
);
3653 switch (domain
->sig_type
) {
3654 case IB_SIG_TYPE_NONE
:
3656 case IB_SIG_TYPE_T10_DIF
:
3657 psv_seg
->transient_sig
= cpu_to_be32(domain
->sig
.dif
.bg
<< 16 |
3658 domain
->sig
.dif
.app_tag
);
3659 psv_seg
->ref_tag
= cpu_to_be32(domain
->sig
.dif
.ref_tag
);
3662 pr_err("Bad signature type (%d) is given.\n",
3667 *seg
+= sizeof(*psv_seg
);
3668 *size
+= sizeof(*psv_seg
) / 16;
3673 static int set_reg_wr(struct mlx5_ib_qp
*qp
,
3674 struct ib_reg_wr
*wr
,
3675 void **seg
, int *size
)
3677 struct mlx5_ib_mr
*mr
= to_mmr(wr
->mr
);
3678 struct mlx5_ib_pd
*pd
= to_mpd(qp
->ibqp
.pd
);
3680 if (unlikely(wr
->wr
.send_flags
& IB_SEND_INLINE
)) {
3681 mlx5_ib_warn(to_mdev(qp
->ibqp
.device
),
3682 "Invalid IB_SEND_INLINE send flag\n");
3686 set_reg_umr_seg(*seg
, mr
);
3687 *seg
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
);
3688 *size
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
) / 16;
3689 if (unlikely((*seg
== qp
->sq
.qend
)))
3690 *seg
= mlx5_get_send_wqe(qp
, 0);
3692 set_reg_mkey_seg(*seg
, mr
, wr
->key
, wr
->access
);
3693 *seg
+= sizeof(struct mlx5_mkey_seg
);
3694 *size
+= sizeof(struct mlx5_mkey_seg
) / 16;
3695 if (unlikely((*seg
== qp
->sq
.qend
)))
3696 *seg
= mlx5_get_send_wqe(qp
, 0);
3698 set_reg_data_seg(*seg
, mr
, pd
);
3699 *seg
+= sizeof(struct mlx5_wqe_data_seg
);
3700 *size
+= (sizeof(struct mlx5_wqe_data_seg
) / 16);
3705 static void set_linv_wr(struct mlx5_ib_qp
*qp
, void **seg
, int *size
)
3707 set_linv_umr_seg(*seg
);
3708 *seg
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
);
3709 *size
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
) / 16;
3710 if (unlikely((*seg
== qp
->sq
.qend
)))
3711 *seg
= mlx5_get_send_wqe(qp
, 0);
3712 set_linv_mkey_seg(*seg
);
3713 *seg
+= sizeof(struct mlx5_mkey_seg
);
3714 *size
+= sizeof(struct mlx5_mkey_seg
) / 16;
3715 if (unlikely((*seg
== qp
->sq
.qend
)))
3716 *seg
= mlx5_get_send_wqe(qp
, 0);
3719 static void dump_wqe(struct mlx5_ib_qp
*qp
, int idx
, int size_16
)
3725 pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp
, tidx
));
3726 for (i
= 0, j
= 0; i
< size_16
* 4; i
+= 4, j
+= 4) {
3727 if ((i
& 0xf) == 0) {
3728 void *buf
= mlx5_get_send_wqe(qp
, tidx
);
3729 tidx
= (tidx
+ 1) & (qp
->sq
.wqe_cnt
- 1);
3733 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p
[j
]),
3734 be32_to_cpu(p
[j
+ 1]), be32_to_cpu(p
[j
+ 2]),
3735 be32_to_cpu(p
[j
+ 3]));
3739 static u8
get_fence(u8 fence
, struct ib_send_wr
*wr
)
3741 if (unlikely(wr
->opcode
== IB_WR_LOCAL_INV
&&
3742 wr
->send_flags
& IB_SEND_FENCE
))
3743 return MLX5_FENCE_MODE_STRONG_ORDERING
;
3745 if (unlikely(fence
)) {
3746 if (wr
->send_flags
& IB_SEND_FENCE
)
3747 return MLX5_FENCE_MODE_SMALL_AND_FENCE
;
3750 } else if (unlikely(wr
->send_flags
& IB_SEND_FENCE
)) {
3751 return MLX5_FENCE_MODE_FENCE
;
3757 static int begin_wqe(struct mlx5_ib_qp
*qp
, void **seg
,
3758 struct mlx5_wqe_ctrl_seg
**ctrl
,
3759 struct ib_send_wr
*wr
, unsigned *idx
,
3760 int *size
, int nreq
)
3762 if (unlikely(mlx5_wq_overflow(&qp
->sq
, nreq
, qp
->ibqp
.send_cq
)))
3765 *idx
= qp
->sq
.cur_post
& (qp
->sq
.wqe_cnt
- 1);
3766 *seg
= mlx5_get_send_wqe(qp
, *idx
);
3768 *(uint32_t *)(*seg
+ 8) = 0;
3769 (*ctrl
)->imm
= send_ieth(wr
);
3770 (*ctrl
)->fm_ce_se
= qp
->sq_signal_bits
|
3771 (wr
->send_flags
& IB_SEND_SIGNALED
?
3772 MLX5_WQE_CTRL_CQ_UPDATE
: 0) |
3773 (wr
->send_flags
& IB_SEND_SOLICITED
?
3774 MLX5_WQE_CTRL_SOLICITED
: 0);
3776 *seg
+= sizeof(**ctrl
);
3777 *size
= sizeof(**ctrl
) / 16;
3782 static void finish_wqe(struct mlx5_ib_qp
*qp
,
3783 struct mlx5_wqe_ctrl_seg
*ctrl
,
3784 u8 size
, unsigned idx
, u64 wr_id
,
3785 int nreq
, u8 fence
, u8 next_fence
,
3790 ctrl
->opmod_idx_opcode
= cpu_to_be32(((u32
)(qp
->sq
.cur_post
) << 8) |
3791 mlx5_opcode
| ((u32
)opmod
<< 24));
3792 ctrl
->qpn_ds
= cpu_to_be32(size
| (qp
->trans_qp
.base
.mqp
.qpn
<< 8));
3793 ctrl
->fm_ce_se
|= fence
;
3794 qp
->fm_cache
= next_fence
;
3795 if (unlikely(qp
->wq_sig
))
3796 ctrl
->signature
= wq_sig(ctrl
);
3798 qp
->sq
.wrid
[idx
] = wr_id
;
3799 qp
->sq
.w_list
[idx
].opcode
= mlx5_opcode
;
3800 qp
->sq
.wqe_head
[idx
] = qp
->sq
.head
+ nreq
;
3801 qp
->sq
.cur_post
+= DIV_ROUND_UP(size
* 16, MLX5_SEND_WQE_BB
);
3802 qp
->sq
.w_list
[idx
].next
= qp
->sq
.cur_post
;
3806 int mlx5_ib_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
3807 struct ib_send_wr
**bad_wr
)
3809 struct mlx5_wqe_ctrl_seg
*ctrl
= NULL
; /* compiler warning */
3810 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
3811 struct mlx5_core_dev
*mdev
= dev
->mdev
;
3812 struct mlx5_ib_qp
*qp
;
3813 struct mlx5_ib_mr
*mr
;
3814 struct mlx5_wqe_data_seg
*dpseg
;
3815 struct mlx5_wqe_xrc_seg
*xrc
;
3817 int uninitialized_var(size
);
3819 unsigned long flags
;
3830 if (unlikely(ibqp
->qp_type
== IB_QPT_GSI
))
3831 return mlx5_ib_gsi_post_send(ibqp
, wr
, bad_wr
);
3837 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
3839 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
3846 for (nreq
= 0; wr
; nreq
++, wr
= wr
->next
) {
3847 if (unlikely(wr
->opcode
>= ARRAY_SIZE(mlx5_ib_opcode
))) {
3848 mlx5_ib_warn(dev
, "\n");
3854 fence
= qp
->fm_cache
;
3855 num_sge
= wr
->num_sge
;
3856 if (unlikely(num_sge
> qp
->sq
.max_gs
)) {
3857 mlx5_ib_warn(dev
, "\n");
3863 err
= begin_wqe(qp
, &seg
, &ctrl
, wr
, &idx
, &size
, nreq
);
3865 mlx5_ib_warn(dev
, "\n");
3871 switch (ibqp
->qp_type
) {
3872 case IB_QPT_XRC_INI
:
3874 seg
+= sizeof(*xrc
);
3875 size
+= sizeof(*xrc
) / 16;
3878 switch (wr
->opcode
) {
3879 case IB_WR_RDMA_READ
:
3880 case IB_WR_RDMA_WRITE
:
3881 case IB_WR_RDMA_WRITE_WITH_IMM
:
3882 set_raddr_seg(seg
, rdma_wr(wr
)->remote_addr
,
3884 seg
+= sizeof(struct mlx5_wqe_raddr_seg
);
3885 size
+= sizeof(struct mlx5_wqe_raddr_seg
) / 16;
3888 case IB_WR_ATOMIC_CMP_AND_SWP
:
3889 case IB_WR_ATOMIC_FETCH_AND_ADD
:
3890 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP
:
3891 mlx5_ib_warn(dev
, "Atomic operations are not supported yet\n");
3896 case IB_WR_LOCAL_INV
:
3897 next_fence
= MLX5_FENCE_MODE_INITIATOR_SMALL
;
3898 qp
->sq
.wr_data
[idx
] = IB_WR_LOCAL_INV
;
3899 ctrl
->imm
= cpu_to_be32(wr
->ex
.invalidate_rkey
);
3900 set_linv_wr(qp
, &seg
, &size
);
3905 next_fence
= MLX5_FENCE_MODE_INITIATOR_SMALL
;
3906 qp
->sq
.wr_data
[idx
] = IB_WR_REG_MR
;
3907 ctrl
->imm
= cpu_to_be32(reg_wr(wr
)->key
);
3908 err
= set_reg_wr(qp
, reg_wr(wr
), &seg
, &size
);
3916 case IB_WR_REG_SIG_MR
:
3917 qp
->sq
.wr_data
[idx
] = IB_WR_REG_SIG_MR
;
3918 mr
= to_mmr(sig_handover_wr(wr
)->sig_mr
);
3920 ctrl
->imm
= cpu_to_be32(mr
->ibmr
.rkey
);
3921 err
= set_sig_umr_wr(wr
, qp
, &seg
, &size
);
3923 mlx5_ib_warn(dev
, "\n");
3928 finish_wqe(qp
, ctrl
, size
, idx
, wr
->wr_id
,
3929 nreq
, get_fence(fence
, wr
),
3930 next_fence
, MLX5_OPCODE_UMR
);
3932 * SET_PSV WQEs are not signaled and solicited
3935 wr
->send_flags
&= ~IB_SEND_SIGNALED
;
3936 wr
->send_flags
|= IB_SEND_SOLICITED
;
3937 err
= begin_wqe(qp
, &seg
, &ctrl
, wr
,
3940 mlx5_ib_warn(dev
, "\n");
3946 err
= set_psv_wr(&sig_handover_wr(wr
)->sig_attrs
->mem
,
3947 mr
->sig
->psv_memory
.psv_idx
, &seg
,
3950 mlx5_ib_warn(dev
, "\n");
3955 finish_wqe(qp
, ctrl
, size
, idx
, wr
->wr_id
,
3956 nreq
, get_fence(fence
, wr
),
3957 next_fence
, MLX5_OPCODE_SET_PSV
);
3958 err
= begin_wqe(qp
, &seg
, &ctrl
, wr
,
3961 mlx5_ib_warn(dev
, "\n");
3967 next_fence
= MLX5_FENCE_MODE_INITIATOR_SMALL
;
3968 err
= set_psv_wr(&sig_handover_wr(wr
)->sig_attrs
->wire
,
3969 mr
->sig
->psv_wire
.psv_idx
, &seg
,
3972 mlx5_ib_warn(dev
, "\n");
3977 finish_wqe(qp
, ctrl
, size
, idx
, wr
->wr_id
,
3978 nreq
, get_fence(fence
, wr
),
3979 next_fence
, MLX5_OPCODE_SET_PSV
);
3989 switch (wr
->opcode
) {
3990 case IB_WR_RDMA_WRITE
:
3991 case IB_WR_RDMA_WRITE_WITH_IMM
:
3992 set_raddr_seg(seg
, rdma_wr(wr
)->remote_addr
,
3994 seg
+= sizeof(struct mlx5_wqe_raddr_seg
);
3995 size
+= sizeof(struct mlx5_wqe_raddr_seg
) / 16;
4004 if (unlikely(!mdev
->port_caps
[qp
->port
- 1].has_smi
)) {
4005 mlx5_ib_warn(dev
, "Send SMP MADs is not allowed\n");
4010 case MLX5_IB_QPT_HW_GSI
:
4011 set_datagram_seg(seg
, wr
);
4012 seg
+= sizeof(struct mlx5_wqe_datagram_seg
);
4013 size
+= sizeof(struct mlx5_wqe_datagram_seg
) / 16;
4014 if (unlikely((seg
== qend
)))
4015 seg
= mlx5_get_send_wqe(qp
, 0);
4018 set_datagram_seg(seg
, wr
);
4019 seg
+= sizeof(struct mlx5_wqe_datagram_seg
);
4020 size
+= sizeof(struct mlx5_wqe_datagram_seg
) / 16;
4022 if (unlikely((seg
== qend
)))
4023 seg
= mlx5_get_send_wqe(qp
, 0);
4025 /* handle qp that supports ud offload */
4026 if (qp
->flags
& IB_QP_CREATE_IPOIB_UD_LSO
) {
4027 struct mlx5_wqe_eth_pad
*pad
;
4030 memset(pad
, 0, sizeof(struct mlx5_wqe_eth_pad
));
4031 seg
+= sizeof(struct mlx5_wqe_eth_pad
);
4032 size
+= sizeof(struct mlx5_wqe_eth_pad
) / 16;
4034 seg
= set_eth_seg(seg
, wr
, qend
, qp
, &size
);
4036 if (unlikely((seg
== qend
)))
4037 seg
= mlx5_get_send_wqe(qp
, 0);
4040 case MLX5_IB_QPT_REG_UMR
:
4041 if (wr
->opcode
!= MLX5_IB_WR_UMR
) {
4043 mlx5_ib_warn(dev
, "bad opcode\n");
4046 qp
->sq
.wr_data
[idx
] = MLX5_IB_WR_UMR
;
4047 ctrl
->imm
= cpu_to_be32(umr_wr(wr
)->mkey
);
4048 set_reg_umr_segment(seg
, wr
, !!(MLX5_CAP_GEN(mdev
, atomic
)));
4049 seg
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
);
4050 size
+= sizeof(struct mlx5_wqe_umr_ctrl_seg
) / 16;
4051 if (unlikely((seg
== qend
)))
4052 seg
= mlx5_get_send_wqe(qp
, 0);
4053 set_reg_mkey_segment(seg
, wr
);
4054 seg
+= sizeof(struct mlx5_mkey_seg
);
4055 size
+= sizeof(struct mlx5_mkey_seg
) / 16;
4056 if (unlikely((seg
== qend
)))
4057 seg
= mlx5_get_send_wqe(qp
, 0);
4064 if (wr
->send_flags
& IB_SEND_INLINE
&& num_sge
) {
4065 int uninitialized_var(sz
);
4067 err
= set_data_inl_seg(qp
, wr
, seg
, &sz
);
4068 if (unlikely(err
)) {
4069 mlx5_ib_warn(dev
, "\n");
4077 for (i
= 0; i
< num_sge
; i
++) {
4078 if (unlikely(dpseg
== qend
)) {
4079 seg
= mlx5_get_send_wqe(qp
, 0);
4082 if (likely(wr
->sg_list
[i
].length
)) {
4083 set_data_ptr_seg(dpseg
, wr
->sg_list
+ i
);
4084 size
+= sizeof(struct mlx5_wqe_data_seg
) / 16;
4090 finish_wqe(qp
, ctrl
, size
, idx
, wr
->wr_id
, nreq
,
4091 get_fence(fence
, wr
), next_fence
,
4092 mlx5_ib_opcode
[wr
->opcode
]);
4095 dump_wqe(qp
, idx
, size
);
4100 qp
->sq
.head
+= nreq
;
4102 /* Make sure that descriptors are written before
4103 * updating doorbell record and ringing the doorbell
4107 qp
->db
.db
[MLX5_SND_DBR
] = cpu_to_be32(qp
->sq
.cur_post
);
4109 /* Make sure doorbell record is visible to the HCA before
4110 * we hit doorbell */
4113 /* currently we support only regular doorbells */
4114 mlx5_write64((__be32
*)ctrl
, bf
->bfreg
->map
+ bf
->offset
, NULL
);
4115 /* Make sure doorbells don't leak out of SQ spinlock
4116 * and reach the HCA out of order.
4119 bf
->offset
^= bf
->buf_size
;
4122 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
4127 static void set_sig_seg(struct mlx5_rwqe_sig
*sig
, int size
)
4129 sig
->signature
= calc_sig(sig
, size
);
4132 int mlx5_ib_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
4133 struct ib_recv_wr
**bad_wr
)
4135 struct mlx5_ib_qp
*qp
= to_mqp(ibqp
);
4136 struct mlx5_wqe_data_seg
*scat
;
4137 struct mlx5_rwqe_sig
*sig
;
4138 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
4139 struct mlx5_core_dev
*mdev
= dev
->mdev
;
4140 unsigned long flags
;
4146 if (unlikely(ibqp
->qp_type
== IB_QPT_GSI
))
4147 return mlx5_ib_gsi_post_recv(ibqp
, wr
, bad_wr
);
4149 spin_lock_irqsave(&qp
->rq
.lock
, flags
);
4151 if (mdev
->state
== MLX5_DEVICE_STATE_INTERNAL_ERROR
) {
4158 ind
= qp
->rq
.head
& (qp
->rq
.wqe_cnt
- 1);
4160 for (nreq
= 0; wr
; nreq
++, wr
= wr
->next
) {
4161 if (mlx5_wq_overflow(&qp
->rq
, nreq
, qp
->ibqp
.recv_cq
)) {
4167 if (unlikely(wr
->num_sge
> qp
->rq
.max_gs
)) {
4173 scat
= get_recv_wqe(qp
, ind
);
4177 for (i
= 0; i
< wr
->num_sge
; i
++)
4178 set_data_ptr_seg(scat
+ i
, wr
->sg_list
+ i
);
4180 if (i
< qp
->rq
.max_gs
) {
4181 scat
[i
].byte_count
= 0;
4182 scat
[i
].lkey
= cpu_to_be32(MLX5_INVALID_LKEY
);
4187 sig
= (struct mlx5_rwqe_sig
*)scat
;
4188 set_sig_seg(sig
, (qp
->rq
.max_gs
+ 1) << 2);
4191 qp
->rq
.wrid
[ind
] = wr
->wr_id
;
4193 ind
= (ind
+ 1) & (qp
->rq
.wqe_cnt
- 1);
4198 qp
->rq
.head
+= nreq
;
4200 /* Make sure that descriptors are written before
4205 *qp
->db
.db
= cpu_to_be32(qp
->rq
.head
& 0xffff);
4208 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
4213 static inline enum ib_qp_state
to_ib_qp_state(enum mlx5_qp_state mlx5_state
)
4215 switch (mlx5_state
) {
4216 case MLX5_QP_STATE_RST
: return IB_QPS_RESET
;
4217 case MLX5_QP_STATE_INIT
: return IB_QPS_INIT
;
4218 case MLX5_QP_STATE_RTR
: return IB_QPS_RTR
;
4219 case MLX5_QP_STATE_RTS
: return IB_QPS_RTS
;
4220 case MLX5_QP_STATE_SQ_DRAINING
:
4221 case MLX5_QP_STATE_SQD
: return IB_QPS_SQD
;
4222 case MLX5_QP_STATE_SQER
: return IB_QPS_SQE
;
4223 case MLX5_QP_STATE_ERR
: return IB_QPS_ERR
;
4228 static inline enum ib_mig_state
to_ib_mig_state(int mlx5_mig_state
)
4230 switch (mlx5_mig_state
) {
4231 case MLX5_QP_PM_ARMED
: return IB_MIG_ARMED
;
4232 case MLX5_QP_PM_REARM
: return IB_MIG_REARM
;
4233 case MLX5_QP_PM_MIGRATED
: return IB_MIG_MIGRATED
;
4238 static int to_ib_qp_access_flags(int mlx5_flags
)
4242 if (mlx5_flags
& MLX5_QP_BIT_RRE
)
4243 ib_flags
|= IB_ACCESS_REMOTE_READ
;
4244 if (mlx5_flags
& MLX5_QP_BIT_RWE
)
4245 ib_flags
|= IB_ACCESS_REMOTE_WRITE
;
4246 if (mlx5_flags
& MLX5_QP_BIT_RAE
)
4247 ib_flags
|= IB_ACCESS_REMOTE_ATOMIC
;
4252 static void to_ib_ah_attr(struct mlx5_ib_dev
*ibdev
, struct ib_ah_attr
*ib_ah_attr
,
4253 struct mlx5_qp_path
*path
)
4255 struct mlx5_core_dev
*dev
= ibdev
->mdev
;
4257 memset(ib_ah_attr
, 0, sizeof(*ib_ah_attr
));
4258 ib_ah_attr
->port_num
= path
->port
;
4260 if (ib_ah_attr
->port_num
== 0 ||
4261 ib_ah_attr
->port_num
> MLX5_CAP_GEN(dev
, num_ports
))
4264 ib_ah_attr
->sl
= path
->dci_cfi_prio_sl
& 0xf;
4266 ib_ah_attr
->dlid
= be16_to_cpu(path
->rlid
);
4267 ib_ah_attr
->src_path_bits
= path
->grh_mlid
& 0x7f;
4268 ib_ah_attr
->static_rate
= path
->static_rate
? path
->static_rate
- 5 : 0;
4269 ib_ah_attr
->ah_flags
= (path
->grh_mlid
& (1 << 7)) ? IB_AH_GRH
: 0;
4270 if (ib_ah_attr
->ah_flags
) {
4271 ib_ah_attr
->grh
.sgid_index
= path
->mgid_index
;
4272 ib_ah_attr
->grh
.hop_limit
= path
->hop_limit
;
4273 ib_ah_attr
->grh
.traffic_class
=
4274 (be32_to_cpu(path
->tclass_flowlabel
) >> 20) & 0xff;
4275 ib_ah_attr
->grh
.flow_label
=
4276 be32_to_cpu(path
->tclass_flowlabel
) & 0xfffff;
4277 memcpy(ib_ah_attr
->grh
.dgid
.raw
,
4278 path
->rgid
, sizeof(ib_ah_attr
->grh
.dgid
.raw
));
4282 static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev
*dev
,
4283 struct mlx5_ib_sq
*sq
,
4291 inlen
= MLX5_ST_SZ_BYTES(query_sq_out
);
4292 out
= mlx5_vzalloc(inlen
);
4296 err
= mlx5_core_query_sq(dev
->mdev
, sq
->base
.mqp
.qpn
, out
);
4300 sqc
= MLX5_ADDR_OF(query_sq_out
, out
, sq_context
);
4301 *sq_state
= MLX5_GET(sqc
, sqc
, state
);
4302 sq
->state
= *sq_state
;
4309 static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev
*dev
,
4310 struct mlx5_ib_rq
*rq
,
4318 inlen
= MLX5_ST_SZ_BYTES(query_rq_out
);
4319 out
= mlx5_vzalloc(inlen
);
4323 err
= mlx5_core_query_rq(dev
->mdev
, rq
->base
.mqp
.qpn
, out
);
4327 rqc
= MLX5_ADDR_OF(query_rq_out
, out
, rq_context
);
4328 *rq_state
= MLX5_GET(rqc
, rqc
, state
);
4329 rq
->state
= *rq_state
;
4336 static int sqrq_state_to_qp_state(u8 sq_state
, u8 rq_state
,
4337 struct mlx5_ib_qp
*qp
, u8
*qp_state
)
4339 static const u8 sqrq_trans
[MLX5_RQ_NUM_STATE
][MLX5_SQ_NUM_STATE
] = {
4340 [MLX5_RQC_STATE_RST
] = {
4341 [MLX5_SQC_STATE_RST
] = IB_QPS_RESET
,
4342 [MLX5_SQC_STATE_RDY
] = MLX5_QP_STATE_BAD
,
4343 [MLX5_SQC_STATE_ERR
] = MLX5_QP_STATE_BAD
,
4344 [MLX5_SQ_STATE_NA
] = IB_QPS_RESET
,
4346 [MLX5_RQC_STATE_RDY
] = {
4347 [MLX5_SQC_STATE_RST
] = MLX5_QP_STATE_BAD
,
4348 [MLX5_SQC_STATE_RDY
] = MLX5_QP_STATE
,
4349 [MLX5_SQC_STATE_ERR
] = IB_QPS_SQE
,
4350 [MLX5_SQ_STATE_NA
] = MLX5_QP_STATE
,
4352 [MLX5_RQC_STATE_ERR
] = {
4353 [MLX5_SQC_STATE_RST
] = MLX5_QP_STATE_BAD
,
4354 [MLX5_SQC_STATE_RDY
] = MLX5_QP_STATE_BAD
,
4355 [MLX5_SQC_STATE_ERR
] = IB_QPS_ERR
,
4356 [MLX5_SQ_STATE_NA
] = IB_QPS_ERR
,
4358 [MLX5_RQ_STATE_NA
] = {
4359 [MLX5_SQC_STATE_RST
] = IB_QPS_RESET
,
4360 [MLX5_SQC_STATE_RDY
] = MLX5_QP_STATE
,
4361 [MLX5_SQC_STATE_ERR
] = MLX5_QP_STATE
,
4362 [MLX5_SQ_STATE_NA
] = MLX5_QP_STATE_BAD
,
4366 *qp_state
= sqrq_trans
[rq_state
][sq_state
];
4368 if (*qp_state
== MLX5_QP_STATE_BAD
) {
4369 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4370 qp
->raw_packet_qp
.sq
.base
.mqp
.qpn
, sq_state
,
4371 qp
->raw_packet_qp
.rq
.base
.mqp
.qpn
, rq_state
);
4375 if (*qp_state
== MLX5_QP_STATE
)
4376 *qp_state
= qp
->state
;
4381 static int query_raw_packet_qp_state(struct mlx5_ib_dev
*dev
,
4382 struct mlx5_ib_qp
*qp
,
4383 u8
*raw_packet_qp_state
)
4385 struct mlx5_ib_raw_packet_qp
*raw_packet_qp
= &qp
->raw_packet_qp
;
4386 struct mlx5_ib_sq
*sq
= &raw_packet_qp
->sq
;
4387 struct mlx5_ib_rq
*rq
= &raw_packet_qp
->rq
;
4389 u8 sq_state
= MLX5_SQ_STATE_NA
;
4390 u8 rq_state
= MLX5_RQ_STATE_NA
;
4392 if (qp
->sq
.wqe_cnt
) {
4393 err
= query_raw_packet_qp_sq_state(dev
, sq
, &sq_state
);
4398 if (qp
->rq
.wqe_cnt
) {
4399 err
= query_raw_packet_qp_rq_state(dev
, rq
, &rq_state
);
4404 return sqrq_state_to_qp_state(sq_state
, rq_state
, qp
,
4405 raw_packet_qp_state
);
4408 static int query_qp_attr(struct mlx5_ib_dev
*dev
, struct mlx5_ib_qp
*qp
,
4409 struct ib_qp_attr
*qp_attr
)
4411 int outlen
= MLX5_ST_SZ_BYTES(query_qp_out
);
4412 struct mlx5_qp_context
*context
;
4417 outb
= kzalloc(outlen
, GFP_KERNEL
);
4421 err
= mlx5_core_qp_query(dev
->mdev
, &qp
->trans_qp
.base
.mqp
, outb
,
4426 /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
4427 context
= (struct mlx5_qp_context
*)MLX5_ADDR_OF(query_qp_out
, outb
, qpc
);
4429 mlx5_state
= be32_to_cpu(context
->flags
) >> 28;
4431 qp
->state
= to_ib_qp_state(mlx5_state
);
4432 qp_attr
->path_mtu
= context
->mtu_msgmax
>> 5;
4433 qp_attr
->path_mig_state
=
4434 to_ib_mig_state((be32_to_cpu(context
->flags
) >> 11) & 0x3);
4435 qp_attr
->qkey
= be32_to_cpu(context
->qkey
);
4436 qp_attr
->rq_psn
= be32_to_cpu(context
->rnr_nextrecvpsn
) & 0xffffff;
4437 qp_attr
->sq_psn
= be32_to_cpu(context
->next_send_psn
) & 0xffffff;
4438 qp_attr
->dest_qp_num
= be32_to_cpu(context
->log_pg_sz_remote_qpn
) & 0xffffff;
4439 qp_attr
->qp_access_flags
=
4440 to_ib_qp_access_flags(be32_to_cpu(context
->params2
));
4442 if (qp
->ibqp
.qp_type
== IB_QPT_RC
|| qp
->ibqp
.qp_type
== IB_QPT_UC
) {
4443 to_ib_ah_attr(dev
, &qp_attr
->ah_attr
, &context
->pri_path
);
4444 to_ib_ah_attr(dev
, &qp_attr
->alt_ah_attr
, &context
->alt_path
);
4445 qp_attr
->alt_pkey_index
=
4446 be16_to_cpu(context
->alt_path
.pkey_index
);
4447 qp_attr
->alt_port_num
= qp_attr
->alt_ah_attr
.port_num
;
4450 qp_attr
->pkey_index
= be16_to_cpu(context
->pri_path
.pkey_index
);
4451 qp_attr
->port_num
= context
->pri_path
.port
;
4453 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4454 qp_attr
->sq_draining
= mlx5_state
== MLX5_QP_STATE_SQ_DRAINING
;
4456 qp_attr
->max_rd_atomic
= 1 << ((be32_to_cpu(context
->params1
) >> 21) & 0x7);
4458 qp_attr
->max_dest_rd_atomic
=
4459 1 << ((be32_to_cpu(context
->params2
) >> 21) & 0x7);
4460 qp_attr
->min_rnr_timer
=
4461 (be32_to_cpu(context
->rnr_nextrecvpsn
) >> 24) & 0x1f;
4462 qp_attr
->timeout
= context
->pri_path
.ackto_lt
>> 3;
4463 qp_attr
->retry_cnt
= (be32_to_cpu(context
->params1
) >> 16) & 0x7;
4464 qp_attr
->rnr_retry
= (be32_to_cpu(context
->params1
) >> 13) & 0x7;
4465 qp_attr
->alt_timeout
= context
->alt_path
.ackto_lt
>> 3;
4472 int mlx5_ib_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*qp_attr
,
4473 int qp_attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
4475 struct mlx5_ib_dev
*dev
= to_mdev(ibqp
->device
);
4476 struct mlx5_ib_qp
*qp
= to_mqp(ibqp
);
4478 u8 raw_packet_qp_state
;
4480 if (ibqp
->rwq_ind_tbl
)
4483 if (unlikely(ibqp
->qp_type
== IB_QPT_GSI
))
4484 return mlx5_ib_gsi_query_qp(ibqp
, qp_attr
, qp_attr_mask
,
4487 mutex_lock(&qp
->mutex
);
4489 if (qp
->ibqp
.qp_type
== IB_QPT_RAW_PACKET
) {
4490 err
= query_raw_packet_qp_state(dev
, qp
, &raw_packet_qp_state
);
4493 qp
->state
= raw_packet_qp_state
;
4494 qp_attr
->port_num
= 1;
4496 err
= query_qp_attr(dev
, qp
, qp_attr
);
4501 qp_attr
->qp_state
= qp
->state
;
4502 qp_attr
->cur_qp_state
= qp_attr
->qp_state
;
4503 qp_attr
->cap
.max_recv_wr
= qp
->rq
.wqe_cnt
;
4504 qp_attr
->cap
.max_recv_sge
= qp
->rq
.max_gs
;
4506 if (!ibqp
->uobject
) {
4507 qp_attr
->cap
.max_send_wr
= qp
->sq
.max_post
;
4508 qp_attr
->cap
.max_send_sge
= qp
->sq
.max_gs
;
4509 qp_init_attr
->qp_context
= ibqp
->qp_context
;
4511 qp_attr
->cap
.max_send_wr
= 0;
4512 qp_attr
->cap
.max_send_sge
= 0;
4515 qp_init_attr
->qp_type
= ibqp
->qp_type
;
4516 qp_init_attr
->recv_cq
= ibqp
->recv_cq
;
4517 qp_init_attr
->send_cq
= ibqp
->send_cq
;
4518 qp_init_attr
->srq
= ibqp
->srq
;
4519 qp_attr
->cap
.max_inline_data
= qp
->max_inline_data
;
4521 qp_init_attr
->cap
= qp_attr
->cap
;
4523 qp_init_attr
->create_flags
= 0;
4524 if (qp
->flags
& MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK
)
4525 qp_init_attr
->create_flags
|= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK
;
4527 if (qp
->flags
& MLX5_IB_QP_CROSS_CHANNEL
)
4528 qp_init_attr
->create_flags
|= IB_QP_CREATE_CROSS_CHANNEL
;
4529 if (qp
->flags
& MLX5_IB_QP_MANAGED_SEND
)
4530 qp_init_attr
->create_flags
|= IB_QP_CREATE_MANAGED_SEND
;
4531 if (qp
->flags
& MLX5_IB_QP_MANAGED_RECV
)
4532 qp_init_attr
->create_flags
|= IB_QP_CREATE_MANAGED_RECV
;
4533 if (qp
->flags
& MLX5_IB_QP_SQPN_QP1
)
4534 qp_init_attr
->create_flags
|= mlx5_ib_create_qp_sqpn_qp1();
4536 qp_init_attr
->sq_sig_type
= qp
->sq_signal_bits
& MLX5_WQE_CTRL_CQ_UPDATE
?
4537 IB_SIGNAL_ALL_WR
: IB_SIGNAL_REQ_WR
;
4540 mutex_unlock(&qp
->mutex
);
4544 struct ib_xrcd
*mlx5_ib_alloc_xrcd(struct ib_device
*ibdev
,
4545 struct ib_ucontext
*context
,
4546 struct ib_udata
*udata
)
4548 struct mlx5_ib_dev
*dev
= to_mdev(ibdev
);
4549 struct mlx5_ib_xrcd
*xrcd
;
4552 if (!MLX5_CAP_GEN(dev
->mdev
, xrc
))
4553 return ERR_PTR(-ENOSYS
);
4555 xrcd
= kmalloc(sizeof(*xrcd
), GFP_KERNEL
);
4557 return ERR_PTR(-ENOMEM
);
4559 err
= mlx5_core_xrcd_alloc(dev
->mdev
, &xrcd
->xrcdn
);
4562 return ERR_PTR(-ENOMEM
);
4565 return &xrcd
->ibxrcd
;
4568 int mlx5_ib_dealloc_xrcd(struct ib_xrcd
*xrcd
)
4570 struct mlx5_ib_dev
*dev
= to_mdev(xrcd
->device
);
4571 u32 xrcdn
= to_mxrcd(xrcd
)->xrcdn
;
4574 err
= mlx5_core_xrcd_dealloc(dev
->mdev
, xrcdn
);
4576 mlx5_ib_warn(dev
, "failed to dealloc xrcdn 0x%x\n", xrcdn
);
4585 static void mlx5_ib_wq_event(struct mlx5_core_qp
*core_qp
, int type
)
4587 struct mlx5_ib_rwq
*rwq
= to_mibrwq(core_qp
);
4588 struct mlx5_ib_dev
*dev
= to_mdev(rwq
->ibwq
.device
);
4589 struct ib_event event
;
4591 if (rwq
->ibwq
.event_handler
) {
4592 event
.device
= rwq
->ibwq
.device
;
4593 event
.element
.wq
= &rwq
->ibwq
;
4595 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR
:
4596 event
.event
= IB_EVENT_WQ_FATAL
;
4599 mlx5_ib_warn(dev
, "Unexpected event type %d on WQ %06x\n", type
, core_qp
->qpn
);
4603 rwq
->ibwq
.event_handler(&event
, rwq
->ibwq
.wq_context
);
4607 static int create_rq(struct mlx5_ib_rwq
*rwq
, struct ib_pd
*pd
,
4608 struct ib_wq_init_attr
*init_attr
)
4610 struct mlx5_ib_dev
*dev
;
4611 int has_net_offloads
;
4619 dev
= to_mdev(pd
->device
);
4621 inlen
= MLX5_ST_SZ_BYTES(create_rq_in
) + sizeof(u64
) * rwq
->rq_num_pas
;
4622 in
= mlx5_vzalloc(inlen
);
4626 rqc
= MLX5_ADDR_OF(create_rq_in
, in
, ctx
);
4627 MLX5_SET(rqc
, rqc
, mem_rq_type
,
4628 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE
);
4629 MLX5_SET(rqc
, rqc
, user_index
, rwq
->user_index
);
4630 MLX5_SET(rqc
, rqc
, cqn
, to_mcq(init_attr
->cq
)->mcq
.cqn
);
4631 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RST
);
4632 MLX5_SET(rqc
, rqc
, flush_in_error_en
, 1);
4633 wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
4634 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_CYCLIC
);
4635 MLX5_SET(wq
, wq
, end_padding_mode
, MLX5_WQ_END_PAD_MODE_ALIGN
);
4636 MLX5_SET(wq
, wq
, log_wq_stride
, rwq
->log_rq_stride
);
4637 MLX5_SET(wq
, wq
, log_wq_sz
, rwq
->log_rq_size
);
4638 MLX5_SET(wq
, wq
, pd
, to_mpd(pd
)->pdn
);
4639 MLX5_SET(wq
, wq
, page_offset
, rwq
->rq_page_offset
);
4640 MLX5_SET(wq
, wq
, log_wq_pg_sz
, rwq
->log_page_size
);
4641 MLX5_SET(wq
, wq
, wq_signature
, rwq
->wq_sig
);
4642 MLX5_SET64(wq
, wq
, dbr_addr
, rwq
->db
.dma
);
4643 has_net_offloads
= MLX5_CAP_GEN(dev
->mdev
, eth_net_offloads
);
4644 if (init_attr
->create_flags
& IB_WQ_FLAGS_CVLAN_STRIPPING
) {
4645 if (!(has_net_offloads
&& MLX5_CAP_ETH(dev
->mdev
, vlan_cap
))) {
4646 mlx5_ib_dbg(dev
, "VLAN offloads are not supported\n");
4651 MLX5_SET(rqc
, rqc
, vsd
, 1);
4653 if (init_attr
->create_flags
& IB_WQ_FLAGS_SCATTER_FCS
) {
4654 if (!(has_net_offloads
&& MLX5_CAP_ETH(dev
->mdev
, scatter_fcs
))) {
4655 mlx5_ib_dbg(dev
, "Scatter FCS is not supported\n");
4659 MLX5_SET(rqc
, rqc
, scatter_fcs
, 1);
4661 rq_pas0
= (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
);
4662 mlx5_ib_populate_pas(dev
, rwq
->umem
, rwq
->page_shift
, rq_pas0
, 0);
4663 err
= mlx5_core_create_rq_tracked(dev
->mdev
, in
, inlen
, &rwq
->core_qp
);
4669 static int set_user_rq_size(struct mlx5_ib_dev
*dev
,
4670 struct ib_wq_init_attr
*wq_init_attr
,
4671 struct mlx5_ib_create_wq
*ucmd
,
4672 struct mlx5_ib_rwq
*rwq
)
4674 /* Sanity check RQ size before proceeding */
4675 if (wq_init_attr
->max_wr
> (1 << MLX5_CAP_GEN(dev
->mdev
, log_max_wq_sz
)))
4678 if (!ucmd
->rq_wqe_count
)
4681 rwq
->wqe_count
= ucmd
->rq_wqe_count
;
4682 rwq
->wqe_shift
= ucmd
->rq_wqe_shift
;
4683 rwq
->buf_size
= (rwq
->wqe_count
<< rwq
->wqe_shift
);
4684 rwq
->log_rq_stride
= rwq
->wqe_shift
;
4685 rwq
->log_rq_size
= ilog2(rwq
->wqe_count
);
4689 static int prepare_user_rq(struct ib_pd
*pd
,
4690 struct ib_wq_init_attr
*init_attr
,
4691 struct ib_udata
*udata
,
4692 struct mlx5_ib_rwq
*rwq
)
4694 struct mlx5_ib_dev
*dev
= to_mdev(pd
->device
);
4695 struct mlx5_ib_create_wq ucmd
= {};
4697 size_t required_cmd_sz
;
4699 required_cmd_sz
= offsetof(typeof(ucmd
), reserved
) + sizeof(ucmd
.reserved
);
4700 if (udata
->inlen
< required_cmd_sz
) {
4701 mlx5_ib_dbg(dev
, "invalid inlen\n");
4705 if (udata
->inlen
> sizeof(ucmd
) &&
4706 !ib_is_udata_cleared(udata
, sizeof(ucmd
),
4707 udata
->inlen
- sizeof(ucmd
))) {
4708 mlx5_ib_dbg(dev
, "inlen is not supported\n");
4712 if (ib_copy_from_udata(&ucmd
, udata
, min(sizeof(ucmd
), udata
->inlen
))) {
4713 mlx5_ib_dbg(dev
, "copy failed\n");
4717 if (ucmd
.comp_mask
) {
4718 mlx5_ib_dbg(dev
, "invalid comp mask\n");
4722 if (ucmd
.reserved
) {
4723 mlx5_ib_dbg(dev
, "invalid reserved\n");
4727 err
= set_user_rq_size(dev
, init_attr
, &ucmd
, rwq
);
4729 mlx5_ib_dbg(dev
, "err %d\n", err
);
4733 err
= create_user_rq(dev
, pd
, rwq
, &ucmd
);
4735 mlx5_ib_dbg(dev
, "err %d\n", err
);
4740 rwq
->user_index
= ucmd
.user_index
;
4744 struct ib_wq
*mlx5_ib_create_wq(struct ib_pd
*pd
,
4745 struct ib_wq_init_attr
*init_attr
,
4746 struct ib_udata
*udata
)
4748 struct mlx5_ib_dev
*dev
;
4749 struct mlx5_ib_rwq
*rwq
;
4750 struct mlx5_ib_create_wq_resp resp
= {};
4751 size_t min_resp_len
;
4755 return ERR_PTR(-ENOSYS
);
4757 min_resp_len
= offsetof(typeof(resp
), reserved
) + sizeof(resp
.reserved
);
4758 if (udata
->outlen
&& udata
->outlen
< min_resp_len
)
4759 return ERR_PTR(-EINVAL
);
4761 dev
= to_mdev(pd
->device
);
4762 switch (init_attr
->wq_type
) {
4764 rwq
= kzalloc(sizeof(*rwq
), GFP_KERNEL
);
4766 return ERR_PTR(-ENOMEM
);
4767 err
= prepare_user_rq(pd
, init_attr
, udata
, rwq
);
4770 err
= create_rq(rwq
, pd
, init_attr
);
4775 mlx5_ib_dbg(dev
, "unsupported wq type %d\n",
4776 init_attr
->wq_type
);
4777 return ERR_PTR(-EINVAL
);
4780 rwq
->ibwq
.wq_num
= rwq
->core_qp
.qpn
;
4781 rwq
->ibwq
.state
= IB_WQS_RESET
;
4782 if (udata
->outlen
) {
4783 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
4784 sizeof(resp
.response_length
);
4785 err
= ib_copy_to_udata(udata
, &resp
, resp
.response_length
);
4790 rwq
->core_qp
.event
= mlx5_ib_wq_event
;
4791 rwq
->ibwq
.event_handler
= init_attr
->event_handler
;
4795 mlx5_core_destroy_rq_tracked(dev
->mdev
, &rwq
->core_qp
);
4797 destroy_user_rq(pd
, rwq
);
4800 return ERR_PTR(err
);
4803 int mlx5_ib_destroy_wq(struct ib_wq
*wq
)
4805 struct mlx5_ib_dev
*dev
= to_mdev(wq
->device
);
4806 struct mlx5_ib_rwq
*rwq
= to_mrwq(wq
);
4808 mlx5_core_destroy_rq_tracked(dev
->mdev
, &rwq
->core_qp
);
4809 destroy_user_rq(wq
->pd
, rwq
);
4815 struct ib_rwq_ind_table
*mlx5_ib_create_rwq_ind_table(struct ib_device
*device
,
4816 struct ib_rwq_ind_table_init_attr
*init_attr
,
4817 struct ib_udata
*udata
)
4819 struct mlx5_ib_dev
*dev
= to_mdev(device
);
4820 struct mlx5_ib_rwq_ind_table
*rwq_ind_tbl
;
4821 int sz
= 1 << init_attr
->log_ind_tbl_size
;
4822 struct mlx5_ib_create_rwq_ind_tbl_resp resp
= {};
4823 size_t min_resp_len
;
4830 if (udata
->inlen
> 0 &&
4831 !ib_is_udata_cleared(udata
, 0,
4833 return ERR_PTR(-EOPNOTSUPP
);
4835 if (init_attr
->log_ind_tbl_size
>
4836 MLX5_CAP_GEN(dev
->mdev
, log_max_rqt_size
)) {
4837 mlx5_ib_dbg(dev
, "log_ind_tbl_size = %d is bigger than supported = %d\n",
4838 init_attr
->log_ind_tbl_size
,
4839 MLX5_CAP_GEN(dev
->mdev
, log_max_rqt_size
));
4840 return ERR_PTR(-EINVAL
);
4843 min_resp_len
= offsetof(typeof(resp
), reserved
) + sizeof(resp
.reserved
);
4844 if (udata
->outlen
&& udata
->outlen
< min_resp_len
)
4845 return ERR_PTR(-EINVAL
);
4847 rwq_ind_tbl
= kzalloc(sizeof(*rwq_ind_tbl
), GFP_KERNEL
);
4849 return ERR_PTR(-ENOMEM
);
4851 inlen
= MLX5_ST_SZ_BYTES(create_rqt_in
) + sizeof(u32
) * sz
;
4852 in
= mlx5_vzalloc(inlen
);
4858 rqtc
= MLX5_ADDR_OF(create_rqt_in
, in
, rqt_context
);
4860 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
4861 MLX5_SET(rqtc
, rqtc
, rqt_max_size
, sz
);
4863 for (i
= 0; i
< sz
; i
++)
4864 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], init_attr
->ind_tbl
[i
]->wq_num
);
4866 err
= mlx5_core_create_rqt(dev
->mdev
, in
, inlen
, &rwq_ind_tbl
->rqtn
);
4872 rwq_ind_tbl
->ib_rwq_ind_tbl
.ind_tbl_num
= rwq_ind_tbl
->rqtn
;
4873 if (udata
->outlen
) {
4874 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
4875 sizeof(resp
.response_length
);
4876 err
= ib_copy_to_udata(udata
, &resp
, resp
.response_length
);
4881 return &rwq_ind_tbl
->ib_rwq_ind_tbl
;
4884 mlx5_core_destroy_rqt(dev
->mdev
, rwq_ind_tbl
->rqtn
);
4887 return ERR_PTR(err
);
4890 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table
*ib_rwq_ind_tbl
)
4892 struct mlx5_ib_rwq_ind_table
*rwq_ind_tbl
= to_mrwq_ind_table(ib_rwq_ind_tbl
);
4893 struct mlx5_ib_dev
*dev
= to_mdev(ib_rwq_ind_tbl
->device
);
4895 mlx5_core_destroy_rqt(dev
->mdev
, rwq_ind_tbl
->rqtn
);
4901 int mlx5_ib_modify_wq(struct ib_wq
*wq
, struct ib_wq_attr
*wq_attr
,
4902 u32 wq_attr_mask
, struct ib_udata
*udata
)
4904 struct mlx5_ib_dev
*dev
= to_mdev(wq
->device
);
4905 struct mlx5_ib_rwq
*rwq
= to_mrwq(wq
);
4906 struct mlx5_ib_modify_wq ucmd
= {};
4907 size_t required_cmd_sz
;
4915 required_cmd_sz
= offsetof(typeof(ucmd
), reserved
) + sizeof(ucmd
.reserved
);
4916 if (udata
->inlen
< required_cmd_sz
)
4919 if (udata
->inlen
> sizeof(ucmd
) &&
4920 !ib_is_udata_cleared(udata
, sizeof(ucmd
),
4921 udata
->inlen
- sizeof(ucmd
)))
4924 if (ib_copy_from_udata(&ucmd
, udata
, min(sizeof(ucmd
), udata
->inlen
)))
4927 if (ucmd
.comp_mask
|| ucmd
.reserved
)
4930 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
4931 in
= mlx5_vzalloc(inlen
);
4935 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
4937 curr_wq_state
= (wq_attr_mask
& IB_WQ_CUR_STATE
) ?
4938 wq_attr
->curr_wq_state
: wq
->state
;
4939 wq_state
= (wq_attr_mask
& IB_WQ_STATE
) ?
4940 wq_attr
->wq_state
: curr_wq_state
;
4941 if (curr_wq_state
== IB_WQS_ERR
)
4942 curr_wq_state
= MLX5_RQC_STATE_ERR
;
4943 if (wq_state
== IB_WQS_ERR
)
4944 wq_state
= MLX5_RQC_STATE_ERR
;
4945 MLX5_SET(modify_rq_in
, in
, rq_state
, curr_wq_state
);
4946 MLX5_SET(rqc
, rqc
, state
, wq_state
);
4948 if (wq_attr_mask
& IB_WQ_FLAGS
) {
4949 if (wq_attr
->flags_mask
& IB_WQ_FLAGS_CVLAN_STRIPPING
) {
4950 if (!(MLX5_CAP_GEN(dev
->mdev
, eth_net_offloads
) &&
4951 MLX5_CAP_ETH(dev
->mdev
, vlan_cap
))) {
4952 mlx5_ib_dbg(dev
, "VLAN offloads are not "
4957 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
4958 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD
);
4959 MLX5_SET(rqc
, rqc
, vsd
,
4960 (wq_attr
->flags
& IB_WQ_FLAGS_CVLAN_STRIPPING
) ? 0 : 1);
4964 if (curr_wq_state
== IB_WQS_RESET
&& wq_state
== IB_WQS_RDY
) {
4965 if (MLX5_CAP_GEN(dev
->mdev
, modify_rq_counter_set_id
)) {
4966 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
4967 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID
);
4968 MLX5_SET(rqc
, rqc
, counter_set_id
,
4969 dev
->port
->cnts
.set_id
);
4971 pr_info_once("%s: Receive WQ counters are not supported on current FW\n",
4975 err
= mlx5_core_modify_rq(dev
->mdev
, rwq
->core_qp
.qpn
, in
, inlen
);
4977 rwq
->ibwq
.state
= (wq_state
== MLX5_RQC_STATE_ERR
) ? IB_WQS_ERR
: wq_state
;