]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - drivers/infiniband/hw/mlx5/qp.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 6.10-rc1
[thirdparty/kernel/stable.git] / drivers / infiniband / hw / mlx5 / qp.c
CommitLineData
e126ba97 1/*
6cf0a15f 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
b6459415 33#include <linux/etherdevice.h>
e126ba97 34#include <rdma/ib_umem.h>
2811ba51 35#include <rdma/ib_cache.h>
cfb5e088 36#include <rdma/ib_user_verbs.h>
d14133dd 37#include <rdma/rdma_counter.h>
c2e53b2c 38#include <linux/mlx5/fs.h>
e126ba97 39#include "mlx5_ib.h"
b96c9dde 40#include "ib_rep.h"
64825827 41#include "counters.h"
443c1cf9 42#include "cmd.h"
8a8a5d37 43#include "umr.h"
333fbaa0 44#include "qp.h"
029e88fd 45#include "wr.h"
e126ba97 46
e126ba97
EC		 47enum {
48 MLX5_IB_ACK_REQ_FREQ = 8,
49};
50
51enum {
52 MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83,
53 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
54 MLX5_IB_LINK_TYPE_IB = 0,
55 MLX5_IB_LINK_TYPE_ETH = 1
56};
57
eb49ab0c
AV		 58enum raw_qp_set_mask_map {
59 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0,
7d29f349 60 MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
eb49ab0c
AV		 61};
62
742948cc
OHT		 63enum {
64 MLX5_QP_RM_GO_BACK_N = 0x1,
65};
66
0680efa2
AV		 67struct mlx5_modify_raw_qp_param {
68 u16 operation;
eb49ab0c
AV		 69
70 u32 set_mask; /* raw_qp_set_mask_map */
61147f39
BW		 71
72 struct mlx5_rate_limit rl;
73
eb49ab0c 74 u8 rq_q_ctr_id;
1fb7f897 75 u32 port;
0680efa2
AV		 76};
77
312b8f79
MZ		 78struct mlx5_ib_qp_event_work {
79 struct work_struct work;
80 struct mlx5_core_qp *qp;
81 int type;
82};
83
84static struct workqueue_struct *mlx5_ib_qp_event_wq;
85
89ea94a7
MG		 86static void get_cqs(enum ib_qp_type qp_type,
87 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
88 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
89
e126ba97
EC		 90static int is_qp0(enum ib_qp_type qp_type)
91{
92 return qp_type == IB_QPT_SMI;
93}
94
e126ba97
EC		 95static int is_sqp(enum ib_qp_type qp_type)
96{
97 return is_qp0(qp_type) || is_qp1(qp_type);
98}
99
c1395a2a 100/**
fbeb4075
MS		 101 * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
102 * to kernel buffer
c1395a2a 103 *
fbeb4075
MS		 104 * @umem: User space memory where the WQ is
105 * @buffer: buffer to copy to
106 * @buflen: buffer length
107 * @wqe_index: index of WQE to copy from
108 * @wq_offset: offset to start of WQ
109 * @wq_wqe_cnt: number of WQEs in WQ
110 * @wq_wqe_shift: log2 of WQE size
111 * @bcnt: number of bytes to copy
112 * @bytes_copied: number of bytes to copy (return value)
c1395a2a 113 *
fbeb4075
MS		 114 * Copies from start of WQE bcnt or less bytes.
115 * Does not gurantee to copy the entire WQE.
c1395a2a 116 *
fbeb4075 117 * Return: zero on success, or an error code.
c1395a2a 118 */
da9ee9d8
MS		 119static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
120 size_t buflen, int wqe_index,
121 int wq_offset, int wq_wqe_cnt,
122 int wq_wqe_shift, int bcnt,
fbeb4075 123 size_t *bytes_copied)
c1395a2a 124{
fbeb4075
MS		 125 size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
126 size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
127 size_t copy_length;
c1395a2a
HE		 128 int ret;
129
fbeb4075
MS		 130 /* don't copy more than requested, more than buffer length or
131 * beyond WQ end
132 */
133 copy_length = min_t(u32, buflen, wq_end - offset);
134 copy_length = min_t(u32, copy_length, bcnt);
135
136 ret = ib_umem_copy_from(buffer, umem, offset, copy_length);
137 if (ret)
138 return ret;
c1395a2a 139
fbeb4075
MS		 140 if (!ret && bytes_copied)
141 *bytes_copied = copy_length;
c1395a2a 142
fbeb4075
MS		 143 return 0;
144}
c1395a2a 145
da9ee9d8
MS		 146static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
147 void *buffer, size_t buflen, size_t *bc)
148{
149 struct mlx5_wqe_ctrl_seg *ctrl;
150 size_t bytes_copied = 0;
151 size_t wqe_length;
152 void *p;
153 int ds;
154
155 wqe_index = wqe_index & qp->sq.fbc.sz_m1;
156
157 /* read the control segment first */
158 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
159 ctrl = p;
160 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
161 wqe_length = ds * MLX5_WQE_DS_UNITS;
162
163 /* read rest of WQE if it spreads over more than one stride */
164 while (bytes_copied < wqe_length) {
165 size_t copy_length =
166 min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
167
168 if (!copy_length)
169 break;
170
171 memcpy(buffer + bytes_copied, p, copy_length);
172 bytes_copied += copy_length;
173
174 wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
175 p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
176 }
177 *bc = bytes_copied;
178 return 0;
179}
180
181static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
182 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 183{
184 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
185 struct ib_umem *umem = base->ubuffer.umem;
186 struct mlx5_ib_wq *wq = &qp->sq;
187 struct mlx5_wqe_ctrl_seg *ctrl;
188 size_t bytes_copied;
189 size_t bytes_copied2;
190 size_t wqe_length;
191 int ret;
192 int ds;
193
fbeb4075 194 /* at first read as much as possible */
da9ee9d8
MS		 195 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
196 wq->offset, wq->wqe_cnt,
197 wq->wqe_shift, buflen,
fbeb4075 198 &bytes_copied);
c1395a2a
HE		 199 if (ret)
200 return ret;
201
fbeb4075
MS		 202 /* we need at least control segment size to proceed */
203 if (bytes_copied < sizeof(*ctrl))
204 return -EINVAL;
205
206 ctrl = buffer;
207 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
208 wqe_length = ds * MLX5_WQE_DS_UNITS;
c1395a2a 209
fbeb4075
MS		 210 /* if we copied enough then we are done */
211 if (bytes_copied >= wqe_length) {
212 *bc = bytes_copied;
213 return 0;
c1395a2a
HE		 214 }
215
fbeb4075
MS		 216 /* otherwise this a wrapped around wqe
217 * so read the remaining bytes starting
218 * from wqe_index 0
219 */
da9ee9d8
MS		 220 ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,
221 buflen - bytes_copied, 0, wq->offset,
222 wq->wqe_cnt, wq->wqe_shift,
fbeb4075
MS		 223 wqe_length - bytes_copied,
224 &bytes_copied2);
225
226 if (ret)
227 return ret;
228 *bc = bytes_copied + bytes_copied2;
229 return 0;
230}
231
da9ee9d8
MS		 232int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
233 size_t buflen, size_t *bc)
234{
235 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
236 struct ib_umem *umem = base->ubuffer.umem;
237
238 if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
239 return -EINVAL;
240
241 if (!umem)
242 return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
243 buflen, bc);
244
245 return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
246}
247
248static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
249 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 250{
251 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
252 struct ib_umem *umem = base->ubuffer.umem;
253 struct mlx5_ib_wq *wq = &qp->rq;
254 size_t bytes_copied;
255 int ret;
256
da9ee9d8
MS		 257 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
258 wq->offset, wq->wqe_cnt,
259 wq->wqe_shift, buflen,
fbeb4075 260 &bytes_copied);
c1395a2a 261
c1395a2a
HE		 262 if (ret)
263 return ret;
fbeb4075
MS		 264 *bc = bytes_copied;
265 return 0;
266}
267
da9ee9d8
MS		 268int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
269 size_t buflen, size_t *bc)
270{
271 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
272 struct ib_umem *umem = base->ubuffer.umem;
273 struct mlx5_ib_wq *wq = &qp->rq;
274 size_t wqe_size = 1 << wq->wqe_shift;
275
276 if (buflen < wqe_size)
277 return -EINVAL;
278
279 if (!umem)
280 return -EOPNOTSUPP;
281
282 return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
283}
284
285static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
286 void *buffer, size_t buflen, size_t *bc)
fbeb4075
MS		 287{
288 struct ib_umem *umem = srq->umem;
289 size_t bytes_copied;
290 int ret;
c1395a2a 291
da9ee9d8
MS		 292 ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,
293 srq->msrq.max, srq->msrq.wqe_shift,
294 buflen, &bytes_copied);
fbeb4075
MS		 295
296 if (ret)
297 return ret;
298 *bc = bytes_copied;
299 return 0;
c1395a2a
HE		 300}
301
da9ee9d8
MS		 302int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
303 size_t buflen, size_t *bc)
304{
305 struct ib_umem *umem = srq->umem;
306 size_t wqe_size = 1 << srq->msrq.wqe_shift;
307
308 if (buflen < wqe_size)
309 return -EINVAL;
310
311 if (!umem)
312 return -EOPNOTSUPP;
313
314 return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
315}
316
8067fd8b
PH		 317static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
318{
319 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
320 int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
321 struct mlx5_ib_qp *qp = to_mqp(ibqp);
322 void *pas_ext_union, *err_syn;
323 u32 *outb;
324 int err;
325
326 if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) ||
327 !MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
328 return;
329
330 outb = kzalloc(outlen, GFP_KERNEL);
331 if (!outb)
332 return;
333
334 err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
335 true);
336 if (err)
337 goto out;
338
339 pas_ext_union =
340 MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
341 err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
342 qpc_data_extension.error_syndrome);
343
344 pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
345 ibqp->device->name, ibqp->port, ibqp->qp_num,
346 ib_wc_status_msg(
347 MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
348 MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
349 MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
350 MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
351out:
352 kfree(outb);
353}
354
312b8f79
MZ		 355static void mlx5_ib_handle_qp_event(struct work_struct *_work)
356{
357 struct mlx5_ib_qp_event_work *qpe_work =
358 container_of(_work, struct mlx5_ib_qp_event_work, work);
359 struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
360 struct ib_event event = {};
361
362 event.device = ibqp->device;
363 event.element.qp = ibqp;
364 switch (qpe_work->type) {
365 case MLX5_EVENT_TYPE_PATH_MIG:
366 event.event = IB_EVENT_PATH_MIG;
367 break;
368 case MLX5_EVENT_TYPE_COMM_EST:
369 event.event = IB_EVENT_COMM_EST;
370 break;
371 case MLX5_EVENT_TYPE_SQ_DRAINED:
372 event.event = IB_EVENT_SQ_DRAINED;
373 break;
374 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
375 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
376 break;
377 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
378 event.event = IB_EVENT_QP_FATAL;
379 break;
380 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
381 event.event = IB_EVENT_PATH_MIG_ERR;
382 break;
383 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
384 event.event = IB_EVENT_QP_REQ_ERR;
385 break;
386 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
387 event.event = IB_EVENT_QP_ACCESS_ERR;
388 break;
389 default:
390 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
391 qpe_work->type, qpe_work->qp->qpn);
392 goto out;
393 }
394
8067fd8b
PH		 395 if ((event.event == IB_EVENT_QP_FATAL) ||
396 (event.event == IB_EVENT_QP_ACCESS_ERR))
397 mlx5_ib_qp_err_syndrome(ibqp);
398
312b8f79
MZ		 399 ibqp->event_handler(&event, ibqp->qp_context);
400
401out:
402 mlx5_core_res_put(&qpe_work->qp->common);
403 kfree(qpe_work);
404}
405
e126ba97
EC		 406static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
407{
408 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
312b8f79 409 struct mlx5_ib_qp_event_work *qpe_work;
e126ba97 410
19098df2 411 if (type == MLX5_EVENT_TYPE_PATH_MIG) {
412 /* This event is only valid for trans_qps */
413 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
414 }
e126ba97 415
312b8f79
MZ		 416 if (!ibqp->event_handler)
417 goto out_no_handler;
e126ba97 418
312b8f79
MZ		 419 qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
420 if (!qpe_work)
421 goto out_no_handler;
422
423 qpe_work->qp = qp;
424 qpe_work->type = type;
425 INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
426 queue_work(mlx5_ib_qp_event_wq, &qpe_work->work);
427 return;
428
429out_no_handler:
430 mlx5_core_res_put(&qp->common);
e126ba97
EC		 431}
432
433static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
434 int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
435{
436 int wqe_size;
437 int wq_size;
438
439 /* Sanity check RQ size before proceeding */
938fe83c 440 if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
e126ba97
EC		 441 return -EINVAL;
442
443 if (!has_rq) {
444 qp->rq.max_gs = 0;
445 qp->rq.wqe_cnt = 0;
446 qp->rq.wqe_shift = 0;
0540d814
NO		 447 cap->max_recv_wr = 0;
448 cap->max_recv_sge = 0;
e126ba97 449 } else {
c95e6d53
LR		 450 int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
451
e126ba97
EC		 452 if (ucmd) {
453 qp->rq.wqe_cnt = ucmd->rq_wqe_count;
002bf228
LR		 454 if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
455 return -EINVAL;
e126ba97 456 qp->rq.wqe_shift = ucmd->rq_wqe_shift;
c95e6d53
LR		 457 if ((1 << qp->rq.wqe_shift) /
458 sizeof(struct mlx5_wqe_data_seg) <
459 wq_sig)
002bf228 460 return -EINVAL;
c95e6d53
LR		 461 qp->rq.max_gs =
462 (1 << qp->rq.wqe_shift) /
463 sizeof(struct mlx5_wqe_data_seg) -
464 wq_sig;
e126ba97
EC		 465 qp->rq.max_post = qp->rq.wqe_cnt;
466 } else {
c95e6d53
LR		 467 wqe_size =
468 wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
469 0;
e126ba97
EC		 470 wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
471 wqe_size = roundup_pow_of_two(wqe_size);
472 wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
473 wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
474 qp->rq.wqe_cnt = wq_size / wqe_size;
938fe83c 475 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
e126ba97
EC		 476 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
477 wqe_size,
938fe83c
SM		 478 MLX5_CAP_GEN(dev->mdev,
479 max_wqe_sz_rq));
e126ba97
EC		 480 return -EINVAL;
481 }
482 qp->rq.wqe_shift = ilog2(wqe_size);
c95e6d53
LR		 483 qp->rq.max_gs =
484 (1 << qp->rq.wqe_shift) /
485 sizeof(struct mlx5_wqe_data_seg) -
486 wq_sig;
e126ba97
EC		 487 qp->rq.max_post = qp->rq.wqe_cnt;
488 }
489 }
490
491 return 0;
492}
493
f0313965 494static int sq_overhead(struct ib_qp_init_attr *attr)
e126ba97 495{
618af384 496 int size = 0;
e126ba97 497
f0313965 498 switch (attr->qp_type) {
e126ba97 499 case IB_QPT_XRC_INI:
b125a54b 500 size += sizeof(struct mlx5_wqe_xrc_seg);
df561f66 501 fallthrough;
e126ba97
EC		 502 case IB_QPT_RC:
503 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 504 max(sizeof(struct mlx5_wqe_atomic_seg) +
505 sizeof(struct mlx5_wqe_raddr_seg),
506 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
064e5262
IB		 507 sizeof(struct mlx5_mkey_seg) +
508 MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
509 MLX5_IB_UMR_OCTOWORD);
e126ba97
EC		 510 break;
511
b125a54b
EC		 512 case IB_QPT_XRC_TGT:
513 return 0;
514
e126ba97 515 case IB_QPT_UC:
b125a54b 516 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 517 max(sizeof(struct mlx5_wqe_raddr_seg),
518 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
519 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 520 break;
521
522 case IB_QPT_UD:
f0313965
ES		 523 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
524 size += sizeof(struct mlx5_wqe_eth_pad) +
525 sizeof(struct mlx5_wqe_eth_seg);
df561f66 526 fallthrough;
e126ba97 527 case IB_QPT_SMI:
d16e91da 528 case MLX5_IB_QPT_HW_GSI:
b125a54b 529 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 530 sizeof(struct mlx5_wqe_datagram_seg);
531 break;
532
533 case MLX5_IB_QPT_REG_UMR:
b125a54b 534 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 535 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
536 sizeof(struct mlx5_mkey_seg);
537 break;
538
539 default:
540 return -EINVAL;
541 }
542
543 return size;
544}
545
546static int calc_send_wqe(struct ib_qp_init_attr *attr)
547{
548 int inl_size = 0;
549 int size;
550
f0313965 551 size = sq_overhead(attr);
e126ba97
EC		 552 if (size < 0)
553 return size;
554
555 if (attr->cap.max_inline_data) {
556 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
557 attr->cap.max_inline_data;
558 }
559
560 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
c0a6cbb9 561 if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
e1e66cc2 562 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
c0a6cbb9 563 return MLX5_SIG_WQE_SIZE;
e1e66cc2
SG		 564 else
565 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
e126ba97
EC		 566}
567
288c01b7
EC		 568static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
569{
570 int max_sge;
571
572 if (attr->qp_type == IB_QPT_RC)
573 max_sge = (min_t(int, wqe_size, 512) -
574 sizeof(struct mlx5_wqe_ctrl_seg) -
575 sizeof(struct mlx5_wqe_raddr_seg)) /
576 sizeof(struct mlx5_wqe_data_seg);
577 else if (attr->qp_type == IB_QPT_XRC_INI)
578 max_sge = (min_t(int, wqe_size, 512) -
579 sizeof(struct mlx5_wqe_ctrl_seg) -
580 sizeof(struct mlx5_wqe_xrc_seg) -
581 sizeof(struct mlx5_wqe_raddr_seg)) /
582 sizeof(struct mlx5_wqe_data_seg);
583 else
584 max_sge = (wqe_size - sq_overhead(attr)) /
585 sizeof(struct mlx5_wqe_data_seg);
586
587 return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
588 sizeof(struct mlx5_wqe_data_seg));
589}
590
e126ba97
EC		 591static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
592 struct mlx5_ib_qp *qp)
593{
594 int wqe_size;
595 int wq_size;
596
597 if (!attr->cap.max_send_wr)
598 return 0;
599
600 wqe_size = calc_send_wqe(attr);
601 mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
602 if (wqe_size < 0)
603 return wqe_size;
604
938fe83c 605 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
b125a54b 606 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
938fe83c 607 wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 608 return -EINVAL;
609 }
610
f0313965
ES		 611 qp->max_inline_data = wqe_size - sq_overhead(attr) -
612 sizeof(struct mlx5_wqe_inline_seg);
e126ba97
EC		 613 attr->cap.max_inline_data = qp->max_inline_data;
614
615 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
616 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
938fe83c 617 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
1974ab9d
BVA		 618 mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
619 attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
938fe83c
SM		 620 qp->sq.wqe_cnt,
621 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
b125a54b
EC		 622 return -ENOMEM;
623 }
e126ba97 624 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
288c01b7
EC		 625 qp->sq.max_gs = get_send_sge(attr, wqe_size);
626 if (qp->sq.max_gs < attr->cap.max_send_sge)
627 return -ENOMEM;
628
629 attr->cap.max_send_sge = qp->sq.max_gs;
b125a54b
EC		 630 qp->sq.max_post = wq_size / wqe_size;
631 attr->cap.max_send_wr = qp->sq.max_post;
e126ba97
EC		 632
633 return wq_size;
634}
635
636static int set_user_buf_size(struct mlx5_ib_dev *dev,
637 struct mlx5_ib_qp *qp,
19098df2 638 struct mlx5_ib_create_qp *ucmd,
0fb2ed66 639 struct mlx5_ib_qp_base *base,
640 struct ib_qp_init_attr *attr)
e126ba97
EC		 641{
642 int desc_sz = 1 << qp->sq.wqe_shift;
643
938fe83c 644 if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
e126ba97 645 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
938fe83c 646 desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 647 return -EINVAL;
648 }
649
af8b38ed
GP		 650 if (ucmd->sq_wqe_count && !is_power_of_2(ucmd->sq_wqe_count)) {
651 mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
652 ucmd->sq_wqe_count);
e126ba97
EC		 653 return -EINVAL;
654 }
655
656 qp->sq.wqe_cnt = ucmd->sq_wqe_count;
657
938fe83c 658 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
e126ba97 659 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
938fe83c
SM		 660 qp->sq.wqe_cnt,
661 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
e126ba97
EC		 662 return -EINVAL;
663 }
664
c2e53b2c 665 if (attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 666 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0fb2ed66 667 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
668 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
669 } else {
670 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
671 (qp->sq.wqe_cnt << 6);
672 }
e126ba97
EC		 673
674 return 0;
675}
676
677static int qp_has_rq(struct ib_qp_init_attr *attr)
678{
679 if (attr->qp_type == IB_QPT_XRC_INI ||
680 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
681 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
682 !attr->cap.max_recv_wr)
683 return 0;
684
685 return 1;
686}
687
0b80c14f
EC		 688enum {
689 /* this is the first blue flame register in the array of bfregs assigned
690 * to a processes. Since we do not use it for blue flame but rather
691 * regular 64 bit doorbells, we do not need a lock for maintaiing
692 * "odd/even" order
693 */
694 NUM_NON_BLUE_FLAME_BFREGS = 1,
695};
696
b037c29a
EC		 697static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
698{
84aa6c39
LR		 699 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
700 bfregi->num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a
EC		 701}
702
703static int num_med_bfreg(struct mlx5_ib_dev *dev,
704 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 705{
706 int n;
707
b037c29a
EC		 708 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
709 NUM_NON_BLUE_FLAME_BFREGS;
c1be5232
EC		 710
711 return n >= 0 ? n : 0;
712}
713
18b0362e
YH		 714static int first_med_bfreg(struct mlx5_ib_dev *dev,
715 struct mlx5_bfreg_info *bfregi)
716{
717 return num_med_bfreg(dev, bfregi) ? 1 : -ENOMEM;
718}
719
b037c29a
EC		 720static int first_hi_bfreg(struct mlx5_ib_dev *dev,
721 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 722{
723 int med;
c1be5232 724
b037c29a
EC		 725 med = num_med_bfreg(dev, bfregi);
726 return ++med;
c1be5232
EC		 727}
728
b037c29a
EC		 729static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
730 struct mlx5_bfreg_info *bfregi)
e126ba97 731{
e126ba97
EC		 732 int i;
733
b037c29a
EC		 734 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
735 if (!bfregi->count[i]) {
2f5ff264 736 bfregi->count[i]++;
e126ba97
EC		 737 return i;
738 }
739 }
740
741 return -ENOMEM;
742}
743
b037c29a
EC		 744static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
745 struct mlx5_bfreg_info *bfregi)
e126ba97 746{
18b0362e 747 int minidx = first_med_bfreg(dev, bfregi);
e126ba97
EC		 748 int i;
749
18b0362e
YH		 750 if (minidx < 0)
751 return minidx;
752
753 for (i = minidx; i < first_hi_bfreg(dev, bfregi); i++) {
2f5ff264 754 if (bfregi->count[i] < bfregi->count[minidx])
e126ba97 755 minidx = i;
0b80c14f
EC		 756 if (!bfregi->count[minidx])
757 break;
e126ba97
EC		 758 }
759
2f5ff264 760 bfregi->count[minidx]++;
e126ba97
EC		 761 return minidx;
762}
763
b037c29a 764static int alloc_bfreg(struct mlx5_ib_dev *dev,
ffaf58de 765 struct mlx5_bfreg_info *bfregi)
e126ba97 766{
ffaf58de 767 int bfregn = -ENOMEM;
e126ba97 768
0a2fd01c
YH		 769 if (bfregi->lib_uar_dyn)
770 return -EINVAL;
771
2f5ff264 772 mutex_lock(&bfregi->lock);
ffaf58de
LR		 773 if (bfregi->ver >= 2) {
774 bfregn = alloc_high_class_bfreg(dev, bfregi);
775 if (bfregn < 0)
776 bfregn = alloc_med_class_bfreg(dev, bfregi);
777 }
778
779 if (bfregn < 0) {
0b80c14f 780 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
2f5ff264
EC		 781 bfregn = 0;
782 bfregi->count[bfregn]++;
e126ba97 783 }
2f5ff264 784 mutex_unlock(&bfregi->lock);
e126ba97 785
2f5ff264 786 return bfregn;
e126ba97
EC		 787}
788
4ed131d0 789void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
e126ba97 790{
2f5ff264 791 mutex_lock(&bfregi->lock);
b037c29a 792 bfregi->count[bfregn]--;
2f5ff264 793 mutex_unlock(&bfregi->lock);
e126ba97
EC		 794}
795
796static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
797{
798 switch (state) {
799 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
800 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;
801 case IB_QPS_RTR: return MLX5_QP_STATE_RTR;
802 case IB_QPS_RTS: return MLX5_QP_STATE_RTS;
803 case IB_QPS_SQD: return MLX5_QP_STATE_SQD;
804 case IB_QPS_SQE: return MLX5_QP_STATE_SQER;
805 case IB_QPS_ERR: return MLX5_QP_STATE_ERR;
806 default: return -1;
807 }
808}
809
810static int to_mlx5_st(enum ib_qp_type type)
811{
812 switch (type) {
813 case IB_QPT_RC: return MLX5_QP_ST_RC;
814 case IB_QPT_UC: return MLX5_QP_ST_UC;
815 case IB_QPT_UD: return MLX5_QP_ST_UD;
816 case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR;
817 case IB_QPT_XRC_INI:
818 case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC;
819 case IB_QPT_SMI: return MLX5_QP_ST_QP0;
d16e91da 820 case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
c32a4f29 821 case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
3ae7e66a 822 case IB_QPT_RAW_PACKET: return MLX5_QP_ST_RAW_ETHERTYPE;
e126ba97
EC		 823 default: return -EINVAL;
824 }
825}
826
89ea94a7
MG		 827static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
828 struct mlx5_ib_cq *recv_cq);
829static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
830 struct mlx5_ib_cq *recv_cq);
831
7c043e90 832int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
05f58ceb 833 struct mlx5_bfreg_info *bfregi, u32 bfregn,
7c043e90 834 bool dyn_bfreg)
e126ba97 835{
05f58ceb
LR		 836 unsigned int bfregs_per_sys_page;
837 u32 index_of_sys_page;
838 u32 offset;
b037c29a 839
0a2fd01c
YH		 840 if (bfregi->lib_uar_dyn)
841 return -EINVAL;
842
b037c29a
EC		 843 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
844 MLX5_NON_FP_BFREGS_PER_UAR;
845 index_of_sys_page = bfregn / bfregs_per_sys_page;
846
1ee47ab3
YH		 847 if (dyn_bfreg) {
848 index_of_sys_page += bfregi->num_static_sys_pages;
05f58ceb
LR		 849
850 if (index_of_sys_page >= bfregi->num_sys_pages)
851 return -EINVAL;
852
1ee47ab3
YH		 853 if (bfregn > bfregi->num_dyn_bfregs ||
854 bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
855 mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
856 return -EINVAL;
857 }
858 }
b037c29a 859
1ee47ab3 860 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a 861 return bfregi->sys_pages[index_of_sys_page] + offset;
e126ba97
EC		 862}
863
fe248c3a 864static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
bdeacabd 865 struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
79b20a6c 866{
bdeacabd
SR		 867 struct mlx5_ib_ucontext *context =
868 rdma_udata_to_drv_context(
869 udata,
870 struct mlx5_ib_ucontext,
871 ibucontext);
79b20a6c 872
fe248c3a
MG		 873 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
874 atomic_dec(&dev->delay_drop.rqs_cnt);
875
79b20a6c 876 mlx5_ib_db_unmap_user(context, &rwq->db);
836a0fbb 877 ib_umem_release(rwq->umem);
79b20a6c
YH		 878}
879
880static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
b0ea0fa5 881 struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
79b20a6c
YH		 882 struct mlx5_ib_create_wq *ucmd)
883{
89944450
SR		 884 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
885 udata, struct mlx5_ib_ucontext, ibucontext);
ad480ea5 886 unsigned long page_size = 0;
79b20a6c 887 u32 offset = 0;
79b20a6c
YH		 888 int err;
889
890 if (!ucmd->buf_addr)
891 return -EINVAL;
892
c320e527 893 rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0);
79b20a6c
YH		 894 if (IS_ERR(rwq->umem)) {
895 mlx5_ib_dbg(dev, "umem_get failed\n");
896 err = PTR_ERR(rwq->umem);
897 return err;
898 }
899
ad480ea5
JG		 900 page_size = mlx5_umem_find_best_quantized_pgoff(
901 rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
902 page_offset, 64, &rwq->rq_page_offset);
903 if (!page_size) {
79b20a6c 904 mlx5_ib_warn(dev, "bad offset\n");
ad480ea5 905 err = -EINVAL;
79b20a6c
YH		 906 goto err_umem;
907 }
908
ad480ea5
JG		 909 rwq->rq_num_pas = ib_umem_num_dma_blocks(rwq->umem, page_size);
910 rwq->page_shift = order_base_2(page_size);
911 rwq->log_page_size = rwq->page_shift - MLX5_ADAPTER_PAGE_SHIFT;
79b20a6c
YH		 912 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
913
f8fb3110
JG		 914 mlx5_ib_dbg(
915 dev,
ad480ea5 916 "addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",
f8fb3110 917 (unsigned long long)ucmd->buf_addr, rwq->buf_size,
ad480ea5 918 ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas,
f8fb3110 919 offset);
79b20a6c 920
0bedd3d0 921 err = mlx5_ib_db_map_user(ucontext, ucmd->db_addr, &rwq->db);
79b20a6c
YH		 922 if (err) {
923 mlx5_ib_dbg(dev, "map failed\n");
924 goto err_umem;
925 }
926
79b20a6c
YH		 927 return 0;
928
929err_umem:
930 ib_umem_release(rwq->umem);
931 return err;
932}
933
b037c29a
EC		 934static int adjust_bfregn(struct mlx5_ib_dev *dev,
935 struct mlx5_bfreg_info *bfregi, int bfregn)
936{
937 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
938 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
939}
940
98fc1126
LR		 941static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
942 struct mlx5_ib_qp *qp, struct ib_udata *udata,
943 struct ib_qp_init_attr *attr, u32 **in,
944 struct mlx5_ib_create_qp_resp *resp, int *inlen,
945 struct mlx5_ib_qp_base *base,
946 struct mlx5_ib_create_qp *ucmd)
e126ba97
EC		 947{
948 struct mlx5_ib_ucontext *context;
19098df2 949 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
a59b7b05
JG		 950 unsigned int page_offset_quantized = 0;
951 unsigned long page_size = 0;
1ee47ab3 952 int uar_index = 0;
2f5ff264 953 int bfregn;
9e9c47d0 954 int ncont = 0;
09a7d9ec
SM		 955 __be64 *pas;
956 void *qpc;
e126ba97 957 int err;
5aa3771d 958 u16 uid;
ac42a5ee 959 u32 uar_flags;
e126ba97 960
89944450
SR		 961 context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
962 ibucontext);
76883a6c
LR		 963 uar_flags = qp->flags_en &
964 (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
ac42a5ee
YH		 965 switch (uar_flags) {
966 case MLX5_QP_FLAG_UAR_PAGE_INDEX:
76883a6c 967 uar_index = ucmd->bfreg_index;
ac42a5ee
YH		 968 bfregn = MLX5_IB_INVALID_BFREG;
969 break;
970 case MLX5_QP_FLAG_BFREG_INDEX:
1ee47ab3 971 uar_index = bfregn_to_uar_index(dev, &context->bfregi,
76883a6c 972 ucmd->bfreg_index, true);
1ee47ab3
YH		 973 if (uar_index < 0)
974 return uar_index;
1ee47ab3 975 bfregn = MLX5_IB_INVALID_BFREG;
ac42a5ee
YH		 976 break;
977 case 0:
2be08c30 978 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
ac42a5ee 979 return -EINVAL;
ffaf58de
LR		 980 bfregn = alloc_bfreg(dev, &context->bfregi);
981 if (bfregn < 0)
982 return bfregn;
ac42a5ee
YH		 983 break;
984 default:
985 return -EINVAL;
e126ba97
EC		 986 }
987
2f5ff264 988 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
1ee47ab3
YH		 989 if (bfregn != MLX5_IB_INVALID_BFREG)
990 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
991 false);
e126ba97 992
48fea837
HE		 993 qp->rq.offset = 0;
994 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
995 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
996
76883a6c 997 err = set_user_buf_size(dev, qp, ucmd, base, attr);
e126ba97 998 if (err)
2f5ff264 999 goto err_bfreg;
e126ba97 1000
76883a6c
LR		 1001 if (ucmd->buf_addr && ubuffer->buf_size) {
1002 ubuffer->buf_addr = ucmd->buf_addr;
a59b7b05
JG		 1003 ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1004 ubuffer->buf_size, 0);
1005 if (IS_ERR(ubuffer->umem)) {
1006 err = PTR_ERR(ubuffer->umem);
2f5ff264 1007 goto err_bfreg;
a59b7b05
JG		 1008 }
1009 page_size = mlx5_umem_find_best_quantized_pgoff(
1010 ubuffer->umem, qpc, log_page_size,
1011 MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
1012 &page_offset_quantized);
1013 if (!page_size) {
1014 err = -EINVAL;
1015 goto err_umem;
1016 }
1017 ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size);
9e9c47d0 1018 } else {
19098df2 1019 ubuffer->umem = NULL;
e126ba97 1020 }
e126ba97 1021
09a7d9ec
SM		 1022 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1023 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1b9a07ee 1024 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 1025 if (!*in) {
1026 err = -ENOMEM;
1027 goto err_umem;
1028 }
09a7d9ec 1029
04bcc1c2 1030 uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
5aa3771d 1031 MLX5_SET(create_qp_in, *in, uid, uid);
09a7d9ec 1032 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
a59b7b05
JG		 1033 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
1034 if (ubuffer->umem) {
1035 mlx5_ib_populate_pas(ubuffer->umem, page_size, pas, 0);
1036 MLX5_SET(qpc, qpc, log_page_size,
1037 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1038 MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);
1039 }
09a7d9ec 1040 MLX5_SET(qpc, qpc, uar_page, uar_index);
1ee47ab3
YH		 1041 if (bfregn != MLX5_IB_INVALID_BFREG)
1042 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
1043 else
1044 resp->bfreg_index = MLX5_IB_INVALID_BFREG;
2f5ff264 1045 qp->bfregn = bfregn;
e126ba97 1046
0bedd3d0 1047 err = mlx5_ib_db_map_user(context, ucmd->db_addr, &qp->db);
e126ba97
EC		 1048 if (err) {
1049 mlx5_ib_dbg(dev, "map failed\n");
1050 goto err_free;
1051 }
1052
e126ba97
EC		 1053 return 0;
1054
e126ba97 1055err_free:
479163f4 1056 kvfree(*in);
e126ba97
EC		 1057
1058err_umem:
836a0fbb 1059 ib_umem_release(ubuffer->umem);
e126ba97 1060
2f5ff264 1061err_bfreg:
1ee47ab3
YH		 1062 if (bfregn != MLX5_IB_INVALID_BFREG)
1063 mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
e126ba97
EC		 1064 return err;
1065}
1066
747c519c
LR		 1067static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1068 struct mlx5_ib_qp_base *base, struct ib_udata *udata)
e126ba97 1069{
747c519c
LR		 1070 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1071 udata, struct mlx5_ib_ucontext, ibucontext);
e126ba97 1072
747c519c
LR		 1073 if (udata) {
1074 /* User QP */
1075 mlx5_ib_db_unmap_user(context, &qp->db);
1076 ib_umem_release(base->ubuffer.umem);
1077
1078 /*
1079 * Free only the BFREGs which are handled by the kernel.
1080 * BFREGs of UARs allocated dynamically are handled by user.
1081 */
1082 if (qp->bfregn != MLX5_IB_INVALID_BFREG)
1083 mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
1084 return;
1085 }
1ee47ab3 1086
747c519c
LR		 1087 /* Kernel QP */
1088 kvfree(qp->sq.wqe_head);
1089 kvfree(qp->sq.w_list);
1090 kvfree(qp->sq.wrid);
1091 kvfree(qp->sq.wr_data);
1092 kvfree(qp->rq.wrid);
1093 if (qp->db.db)
1094 mlx5_db_free(dev->mdev, &qp->db);
1095 if (qp->buf.frags)
1096 mlx5_frag_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1097}
1098
98fc1126
LR		 1099static int _create_kernel_qp(struct mlx5_ib_dev *dev,
1100 struct ib_qp_init_attr *init_attr,
1101 struct mlx5_ib_qp *qp, u32 **in, int *inlen,
1102 struct mlx5_ib_qp_base *base)
e126ba97 1103{
e126ba97 1104 int uar_index;
09a7d9ec 1105 void *qpc;
e126ba97
EC		 1106 int err;
1107
e126ba97 1108 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
5fe9dec0 1109 qp->bf.bfreg = &dev->fp_bfreg;
2978975c 1110 else if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
11f552e2 1111 qp->bf.bfreg = &dev->wc_bfreg;
5fe9dec0
EC		 1112 else
1113 qp->bf.bfreg = &dev->bfreg;
e126ba97 1114
d8030b0d
EC		 1115 /* We need to divide by two since each register is comprised of
1116 * two buffers of identical size, namely odd and even
1117 */
1118 qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
5fe9dec0 1119 uar_index = qp->bf.bfreg->index;
e126ba97
EC		 1120
1121 err = calc_sq_size(dev, init_attr, qp);
1122 if (err < 0) {
1123 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 1124 return err;
e126ba97
EC		 1125 }
1126
1127 qp->rq.offset = 0;
1128 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
19098df2 1129 base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
e126ba97 1130
34f4c955
GL		 1131 err = mlx5_frag_buf_alloc_node(dev->mdev, base->ubuffer.buf_size,
1132 &qp->buf, dev->mdev->priv.numa_node);
e126ba97
EC		 1133 if (err) {
1134 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 1135 return err;
e126ba97
EC		 1136 }
1137
34f4c955
GL		 1138 if (qp->rq.wqe_cnt)
1139 mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift,
1140 ilog2(qp->rq.wqe_cnt), &qp->rq.fbc);
1141
1142 if (qp->sq.wqe_cnt) {
1143 int sq_strides_offset = (qp->sq.offset & (PAGE_SIZE - 1)) /
1144 MLX5_SEND_WQE_BB;
1145 mlx5_init_fbc_offset(qp->buf.frags +
1146 (qp->sq.offset / PAGE_SIZE),
1147 ilog2(MLX5_SEND_WQE_BB),
1148 ilog2(qp->sq.wqe_cnt),
1149 sq_strides_offset, &qp->sq.fbc);
1150
1151 qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
1152 }
1153
09a7d9ec
SM		 1154 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1155 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1b9a07ee 1156 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 1157 if (!*in) {
1158 err = -ENOMEM;
1159 goto err_buf;
1160 }
09a7d9ec
SM		 1161
1162 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1163 MLX5_SET(qpc, qpc, uar_page, uar_index);
8256c69b 1164 MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
09a7d9ec
SM		 1165 MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1166
e126ba97 1167 /* Set "fast registration enabled" for all kernel QPs */
09a7d9ec
SM		 1168 MLX5_SET(qpc, qpc, fre, 1);
1169 MLX5_SET(qpc, qpc, rlky, 1);
e126ba97 1170
2978975c 1171 if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
09a7d9ec 1172 MLX5_SET(qpc, qpc, deth_sqpn, 1);
b11a4f9c 1173
34f4c955
GL		 1174 mlx5_fill_page_frag_array(&qp->buf,
1175 (__be64 *)MLX5_ADDR_OF(create_qp_in,
1176 *in, pas));
e126ba97 1177
9603b61d 1178 err = mlx5_db_alloc(dev->mdev, &qp->db);
e126ba97
EC		 1179 if (err) {
1180 mlx5_ib_dbg(dev, "err %d\n", err);
1181 goto err_free;
1182 }
1183
b5883008
LD		 1184 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1185 sizeof(*qp->sq.wrid), GFP_KERNEL);
1186 qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1187 sizeof(*qp->sq.wr_data), GFP_KERNEL);
1188 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1189 sizeof(*qp->rq.wrid), GFP_KERNEL);
1190 qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1191 sizeof(*qp->sq.w_list), GFP_KERNEL);
1192 qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1193 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
e126ba97
EC		 1194
1195 if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1196 !qp->sq.w_list || !qp->sq.wqe_head) {
1197 err = -ENOMEM;
1198 goto err_wrid;
1199 }
e126ba97
EC		 1200
1201 return 0;
1202
1203err_wrid:
b5883008
LD		 1204 kvfree(qp->sq.wqe_head);
1205 kvfree(qp->sq.w_list);
1206 kvfree(qp->sq.wrid);
1207 kvfree(qp->sq.wr_data);
1208 kvfree(qp->rq.wrid);
f4044dac 1209 mlx5_db_free(dev->mdev, &qp->db);
e126ba97
EC		 1210
1211err_free:
479163f4 1212 kvfree(*in);
e126ba97
EC		 1213
1214err_buf:
34f4c955 1215 mlx5_frag_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1216 return err;
1217}
1218
09a7d9ec 1219static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
e126ba97 1220{
7aede1a2
LR		 1221 if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
1222 (qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
09a7d9ec 1223 return MLX5_SRQ_RQ;
e126ba97 1224 else if (!qp->has_rq)
09a7d9ec 1225 return MLX5_ZERO_LEN_RQ;
7aede1a2
LR		 1226
1227 return MLX5_NON_ZERO_RQ;
e126ba97
EC		 1228}
1229
0fb2ed66 1230static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
c2e53b2c 1231 struct mlx5_ib_qp *qp,
1cd6dbd3
YH		 1232 struct mlx5_ib_sq *sq, u32 tdn,
1233 struct ib_pd *pd)
0fb2ed66 1234{
e0b4b472 1235 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
0fb2ed66 1236 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1237
1cd6dbd3 1238 MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1239 MLX5_SET(tisc, tisc, transport_domain, tdn);
617f5db1
MB		 1240 if (!mlx5_ib_lag_should_assign_affinity(dev) &&
1241 mlx5_lag_is_lacp_owner(dev->mdev))
1242 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
2be08c30 1243 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
c2e53b2c
YH		 1244 MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1245
e0b4b472 1246 return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
0fb2ed66 1247}
1248
1249static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1cd6dbd3 1250 struct mlx5_ib_sq *sq, struct ib_pd *pd)
0fb2ed66 1251{
1cd6dbd3 1252 mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
0fb2ed66 1253}
1254
d5ed8ac3 1255static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
b96c9dde
MB		 1256{
1257 if (sq->flow_rule)
1258 mlx5_del_flow_rules(sq->flow_rule);
d5ed8ac3 1259 sq->flow_rule = NULL;
b96c9dde
MB		 1260}
1261
9a1ac95a 1262static bool fr_supported(int ts_cap)
2fe8d4b8 1263{
9a1ac95a
AL		 1264 return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING ||
1265 ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;
1266}
1267
1268static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
33652951 1269 bool fr_sup, bool rt_sup)
9a1ac95a 1270{
33652951
AL		 1271 if (cq->private_flags & MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) {
1272 if (!rt_sup) {
1273 mlx5_ib_dbg(dev,
1274 "Real time TS format is not supported\n");
2fe8d4b8
AL		 1275 return -EOPNOTSUPP;
1276 }
33652951 1277 return MLX5_TIMESTAMP_FORMAT_REAL_TIME;
2fe8d4b8 1278 }
9a1ac95a
AL		 1279 if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
1280 if (!fr_sup) {
1281 mlx5_ib_dbg(dev,
1282 "Free running TS format is not supported\n");
2fe8d4b8
AL		 1283 return -EOPNOTSUPP;
1284 }
9a1ac95a 1285 return MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
2fe8d4b8 1286 }
9a1ac95a
AL		 1287 return fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1288 MLX5_TIMESTAMP_FORMAT_DEFAULT;
1289}
1290
1291static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)
1292{
1293 u8 ts_cap = MLX5_CAP_GEN(dev->mdev, rq_ts_format);
1294
33652951
AL		 1295 return get_ts_format(dev, recv_cq, fr_supported(ts_cap),
1296 rt_supported(ts_cap));
2fe8d4b8
AL		 1297}
1298
1299static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)
1300{
9a1ac95a
AL		 1301 u8 ts_cap = MLX5_CAP_GEN(dev->mdev, sq_ts_format);
1302
33652951
AL		 1303 return get_ts_format(dev, send_cq, fr_supported(ts_cap),
1304 rt_supported(ts_cap));
2fe8d4b8
AL		 1305}
1306
1307static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,
1308 struct mlx5_ib_cq *recv_cq)
1309{
9a1ac95a
AL		 1310 u8 ts_cap = MLX5_CAP_ROCE(dev->mdev, qp_ts_format);
1311 bool fr_sup = fr_supported(ts_cap);
33652951 1312 bool rt_sup = rt_supported(ts_cap);
9a1ac95a
AL		 1313 u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1314 MLX5_TIMESTAMP_FORMAT_DEFAULT;
1315 int send_ts_format =
33652951 1316 send_cq ? get_ts_format(dev, send_cq, fr_sup, rt_sup) :
9a1ac95a
AL		 1317 default_ts;
1318 int recv_ts_format =
33652951 1319 recv_cq ? get_ts_format(dev, recv_cq, fr_sup, rt_sup) :
9a1ac95a
AL		 1320 default_ts;
1321
1322 if (send_ts_format < 0 || recv_ts_format < 0)
2fe8d4b8 1323 return -EOPNOTSUPP;
9a1ac95a 1324
33652951
AL		 1325 if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1326 recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1327 send_ts_format != recv_ts_format) {
1328 mlx5_ib_dbg(
1329 dev,
1330 "The send ts_format does not match the receive ts_format\n");
2fe8d4b8
AL		 1331 return -EOPNOTSUPP;
1332 }
33652951 1333
9a1ac95a 1334 return send_ts_format == default_ts ? recv_ts_format : send_ts_format;
2fe8d4b8
AL		 1335}
1336
0fb2ed66 1337static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
b0ea0fa5 1338 struct ib_udata *udata,
0fb2ed66 1339 struct mlx5_ib_sq *sq, void *qpin,
2fe8d4b8 1340 struct ib_pd *pd, struct mlx5_ib_cq *cq)
0fb2ed66 1341{
1342 struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1343 __be64 *pas;
1344 void *in;
1345 void *sqc;
1346 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1347 void *wq;
1348 int inlen;
1349 int err;
ad480ea5
JG		 1350 unsigned int page_offset_quantized;
1351 unsigned long page_size;
2fe8d4b8
AL		 1352 int ts_format;
1353
1354 ts_format = get_sq_ts_format(dev, cq);
1355 if (ts_format < 0)
1356 return ts_format;
ad480ea5
JG		 1357
1358 sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1359 ubuffer->buf_size, 0);
1360 if (IS_ERR(sq->ubuffer.umem))
1361 return PTR_ERR(sq->ubuffer.umem);
1362 page_size = mlx5_umem_find_best_quantized_pgoff(
1363 ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
1364 page_offset, 64, &page_offset_quantized);
1365 if (!page_size) {
1366 err = -EINVAL;
1367 goto err_umem;
1368 }
0fb2ed66 1369
7db0eea9 1370 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
ad480ea5
JG		 1371 sizeof(u64) *
1372 ib_umem_num_dma_blocks(sq->ubuffer.umem, page_size);
1b9a07ee 1373 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1374 if (!in) {
1375 err = -ENOMEM;
1376 goto err_umem;
1377 }
1378
c14003f0 1379 MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1380 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1381 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
795b609c
BW		 1382 if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1383 MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
0fb2ed66 1384 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
2fe8d4b8 1385 MLX5_SET(sqc, sqc, ts_format, ts_format);
0fb2ed66 1386 MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1387 MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1388 MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1389 MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
96dc3fc5
NO		 1390 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1391 MLX5_CAP_ETH(dev->mdev, swp))
1392 MLX5_SET(sqc, sqc, allow_swp, 1);
0fb2ed66 1393
1394 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1395 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1396 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1397 MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1398 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1399 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1400 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
ad480ea5
JG		 1401 MLX5_SET(wq, wq, log_wq_pg_sz,
1402 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1403 MLX5_SET(wq, wq, page_offset, page_offset_quantized);
0fb2ed66 1404
1405 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
ad480ea5 1406 mlx5_ib_populate_pas(sq->ubuffer.umem, page_size, pas, 0);
0fb2ed66 1407
333fbaa0 1408 err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
0fb2ed66 1409
1410 kvfree(in);
1411
1412 if (err)
1413 goto err_umem;
1414
1415 return 0;
1416
1417err_umem:
1418 ib_umem_release(sq->ubuffer.umem);
1419 sq->ubuffer.umem = NULL;
1420
1421 return err;
1422}
1423
1424static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1425 struct mlx5_ib_sq *sq)
1426{
d5ed8ac3 1427 destroy_flow_rule_vport_sq(sq);
333fbaa0 1428 mlx5_core_destroy_sq_tracked(dev, &sq->base.mqp);
0fb2ed66 1429 ib_umem_release(sq->ubuffer.umem);
1430}
1431
0fb2ed66 1432static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2c292dbb 1433 struct mlx5_ib_rq *rq, void *qpin,
2fe8d4b8 1434 struct ib_pd *pd, struct mlx5_ib_cq *cq)
0fb2ed66 1435{
358e42ea 1436 struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
0fb2ed66 1437 __be64 *pas;
0fb2ed66 1438 void *in;
1439 void *rqc;
1440 void *wq;
1441 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
7579dcdf
JG		 1442 struct ib_umem *umem = rq->base.ubuffer.umem;
1443 unsigned int page_offset_quantized;
1444 unsigned long page_size = 0;
2fe8d4b8 1445 int ts_format;
2c292dbb 1446 size_t inlen;
0fb2ed66 1447 int err;
2c292dbb 1448
2fe8d4b8
AL		 1449 ts_format = get_rq_ts_format(dev, cq);
1450 if (ts_format < 0)
1451 return ts_format;
1452
7579dcdf
JG		 1453 page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,
1454 MLX5_ADAPTER_PAGE_SHIFT,
1455 page_offset, 64,
1456 &page_offset_quantized);
1457 if (!page_size)
2c292dbb 1458 return -EINVAL;
0fb2ed66 1459
7579dcdf
JG		 1460 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1461 sizeof(u64) * ib_umem_num_dma_blocks(umem, page_size);
1b9a07ee 1462 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1463 if (!in)
1464 return -ENOMEM;
1465
34d57585 1466 MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1467 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
e4cc4fa7
NO		 1468 if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1469 MLX5_SET(rqc, rqc, vsd, 1);
0fb2ed66 1470 MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1471 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
2fe8d4b8 1472 MLX5_SET(rqc, rqc, ts_format, ts_format);
0fb2ed66 1473 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1474 MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1475 MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1476
2be08c30 1477 if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
358e42ea
MD		 1478 MLX5_SET(rqc, rqc, scatter_fcs, 1);
1479
0fb2ed66 1480 wq = MLX5_ADDR_OF(rqc, rqc, wq);
1481 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 1482 if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1483 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
7579dcdf 1484 MLX5_SET(wq, wq, page_offset, page_offset_quantized);
0fb2ed66 1485 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1486 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1487 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
7579dcdf
JG		 1488 MLX5_SET(wq, wq, log_wq_pg_sz,
1489 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
0fb2ed66 1490 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1491
1492 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
7579dcdf 1493 mlx5_ib_populate_pas(umem, page_size, pas, 0);
0fb2ed66 1494
333fbaa0 1495 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
0fb2ed66 1496
1497 kvfree(in);
1498
1499 return err;
1500}
1501
1502static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1503 struct mlx5_ib_rq *rq)
1504{
333fbaa0 1505 mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
0fb2ed66 1506}
1507
0042f9e4
MB		 1508static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1509 struct mlx5_ib_rq *rq,
443c1cf9
YH		 1510 u32 qp_flags_en,
1511 struct ib_pd *pd)
0042f9e4
MB		 1512{
1513 if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1514 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1515 mlx5_ib_disable_lb(dev, false, true);
443c1cf9 1516 mlx5_cmd_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
0042f9e4
MB		 1517}
1518
0fb2ed66 1519static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
f95ef6cb 1520 struct mlx5_ib_rq *rq, u32 tdn,
e0b4b472
LR		 1521 u32 *qp_flags_en, struct ib_pd *pd,
1522 u32 *out)
0fb2ed66 1523{
175edba8 1524 u8 lb_flag = 0;
0fb2ed66 1525 u32 *in;
1526 void *tirc;
1527 int inlen;
1528 int err;
1529
1530 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1531 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1532 if (!in)
1533 return -ENOMEM;
1534
443c1cf9 1535 MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
0fb2ed66 1536 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1537 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1538 MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1539 MLX5_SET(tirc, tirc, transport_domain, tdn);
175edba8 1540 if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
f95ef6cb 1541 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
0fb2ed66 1542
175edba8
MB		 1543 if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1544 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1545
1546 if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1547 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1548
6a4d00be 1549 if (dev->is_rep) {
175edba8
MB		 1550 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1551 *qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1552 }
1553
1554 MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
e0b4b472
LR		 1555 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1556 err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1f1d6abb 1557 rq->tirn = MLX5_GET(create_tir_out, out, tirn);
0042f9e4
MB		 1558 if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1559 err = mlx5_ib_enable_lb(dev, false, true);
1560
1561 if (err)
443c1cf9 1562 destroy_raw_packet_qp_tir(dev, rq, 0, pd);
0042f9e4 1563 }
0fb2ed66 1564 kvfree(in);
1565
1566 return err;
1567}
1568
0fb2ed66 1569static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2fe8d4b8 1570 u32 *in, size_t inlen, struct ib_pd *pd,
7f72052c 1571 struct ib_udata *udata,
2fe8d4b8
AL		 1572 struct mlx5_ib_create_qp_resp *resp,
1573 struct ib_qp_init_attr *init_attr)
0fb2ed66 1574{
1575 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1576 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1577 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
89944450
SR		 1578 struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1579 udata, struct mlx5_ib_ucontext, ibucontext);
0fb2ed66 1580 int err;
1581 u32 tdn = mucontext->tdn;
7f72052c 1582 u16 uid = to_mpd(pd)->uid;
1f1d6abb 1583 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
0fb2ed66 1584
0eacc574
AL		 1585 if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
1586 return -EINVAL;
0fb2ed66 1587 if (qp->sq.wqe_cnt) {
1cd6dbd3 1588 err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
0fb2ed66 1589 if (err)
1590 return err;
1591
2fe8d4b8
AL		 1592 err = create_raw_packet_qp_sq(dev, udata, sq, in, pd,
1593 to_mcq(init_attr->send_cq));
0fb2ed66 1594 if (err)
1595 goto err_destroy_tis;
1596
7f72052c
YH		 1597 if (uid) {
1598 resp->tisn = sq->tisn;
1599 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
1600 resp->sqn = sq->base.mqp.qpn;
1601 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
1602 }
1603
0fb2ed66 1604 sq->base.container_mibqp = qp;
1d31e9c0 1605 sq->base.mqp.event = mlx5_ib_qp_event;
0fb2ed66 1606 }
1607
1608 if (qp->rq.wqe_cnt) {
358e42ea
MD		 1609 rq->base.container_mibqp = qp;
1610
2be08c30 1611 if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
e4cc4fa7 1612 rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
2be08c30 1613 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
b1383aa6 1614 rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
2fe8d4b8
AL		 1615 err = create_raw_packet_qp_rq(dev, rq, in, pd,
1616 to_mcq(init_attr->recv_cq));
0fb2ed66 1617 if (err)
1618 goto err_destroy_sq;
1619
e0b4b472
LR		 1620 err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd,
1621 out);
0fb2ed66 1622 if (err)
1623 goto err_destroy_rq;
7f72052c
YH		 1624
1625 if (uid) {
1626 resp->rqn = rq->base.mqp.qpn;
1627 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
1628 resp->tirn = rq->tirn;
1629 resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
54a38b66
AV		 1630 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1631 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1f1d6abb
AL		 1632 resp->tir_icm_addr = MLX5_GET(
1633 create_tir_out, out, icm_address_31_0);
1634 resp->tir_icm_addr |=
1635 (u64)MLX5_GET(create_tir_out, out,
1636 icm_address_39_32)
1637 << 32;
1638 resp->tir_icm_addr |=
1639 (u64)MLX5_GET(create_tir_out, out,
1640 icm_address_63_40)
1641 << 40;
1642 resp->comp_mask |=
1643 MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1644 }
7f72052c 1645 }
0fb2ed66 1646 }
1647
1648 qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1649 rq->base.mqp.qpn;
0fb2ed66 1650 return 0;
1651
1652err_destroy_rq:
1653 destroy_raw_packet_qp_rq(dev, rq);
1654err_destroy_sq:
1655 if (!qp->sq.wqe_cnt)
1656 return err;
1657 destroy_raw_packet_qp_sq(dev, sq);
1658err_destroy_tis:
1cd6dbd3 1659 destroy_raw_packet_qp_tis(dev, sq, pd);
0fb2ed66 1660
1661 return err;
1662}
1663
1664static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1665 struct mlx5_ib_qp *qp)
1666{
1667 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1668 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1669 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1670
1671 if (qp->rq.wqe_cnt) {
443c1cf9 1672 destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd);
0fb2ed66 1673 destroy_raw_packet_qp_rq(dev, rq);
1674 }
1675
1676 if (qp->sq.wqe_cnt) {
1677 destroy_raw_packet_qp_sq(dev, sq);
1cd6dbd3 1678 destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
0fb2ed66 1679 }
1680}
1681
1682static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1683 struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1684{
1685 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1686 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1687
1688 sq->sq = &qp->sq;
1689 rq->rq = &qp->rq;
1690 sq->doorbell = &qp->db;
1691 rq->doorbell = &qp->db;
1692}
1693
28d61370
YH		 1694static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1695{
0042f9e4
MB		 1696 if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1697 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1698 mlx5_ib_disable_lb(dev, false, true);
443c1cf9
YH		 1699 mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1700 to_mpd(qp->ibqp.pd)->uid);
28d61370
YH		 1701}
1702
f78d358c
LR		 1703struct mlx5_create_qp_params {
1704 struct ib_udata *udata;
1705 size_t inlen;
6f2cf76e 1706 size_t outlen;
e383085c 1707 size_t ucmd_size;
f78d358c
LR		 1708 void *ucmd;
1709 u8 is_rss_raw : 1;
1710 struct ib_qp_init_attr *attr;
1711 u32 uidx;
08d53976 1712 struct mlx5_ib_create_qp_resp resp;
f78d358c
LR		 1713};
1714
1715static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1716 struct mlx5_ib_qp *qp,
1717 struct mlx5_create_qp_params *params)
28d61370 1718{
f78d358c
LR		 1719 struct ib_qp_init_attr *init_attr = params->attr;
1720 struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
1721 struct ib_udata *udata = params->udata;
89944450
SR		 1722 struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1723 udata, struct mlx5_ib_ucontext, ibucontext);
28d61370 1724 int inlen;
1f1d6abb 1725 int outlen;
28d61370
YH		 1726 int err;
1727 u32 *in;
1f1d6abb 1728 u32 *out;
28d61370
YH		 1729 void *tirc;
1730 void *hfso;
1731 u32 selected_fields = 0;
2d93fc85 1732 u32 outer_l4;
28d61370 1733 u32 tdn = mucontext->tdn;
175edba8 1734 u8 lb_flag = 0;
28d61370 1735
5ce0592b 1736 if (ucmd->comp_mask) {
28d61370
YH		 1737 mlx5_ib_dbg(dev, "invalid comp mask\n");
1738 return -EOPNOTSUPP;
1739 }
1740
5ce0592b
LR		 1741 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1742 !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
309fa347
MG		 1743 mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1744 return -EOPNOTSUPP;
1745 }
1746
37518fa4 1747 if (dev->is_rep)
175edba8 1748 qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
175edba8 1749
37518fa4
LR		 1750 if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1751 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1752
1753 if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
175edba8 1754 lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
175edba8 1755
28d61370 1756 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1f1d6abb
AL		 1757 outlen = MLX5_ST_SZ_BYTES(create_tir_out);
1758 in = kvzalloc(inlen + outlen, GFP_KERNEL);
28d61370
YH		 1759 if (!in)
1760 return -ENOMEM;
1761
1f1d6abb 1762 out = in + MLX5_ST_SZ_DW(create_tir_in);
443c1cf9 1763 MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
28d61370
YH		 1764 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1765 MLX5_SET(tirc, tirc, disp_type,
1766 MLX5_TIRC_DISP_TYPE_INDIRECT);
1767 MLX5_SET(tirc, tirc, indirect_table,
1768 init_attr->rwq_ind_tbl->ind_tbl_num);
1769 MLX5_SET(tirc, tirc, transport_domain, tdn);
1770
1771 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
f95ef6cb 1772
5ce0592b 1773 if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
f95ef6cb
MG		 1774 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1775
175edba8
MB		 1776 MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1777
5ce0592b 1778 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
309fa347
MG		 1779 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1780 else
1781 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1782
5ce0592b 1783 switch (ucmd->rx_hash_function) {
28d61370
YH		 1784 case MLX5_RX_HASH_FUNC_TOEPLITZ:
1785 {
1786 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1787 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1788
5ce0592b 1789 if (len != ucmd->rx_key_len) {
28d61370
YH		 1790 err = -EINVAL;
1791 goto err;
1792 }
1793
1794 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
5ce0592b 1795 memcpy(rss_key, ucmd->rx_hash_key, len);
28d61370
YH		 1796 break;
1797 }
1798 default:
1799 err = -EOPNOTSUPP;
1800 goto err;
1801 }
1802
5ce0592b 1803 if (!ucmd->rx_hash_fields_mask) {
28d61370
YH		 1804 /* special case when this TIR serves as steering entry without hashing */
1805 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1806 goto create_tir;
1807 err = -EINVAL;
1808 goto err;
1809 }
1810
5ce0592b
LR		 1811 if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1812 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1813 ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1814 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
28d61370
YH		 1815 err = -EINVAL;
1816 goto err;
1817 }
1818
1819 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
5ce0592b
LR		 1820 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1821 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
28d61370
YH		 1822 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1823 MLX5_L3_PROT_TYPE_IPV4);
5ce0592b
LR		 1824 else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1825 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
28d61370
YH		 1826 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1827 MLX5_L3_PROT_TYPE_IPV6);
1828
5ce0592b
LR		 1829 outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1830 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1831 << 0 |
1832 ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1833 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1834 << 1 |
1835 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
2d93fc85
MB		 1836
1837 /* Check that only one l4 protocol is set */
1838 if (outer_l4 & (outer_l4 - 1)) {
28d61370
YH		 1839 err = -EINVAL;
1840 goto err;
1841 }
1842
1843 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
5ce0592b
LR		 1844 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1845 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
28d61370
YH		 1846 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1847 MLX5_L4_PROT_TYPE_TCP);
5ce0592b
LR		 1848 else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1849 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
28d61370
YH		 1850 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1851 MLX5_L4_PROT_TYPE_UDP);
1852
5ce0592b
LR		 1853 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1854 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
28d61370
YH		 1855 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1856
5ce0592b
LR		 1857 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1858 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
28d61370
YH		 1859 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1860
5ce0592b
LR		 1861 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1862 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
28d61370
YH		 1863 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1864
5ce0592b
LR		 1865 if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1866 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
28d61370
YH		 1867 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1868
5ce0592b 1869 if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
2d93fc85
MB		 1870 selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1871
28d61370
YH		 1872 MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1873
1874create_tir:
e0b4b472
LR		 1875 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1876 err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
28d61370 1877
1f1d6abb 1878 qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
0042f9e4
MB		 1879 if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1880 err = mlx5_ib_enable_lb(dev, false, true);
1881
1882 if (err)
443c1cf9
YH		 1883 mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1884 to_mpd(pd)->uid);
0042f9e4
MB		 1885 }
1886
28d61370
YH		 1887 if (err)
1888 goto err;
1889
7f72052c 1890 if (mucontext->devx_uid) {
08d53976
LR		 1891 params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1892 params->resp.tirn = qp->rss_qp.tirn;
54a38b66
AV		 1893 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1894 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
08d53976 1895 params->resp.tir_icm_addr =
1f1d6abb 1896 MLX5_GET(create_tir_out, out, icm_address_31_0);
08d53976
LR		 1897 params->resp.tir_icm_addr |=
1898 (u64)MLX5_GET(create_tir_out, out,
1899 icm_address_39_32)
1900 << 32;
1901 params->resp.tir_icm_addr |=
1902 (u64)MLX5_GET(create_tir_out, out,
1903 icm_address_63_40)
1904 << 40;
1905 params->resp.comp_mask |=
1f1d6abb
AL		 1906 MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1907 }
7f72052c
YH		 1908 }
1909
28d61370
YH		 1910 kvfree(in);
1911 /* qpn is reserved for that QP */
1912 qp->trans_qp.base.mqp.qpn = 0;
2be08c30 1913 qp->is_rss = true;
28d61370
YH		 1914 return 0;
1915
1916err:
1917 kvfree(in);
1918 return err;
1919}
1920
5d6ff1ba 1921static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
81530ab0 1922 struct mlx5_ib_qp *qp,
5d6ff1ba
YC		 1923 struct ib_qp_init_attr *init_attr,
1924 void *qpc)
1925{
5d6ff1ba 1926 int scqe_sz;
2ab367a7 1927 bool allow_scat_cqe = false;
5d6ff1ba 1928
81530ab0 1929 allow_scat_cqe = qp->flags_en & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
6f4bc0ea
YC		 1930
1931 if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
5d6ff1ba
YC		 1932 return;
1933
1934 scqe_sz = mlx5_ib_get_cqe_size(init_attr->send_cq);
1935 if (scqe_sz == 128) {
1936 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1937 return;
1938 }
1939
1940 if (init_attr->qp_type != MLX5_IB_QPT_DCI ||
1941 MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
1942 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1943}
1944
a60109dc
YC		 1945static int atomic_size_to_mode(int size_mask)
1946{
1947 /* driver does not support atomic_size > 256B
1948 * and does not know how to translate bigger sizes
1949 */
1950 int supported_size_mask = size_mask & 0x1ff;
1951 int log_max_size;
1952
1953 if (!supported_size_mask)
1954 return -EOPNOTSUPP;
1955
1956 log_max_size = __fls(supported_size_mask);
1957
1958 if (log_max_size > 3)
1959 return log_max_size;
1960
1961 return MLX5_ATOMIC_MODE_8B;
1962}
1963
1964static int get_atomic_mode(struct mlx5_ib_dev *dev,
1965 enum ib_qp_type qp_type)
1966{
1967 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
1968 u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
1969 int atomic_mode = -EOPNOTSUPP;
1970 int atomic_size_mask;
1971
1972 if (!atomic)
1973 return -EOPNOTSUPP;
1974
1975 if (qp_type == MLX5_IB_QPT_DCT)
1976 atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
1977 else
1978 atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
1979
1980 if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) ||
1981 (atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
1982 atomic_mode = atomic_size_to_mode(atomic_size_mask);
1983
1984 if (atomic_mode <= 0 &&
1985 (atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
1986 atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
1987 atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
1988
1989 return atomic_mode;
1990}
1991
f78d358c
LR		 1992static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1993 struct mlx5_create_qp_params *params)
04bcc1c2 1994{
f78d358c 1995 struct ib_qp_init_attr *attr = params->attr;
f78d358c 1996 u32 uidx = params->uidx;
04bcc1c2 1997 struct mlx5_ib_resources *devr = &dev->devr;
3e09a427 1998 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
04bcc1c2
LR		 1999 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2000 struct mlx5_core_dev *mdev = dev->mdev;
2001 struct mlx5_ib_qp_base *base;
2002 unsigned long flags;
2003 void *qpc;
2004 u32 *in;
2005 int err;
2006
04bcc1c2
LR		 2007 if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2008 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2009
2010 in = kvzalloc(inlen, GFP_KERNEL);
2011 if (!in)
2012 return -ENOMEM;
2013
2014 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2015
2016 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
2017 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2018 MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
2019
2020 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2021 MLX5_SET(qpc, qpc, block_lb_mc, 1);
2022 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2023 MLX5_SET(qpc, qpc, cd_master, 1);
2024 if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2025 MLX5_SET(qpc, qpc, cd_slave_send, 1);
2026 if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2027 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2028
8256c69b 2029 MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
04bcc1c2
LR		 2030 MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
2031 MLX5_SET(qpc, qpc, no_sq, 1);
2032 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2033 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
2034 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2035 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
2036 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2037
2038 /* 0xffffff means we ask to work with cqe version 0 */
2039 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2040 MLX5_SET(qpc, qpc, user_index, uidx);
2041
2042 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2043 MLX5_SET(qpc, qpc, end_padding_mode,
2044 MLX5_WQ_END_PAD_MODE_ALIGN);
2045 /* Special case to clean flag */
2046 qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2047 }
2048
2049 base = &qp->trans_qp.base;
3e09a427 2050 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
04bcc1c2 2051 kvfree(in);
6367da46 2052 if (err)
04bcc1c2 2053 return err;
04bcc1c2
LR		 2054
2055 base->container_mibqp = qp;
2056 base->mqp.event = mlx5_ib_qp_event;
92cd667c
LR		 2057 if (MLX5_CAP_GEN(mdev, ece_support))
2058 params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
04bcc1c2
LR		 2059
2060 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2061 list_add_tail(&qp->qps_list, &dev->qp_list);
2062 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2063
968f0b6f 2064 qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
04bcc1c2
LR		 2065 return 0;
2066}
2067
2013b4d5
LN		 2068static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2069 struct mlx5_ib_qp *qp,
2070 struct mlx5_create_qp_params *params)
2071{
2072 struct ib_qp_init_attr *init_attr = params->attr;
2073 struct mlx5_ib_create_qp *ucmd = params->ucmd;
2074 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2075 struct ib_udata *udata = params->udata;
2076 u32 uidx = params->uidx;
2077 struct mlx5_ib_resources *devr = &dev->devr;
2078 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2079 struct mlx5_core_dev *mdev = dev->mdev;
2080 struct mlx5_ib_cq *send_cq;
2081 struct mlx5_ib_cq *recv_cq;
2082 unsigned long flags;
2083 struct mlx5_ib_qp_base *base;
2084 int ts_format;
2085 int mlx5_st;
2086 void *qpc;
2087 u32 *in;
2088 int err;
2089
2090 spin_lock_init(&qp->sq.lock);
2091 spin_lock_init(&qp->rq.lock);
2092
2093 mlx5_st = to_mlx5_st(qp->type);
2094 if (mlx5_st < 0)
2095 return -EINVAL;
2096
2097 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2098 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2099
2100 base = &qp->trans_qp.base;
2101
2102 qp->has_rq = qp_has_rq(init_attr);
2103 err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
2104 if (err) {
2105 mlx5_ib_dbg(dev, "err %d\n", err);
2106 return err;
2107 }
2108
2109 if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2110 ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2111 return -EINVAL;
2112
2113 if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2114 return -EINVAL;
2115
2116 ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2117 to_mcq(init_attr->recv_cq));
2118
2119 if (ts_format < 0)
2120 return ts_format;
2121
2122 err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2123 &inlen, base, ucmd);
2124 if (err)
2125 return err;
2126
2127 if (MLX5_CAP_GEN(mdev, ece_support))
2128 MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2129 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2130
2131 MLX5_SET(qpc, qpc, st, mlx5_st);
2132 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2133 MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2134
2135 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2136 MLX5_SET(qpc, qpc, wq_signature, 1);
2137
2138 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2139 MLX5_SET(qpc, qpc, cd_master, 1);
2140 if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2141 MLX5_SET(qpc, qpc, cd_slave_send, 1);
2142 if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE)
2143 configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2144
2145 if (qp->rq.wqe_cnt) {
2146 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2147 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2148 }
2149
11656f59
LN		 2150 if (qp->flags_en & MLX5_QP_FLAG_DCI_STREAM) {
2151 MLX5_SET(qpc, qpc, log_num_dci_stream_channels,
2152 ucmd->dci_streams.log_num_concurent);
2153 MLX5_SET(qpc, qpc, log_num_dci_errored_streams,
2154 ucmd->dci_streams.log_num_errored);
2155 }
2156
2013b4d5
LN		 2157 MLX5_SET(qpc, qpc, ts_format, ts_format);
2158 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2159
2160 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2161
2162 /* Set default resources */
2163 if (init_attr->srq) {
2164 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2165 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2166 to_msrq(init_attr->srq)->msrq.srqn);
2167 } else {
2168 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2169 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2170 to_msrq(devr->s1)->msrq.srqn);
2171 }
2172
2173 if (init_attr->send_cq)
2174 MLX5_SET(qpc, qpc, cqn_snd,
2175 to_mcq(init_attr->send_cq)->mcq.cqn);
2176
2177 if (init_attr->recv_cq)
2178 MLX5_SET(qpc, qpc, cqn_rcv,
2179 to_mcq(init_attr->recv_cq)->mcq.cqn);
2180
2181 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2182
2183 /* 0xffffff means we ask to work with cqe version 0 */
2184 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2185 MLX5_SET(qpc, qpc, user_index, uidx);
2186
2187 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2188 MLX5_SET(qpc, qpc, end_padding_mode,
2189 MLX5_WQ_END_PAD_MODE_ALIGN);
2190 /* Special case to clean flag */
2191 qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2192 }
2193
2194 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2195
2196 kvfree(in);
2197 if (err)
2198 goto err_create;
2199
2200 base->container_mibqp = qp;
2201 base->mqp.event = mlx5_ib_qp_event;
2202 if (MLX5_CAP_GEN(mdev, ece_support))
2203 params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2204
2205 get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
2206 &send_cq, &recv_cq);
2207 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2208 mlx5_ib_lock_cqs(send_cq, recv_cq);
2209 /* Maintain device to QPs access, needed for further handling via reset
2210 * flow
2211 */
2212 list_add_tail(&qp->qps_list, &dev->qp_list);
2213 /* Maintain CQ to QPs access, needed for further handling via reset flow
2214 */
2215 if (send_cq)
2216 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2217 if (recv_cq)
2218 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2219 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2220 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2221
2222 return 0;
2223
2224err_create:
2225 destroy_qp(dev, qp, base, udata);
2226 return err;
2227}
2228
98fc1126 2229static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
f78d358c
LR		 2230 struct mlx5_ib_qp *qp,
2231 struct mlx5_create_qp_params *params)
e126ba97 2232{
f78d358c
LR		 2233 struct ib_qp_init_attr *init_attr = params->attr;
2234 struct mlx5_ib_create_qp *ucmd = params->ucmd;
3e09a427 2235 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
f78d358c
LR		 2236 struct ib_udata *udata = params->udata;
2237 u32 uidx = params->uidx;
e126ba97 2238 struct mlx5_ib_resources *devr = &dev->devr;
09a7d9ec 2239 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
938fe83c 2240 struct mlx5_core_dev *mdev = dev->mdev;
89ea94a7
MG		 2241 struct mlx5_ib_cq *send_cq;
2242 struct mlx5_ib_cq *recv_cq;
2243 unsigned long flags;
09a7d9ec 2244 struct mlx5_ib_qp_base *base;
2fe8d4b8 2245 int ts_format;
e7b169f3 2246 int mlx5_st;
cfb5e088 2247 void *qpc;
09a7d9ec
SM		 2248 u32 *in;
2249 int err;
e126ba97 2250
e126ba97
EC		 2251 spin_lock_init(&qp->sq.lock);
2252 spin_lock_init(&qp->rq.lock);
2253
7aede1a2 2254 mlx5_st = to_mlx5_st(qp->type);
e7b169f3
NO		 2255 if (mlx5_st < 0)
2256 return -EINVAL;
2257
e126ba97
EC		 2258 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2259 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2260
2978975c
LR		 2261 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
2262 qp->underlay_qpn = init_attr->source_qpn;
2263
c2e53b2c 2264 base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 2265 qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
c2e53b2c
YH		 2266 &qp->raw_packet_qp.rq.base :
2267 &qp->trans_qp.base;
2268
e126ba97 2269 qp->has_rq = qp_has_rq(init_attr);
2dfac92d 2270 err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
e126ba97
EC		 2271 if (err) {
2272 mlx5_ib_dbg(dev, "err %d\n", err);
2273 return err;
2274 }
2275
98fc1126
LR		 2276 if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2277 ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2278 return -EINVAL;
04bcc1c2 2279
98fc1126
LR		 2280 if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2281 return -EINVAL;
e126ba97 2282
2fe8d4b8
AL		 2283 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
2284 ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2285 to_mcq(init_attr->recv_cq));
2286 if (ts_format < 0)
2287 return ts_format;
2288 }
2289
08d53976
LR		 2290 err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2291 &inlen, base, ucmd);
04bcc1c2
LR		 2292 if (err)
2293 return err;
e126ba97
EC		 2294
2295 if (is_sqp(init_attr->qp_type))
2296 qp->port = init_attr->port_num;
2297
e383085c
LR		 2298 if (MLX5_CAP_GEN(mdev, ece_support))
2299 MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
09a7d9ec
SM		 2300 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2301
e7b169f3 2302 MLX5_SET(qpc, qpc, st, mlx5_st);
09a7d9ec 2303 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
98fc1126 2304 MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
e126ba97 2305
c95e6d53 2306 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
09a7d9ec 2307 MLX5_SET(qpc, qpc, wq_signature, 1);
e126ba97 2308
2be08c30 2309 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
09a7d9ec 2310 MLX5_SET(qpc, qpc, block_lb_mc, 1);
f360d88a 2311
2be08c30 2312 if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
09a7d9ec 2313 MLX5_SET(qpc, qpc, cd_master, 1);
2be08c30 2314 if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
09a7d9ec 2315 MLX5_SET(qpc, qpc, cd_slave_send, 1);
2be08c30 2316 if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
09a7d9ec 2317 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2be08c30 2318 if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
569c6651 2319 MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
90ecb37a
LR		 2320 if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2321 (init_attr->qp_type == IB_QPT_RC ||
2322 init_attr->qp_type == IB_QPT_UC)) {
52c81f47 2323 int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
8bde2c50
LR		 2324
2325 MLX5_SET(qpc, qpc, cs_res,
2326 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
2327 MLX5_RES_SCAT_DATA32_CQE);
2328 }
90ecb37a 2329 if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
7aede1a2 2330 (qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
81530ab0 2331 configure_requester_scat_cqe(dev, qp, init_attr, qpc);
e126ba97
EC		 2332
2333 if (qp->rq.wqe_cnt) {
09a7d9ec
SM		 2334 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2335 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
e126ba97
EC		 2336 }
2337
2fe8d4b8
AL		 2338 if (init_attr->qp_type != IB_QPT_RAW_PACKET)
2339 MLX5_SET(qpc, qpc, ts_format, ts_format);
2340
09a7d9ec 2341 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
e126ba97 2342
3fd3307e 2343 if (qp->sq.wqe_cnt) {
09a7d9ec 2344 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
3fd3307e 2345 } else {
09a7d9ec 2346 MLX5_SET(qpc, qpc, no_sq, 1);
3fd3307e
AK		 2347 if (init_attr->srq &&
2348 init_attr->srq->srq_type == IB_SRQT_TM)
2349 MLX5_SET(qpc, qpc, offload_type,
2350 MLX5_QPC_OFFLOAD_TYPE_RNDV);
2351 }
e126ba97
EC		 2352
2353 /* Set default resources */
2354 switch (init_attr->qp_type) {
e126ba97 2355 case IB_QPT_XRC_INI:
09a7d9ec 2356 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
f4375443 2357 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
09a7d9ec 2358 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
e126ba97
EC		 2359 break;
2360 default:
2361 if (init_attr->srq) {
f4375443 2362 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
09a7d9ec 2363 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
e126ba97 2364 } else {
f4375443 2365 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
09a7d9ec 2366 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
e126ba97
EC		 2367 }
2368 }
2369
2370 if (init_attr->send_cq)
09a7d9ec 2371 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
e126ba97
EC		 2372
2373 if (init_attr->recv_cq)
09a7d9ec 2374 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
e126ba97 2375
09a7d9ec 2376 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
e126ba97 2377
09a7d9ec
SM		 2378 /* 0xffffff means we ask to work with cqe version 0 */
2379 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
cfb5e088 2380 MLX5_SET(qpc, qpc, user_index, uidx);
09a7d9ec 2381
2978975c
LR		 2382 if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
2383 init_attr->qp_type != IB_QPT_RAW_PACKET) {
2384 MLX5_SET(qpc, qpc, end_padding_mode,
2385 MLX5_WQ_END_PAD_MODE_ALIGN);
2386 /* Special case to clean flag */
2387 qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
b1383aa6
NO		 2388 }
2389
c2e53b2c 2390 if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2be08c30 2391 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2dfac92d 2392 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
0fb2ed66 2393 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
7f72052c 2394 err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
2fe8d4b8 2395 &params->resp, init_attr);
04bcc1c2 2396 } else
3e09a427 2397 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
e126ba97 2398
479163f4 2399 kvfree(in);
04bcc1c2
LR		 2400 if (err)
2401 goto err_create;
e126ba97 2402
19098df2 2403 base->container_mibqp = qp;
2404 base->mqp.event = mlx5_ib_qp_event;
92cd667c
LR		 2405 if (MLX5_CAP_GEN(mdev, ece_support))
2406 params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
e126ba97 2407
7aede1a2 2408 get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
89ea94a7
MG		 2409 &send_cq, &recv_cq);
2410 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2411 mlx5_ib_lock_cqs(send_cq, recv_cq);
2412 /* Maintain device to QPs access, needed for further handling via reset
2413 * flow
2414 */
2415 list_add_tail(&qp->qps_list, &dev->qp_list);
2416 /* Maintain CQ to QPs access, needed for further handling via reset flow
2417 */
2418 if (send_cq)
2419 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2420 if (recv_cq)
2421 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2422 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2423 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2424
e126ba97
EC		 2425 return 0;
2426
2427err_create:
747c519c 2428 destroy_qp(dev, qp, base, udata);
e126ba97
EC		 2429 return err;
2430}
2431
98fc1126 2432static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
f78d358c
LR		 2433 struct mlx5_ib_qp *qp,
2434 struct mlx5_create_qp_params *params)
98fc1126 2435{
f78d358c
LR		 2436 struct ib_qp_init_attr *attr = params->attr;
2437 u32 uidx = params->uidx;
98fc1126 2438 struct mlx5_ib_resources *devr = &dev->devr;
3e09a427 2439 u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
98fc1126
LR		 2440 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2441 struct mlx5_core_dev *mdev = dev->mdev;
2442 struct mlx5_ib_cq *send_cq;
2443 struct mlx5_ib_cq *recv_cq;
2444 unsigned long flags;
2445 struct mlx5_ib_qp_base *base;
2446 int mlx5_st;
2447 void *qpc;
2448 u32 *in;
2449 int err;
2450
98fc1126
LR		 2451 spin_lock_init(&qp->sq.lock);
2452 spin_lock_init(&qp->rq.lock);
2453
2454 mlx5_st = to_mlx5_st(qp->type);
2455 if (mlx5_st < 0)
2456 return -EINVAL;
2457
2458 if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2459 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2460
2461 base = &qp->trans_qp.base;
2462
2463 qp->has_rq = qp_has_rq(attr);
2464 err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
2465 if (err) {
2466 mlx5_ib_dbg(dev, "err %d\n", err);
2467 return err;
2468 }
2469
2470 err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
2471 if (err)
2472 return err;
2473
2474 if (is_sqp(attr->qp_type))
2475 qp->port = attr->port_num;
2476
2477 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2478
2479 MLX5_SET(qpc, qpc, st, mlx5_st);
2480 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2481
2482 if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
2483 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
2484 else
2485 MLX5_SET(qpc, qpc, latency_sensitive, 1);
2486
2487
2488 if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2489 MLX5_SET(qpc, qpc, block_lb_mc, 1);
2490
2491 if (qp->rq.wqe_cnt) {
2492 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2493 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2494 }
2495
2496 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
2497
2498 if (qp->sq.wqe_cnt)
2499 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2500 else
2501 MLX5_SET(qpc, qpc, no_sq, 1);
2502
2503 if (attr->srq) {
f4375443 2504 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
98fc1126
LR		 2505 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2506 to_msrq(attr->srq)->msrq.srqn);
2507 } else {
f4375443 2508 MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
98fc1126
LR		 2509 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2510 to_msrq(devr->s1)->msrq.srqn);
2511 }
2512
2513 if (attr->send_cq)
2514 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
2515
2516 if (attr->recv_cq)
2517 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
2518
2519 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2520
2521 /* 0xffffff means we ask to work with cqe version 0 */
2522 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2523 MLX5_SET(qpc, qpc, user_index, uidx);
2524
2525 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
2526 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
2527 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
2528
742948cc
OHT		 2529 if (qp->flags & IB_QP_CREATE_INTEGRITY_EN &&
2530 MLX5_CAP_GEN(mdev, go_back_n))
2531 MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
2532
3e09a427 2533 err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
98fc1126
LR		 2534 kvfree(in);
2535 if (err)
2536 goto err_create;
2537
2538 base->container_mibqp = qp;
2539 base->mqp.event = mlx5_ib_qp_event;
2540
2541 get_cqs(qp->type, attr->send_cq, attr->recv_cq,
2542 &send_cq, &recv_cq);
2543 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2544 mlx5_ib_lock_cqs(send_cq, recv_cq);
2545 /* Maintain device to QPs access, needed for further handling via reset
2546 * flow
2547 */
2548 list_add_tail(&qp->qps_list, &dev->qp_list);
2549 /* Maintain CQ to QPs access, needed for further handling via reset flow
2550 */
2551 if (send_cq)
2552 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2553 if (recv_cq)
2554 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2555 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2556 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2557
2558 return 0;
2559
2560err_create:
747c519c 2561 destroy_qp(dev, qp, base, NULL);
98fc1126
LR		 2562 return err;
2563}
2564
e126ba97
EC		 2565static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2566 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
2567{
2568 if (send_cq) {
2569 if (recv_cq) {
2570 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
89ea94a7 2571 spin_lock(&send_cq->lock);
e126ba97
EC		 2572 spin_lock_nested(&recv_cq->lock,
2573 SINGLE_DEPTH_NESTING);
2574 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
89ea94a7 2575 spin_lock(&send_cq->lock);
e126ba97
EC		 2576 __acquire(&recv_cq->lock);
2577 } else {
89ea94a7 2578 spin_lock(&recv_cq->lock);
e126ba97
EC		 2579 spin_lock_nested(&send_cq->lock,
2580 SINGLE_DEPTH_NESTING);
2581 }
2582 } else {
89ea94a7 2583 spin_lock(&send_cq->lock);
6a4f139a 2584 __acquire(&recv_cq->lock);
e126ba97
EC		 2585 }
2586 } else if (recv_cq) {
89ea94a7 2587 spin_lock(&recv_cq->lock);
6a4f139a
EC		 2588 __acquire(&send_cq->lock);
2589 } else {
2590 __acquire(&send_cq->lock);
2591 __acquire(&recv_cq->lock);
e126ba97
EC		 2592 }
2593}
2594
2595static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2596 __releases(&send_cq->lock) __releases(&recv_cq->lock)
2597{
2598 if (send_cq) {
2599 if (recv_cq) {
2600 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
2601 spin_unlock(&recv_cq->lock);
89ea94a7 2602 spin_unlock(&send_cq->lock);
e126ba97
EC		 2603 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2604 __release(&recv_cq->lock);
89ea94a7 2605 spin_unlock(&send_cq->lock);
e126ba97
EC		 2606 } else {
2607 spin_unlock(&send_cq->lock);
89ea94a7 2608 spin_unlock(&recv_cq->lock);
e126ba97
EC		 2609 }
2610 } else {
6a4f139a 2611 __release(&recv_cq->lock);
89ea94a7 2612 spin_unlock(&send_cq->lock);
e126ba97
EC		 2613 }
2614 } else if (recv_cq) {
6a4f139a 2615 __release(&send_cq->lock);
89ea94a7 2616 spin_unlock(&recv_cq->lock);
6a4f139a
EC		 2617 } else {
2618 __release(&recv_cq->lock);
2619 __release(&send_cq->lock);
e126ba97
EC		 2620 }
2621}
2622
89ea94a7
MG		 2623static void get_cqs(enum ib_qp_type qp_type,
2624 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
e126ba97
EC		 2625 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2626{
89ea94a7 2627 switch (qp_type) {
e126ba97
EC		 2628 case IB_QPT_XRC_TGT:
2629 *send_cq = NULL;
2630 *recv_cq = NULL;
2631 break;
2632 case MLX5_IB_QPT_REG_UMR:
2633 case IB_QPT_XRC_INI:
89ea94a7 2634 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
e126ba97
EC		 2635 *recv_cq = NULL;
2636 break;
2637
2638 case IB_QPT_SMI:
d16e91da 2639 case MLX5_IB_QPT_HW_GSI:
e126ba97
EC		 2640 case IB_QPT_RC:
2641 case IB_QPT_UC:
2642 case IB_QPT_UD:
0fb2ed66 2643 case IB_QPT_RAW_PACKET:
89ea94a7
MG		 2644 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2645 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
e126ba97 2646 break;
e126ba97
EC		 2647 default:
2648 *send_cq = NULL;
2649 *recv_cq = NULL;
2650 break;
2651 }
2652}
2653
ad5f8e96 2654static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2655 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2656 u8 lag_tx_affinity);
ad5f8e96 2657
bdeacabd
SR		 2658static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2659 struct ib_udata *udata)
e126ba97
EC		 2660{
2661 struct mlx5_ib_cq *send_cq, *recv_cq;
c2e53b2c 2662 struct mlx5_ib_qp_base *base;
89ea94a7 2663 unsigned long flags;
e126ba97
EC		 2664 int err;
2665
6c41965d 2666 if (qp->is_rss) {
28d61370
YH		 2667 destroy_rss_raw_qp_tir(dev, qp);
2668 return;
2669 }
2670
6c41965d 2671 base = (qp->type == IB_QPT_RAW_PACKET ||
2be08c30 2672 qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
6c41965d
LR		 2673 &qp->raw_packet_qp.rq.base :
2674 &qp->trans_qp.base;
0fb2ed66 2675
6aec21f6 2676 if (qp->state != IB_QPS_RESET) {
6c41965d 2677 if (qp->type != IB_QPT_RAW_PACKET &&
2be08c30 2678 !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
333fbaa0 2679 err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
5f62a521 2680 NULL, &base->mqp, NULL);
ad5f8e96 2681 } else {
0680efa2
AV		 2682 struct mlx5_modify_raw_qp_param raw_qp_param = {
2683 .operation = MLX5_CMD_OP_2RST_QP
2684 };
2685
13eab21f 2686 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
ad5f8e96 2687 }
2688 if (err)
427c1e7b 2689 mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
19098df2 2690 base->mqp.qpn);
6aec21f6 2691 }
e126ba97 2692
6c41965d
LR		 2693 get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq,
2694 &recv_cq);
89ea94a7
MG		 2695
2696 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2697 mlx5_ib_lock_cqs(send_cq, recv_cq);
2698 /* del from lists under both locks above to protect reset flow paths */
2699 list_del(&qp->qps_list);
2700 if (send_cq)
2701 list_del(&qp->cq_send_list);
2702
2703 if (recv_cq)
2704 list_del(&qp->cq_recv_list);
e126ba97 2705
03c4077b 2706 if (!udata) {
19098df2 2707 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 2708 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2709 if (send_cq != recv_cq)
19098df2 2710 __mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2711 NULL);
e126ba97 2712 }
89ea94a7
MG		 2713 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2714 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
e126ba97 2715
6c41965d 2716 if (qp->type == IB_QPT_RAW_PACKET ||
2be08c30 2717 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0fb2ed66 2718 destroy_raw_packet_qp(dev, qp);
2719 } else {
333fbaa0 2720 err = mlx5_core_destroy_qp(dev, &base->mqp);
0fb2ed66 2721 if (err)
2722 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2723 base->mqp.qpn);
2724 }
e126ba97 2725
747c519c 2726 destroy_qp(dev, qp, base, udata);
e126ba97
EC		 2727}
2728
a645a89d
LR		 2729static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2730 struct mlx5_ib_qp *qp,
f78d358c 2731 struct mlx5_create_qp_params *params)
b4aaa1f0 2732{
f78d358c
LR		 2733 struct ib_qp_init_attr *attr = params->attr;
2734 struct mlx5_ib_create_qp *ucmd = params->ucmd;
2735 u32 uidx = params->uidx;
b4aaa1f0
MS		 2736 void *dctc;
2737
7c4b1ab9
MZ		 2738 if (mlx5_lag_is_active(dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct))
2739 return -EOPNOTSUPP;
2740
b4aaa1f0 2741 qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
9c2ba4ed 2742 if (!qp->dct.in)
47c80612 2743 return -ENOMEM;
b4aaa1f0 2744
a01a5860 2745 MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
b4aaa1f0 2746 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
b4aaa1f0
MS		 2747 MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2748 MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2749 MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2750 MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2751 MLX5_SET(dctc, dctc, user_index, uidx);
a645a89d
LR		 2752 if (MLX5_CAP_GEN(dev->mdev, ece_support))
2753 MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
b4aaa1f0 2754
37518fa4 2755 if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
fd9dab7e
LR		 2756 int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
2757
2758 if (rcqe_sz == 128)
2759 MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
2760 }
5d6ff1ba 2761
b4aaa1f0 2762 qp->state = IB_QPS_RESET;
47c80612 2763 return 0;
b4aaa1f0
MS		 2764}
2765
7aede1a2
LR		 2766static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
2767 enum ib_qp_type *type)
6eb7edff
LR		 2768{
2769 if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
2770 goto out;
2771
2772 switch (attr->qp_type) {
2773 case IB_QPT_XRC_TGT:
2774 case IB_QPT_XRC_INI:
2775 if (!MLX5_CAP_GEN(dev->mdev, xrc))
2776 goto out;
2777 fallthrough;
6eb7edff
LR		 2778 case IB_QPT_RC:
2779 case IB_QPT_UC:
6eb7edff
LR		 2780 case IB_QPT_SMI:
2781 case MLX5_IB_QPT_HW_GSI:
6eb7edff
LR		 2782 case IB_QPT_DRIVER:
2783 case IB_QPT_GSI:
42caf9cb
MB		 2784 case IB_QPT_RAW_PACKET:
2785 case IB_QPT_UD:
2786 case MLX5_IB_QPT_REG_UMR:
7aede1a2 2787 break;
6eb7edff
LR		 2788 default:
2789 goto out;
b4aaa1f0
MS		 2790 }
2791
7aede1a2 2792 *type = attr->qp_type;
b4aaa1f0 2793 return 0;
6eb7edff
LR		 2794
2795out:
2796 mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
2797 return -EOPNOTSUPP;
b4aaa1f0
MS		 2798}
2799
2242cc25
LR		 2800static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2801 struct ib_qp_init_attr *attr,
2802 struct ib_udata *udata)
2803{
2804 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2805 udata, struct mlx5_ib_ucontext, ibucontext);
2806
2807 if (!udata) {
2808 /* Kernel create_qp callers */
2809 if (attr->rwq_ind_tbl)
2810 return -EOPNOTSUPP;
2811
2812 switch (attr->qp_type) {
2813 case IB_QPT_RAW_PACKET:
2814 case IB_QPT_DRIVER:
2815 return -EOPNOTSUPP;
2816 default:
2817 return 0;
2818 }
2819 }
2820
2821 /* Userspace create_qp callers */
2822 if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
2823 mlx5_ib_dbg(dev,
2824 "Raw Packet QP is only supported for CQE version > 0\n");
2825 return -EINVAL;
2826 }
2827
2828 if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
2829 mlx5_ib_dbg(dev,
2830 "Wrong QP type %d for the RWQ indirect table\n",
2831 attr->qp_type);
2832 return -EINVAL;
2833 }
2834
2242cc25
LR		 2835 /*
2836 * We don't need to see this warning, it means that kernel code
2837 * missing ib_pd. Placed here to catch developer's mistakes.
2838 */
2839 WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
2840 "There is a missing PD pointer assignment\n");
2841 return 0;
2842}
2843
37518fa4
LR		 2844static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2845 bool cond, struct mlx5_ib_qp *qp)
2846{
2847 if (!(*flags & flag))
2848 return;
2849
2850 if (cond) {
2851 qp->flags_en |= flag;
2852 *flags &= ~flag;
2853 return;
2854 }
2855
81530ab0
LR		 2856 switch (flag) {
2857 case MLX5_QP_FLAG_SCATTER_CQE:
2858 case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
37518fa4 2859 /*
c4526fe2
RC		 2860 * We don't return error if these flags were provided,
2861 * and mlx5 doesn't have right capability.
2862 */
81530ab0
LR		 2863 *flags &= ~(MLX5_QP_FLAG_SCATTER_CQE |
2864 MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
37518fa4 2865 return;
81530ab0
LR		 2866 default:
2867 break;
37518fa4
LR		 2868 }
2869 mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
2870}
2871
2872static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
5ce0592b 2873 void *ucmd, struct ib_qp_init_attr *attr)
2fdddbd5 2874{
37518fa4 2875 struct mlx5_core_dev *mdev = dev->mdev;
37518fa4 2876 bool cond;
5ce0592b
LR		 2877 int flags;
2878
2879 if (attr->rwq_ind_tbl)
2880 flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
2881 else
2882 flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
37518fa4
LR		 2883
2884 switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
2fdddbd5 2885 case MLX5_QP_FLAG_TYPE_DCI:
7aede1a2 2886 qp->type = MLX5_IB_QPT_DCI;
2fdddbd5
LR		 2887 break;
2888 case MLX5_QP_FLAG_TYPE_DCT:
7aede1a2 2889 qp->type = MLX5_IB_QPT_DCT;
37518fa4 2890 break;
7aede1a2
LR		 2891 default:
2892 if (qp->type != IB_QPT_DRIVER)
2893 break;
2894 /*
2895 * It is IB_QPT_DRIVER and or no subtype or
2896 * wrong subtype were provided.
2897 */
2fdddbd5 2898 return -EINVAL;
7aede1a2 2899 }
37518fa4
LR		 2900
2901 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
2902 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
11656f59 2903 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_DCI_STREAM,
65f90c8e 2904 MLX5_CAP_GEN(mdev, log_max_dci_stream_channels),
11656f59 2905 qp);
37518fa4
LR		 2906
2907 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
2908 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
2909 MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
81530ab0
LR		 2910 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_ALLOW_SCATTER_CQE,
2911 MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
37518fa4 2912
7aede1a2 2913 if (qp->type == IB_QPT_RAW_PACKET) {
37518fa4
LR		 2914 cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
2915 MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
2916 MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
2917 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
2918 cond, qp);
2919 process_vendor_flag(dev, &flags,
2920 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
2921 qp);
2922 process_vendor_flag(dev, &flags,
2923 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
2924 qp);
2fdddbd5
LR		 2925 }
2926
7aede1a2 2927 if (qp->type == IB_QPT_RC)
37518fa4
LR		 2928 process_vendor_flag(dev, &flags,
2929 MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
2930 MLX5_CAP_GEN(mdev, qp_packet_based), qp);
2931
76883a6c
LR		 2932 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
2933 process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
2934
5d6fffed
LR		 2935 cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
2936 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
2937 MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
2938 if (attr->rwq_ind_tbl && cond) {
2939 mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
2940 cond);
2941 return -EINVAL;
2942 }
2943
37518fa4
LR		 2944 if (flags)
2945 mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
2946
2947 return (flags) ? -EINVAL : 0;
5d6fffed 2948 }
2fdddbd5 2949
2978975c
LR		 2950static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2951 bool cond, struct mlx5_ib_qp *qp)
2952{
2953 if (!(*flags & flag))
2954 return;
2955
2956 if (cond) {
2957 qp->flags |= flag;
2958 *flags &= ~flag;
2959 return;
2960 }
2961
2962 if (flag == MLX5_IB_QP_CREATE_WC_TEST) {
2963 /*
2964 * Special case, if condition didn't meet, it won't be error,
2965 * just different in-kernel flow.
2966 */
2967 *flags &= ~MLX5_IB_QP_CREATE_WC_TEST;
2968 return;
2969 }
2970 mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
2971}
2972
2973static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2974 struct ib_qp_init_attr *attr)
2975{
7aede1a2 2976 enum ib_qp_type qp_type = qp->type;
2978975c
LR		 2977 struct mlx5_core_dev *mdev = dev->mdev;
2978 int create_flags = attr->create_flags;
2979 bool cond;
2980
7aede1a2 2981 if (qp_type == MLX5_IB_QPT_DCT)
2978975c
LR		 2982 return (create_flags) ? -EINVAL : 0;
2983
2984 if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
2985 return (create_flags) ? -EINVAL : 0;
2986
f81b4565
LR		 2987 process_create_flag(dev, &create_flags, IB_QP_CREATE_NETIF_QP,
2988 mlx5_get_flow_namespace(dev->mdev,
2989 MLX5_FLOW_NAMESPACE_BYPASS),
2990 qp);
9e0dc7b9
MG		 2991 process_create_flag(dev, &create_flags,
2992 IB_QP_CREATE_INTEGRITY_EN,
2993 MLX5_CAP_GEN(mdev, sho), qp);
2978975c
LR		 2994 process_create_flag(dev, &create_flags,
2995 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
2996 MLX5_CAP_GEN(mdev, block_lb_mc), qp);
2997 process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
2998 MLX5_CAP_GEN(mdev, cd), qp);
2999 process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
3000 MLX5_CAP_GEN(mdev, cd), qp);
3001 process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
3002 MLX5_CAP_GEN(mdev, cd), qp);
3003
3004 if (qp_type == IB_QPT_UD) {
3005 process_create_flag(dev, &create_flags,
3006 IB_QP_CREATE_IPOIB_UD_LSO,
3007 MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
3008 qp);
3009 cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
3010 process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
3011 cond, qp);
3012 }
3013
3014 if (qp_type == IB_QPT_RAW_PACKET) {
3015 cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3016 MLX5_CAP_ETH(mdev, scatter_fcs);
3017 process_create_flag(dev, &create_flags,
3018 IB_QP_CREATE_SCATTER_FCS, cond, qp);
3019
3020 cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3021 MLX5_CAP_ETH(mdev, vlan_cap);
3022 process_create_flag(dev, &create_flags,
3023 IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
3024 }
3025
3026 process_create_flag(dev, &create_flags,
3027 IB_QP_CREATE_PCI_WRITE_END_PADDING,
3028 MLX5_CAP_GEN(mdev, end_pad), qp);
3029
3030 process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_WC_TEST,
3031 qp_type != MLX5_IB_QPT_REG_UMR, qp);
3032 process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
3033 true, qp);
3034
1f11a761 3035 if (create_flags) {
2978975c
LR		 3036 mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
3037 create_flags);
1f11a761
JG		 3038 return -EOPNOTSUPP;
3039 }
3040 return 0;
2978975c
LR		 3041}
3042
6f2cf76e
LR		 3043static int process_udata_size(struct mlx5_ib_dev *dev,
3044 struct mlx5_create_qp_params *params)
2fdddbd5
LR		 3045{
3046 size_t ucmd = sizeof(struct mlx5_ib_create_qp);
6f2cf76e
LR		 3047 struct ib_udata *udata = params->udata;
3048 size_t outlen = udata->outlen;
5ce0592b 3049 size_t inlen = udata->inlen;
2fdddbd5 3050
6f2cf76e 3051 params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
e383085c 3052 params->ucmd_size = ucmd;
6f2cf76e 3053 if (!params->is_rss_raw) {
e383085c
LR		 3054 /* User has old rdma-core, which doesn't support ECE */
3055 size_t min_inlen =
3056 offsetof(struct mlx5_ib_create_qp, ece_options);
3057
3058 /*
3059 * We will check in check_ucmd_data() that user
3060 * cleared everything after inlen.
3061 */
3062 params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
6f2cf76e
LR		 3063 goto out;
3064 }
5ce0592b 3065
6f2cf76e 3066 /* RSS RAW QP */
5ce0592b 3067 if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
6f2cf76e
LR		 3068 return -EINVAL;
3069
3070 if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
3071 return -EINVAL;
5ce0592b
LR		 3072
3073 ucmd = sizeof(struct mlx5_ib_create_qp_rss);
e383085c 3074 params->ucmd_size = ucmd;
5ce0592b 3075 if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
6f2cf76e
LR		 3076 return -EINVAL;
3077
3078 params->inlen = min(ucmd, inlen);
3079out:
3080 if (!params->inlen)
e383085c 3081 mlx5_ib_dbg(dev, "udata is too small\n");
2dfac92d 3082
6f2cf76e 3083 return (params->inlen) ? 0 : -EINVAL;
2fdddbd5
LR		 3084}
3085
968f0b6f
LR		 3086static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
3087 struct mlx5_ib_qp *qp,
3088 struct mlx5_create_qp_params *params)
5d0dc3d9 3089{
968f0b6f
LR		 3090 int err;
3091
3092 if (params->is_rss_raw) {
3093 err = create_rss_raw_qp_tir(dev, pd, qp, params);
3094 goto out;
3095 }
3096
2dc4d672
LR		 3097 switch (qp->type) {
3098 case MLX5_IB_QPT_DCT:
a645a89d 3099 err = create_dct(dev, pd, qp, params);
2dc4d672 3100 break;
2013b4d5
LN		 3101 case MLX5_IB_QPT_DCI:
3102 err = create_dci(dev, pd, qp, params);
3103 break;
2dc4d672 3104 case IB_QPT_XRC_TGT:
968f0b6f 3105 err = create_xrc_tgt_qp(dev, qp, params);
2dc4d672
LR		 3106 break;
3107 case IB_QPT_GSI:
3108 err = mlx5_ib_create_gsi(pd, qp, params->attr);
3109 break;
0dc0da15 3110 case MLX5_IB_QPT_HW_GSI:
0dc0da15
LR		 3111 rdma_restrack_no_track(&qp->ibqp.res);
3112 fallthrough;
fd3af5e2 3113 case MLX5_IB_QPT_REG_UMR:
2dc4d672
LR		 3114 default:
3115 if (params->udata)
3116 err = create_user_qp(dev, pd, qp, params);
3117 else
3118 err = create_kernel_qp(dev, pd, qp, params);
968f0b6f 3119 }
5d0dc3d9 3120
968f0b6f
LR		 3121out:
3122 if (err) {
3123 mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
3124 return err;
3125 }
3126
3127 if (is_qp0(qp->type))
3128 qp->ibqp.qp_num = 0;
3129 else if (is_qp1(qp->type))
3130 qp->ibqp.qp_num = 1;
3131 else
3132 qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
3133
3134 mlx5_ib_dbg(dev,
3e09a427 3135 "QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
968f0b6f
LR		 3136 qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
3137 params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
3138 -1,
3139 params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
3e09a427
LR		 3140 -1,
3141 params->resp.ece_options);
968f0b6f
LR		 3142
3143 return 0;
5d0dc3d9
LR		 3144}
3145
7aede1a2
LR		 3146static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3147 struct ib_qp_init_attr *attr)
3148{
3149 int ret = 0;
3150
3151 switch (qp->type) {
3152 case MLX5_IB_QPT_DCT:
3153 ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
3154 break;
3155 case MLX5_IB_QPT_DCI:
3156 ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
3157 -EINVAL :
3158 0;
3159 break;
266424eb
LR		 3160 case IB_QPT_RAW_PACKET:
3161 ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
3162 break;
7aede1a2
LR		 3163 default:
3164 break;
3165 }
3166
3167 if (ret)
3168 mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
3169
3170 return ret;
3171}
3172
f78d358c
LR		 3173static int get_qp_uidx(struct mlx5_ib_qp *qp,
3174 struct mlx5_create_qp_params *params)
21aad80b 3175{
f78d358c
LR		 3176 struct mlx5_ib_create_qp *ucmd = params->ucmd;
3177 struct ib_udata *udata = params->udata;
21aad80b
LR		 3178 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3179 udata, struct mlx5_ib_ucontext, ibucontext);
3180
f78d358c 3181 if (params->is_rss_raw)
21aad80b
LR		 3182 return 0;
3183
f78d358c 3184 return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
21aad80b
LR		 3185}
3186
08d53976
LR		 3187static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
3188{
3189 struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
3190
3191 if (mqp->state == IB_QPS_RTR) {
3192 int err;
3193
3194 err = mlx5_core_destroy_dct(dev, &mqp->dct.mdct);
3195 if (err) {
3196 mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
3197 return err;
3198 }
3199 }
3200
3201 kfree(mqp->dct.in);
08d53976
LR		 3202 return 0;
3203}
3204
e383085c
LR		 3205static int check_ucmd_data(struct mlx5_ib_dev *dev,
3206 struct mlx5_create_qp_params *params)
3207{
e383085c
LR		 3208 struct ib_udata *udata = params->udata;
3209 size_t size, last;
3210 int ret;
3211
3212 if (params->is_rss_raw)
3213 /*
3214 * These QPs don't have "reserved" field in their
3215 * create_qp input struct, so their data is always valid.
3216 */
3217 last = sizeof(struct mlx5_ib_create_qp_rss);
3218 else
2c0f5292 3219 last = offsetof(struct mlx5_ib_create_qp, reserved);
e383085c
LR		 3220
3221 if (udata->inlen <= last)
3222 return 0;
3223
3224 /*
3225 * User provides different create_qp structures based on the
3226 * flow and we need to know if he cleared memory after our
3227 * struct create_qp ends.
3228 */
3229 size = udata->inlen - last;
3230 ret = ib_is_udata_cleared(params->udata, last, size);
3231 if (!ret)
3232 mlx5_ib_dbg(
3233 dev,
4f5747cf 3234 "udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",
e383085c
LR		 3235 udata->inlen, params->ucmd_size, last, size);
3236 return ret ? 0 : -EINVAL;
3237}
3238
514aee66
LR		 3239int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
3240 struct ib_udata *udata)
e126ba97 3241{
f78d358c 3242 struct mlx5_create_qp_params params = {};
514aee66
LR		 3243 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3244 struct mlx5_ib_qp *qp = to_mqp(ibqp);
3245 struct ib_pd *pd = ibqp->pd;
7aede1a2 3246 enum ib_qp_type type;
e126ba97
EC		 3247 int err;
3248
f78d358c
LR		 3249 err = check_qp_type(dev, attr, &type);
3250 if (err)
514aee66 3251 return err;
6eb7edff 3252
f78d358c 3253 err = check_valid_flow(dev, pd, attr, udata);
2242cc25 3254 if (err)
514aee66 3255 return err;
e126ba97 3256
f78d358c
LR		 3257 params.udata = udata;
3258 params.uidx = MLX5_IB_DEFAULT_UIDX;
3259 params.attr = attr;
3260 params.is_rss_raw = !!attr->rwq_ind_tbl;
2fdddbd5 3261
f78d358c 3262 if (udata) {
6f2cf76e
LR		 3263 err = process_udata_size(dev, &params);
3264 if (err)
514aee66 3265 return err;
2fdddbd5 3266
e383085c
LR		 3267 err = check_ucmd_data(dev, &params);
3268 if (err)
514aee66 3269 return err;
e383085c
LR		 3270
3271 params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
f78d358c 3272 if (!params.ucmd)
514aee66 3273 return -ENOMEM;
5ce0592b 3274
f78d358c 3275 err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
2fdddbd5 3276 if (err)
5ce0592b 3277 goto free_ucmd;
2fdddbd5
LR		 3278 }
3279
7fa84b57 3280 mutex_init(&qp->mutex);
7aede1a2 3281 qp->type = type;
37518fa4 3282 if (udata) {
f78d358c 3283 err = process_vendor_flags(dev, qp, params.ucmd, attr);
b4aaa1f0 3284 if (err)
514aee66 3285 goto free_ucmd;
21aad80b 3286
f78d358c 3287 err = get_qp_uidx(qp, &params);
21aad80b 3288 if (err)
514aee66 3289 goto free_ucmd;
b4aaa1f0 3290 }
f78d358c 3291 err = process_create_flags(dev, qp, attr);
2978975c 3292 if (err)
514aee66 3293 goto free_ucmd;
b4aaa1f0 3294
f78d358c 3295 err = check_qp_attr(dev, qp, attr);
7aede1a2 3296 if (err)
514aee66 3297 goto free_ucmd;
7aede1a2 3298
968f0b6f
LR		 3299 err = create_qp(dev, pd, qp, &params);
3300 if (err)
514aee66 3301 goto free_ucmd;
e126ba97 3302
f78d358c 3303 kfree(params.ucmd);
08d53976 3304 params.ucmd = NULL;
5ce0592b 3305
08d53976
LR		 3306 if (udata)
3307 /*
3308 * It is safe to copy response for all user create QP flows,
3309 * including MLX5_IB_QPT_DCT, which doesn't need it.
3310 * In that case, resp will be filled with zeros.
3311 */
3312 err = ib_copy_to_udata(udata, &params.resp, params.outlen);
3313 if (err)
3314 goto destroy_qp;
3315
514aee66 3316 return 0;
9c2ba4ed 3317
08d53976 3318destroy_qp:
2dc4d672
LR		 3319 switch (qp->type) {
3320 case MLX5_IB_QPT_DCT:
08d53976 3321 mlx5_ib_destroy_dct(qp);
2dc4d672
LR		 3322 break;
3323 case IB_QPT_GSI:
3324 mlx5_ib_destroy_gsi(qp);
3325 break;
3326 default:
08d53976 3327 destroy_qp_common(dev, qp, udata);
6c41965d
LR		 3328 }
3329
5ce0592b 3330free_ucmd:
f78d358c 3331 kfree(params.ucmd);
514aee66 3332 return err;
e126ba97
EC		 3333}
3334
c4367a26 3335int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
e126ba97
EC		 3336{
3337 struct mlx5_ib_dev *dev = to_mdev(qp->device);
3338 struct mlx5_ib_qp *mqp = to_mqp(qp);
3339
9ecf6ac1 3340 if (mqp->type == IB_QPT_GSI)
0d9aef86 3341 return mlx5_ib_destroy_gsi(mqp);
d16e91da 3342
7aede1a2 3343 if (mqp->type == MLX5_IB_QPT_DCT)
776a3906
MS		 3344 return mlx5_ib_destroy_dct(mqp);
3345
bdeacabd 3346 destroy_qp_common(dev, mqp, udata);
e126ba97
EC		 3347 return 0;
3348}
3349
f18e26af
LR		 3350static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
3351 const struct ib_qp_attr *attr, int attr_mask,
3352 void *qpc)
e126ba97 3353{
a60109dc 3354 struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
f18e26af
LR		 3355 u8 dest_rd_atomic;
3356 u32 access_flags;
a60109dc 3357
e126ba97
EC		 3358 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3359 dest_rd_atomic = attr->max_dest_rd_atomic;
3360 else
19098df2 3361 dest_rd_atomic = qp->trans_qp.resp_depth;
e126ba97
EC		 3362
3363 if (attr_mask & IB_QP_ACCESS_FLAGS)
3364 access_flags = attr->qp_access_flags;
3365 else
19098df2 3366 access_flags = qp->trans_qp.atomic_rd_en;
e126ba97
EC		 3367
3368 if (!dest_rd_atomic)
3369 access_flags &= IB_ACCESS_REMOTE_WRITE;
3370
f18e26af
LR		 3371 MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
3372
13f8d9c1 3373 if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
a60109dc
YC		 3374 int atomic_mode;
3375
9ecf6ac1 3376 atomic_mode = get_atomic_mode(dev, qp->type);
a60109dc
YC		 3377 if (atomic_mode < 0)
3378 return -EOPNOTSUPP;
3379
f18e26af
LR		 3380 MLX5_SET(qpc, qpc, rae, 1);
3381 MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
a60109dc
YC		 3382 }
3383
f18e26af 3384 MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
a60109dc 3385 return 0;
e126ba97
EC		 3386}
3387
3388enum {
3389 MLX5_PATH_FLAG_FL = 1 << 0,
3390 MLX5_PATH_FLAG_FREE_AR = 1 << 1,
3391 MLX5_PATH_FLAG_COUNTER = 1 << 2,
3392};
3393
6fe6e568
MZ		 3394static int mlx5_to_ib_rate_map(u8 rate)
3395{
3396 static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,
3397 IB_RATE_25_GBPS, IB_RATE_100_GBPS,
3398 IB_RATE_200_GBPS, IB_RATE_50_GBPS,
3399 IB_RATE_400_GBPS };
3400
3401 if (rate < ARRAY_SIZE(rates))
3402 return rates[rate];
3403
3404 return rate - MLX5_STAT_RATE_OFFSET;
3405}
3406
c531024b
MZ		 3407static int ib_to_mlx5_rate_map(u8 rate)
3408{
3409 switch (rate) {
3410 case IB_RATE_PORT_CURRENT:
3411 return 0;
3412 case IB_RATE_56_GBPS:
3413 return 1;
3414 case IB_RATE_25_GBPS:
3415 return 2;
3416 case IB_RATE_100_GBPS:
3417 return 3;
3418 case IB_RATE_200_GBPS:
3419 return 4;
3420 case IB_RATE_50_GBPS:
3421 return 5;
c70f51de
PH		 3422 case IB_RATE_400_GBPS:
3423 return 6;
c531024b
MZ		 3424 default:
3425 return rate + MLX5_STAT_RATE_OFFSET;
7f1d2dfa 3426 }
c531024b
MZ		 3427
3428 return 0;
3429}
3430
e126ba97
EC		 3431static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
3432{
c531024b
MZ		 3433 u32 stat_rate_support;
3434
4f32ac2e 3435 if (rate == IB_RATE_PORT_CURRENT)
e126ba97 3436 return 0;
4f32ac2e 3437
4f4db190 3438 if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_800_GBPS)
e126ba97 3439 return -EINVAL;
e126ba97 3440
c531024b 3441 stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support);
4f32ac2e 3442 while (rate != IB_RATE_PORT_CURRENT &&
c531024b 3443 !(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support))
4f32ac2e
DG		 3444 --rate;
3445
c531024b 3446 return ib_to_mlx5_rate_map(rate);
e126ba97
EC		 3447}
3448
75850d0b 3449static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
1cd6dbd3
YH		 3450 struct mlx5_ib_sq *sq, u8 sl,
3451 struct ib_pd *pd)
75850d0b 3452{
3453 void *in;
3454 void *tisc;
3455 int inlen;
3456 int err;
3457
3458 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 3459 in = kvzalloc(inlen, GFP_KERNEL);
75850d0b 3460 if (!in)
3461 return -ENOMEM;
3462
3463 MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
1cd6dbd3 3464 MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
75850d0b 3465
3466 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3467 MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
3468
e0b4b472 3469 err = mlx5_core_modify_tis(dev, sq->tisn, in);
75850d0b 3470
3471 kvfree(in);
3472
3473 return err;
3474}
3475
13eab21f 3476static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
1cd6dbd3
YH		 3477 struct mlx5_ib_sq *sq, u8 tx_affinity,
3478 struct ib_pd *pd)
13eab21f
AH		 3479{
3480 void *in;
3481 void *tisc;
3482 int inlen;
3483 int err;
3484
3485 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 3486 in = kvzalloc(inlen, GFP_KERNEL);
13eab21f
AH		 3487 if (!in)
3488 return -ENOMEM;
3489
3490 MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
1cd6dbd3 3491 MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
13eab21f
AH		 3492
3493 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3494 MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
3495
e0b4b472 3496 err = mlx5_core_modify_tis(dev, sq->tisn, in);
13eab21f
AH		 3497
3498 kvfree(in);
3499
3500 return err;
3501}
3502
f18e26af 3503static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
2b880b2e
MZ		 3504 u32 lqpn, u32 rqpn)
3505
3506{
3507 u32 fl = ah->grh.flow_label;
2b880b2e
MZ		 3508
3509 if (!fl)
3510 fl = rdma_calc_flow_label(lqpn, rqpn);
3511
f18e26af 3512 MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
2b880b2e
MZ		 3513}
3514
75850d0b 3515static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
f18e26af
LR		 3516 const struct rdma_ah_attr *ah, void *path, u8 port,
3517 int attr_mask, u32 path_flags,
3518 const struct ib_qp_attr *attr, bool alt)
e126ba97 3519{
d8966fcd 3520 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
e126ba97 3521 int err;
ed88451e 3522 enum ib_gid_type gid_type;
d8966fcd
DC		 3523 u8 ah_flags = rdma_ah_get_ah_flags(ah);
3524 u8 sl = rdma_ah_get_sl(ah);
e126ba97 3525
e126ba97 3526 if (attr_mask & IB_QP_PKEY_INDEX)
f18e26af
LR		 3527 MLX5_SET(ads, path, pkey_index,
3528 alt ? attr->alt_pkey_index : attr->pkey_index);
e126ba97 3529
d8966fcd 3530 if (ah_flags & IB_AH_GRH) {
7416790e
PP		 3531 const struct ib_port_immutable *immutable;
3532
3533 immutable = ib_port_immutable_read(&dev->ib_dev, port);
3534 if (grh->sgid_index >= immutable->gid_tbl_len) {
f4f01b54 3535 pr_err("sgid_index (%u) too large. max is %d\n",
d8966fcd 3536 grh->sgid_index,
7416790e 3537 immutable->gid_tbl_len);
f83b4263
EC		 3538 return -EINVAL;
3539 }
2811ba51 3540 }
44c58487
DC		 3541
3542 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
d8966fcd 3543 if (!(ah_flags & IB_AH_GRH))
2811ba51 3544 return -EINVAL;
47ec3866 3545
f18e26af
LR		 3546 ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
3547 ah->roce.dmac);
9ecf6ac1
MG		 3548 if ((qp->type == IB_QPT_RC ||
3549 qp->type == IB_QPT_UC ||
3550 qp->type == IB_QPT_XRC_INI ||
3551 qp->type == IB_QPT_XRC_TGT) &&
2b880b2e
MZ		 3552 (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
3553 (attr_mask & IB_QP_DEST_QPN))
3554 mlx5_set_path_udp_sport(path, ah,
3555 qp->ibqp.qp_num,
3556 attr->dest_qp_num);
f18e26af 3557 MLX5_SET(ads, path, eth_prio, sl & 0x7);
47ec3866 3558 gid_type = ah->grh.sgid_attr->gid_type;
ed88451e 3559 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
f18e26af 3560 MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
2811ba51 3561 } else {
f18e26af
LR		 3562 MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
3563 MLX5_SET(ads, path, free_ar,
3564 !!(path_flags & MLX5_PATH_FLAG_FREE_AR));
3565 MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
3566 MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
3567 MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
3568 MLX5_SET(ads, path, sl, sl);
2811ba51
AS		 3569 }
3570
d8966fcd 3571 if (ah_flags & IB_AH_GRH) {
f18e26af
LR		 3572 MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
3573 MLX5_SET(ads, path, hop_limit, grh->hop_limit);
3574 MLX5_SET(ads, path, tclass, grh->traffic_class);
3575 MLX5_SET(ads, path, flow_label, grh->flow_label);
3576 memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
3577 sizeof(grh->dgid.raw));
e126ba97
EC		 3578 }
3579
d8966fcd 3580 err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
e126ba97
EC		 3581 if (err < 0)
3582 return err;
f18e26af
LR		 3583 MLX5_SET(ads, path, stat_rate, err);
3584 MLX5_SET(ads, path, vhca_port_num, port);
e126ba97 3585
e126ba97 3586 if (attr_mask & IB_QP_TIMEOUT)
f18e26af
LR		 3587 MLX5_SET(ads, path, ack_timeout,
3588 alt ? attr->alt_timeout : attr->timeout);
e126ba97 3589
9ecf6ac1 3590 if ((qp->type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
75850d0b 3591 return modify_raw_packet_eth_prio(dev->mdev,
3592 &qp->raw_packet_qp.sq,
1cd6dbd3 3593 sl & 0xf, qp->ibqp.pd);
75850d0b 3594
e126ba97
EC		 3595 return 0;
3596}
3597
3598static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
3599 [MLX5_QP_STATE_INIT] = {
3600 [MLX5_QP_STATE_INIT] = {
3601 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3602 MLX5_QP_OPTPAR_RAE |
3603 MLX5_QP_OPTPAR_RWE |
3604 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3605 MLX5_QP_OPTPAR_PRI_PORT |
3606 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3607 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
3608 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3609 MLX5_QP_OPTPAR_PRI_PORT |
3610 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3611 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3612 MLX5_QP_OPTPAR_Q_KEY |
3613 MLX5_QP_OPTPAR_PRI_PORT,
8f4426aa
JM		 3614 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3615 MLX5_QP_OPTPAR_RAE |
3616 MLX5_QP_OPTPAR_RWE |
3617 MLX5_QP_OPTPAR_PKEY_INDEX |
cfc1a89e
MG		 3618 MLX5_QP_OPTPAR_PRI_PORT |
3619 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3620 },
3621 [MLX5_QP_STATE_RTR] = {
3622 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3623 MLX5_QP_OPTPAR_RRE |
3624 MLX5_QP_OPTPAR_RAE |
3625 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3626 MLX5_QP_OPTPAR_PKEY_INDEX |
3627 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3628 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3629 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3630 MLX5_QP_OPTPAR_PKEY_INDEX |
3631 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3632 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
3633 MLX5_QP_OPTPAR_Q_KEY,
3634 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX |
3635 MLX5_QP_OPTPAR_Q_KEY,
a4774e90
EC		 3636 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3637 MLX5_QP_OPTPAR_RRE |
3638 MLX5_QP_OPTPAR_RAE |
3639 MLX5_QP_OPTPAR_RWE |
cfc1a89e
MG		 3640 MLX5_QP_OPTPAR_PKEY_INDEX |
3641 MLX5_QP_OPTPAR_LAG_TX_AFF,
e126ba97
EC		 3642 },
3643 },
3644 [MLX5_QP_STATE_RTR] = {
3645 [MLX5_QP_STATE_RTS] = {
3646 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3647 MLX5_QP_OPTPAR_RRE |
3648 MLX5_QP_OPTPAR_RAE |
3649 MLX5_QP_OPTPAR_RWE |
3650 MLX5_QP_OPTPAR_PM_STATE |
3651 MLX5_QP_OPTPAR_RNR_TIMEOUT,
3652 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3653 MLX5_QP_OPTPAR_RWE |
3654 MLX5_QP_OPTPAR_PM_STATE,
3655 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
8f4426aa
JM		 3656 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3657 MLX5_QP_OPTPAR_RRE |
3658 MLX5_QP_OPTPAR_RAE |
3659 MLX5_QP_OPTPAR_RWE |
3660 MLX5_QP_OPTPAR_PM_STATE |
3661 MLX5_QP_OPTPAR_RNR_TIMEOUT,
e126ba97
EC		 3662 },
3663 },
3664 [MLX5_QP_STATE_RTS] = {
3665 [MLX5_QP_STATE_RTS] = {
3666 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
3667 MLX5_QP_OPTPAR_RAE |
3668 MLX5_QP_OPTPAR_RWE |
3669 MLX5_QP_OPTPAR_RNR_TIMEOUT |
c2a3431e
EC		 3670 MLX5_QP_OPTPAR_PM_STATE |
3671 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97 3672 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
c2a3431e
EC		 3673 MLX5_QP_OPTPAR_PM_STATE |
3674 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 3675 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY |
3676 MLX5_QP_OPTPAR_SRQN |
3677 MLX5_QP_OPTPAR_CQN_RCV,
8f4426aa
JM		 3678 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE |
3679 MLX5_QP_OPTPAR_RAE |
3680 MLX5_QP_OPTPAR_RWE |
3681 MLX5_QP_OPTPAR_RNR_TIMEOUT |
3682 MLX5_QP_OPTPAR_PM_STATE |
3683 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 3684 },
3685 },
3686 [MLX5_QP_STATE_SQER] = {
3687 [MLX5_QP_STATE_RTS] = {
3688 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3689 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
75959f56 3690 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
a4774e90
EC		 3691 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3692 MLX5_QP_OPTPAR_RWE |
3693 MLX5_QP_OPTPAR_RAE |
3694 MLX5_QP_OPTPAR_RRE,
8f4426aa
JM		 3695 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3696 MLX5_QP_OPTPAR_RWE |
3697 MLX5_QP_OPTPAR_RAE |
3698 MLX5_QP_OPTPAR_RRE,
e126ba97
EC		 3699 },
3700 },
021c1f24
SG		 3701 [MLX5_QP_STATE_SQD] = {
3702 [MLX5_QP_STATE_RTS] = {
3703 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3704 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3705 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
3706 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
3707 MLX5_QP_OPTPAR_RWE |
3708 MLX5_QP_OPTPAR_RAE |
3709 MLX5_QP_OPTPAR_RRE,
3710 },
3711 },
e126ba97
EC		 3712};
3713
3714static int ib_nr_to_mlx5_nr(int ib_mask)
3715{
3716 switch (ib_mask) {
3717 case IB_QP_STATE:
3718 return 0;
3719 case IB_QP_CUR_STATE:
3720 return 0;
3721 case IB_QP_EN_SQD_ASYNC_NOTIFY:
3722 return 0;
3723 case IB_QP_ACCESS_FLAGS:
3724 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
3725 MLX5_QP_OPTPAR_RAE;
3726 case IB_QP_PKEY_INDEX:
3727 return MLX5_QP_OPTPAR_PKEY_INDEX;
3728 case IB_QP_PORT:
3729 return MLX5_QP_OPTPAR_PRI_PORT;
3730 case IB_QP_QKEY:
3731 return MLX5_QP_OPTPAR_Q_KEY;
3732 case IB_QP_AV:
3733 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
3734 MLX5_QP_OPTPAR_PRI_PORT;
3735 case IB_QP_PATH_MTU:
3736 return 0;
3737 case IB_QP_TIMEOUT:
3738 return MLX5_QP_OPTPAR_ACK_TIMEOUT;
3739 case IB_QP_RETRY_CNT:
3740 return MLX5_QP_OPTPAR_RETRY_COUNT;
3741 case IB_QP_RNR_RETRY:
3742 return MLX5_QP_OPTPAR_RNR_RETRY;
3743 case IB_QP_RQ_PSN:
3744 return 0;
3745 case IB_QP_MAX_QP_RD_ATOMIC:
3746 return MLX5_QP_OPTPAR_SRA_MAX;
3747 case IB_QP_ALT_PATH:
3748 return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
3749 case IB_QP_MIN_RNR_TIMER:
3750 return MLX5_QP_OPTPAR_RNR_TIMEOUT;
3751 case IB_QP_SQ_PSN:
3752 return 0;
3753 case IB_QP_MAX_DEST_RD_ATOMIC:
3754 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
3755 MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
3756 case IB_QP_PATH_MIG_STATE:
3757 return MLX5_QP_OPTPAR_PM_STATE;
3758 case IB_QP_CAP:
3759 return 0;
3760 case IB_QP_DEST_QPN:
3761 return 0;
3762 }
3763 return 0;
3764}
3765
3766static int ib_mask_to_mlx5_opt(int ib_mask)
3767{
3768 int result = 0;
3769 int i;
3770
3771 for (i = 0; i < 8 * sizeof(int); i++) {
3772 if ((1 << i) & ib_mask)
3773 result |= ib_nr_to_mlx5_nr(1 << i);
3774 }
3775
3776 return result;
3777}
3778
34d57585
YH		 3779static int modify_raw_packet_qp_rq(
3780 struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
3781 const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
ad5f8e96 3782{
3783 void *in;
3784 void *rqc;
3785 int inlen;
3786 int err;
3787
3788 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 3789 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 3790 if (!in)
3791 return -ENOMEM;
3792
3793 MLX5_SET(modify_rq_in, in, rq_state, rq->state);
34d57585 3794 MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
ad5f8e96 3795
3796 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
3797 MLX5_SET(rqc, rqc, state, new_state);
3798
eb49ab0c
AV		 3799 if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
3800 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
3801 MLX5_SET64(modify_rq_in, in, modify_bitmask,
23a6964e 3802 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
eb49ab0c
AV		 3803 MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
3804 } else
5a738b5d
JG		 3805 dev_info_once(
3806 &dev->ib_dev.dev,
3807 "RAW PACKET QP counters are not supported on current FW\n");
eb49ab0c
AV		 3808 }
3809
e0b4b472 3810 err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in);
ad5f8e96 3811 if (err)
3812 goto out;
3813
3814 rq->state = new_state;
3815
3816out:
3817 kvfree(in);
3818 return err;
3819}
3820
c14003f0
YH		 3821static int modify_raw_packet_qp_sq(
3822 struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
3823 const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
ad5f8e96 3824{
7d29f349 3825 struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
61147f39
BW		 3826 struct mlx5_rate_limit old_rl = ibqp->rl;
3827 struct mlx5_rate_limit new_rl = old_rl;
3828 bool new_rate_added = false;
7d29f349 3829 u16 rl_index = 0;
ad5f8e96 3830 void *in;
3831 void *sqc;
3832 int inlen;
3833 int err;
3834
3835 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 3836 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 3837 if (!in)
3838 return -ENOMEM;
3839
c14003f0 3840 MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
ad5f8e96 3841 MLX5_SET(modify_sq_in, in, sq_state, sq->state);
3842
3843 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
3844 MLX5_SET(sqc, sqc, state, new_state);
3845
7d29f349
BW		 3846 if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
3847 if (new_state != MLX5_SQC_STATE_RDY)
3848 pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
3849 __func__);
3850 else
61147f39 3851 new_rl = raw_qp_param->rl;
7d29f349
BW		 3852 }
3853
61147f39
BW		 3854 if (!mlx5_rl_are_equal(&old_rl, &new_rl)) {
3855 if (new_rl.rate) {
3856 err = mlx5_rl_add_rate(dev, &rl_index, &new_rl);
7d29f349 3857 if (err) {
61147f39
BW		 3858 pr_err("Failed configuring rate limit(err %d): \
3859 rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
3860 err, new_rl.rate, new_rl.max_burst_sz,
3861 new_rl.typical_pkt_sz);
3862
7d29f349
BW		 3863 goto out;
3864 }
61147f39 3865 new_rate_added = true;
7d29f349
BW		 3866 }
3867
3868 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
61147f39 3869 /* index 0 means no limit */
7d29f349
BW		 3870 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
3871 }
3872
e0b4b472 3873 err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in);
7d29f349
BW		 3874 if (err) {
3875 /* Remove new rate from table if failed */
61147f39
BW		 3876 if (new_rate_added)
3877 mlx5_rl_remove_rate(dev, &new_rl);
ad5f8e96 3878 goto out;
7d29f349
BW		 3879 }
3880
3881 /* Only remove the old rate after new rate was set */
c8973df2
RW		 3882 if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) ||
3883 (new_state != MLX5_SQC_STATE_RDY)) {
61147f39 3884 mlx5_rl_remove_rate(dev, &old_rl);
c8973df2
RW		 3885 if (new_state != MLX5_SQC_STATE_RDY)
3886 memset(&new_rl, 0, sizeof(new_rl));
3887 }
ad5f8e96 3888
61147f39 3889 ibqp->rl = new_rl;
ad5f8e96 3890 sq->state = new_state;
3891
3892out:
3893 kvfree(in);
3894 return err;
3895}
3896
3897static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 3898 const struct mlx5_modify_raw_qp_param *raw_qp_param,
3899 u8 tx_affinity)
ad5f8e96 3900{
3901 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
3902 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
3903 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
7d29f349
BW		 3904 int modify_rq = !!qp->rq.wqe_cnt;
3905 int modify_sq = !!qp->sq.wqe_cnt;
ad5f8e96 3906 int rq_state;
3907 int sq_state;
3908 int err;
3909
0680efa2 3910 switch (raw_qp_param->operation) {
ad5f8e96 3911 case MLX5_CMD_OP_RST2INIT_QP:
3912 rq_state = MLX5_RQC_STATE_RDY;
c94e272b 3913 sq_state = MLX5_SQC_STATE_RST;
ad5f8e96 3914 break;
3915 case MLX5_CMD_OP_2ERR_QP:
3916 rq_state = MLX5_RQC_STATE_ERR;
3917 sq_state = MLX5_SQC_STATE_ERR;
3918 break;
3919 case MLX5_CMD_OP_2RST_QP:
3920 rq_state = MLX5_RQC_STATE_RST;
3921 sq_state = MLX5_SQC_STATE_RST;
3922 break;
ad5f8e96 3923 case MLX5_CMD_OP_RTR2RTS_QP:
3924 case MLX5_CMD_OP_RTS2RTS_QP:
c94e272b
MG		 3925 if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
3926 return -EINVAL;
3927
3928 modify_rq = 0;
3929 sq_state = MLX5_SQC_STATE_RDY;
7d29f349
BW		 3930 break;
3931 case MLX5_CMD_OP_INIT2INIT_QP:
3932 case MLX5_CMD_OP_INIT2RTR_QP:
eb49ab0c
AV		 3933 if (raw_qp_param->set_mask)
3934 return -EINVAL;
3935 else
3936 return 0;
ad5f8e96 3937 default:
3938 WARN_ON(1);
3939 return -EINVAL;
3940 }
3941
7d29f349 3942 if (modify_rq) {
34d57585
YH		 3943 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
3944 qp->ibqp.pd);
ad5f8e96 3945 if (err)
3946 return err;
3947 }
3948
7d29f349 3949 if (modify_sq) {
d5ed8ac3
MB		 3950 struct mlx5_flow_handle *flow_rule;
3951
13eab21f
AH		 3952 if (tx_affinity) {
3953 err = modify_raw_packet_tx_affinity(dev->mdev, sq,
1cd6dbd3
YH		 3954 tx_affinity,
3955 qp->ibqp.pd);
13eab21f
AH		 3956 if (err)
3957 return err;
3958 }
3959
d5ed8ac3
MB		 3960 flow_rule = create_flow_rule_vport_sq(dev, sq,
3961 raw_qp_param->port);
3962 if (IS_ERR(flow_rule))
1db86318 3963 return PTR_ERR(flow_rule);
d5ed8ac3
MB		 3964
3965 err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
3966 raw_qp_param, qp->ibqp.pd);
3967 if (err) {
3968 if (flow_rule)
3969 mlx5_del_flow_rules(flow_rule);
3970 return err;
3971 }
3972
3973 if (flow_rule) {
3974 destroy_flow_rule_vport_sq(sq);
3975 sq->flow_rule = flow_rule;
3976 }
3977
3978 return err;
13eab21f 3979 }
ad5f8e96 3980
3981 return 0;
3982}
3983
5163b274
MG		 3984static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
3985 struct ib_udata *udata)
c6a21c38 3986{
89944450
SR		 3987 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3988 udata, struct mlx5_ib_ucontext, ibucontext);
5163b274
MG		 3989 u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3990 atomic_t *tx_port_affinity;
c6a21c38 3991
5163b274
MG		 3992 if (ucontext)
3993 tx_port_affinity = &ucontext->tx_port_affinity;
3994 else
3995 tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
3996
3997 return (unsigned int)atomic_add_return(1, tx_port_affinity) %
34a30d76 3998 (dev->lag_active ? dev->lag_ports : MLX5_CAP_GEN(dev->mdev, num_lag_ports)) + 1;
5163b274
MG		 3999}
4000
8f3243a0
MZ		 4001static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
4002{
4003 if ((qp->type == IB_QPT_RC) || (qp->type == IB_QPT_UD) ||
4004 (qp->type == IB_QPT_UC) || (qp->type == IB_QPT_RAW_PACKET) ||
4005 (qp->type == IB_QPT_XRC_INI) || (qp->type == IB_QPT_XRC_TGT) ||
4006 (qp->type == MLX5_IB_QPT_DCI))
5163b274
MG		 4007 return true;
4008 return false;
4009}
4010
cfc1a89e
MG		 4011static unsigned int get_tx_affinity(struct ib_qp *qp,
4012 const struct ib_qp_attr *attr,
4013 int attr_mask, u8 init,
5163b274
MG		 4014 struct ib_udata *udata)
4015{
4016 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4017 udata, struct mlx5_ib_ucontext, ibucontext);
4018 struct mlx5_ib_dev *dev = to_mdev(qp->device);
4019 struct mlx5_ib_qp *mqp = to_mqp(qp);
4020 struct mlx5_ib_qp_base *qp_base;
4021 unsigned int tx_affinity;
4022
802dcc7f 4023 if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
8f3243a0 4024 qp_supports_affinity(mqp)))
5163b274
MG		 4025 return 0;
4026
cfc1a89e
MG		 4027 if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4028 tx_affinity = mqp->gsi_lag_port;
4029 else if (init)
4030 tx_affinity = get_tx_affinity_rr(dev, udata);
4031 else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
4032 tx_affinity =
4033 mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
4034 else
4035 return 0;
5163b274
MG		 4036
4037 qp_base = &mqp->trans_qp.base;
4038 if (ucontext)
c6a21c38 4039 mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
5163b274
MG		 4040 tx_affinity, qp_base->mqp.qpn, ucontext);
4041 else
c6a21c38 4042 mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
5163b274
MG		 4043 tx_affinity, qp_base->mqp.qpn);
4044 return tx_affinity;
c6a21c38
MD		 4045}
4046
c1336bb4
PH		 4047static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,
4048 struct mlx5_core_dev *mdev)
4049{
4050 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4051 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4052 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
4053 void *rqc;
4054
4055 if (!qp->rq.wqe_cnt)
4056 return 0;
4057
4058 MLX5_SET(modify_rq_in, in, rq_state, rq->state);
4059 MLX5_SET(modify_rq_in, in, uid, to_mpd(qp->ibqp.pd)->uid);
4060
4061 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
4062 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
4063
4064 MLX5_SET64(modify_rq_in, in, modify_bitmask,
4065 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
4066 MLX5_SET(rqc, rqc, counter_set_id, set_id);
4067
4068 return mlx5_core_modify_rq(mdev, rq->base.mqp.qpn, in);
4069}
4070
d14133dd
MZ		 4071static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
4072 struct rdma_counter *counter)
4073{
4074 struct mlx5_ib_dev *dev = to_mdev(qp->device);
64bae2d4 4075 u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
d14133dd 4076 struct mlx5_ib_qp *mqp = to_mqp(qp);
d14133dd
MZ		 4077 struct mlx5_ib_qp_base *base;
4078 u32 set_id;
64bae2d4 4079 u32 *qpc;
d14133dd 4080
3e1f000f 4081 if (counter)
d14133dd 4082 set_id = counter->id;
3e1f000f
PP		 4083 else
4084 set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
d14133dd 4085
c1336bb4
PH		 4086 if (mqp->type == IB_QPT_RAW_PACKET)
4087 return __mlx5_ib_qp_set_raw_qp_counter(mqp, set_id, dev->mdev);
4088
d14133dd 4089 base = &mqp->trans_qp.base;
64bae2d4
LR		 4090 MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
4091 MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
4092 MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
4093 MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
4094 MLX5_QP_OPTPAR_COUNTER_SET_ID);
4095
4096 qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
4097 MLX5_SET(qpc, qpc, counter_set_id, set_id);
4098 return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
d14133dd
MZ		 4099}
4100
e126ba97
EC		 4101static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
4102 const struct ib_qp_attr *attr, int attr_mask,
89944450
SR		 4103 enum ib_qp_state cur_state,
4104 enum ib_qp_state new_state,
4105 const struct mlx5_ib_modify_qp *ucmd,
50aec2c3 4106 struct mlx5_ib_modify_qp_resp *resp,
89944450 4107 struct ib_udata *udata)
e126ba97 4108{
427c1e7b 4109 static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
4110 [MLX5_QP_STATE_RST] = {
4111 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4112 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4113 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP,
4114 },
4115 [MLX5_QP_STATE_INIT] = {
4116 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4117 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4118 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP,
4119 [MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP,
4120 },
4121 [MLX5_QP_STATE_RTR] = {
4122 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4123 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4124 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP,
4125 },
4126 [MLX5_QP_STATE_RTS] = {
4127 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4128 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4129 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP,
4130 },
4131 [MLX5_QP_STATE_SQD] = {
4132 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4133 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
021c1f24 4134 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQD_RTS_QP,
427c1e7b 4135 },
4136 [MLX5_QP_STATE_SQER] = {
4137 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4138 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4139 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP,
4140 },
4141 [MLX5_QP_STATE_ERR] = {
4142 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
4143 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
4144 }
4145 };
4146
e126ba97
EC		 4147 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4148 struct mlx5_ib_qp *qp = to_mqp(ibqp);
19098df2 4149 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
e126ba97 4150 struct mlx5_ib_cq *send_cq, *recv_cq;
e126ba97
EC		 4151 struct mlx5_ib_pd *pd;
4152 enum mlx5_qp_state mlx5_cur, mlx5_new;
f18e26af 4153 void *qpc, *pri_path, *alt_path;
cfc1a89e 4154 enum mlx5_qp_optpar optpar = 0;
d14133dd 4155 u32 set_id = 0;
e126ba97
EC		 4156 int mlx5_st;
4157 int err;
427c1e7b 4158 u16 op;
13eab21f 4159 u8 tx_affinity = 0;
e126ba97 4160
7aede1a2 4161 mlx5_st = to_mlx5_st(qp->type);
55de9a77
LR		 4162 if (mlx5_st < 0)
4163 return -EINVAL;
4164
f18e26af
LR		 4165 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
4166 if (!qpc)
e126ba97
EC		 4167 return -ENOMEM;
4168
029e88fd 4169 pd = to_mpd(qp->ibqp.pd);
f18e26af 4170 MLX5_SET(qpc, qpc, st, mlx5_st);
e126ba97
EC		 4171
4172 if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
f18e26af 4173 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
e126ba97
EC		 4174 } else {
4175 switch (attr->path_mig_state) {
4176 case IB_MIG_MIGRATED:
f18e26af 4177 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
e126ba97
EC		 4178 break;
4179 case IB_MIG_REARM:
f18e26af 4180 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
e126ba97
EC		 4181 break;
4182 case IB_MIG_ARMED:
f18e26af 4183 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
e126ba97
EC		 4184 break;
4185 }
4186 }
4187
cfc1a89e 4188 tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
5163b274
MG		 4189 cur_state == IB_QPS_RESET &&
4190 new_state == IB_QPS_INIT, udata);
f18e26af
LR		 4191
4192 MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
4193 if (tx_affinity && new_state == IB_QPS_RTR &&
4194 MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
4195 optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
13eab21f 4196
9ecf6ac1 4197 if (is_sqp(qp->type)) {
f18e26af
LR		 4198 MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
4199 MLX5_SET(qpc, qpc, log_msg_max, 8);
9ecf6ac1 4200 } else if ((qp->type == IB_QPT_UD &&
2be08c30 4201 !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
9ecf6ac1 4202 qp->type == MLX5_IB_QPT_REG_UMR) {
f18e26af
LR		 4203 MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
4204 MLX5_SET(qpc, qpc, log_msg_max, 12);
e126ba97
EC		 4205 } else if (attr_mask & IB_QP_PATH_MTU) {
4206 if (attr->path_mtu < IB_MTU_256 ||
4207 attr->path_mtu > IB_MTU_4096) {
4208 mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
4209 err = -EINVAL;
4210 goto out;
4211 }
f18e26af
LR		 4212 MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
4213 MLX5_SET(qpc, qpc, log_msg_max,
4214 MLX5_CAP_GEN(dev->mdev, log_max_msg));
e126ba97
EC		 4215 }
4216
4217 if (attr_mask & IB_QP_DEST_QPN)
f18e26af
LR		 4218 MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
4219
4220 pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4221 alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
e126ba97
EC		 4222
4223 if (attr_mask & IB_QP_PKEY_INDEX)
f18e26af 4224 MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
e126ba97
EC		 4225
4226 /* todo implement counter_index functionality */
4227
9ecf6ac1 4228 if (is_sqp(qp->type))
f18e26af 4229 MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
e126ba97
EC		 4230
4231 if (attr_mask & IB_QP_PORT)
f18e26af 4232 MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
e126ba97
EC		 4233
4234 if (attr_mask & IB_QP_AV) {
f18e26af
LR		 4235 err = mlx5_set_path(dev, qp, &attr->ah_attr, pri_path,
4236 attr_mask & IB_QP_PORT ? attr->port_num :
4237 qp->port,
f879ee8d 4238 attr_mask, 0, attr, false);
e126ba97
EC		 4239 if (err)
4240 goto out;
4241 }
4242
4243 if (attr_mask & IB_QP_TIMEOUT)
f18e26af 4244 MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
e126ba97
EC		 4245
4246 if (attr_mask & IB_QP_ALT_PATH) {
f18e26af 4247 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, alt_path,
f879ee8d 4248 attr->alt_port_num,
f18e26af
LR		 4249 attr_mask | IB_QP_PKEY_INDEX |
4250 IB_QP_TIMEOUT,
f879ee8d 4251 0, attr, true);
e126ba97
EC		 4252 if (err)
4253 goto out;
4254 }
4255
9ecf6ac1 4256 get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
89ea94a7 4257 &send_cq, &recv_cq);
e126ba97 4258
f18e26af
LR		 4259 MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
4260 if (send_cq)
4261 MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
4262 if (recv_cq)
4263 MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
4264
4265 MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
e126ba97
EC		 4266
4267 if (attr_mask & IB_QP_RNR_RETRY)
f18e26af 4268 MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
e126ba97
EC		 4269
4270 if (attr_mask & IB_QP_RETRY_CNT)
f18e26af 4271 MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
e126ba97 4272
f18e26af
LR		 4273 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
4274 MLX5_SET(qpc, qpc, log_sra_max, ilog2(attr->max_rd_atomic));
e126ba97
EC		 4275
4276 if (attr_mask & IB_QP_SQ_PSN)
f18e26af 4277 MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
e126ba97 4278
f18e26af
LR		 4279 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
4280 MLX5_SET(qpc, qpc, log_rra_max,
4281 ilog2(attr->max_dest_rd_atomic));
e126ba97 4282
a60109dc 4283 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
f18e26af 4284 err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
a60109dc
YC		 4285 if (err)
4286 goto out;
a60109dc 4287 }
e126ba97
EC		 4288
4289 if (attr_mask & IB_QP_MIN_RNR_TIMER)
f18e26af 4290 MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
e126ba97
EC		 4291
4292 if (attr_mask & IB_QP_RQ_PSN)
f18e26af 4293 MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
e126ba97
EC		 4294
4295 if (attr_mask & IB_QP_QKEY)
f18e26af 4296 MLX5_SET(qpc, qpc, q_key, attr->qkey);
e126ba97
EC		 4297
4298 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
f18e26af 4299 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
e126ba97 4300
0837e86a
MB		 4301 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4302 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
4303 qp->port) - 1;
c2e53b2c
YH		 4304
4305 /* Underlay port should be used - index 0 function per port */
2be08c30 4306 if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
c2e53b2c
YH		 4307 port_num = 0;
4308
d14133dd
MZ		 4309 if (ibqp->counter)
4310 set_id = ibqp->counter->id;
4311 else
3e1f000f 4312 set_id = mlx5_ib_get_counters_id(dev, port_num);
f18e26af 4313 MLX5_SET(qpc, qpc, counter_set_id, set_id);
0837e86a
MB		 4314 }
4315
e126ba97 4316 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
f18e26af 4317 MLX5_SET(qpc, qpc, rlky, 1);
e126ba97 4318
2be08c30 4319 if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
f18e26af 4320 MLX5_SET(qpc, qpc, deth_sqpn, 1);
e126ba97
EC		 4321
4322 mlx5_cur = to_mlx5_state(cur_state);
4323 mlx5_new = to_mlx5_state(new_state);
e126ba97 4324
427c1e7b 4325 if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
5d414b17
DC		 4326 !optab[mlx5_cur][mlx5_new]) {
4327 err = -EINVAL;
427c1e7b 4328 goto out;
5d414b17 4329 }
427c1e7b 4330
4331 op = optab[mlx5_cur][mlx5_new];
cfc1a89e 4332 optpar |= ib_mask_to_mlx5_opt(attr_mask);
e126ba97 4333 optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
ad5f8e96 4334
9ecf6ac1 4335 if (qp->type == IB_QPT_RAW_PACKET ||
2be08c30 4336 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
0680efa2
AV		 4337 struct mlx5_modify_raw_qp_param raw_qp_param = {};
4338
4339 raw_qp_param.operation = op;
eb49ab0c 4340 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
d14133dd 4341 raw_qp_param.rq_q_ctr_id = set_id;
eb49ab0c
AV		 4342 raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
4343 }
7d29f349 4344
d5ed8ac3
MB		 4345 if (attr_mask & IB_QP_PORT)
4346 raw_qp_param.port = attr->port_num;
4347
7d29f349 4348 if (attr_mask & IB_QP_RATE_LIMIT) {
61147f39
BW		 4349 raw_qp_param.rl.rate = attr->rate_limit;
4350
4351 if (ucmd->burst_info.max_burst_sz) {
4352 if (attr->rate_limit &&
4353 MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
4354 raw_qp_param.rl.max_burst_sz =
4355 ucmd->burst_info.max_burst_sz;
4356 } else {
4357 err = -EINVAL;
4358 goto out;
4359 }
4360 }
4361
4362 if (ucmd->burst_info.typical_pkt_sz) {
4363 if (attr->rate_limit &&
4364 MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
4365 raw_qp_param.rl.typical_pkt_sz =
4366 ucmd->burst_info.typical_pkt_sz;
4367 } else {
4368 err = -EINVAL;
4369 goto out;
4370 }
4371 }
4372
7d29f349
BW		 4373 raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
4374 }
4375
13eab21f 4376 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
0680efa2 4377 } else {
50aec2c3
LR		 4378 if (udata) {
4379 /* For the kernel flows, the resp will stay zero */
4380 resp->ece_options =
4381 MLX5_CAP_GEN(dev->mdev, ece_support) ?
4382 ucmd->ece_options : 0;
4383 resp->response_length = sizeof(*resp);
4384 }
5f62a521 4385 err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
50aec2c3 4386 &resp->ece_options);
0680efa2
AV		 4387 }
4388
e126ba97
EC		 4389 if (err)
4390 goto out;
4391
4392 qp->state = new_state;
4393
4394 if (attr_mask & IB_QP_ACCESS_FLAGS)
19098df2 4395 qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
e126ba97 4396 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
19098df2 4397 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
e126ba97
EC		 4398 if (attr_mask & IB_QP_PORT)
4399 qp->port = attr->port_num;
4400 if (attr_mask & IB_QP_ALT_PATH)
19098df2 4401 qp->trans_qp.alt_port = attr->alt_port_num;
e126ba97
EC		 4402
4403 /*
4404 * If we moved a kernel QP to RESET, clean up all old CQ
4405 * entries and reinitialize the QP.
4406 */
75a45982 4407 if (new_state == IB_QPS_RESET &&
9ecf6ac1 4408 !ibqp->uobject && qp->type != IB_QPT_XRC_TGT) {
19098df2 4409 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 4410 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
4411 if (send_cq != recv_cq)
19098df2 4412 mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
e126ba97
EC		 4413
4414 qp->rq.head = 0;
4415 qp->rq.tail = 0;
4416 qp->sq.head = 0;
4417 qp->sq.tail = 0;
4418 qp->sq.cur_post = 0;
34f4c955
GL		 4419 if (qp->sq.wqe_cnt)
4420 qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
950bf4f1 4421 qp->sq.last_poll = 0;
e126ba97
EC		 4422 qp->db.db[MLX5_RCV_DBR] = 0;
4423 qp->db.db[MLX5_SND_DBR] = 0;
4424 }
4425
d14133dd
MZ		 4426 if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
4427 err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
4428 if (!err)
4429 qp->counter_pending = 0;
4430 }
4431
e126ba97 4432out:
f18e26af 4433 kfree(qpc);
e126ba97
EC		 4434 return err;
4435}
4436
c32a4f29
MS		 4437static inline bool is_valid_mask(int mask, int req, int opt)
4438{
4439 if ((mask & req) != req)
4440 return false;
4441
4442 if (mask & ~(req | opt))
4443 return false;
4444
4445 return true;
4446}
4447
4448/* check valid transition for driver QP types
4449 * for now the only QP type that this function supports is DCI
4450 */
4451static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
4452 enum ib_qp_attr_mask attr_mask)
4453{
4454 int req = IB_QP_STATE;
4455 int opt = 0;
4456
99ed748e
MS		 4457 if (new_state == IB_QPS_RESET) {
4458 return is_valid_mask(attr_mask, req, opt);
4459 } else if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
c32a4f29
MS		 4460 req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
4461 return is_valid_mask(attr_mask, req, opt);
4462 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4463 opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
4464 return is_valid_mask(attr_mask, req, opt);
4465 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4466 req |= IB_QP_PATH_MTU;
5ec0304c 4467 opt = IB_QP_PKEY_INDEX | IB_QP_AV;
c32a4f29
MS		 4468 return is_valid_mask(attr_mask, req, opt);
4469 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4470 req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4471 IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
4472 opt = IB_QP_MIN_RNR_TIMER;
4473 return is_valid_mask(attr_mask, req, opt);
4474 } else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
4475 opt = IB_QP_MIN_RNR_TIMER;
4476 return is_valid_mask(attr_mask, req, opt);
4477 } else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
4478 return is_valid_mask(attr_mask, req, opt);
4479 }
4480 return false;
4481}
4482
776a3906
MS		 4483/* mlx5_ib_modify_dct: modify a DCT QP
4484 * valid transitions are:
4485 * RESET to INIT: must set access_flags, pkey_index and port
4486 * INIT to RTR : must set min_rnr_timer, tclass, flow_label,
4487 * mtu, gid_index and hop_limit
4488 * Other transitions and attributes are illegal
4489 */
4490static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
a645a89d
LR		 4491 int attr_mask, struct mlx5_ib_modify_qp *ucmd,
4492 struct ib_udata *udata)
776a3906
MS		 4493{
4494 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4495 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4496 enum ib_qp_state cur_state, new_state;
776a3906
MS		 4497 int required = IB_QP_STATE;
4498 void *dctc;
71cab8ef 4499 int err;
776a3906
MS		 4500
4501 if (!(attr_mask & IB_QP_STATE))
4502 return -EINVAL;
4503
4504 cur_state = qp->state;
4505 new_state = attr->qp_state;
4506
4507 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
a645a89d
LR		 4508 if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
4509 /*
4510 * DCT doesn't initialize QP till modify command is executed,
4511 * so we need to overwrite previously set ECE field if user
4512 * provided any value except zero, which means not set/not
4513 * valid.
4514 */
4515 MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
4516
776a3906 4517 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3e1f000f
PP		 4518 u16 set_id;
4519
776a3906
MS		 4520 required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
4521 if (!is_valid_mask(attr_mask, required, 0))
4522 return -EINVAL;
4523
4524 if (attr->port_num == 0 ||
746aa3c8 4525 attr->port_num > dev->num_ports) {
776a3906
MS		 4526 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4527 attr->port_num, dev->num_ports);
4528 return -EINVAL;
4529 }
4530 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
4531 MLX5_SET(dctc, dctc, rre, 1);
4532 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
4533 MLX5_SET(dctc, dctc, rwe, 1);
4534 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
a60109dc
YC		 4535 int atomic_mode;
4536
4537 atomic_mode = get_atomic_mode(dev, MLX5_IB_QPT_DCT);
4538 if (atomic_mode < 0)
776a3906 4539 return -EOPNOTSUPP;
a60109dc
YC		 4540
4541 MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
776a3906 4542 MLX5_SET(dctc, dctc, rae, 1);
776a3906
MS		 4543 }
4544 MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
7c4b1ab9
MZ		 4545 if (mlx5_lag_is_active(dev->mdev))
4546 MLX5_SET(dctc, dctc, port,
4547 get_tx_affinity_rr(dev, udata));
4548 else
4549 MLX5_SET(dctc, dctc, port, attr->port_num);
3e1f000f
PP		 4550
4551 set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
4552 MLX5_SET(dctc, dctc, counter_set_id, set_id);
776a3906
MS		 4553 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4554 struct mlx5_ib_modify_qp_resp resp = {};
a645a89d
LR		 4555 u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
4556 u32 min_resp_len = offsetofend(typeof(resp), dctn);
776a3906
MS		 4557
4558 if (udata->outlen < min_resp_len)
4559 return -EINVAL;
a645a89d
LR		 4560 /*
4561 * If we don't have enough space for the ECE options,
4562 * simply indicate it with resp.response_length.
4563 */
4564 resp.response_length = (udata->outlen < sizeof(resp)) ?
4565 min_resp_len :
4566 sizeof(resp);
4567
776a3906
MS		 4568 required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
4569 if (!is_valid_mask(attr_mask, required, 0))
4570 return -EINVAL;
4571 MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
4572 MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
4573 MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
4574 MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
4575 MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
4576 MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
1ab52ac1
PH		 4577 if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
4578 MLX5_SET(dctc, dctc, eth_prio, attr->ah_attr.sl & 0x7);
776a3906 4579
333fbaa0 4580 err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,
c5ae1954
YH		 4581 MLX5_ST_SZ_BYTES(create_dct_in), out,
4582 sizeof(out));
31803e59 4583 err = mlx5_cmd_check(dev->mdev, err, qp->dct.in, out);
776a3906
MS		 4584 if (err)
4585 return err;
4586 resp.dctn = qp->dct.mdct.mqp.qpn;
a645a89d
LR		 4587 if (MLX5_CAP_GEN(dev->mdev, ece_support))
4588 resp.ece_options = MLX5_GET(create_dct_out, out, ece);
776a3906
MS		 4589 err = ib_copy_to_udata(udata, &resp, resp.response_length);
4590 if (err) {
333fbaa0 4591 mlx5_core_destroy_dct(dev, &qp->dct.mdct);
776a3906
MS		 4592 return err;
4593 }
4594 } else {
4595 mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4596 return -EINVAL;
4597 }
71cab8ef
LR		 4598
4599 qp->state = new_state;
4600 return 0;
776a3906
MS		 4601}
4602
2614488d 4603static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
9ecf6ac1 4604 struct mlx5_ib_qp *qp)
2614488d
MB		 4605{
4606 if (dev->profile != &raw_eth_profile)
4607 return true;
4608
9ecf6ac1 4609 if (qp->type == IB_QPT_RAW_PACKET || qp->type == MLX5_IB_QPT_REG_UMR)
2614488d
MB		 4610 return true;
4611
4612 /* Internal QP used for wc testing, with NOPs in wq */
4613 if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
4614 return true;
4615
4616 return false;
4617}
4618
8de8482f
MG		 4619static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,
4620 int attr_mask, enum ib_qp_type qp_type)
4621{
4622 int log_max_ra_res;
4623 int log_max_ra_req;
4624
4625 if (qp_type == MLX5_IB_QPT_DCI) {
4626 log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4627 log_max_ra_res_dc);
4628 log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4629 log_max_ra_req_dc);
4630 } else {
4631 log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4632 log_max_ra_res_qp);
4633 log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4634 log_max_ra_req_qp);
4635 }
4636
4637 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
4638 attr->max_rd_atomic > log_max_ra_res) {
4639 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
4640 attr->max_rd_atomic);
4641 return false;
4642 }
4643
4644 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
4645 attr->max_dest_rd_atomic > log_max_ra_req) {
4646 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
4647 attr->max_dest_rd_atomic);
4648 return false;
4649 }
4650 return true;
4651}
4652
e126ba97
EC		 4653int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4654 int attr_mask, struct ib_udata *udata)
4655{
4656 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
50aec2c3 4657 struct mlx5_ib_modify_qp_resp resp = {};
e126ba97 4658 struct mlx5_ib_qp *qp = to_mqp(ibqp);
61147f39 4659 struct mlx5_ib_modify_qp ucmd = {};
d16e91da 4660 enum ib_qp_type qp_type;
e126ba97
EC		 4661 enum ib_qp_state cur_state, new_state;
4662 int err = -EINVAL;
e126ba97 4663
9ecf6ac1 4664 if (!mlx5_ib_modify_qp_allowed(dev, qp))
2614488d
MB		 4665 return -EOPNOTSUPP;
4666
26e990ba
JG		 4667 if (attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
4668 return -EOPNOTSUPP;
4669
28d61370
YH		 4670 if (ibqp->rwq_ind_tbl)
4671 return -ENOSYS;
4672
61147f39 4673 if (udata && udata->inlen) {
5f62a521 4674 if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
61147f39
BW		 4675 return -EINVAL;
4676
4677 if (udata->inlen > sizeof(ucmd) &&
4678 !ib_is_udata_cleared(udata, sizeof(ucmd),
4679 udata->inlen - sizeof(ucmd)))
4680 return -EOPNOTSUPP;
4681
4682 if (ib_copy_from_udata(&ucmd, udata,
4683 min(udata->inlen, sizeof(ucmd))))
4684 return -EFAULT;
4685
4686 if (ucmd.comp_mask ||
61147f39
BW		 4687 memchr_inv(&ucmd.burst_info.reserved, 0,
4688 sizeof(ucmd.burst_info.reserved)))
4689 return -EOPNOTSUPP;
5f62a521 4690
61147f39
BW		 4691 }
4692
9ecf6ac1 4693 if (qp->type == IB_QPT_GSI)
d16e91da
HE		 4694 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
4695
9ecf6ac1 4696 qp_type = (qp->type == MLX5_IB_QPT_HW_GSI) ? IB_QPT_GSI : qp->type;
c32a4f29 4697
a645a89d
LR		 4698 if (qp_type == MLX5_IB_QPT_DCT)
4699 return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &ucmd, udata);
d16e91da 4700
e126ba97
EC		 4701 mutex_lock(&qp->mutex);
4702
4703 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
4704 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
4705
2be08c30 4706 if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
c2e53b2c
YH		 4707 if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
4708 mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
4709 attr_mask);
4710 goto out;
4711 }
4712 } else if (qp_type != MLX5_IB_QPT_REG_UMR &&
c32a4f29 4713 qp_type != MLX5_IB_QPT_DCI &&
d31131bb
KH		 4714 !ib_modify_qp_is_ok(cur_state, new_state, qp_type,
4715 attr_mask)) {
158abf86 4716 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
9ecf6ac1 4717 cur_state, new_state, qp->type, attr_mask);
e126ba97 4718 goto out;
c32a4f29
MS		 4719 } else if (qp_type == MLX5_IB_QPT_DCI &&
4720 !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
4721 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4722 cur_state, new_state, qp_type, attr_mask);
4723 goto out;
158abf86 4724 }
e126ba97
EC		 4725
4726 if ((attr_mask & IB_QP_PORT) &&
938fe83c 4727 (attr->port_num == 0 ||
508562d6 4728 attr->port_num > dev->num_ports)) {
158abf86
HE		 4729 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4730 attr->port_num, dev->num_ports);
e126ba97 4731 goto out;
158abf86 4732 }
e126ba97 4733
2019d70e
PP		 4734 if ((attr_mask & IB_QP_PKEY_INDEX) &&
4735 attr->pkey_index >= dev->pkey_table_len) {
4736 mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
4737 goto out;
e126ba97
EC		 4738 }
4739
8de8482f 4740 if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))
e126ba97
EC		 4741 goto out;
4742
4743 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
4744 err = 0;
4745 goto out;
4746 }
4747
61147f39 4748 err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
50aec2c3
LR		 4749 new_state, &ucmd, &resp, udata);
4750
4751 /* resp.response_length is set in ECE supported flows only */
4752 if (!err && resp.response_length &&
4753 udata->outlen >= resp.response_length)
6512f11d
LR		 4754 /* Return -EFAULT to the user and expect him to destroy QP. */
4755 err = ib_copy_to_udata(udata, &resp, resp.response_length);
e126ba97
EC		 4756
4757out:
4758 mutex_unlock(&qp->mutex);
4759 return err;
4760}
4761
e126ba97
EC		 4762static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4763{
4764 switch (mlx5_state) {
4765 case MLX5_QP_STATE_RST: return IB_QPS_RESET;
4766 case MLX5_QP_STATE_INIT: return IB_QPS_INIT;
4767 case MLX5_QP_STATE_RTR: return IB_QPS_RTR;
4768 case MLX5_QP_STATE_RTS: return IB_QPS_RTS;
4769 case MLX5_QP_STATE_SQ_DRAINING:
4770 case MLX5_QP_STATE_SQD: return IB_QPS_SQD;
4771 case MLX5_QP_STATE_SQER: return IB_QPS_SQE;
4772 case MLX5_QP_STATE_ERR: return IB_QPS_ERR;
4773 default: return -1;
4774 }
4775}
4776
4777static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4778{
4779 switch (mlx5_mig_state) {
4780 case MLX5_QP_PM_ARMED: return IB_MIG_ARMED;
4781 case MLX5_QP_PM_REARM: return IB_MIG_REARM;
4782 case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4783 default: return -1;
4784 }
4785}
4786
38349389 4787static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
70bd7fb8 4788 struct rdma_ah_attr *ah_attr, void *path)
e126ba97 4789{
70bd7fb8
LR		 4790 int port = MLX5_GET(ads, path, vhca_port_num);
4791 int static_rate;
e126ba97 4792
d8966fcd 4793 memset(ah_attr, 0, sizeof(*ah_attr));
e126ba97 4794
70bd7fb8 4795 if (!port || port > ibdev->num_ports)
e126ba97
EC		 4796 return;
4797
70bd7fb8 4798 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port);
ae59c3f0 4799
70bd7fb8
LR		 4800 rdma_ah_set_port_num(ah_attr, port);
4801 rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));
d8966fcd 4802
70bd7fb8
LR		 4803 rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));
4804 rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));
2d7e3ff7 4805
70bd7fb8 4806 static_rate = MLX5_GET(ads, path, stat_rate);
6fe6e568 4807 rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate));
70bd7fb8 4808 if (MLX5_GET(ads, path, grh) ||
2d7e3ff7 4809 ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
70bd7fb8
LR		 4810 rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),
4811 MLX5_GET(ads, path, src_addr_index),
4812 MLX5_GET(ads, path, hop_limit),
4813 MLX5_GET(ads, path, tclass));
d4433557 4814 rdma_ah_set_dgid_raw(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));
e126ba97
EC		 4815 }
4816}
4817
6d2f89df 4818static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4819 struct mlx5_ib_sq *sq,
4820 u8 *sq_state)
4821{
6d2f89df 4822 int err;
4823
28160771 4824 err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state);
6d2f89df 4825 if (err)
4826 goto out;
6d2f89df 4827 sq->state = *sq_state;
4828
4829out:
6d2f89df 4830 return err;
4831}
4832
4833static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4834 struct mlx5_ib_rq *rq,
4835 u8 *rq_state)
4836{
4837 void *out;
4838 void *rqc;
4839 int inlen;
4840 int err;
4841
4842 inlen = MLX5_ST_SZ_BYTES(query_rq_out);
1b9a07ee 4843 out = kvzalloc(inlen, GFP_KERNEL);
6d2f89df 4844 if (!out)
4845 return -ENOMEM;
4846
4847 err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4848 if (err)
4849 goto out;
4850
4851 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4852 *rq_state = MLX5_GET(rqc, rqc, state);
4853 rq->state = *rq_state;
4854
4855out:
4856 kvfree(out);
4857 return err;
4858}
4859
4860static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4861 struct mlx5_ib_qp *qp, u8 *qp_state)
4862{
4863 static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4864 [MLX5_RQC_STATE_RST] = {
4865 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4866 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4867 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD,
4868 [MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4869 },
4870 [MLX5_RQC_STATE_RDY] = {
c94e272b 4871 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
6d2f89df 4872 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4873 [MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4874 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
4875 },
4876 [MLX5_RQC_STATE_ERR] = {
4877 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4878 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4879 [MLX5_SQC_STATE_ERR] = IB_QPS_ERR,
4880 [MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4881 },
4882 [MLX5_RQ_STATE_NA] = {
c94e272b 4883 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE,
6d2f89df 4884 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4885 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4886 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
4887 },
4888 };
4889
4890 *qp_state = sqrq_trans[rq_state][sq_state];
4891
4892 if (*qp_state == MLX5_QP_STATE_BAD) {
4893 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4894 qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4895 qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4896 return -EINVAL;
4897 }
4898
4899 if (*qp_state == MLX5_QP_STATE)
4900 *qp_state = qp->state;
4901
4902 return 0;
4903}
4904
4905static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4906 struct mlx5_ib_qp *qp,
4907 u8 *raw_packet_qp_state)
4908{
4909 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4910 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4911 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4912 int err;
4913 u8 sq_state = MLX5_SQ_STATE_NA;
4914 u8 rq_state = MLX5_RQ_STATE_NA;
4915
4916 if (qp->sq.wqe_cnt) {
4917 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4918 if (err)
4919 return err;
4920 }
4921
4922 if (qp->rq.wqe_cnt) {
4923 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4924 if (err)
4925 return err;
4926 }
4927
4928 return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4929 raw_packet_qp_state);
4930}
4931
4932static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4933 struct ib_qp_attr *qp_attr)
e126ba97 4934{
09a7d9ec 4935 int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
70bd7fb8 4936 void *qpc, *pri_path, *alt_path;
09a7d9ec 4937 u32 *outb;
70bd7fb8 4938 int err;
e126ba97 4939
09a7d9ec 4940 outb = kzalloc(outlen, GFP_KERNEL);
6d2f89df 4941 if (!outb)
4942 return -ENOMEM;
4943
8067fd8b
PH		 4944 err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
4945 false);
e126ba97 4946 if (err)
6d2f89df 4947 goto out;
e126ba97 4948
70bd7fb8
LR		 4949 qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
4950
4951 qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
4952 if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
4953 qp_attr->sq_draining = 1;
4954
4955 qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
4956 qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
4957 qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
4958 qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
4959 qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
4960 qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
09a7d9ec 4961
70bd7fb8
LR		 4962 if (MLX5_GET(qpc, qpc, rre))
4963 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
4964 if (MLX5_GET(qpc, qpc, rwe))
4965 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_WRITE;
4966 if (MLX5_GET(qpc, qpc, rae))
4967 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;
e126ba97 4968
70bd7fb8
LR		 4969 qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
4970 qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
4971 qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
4972 qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
4973 qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
4974
4975 pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4976 alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
e126ba97 4977
9ecf6ac1
MG		 4978 if (qp->type == IB_QPT_RC || qp->type == IB_QPT_UC ||
4979 qp->type == IB_QPT_XRC_INI || qp->type == IB_QPT_XRC_TGT) {
70bd7fb8
LR		 4980 to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
4981 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
4982 qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
4983 qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
4984 }
4985
4986 qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
4987 qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
4988 qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
4989 qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
6d2f89df 4990
4991out:
4992 kfree(outb);
4993 return err;
4994}
4995
776a3906
MS		 4996static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
4997 struct ib_qp_attr *qp_attr, int qp_attr_mask,
4998 struct ib_qp_init_attr *qp_init_attr)
4999{
5000 struct mlx5_core_dct *dct = &mqp->dct.mdct;
5001 u32 *out;
5002 u32 access_flags = 0;
5003 int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
5004 void *dctc;
5005 int err;
5006 int supported_mask = IB_QP_STATE |
5007 IB_QP_ACCESS_FLAGS |
5008 IB_QP_PORT |
5009 IB_QP_MIN_RNR_TIMER |
5010 IB_QP_AV |
5011 IB_QP_PATH_MTU |
5012 IB_QP_PKEY_INDEX;
5013
5014 if (qp_attr_mask & ~supported_mask)
5015 return -EINVAL;
5016 if (mqp->state != IB_QPS_RTR)
5017 return -EINVAL;
5018
5019 out = kzalloc(outlen, GFP_KERNEL);
5020 if (!out)
5021 return -ENOMEM;
5022
333fbaa0 5023 err = mlx5_core_dct_query(dev, dct, out, outlen);
776a3906
MS		 5024 if (err)
5025 goto out;
5026
5027 dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
5028
5029 if (qp_attr_mask & IB_QP_STATE)
5030 qp_attr->qp_state = IB_QPS_RTR;
5031
5032 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
5033 if (MLX5_GET(dctc, dctc, rre))
5034 access_flags |= IB_ACCESS_REMOTE_READ;
5035 if (MLX5_GET(dctc, dctc, rwe))
5036 access_flags |= IB_ACCESS_REMOTE_WRITE;
5037 if (MLX5_GET(dctc, dctc, rae))
5038 access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5039 qp_attr->qp_access_flags = access_flags;
5040 }
5041
5042 if (qp_attr_mask & IB_QP_PORT)
5043 qp_attr->port_num = MLX5_GET(dctc, dctc, port);
5044 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
5045 qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
5046 if (qp_attr_mask & IB_QP_AV) {
5047 qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
5048 qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
5049 qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
5050 qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
5051 }
5052 if (qp_attr_mask & IB_QP_PATH_MTU)
5053 qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
5054 if (qp_attr_mask & IB_QP_PKEY_INDEX)
5055 qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
5056out:
5057 kfree(out);
5058 return err;
5059}
5060
6d2f89df 5061int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5062 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
5063{
5064 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
5065 struct mlx5_ib_qp *qp = to_mqp(ibqp);
5066 int err = 0;
5067 u8 raw_packet_qp_state;
5068
28d61370
YH		 5069 if (ibqp->rwq_ind_tbl)
5070 return -ENOSYS;
5071
9ecf6ac1 5072 if (qp->type == IB_QPT_GSI)
d16e91da
HE		 5073 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
5074 qp_init_attr);
5075
c2e53b2c
YH		 5076 /* Not all of output fields are applicable, make sure to zero them */
5077 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5078 memset(qp_attr, 0, sizeof(*qp_attr));
5079
7aede1a2 5080 if (unlikely(qp->type == MLX5_IB_QPT_DCT))
776a3906
MS		 5081 return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5082 qp_attr_mask, qp_init_attr);
5083
6d2f89df 5084 mutex_lock(&qp->mutex);
5085
9ecf6ac1 5086 if (qp->type == IB_QPT_RAW_PACKET ||
2be08c30 5087 qp->flags & IB_QP_CREATE_SOURCE_QPN) {
6d2f89df 5088 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5089 if (err)
5090 goto out;
5091 qp->state = raw_packet_qp_state;
5092 qp_attr->port_num = 1;
5093 } else {
5094 err = query_qp_attr(dev, qp, qp_attr);
5095 if (err)
5096 goto out;
5097 }
5098
5099 qp_attr->qp_state = qp->state;
e126ba97
EC		 5100 qp_attr->cur_qp_state = qp_attr->qp_state;
5101 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
5102 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
5103
5104 if (!ibqp->uobject) {
0540d814 5105 qp_attr->cap.max_send_wr = qp->sq.max_post;
e126ba97 5106 qp_attr->cap.max_send_sge = qp->sq.max_gs;
0540d814 5107 qp_init_attr->qp_context = ibqp->qp_context;
e126ba97
EC		 5108 } else {
5109 qp_attr->cap.max_send_wr = 0;
5110 qp_attr->cap.max_send_sge = 0;
5111 }
5112
9ecf6ac1 5113 qp_init_attr->qp_type = qp->type;
0540d814
NO		 5114 qp_init_attr->recv_cq = ibqp->recv_cq;
5115 qp_init_attr->send_cq = ibqp->send_cq;
5116 qp_init_attr->srq = ibqp->srq;
5117 qp_attr->cap.max_inline_data = qp->max_inline_data;
e126ba97
EC		 5118
5119 qp_init_attr->cap = qp_attr->cap;
5120
a8f3ea61 5121 qp_init_attr->create_flags = qp->flags;
051f2630 5122
e126ba97
EC		 5123 qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5124 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5125
e126ba97
EC		 5126out:
5127 mutex_unlock(&qp->mutex);
5128 return err;
5129}
5130
28ad5f65 5131int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
e126ba97 5132{
28ad5f65
LR		 5133 struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
5134 struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
e126ba97 5135
938fe83c 5136 if (!MLX5_CAP_GEN(dev->mdev, xrc))
28ad5f65 5137 return -EOPNOTSUPP;
e126ba97 5138
28ad5f65 5139 return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
e126ba97
EC		 5140}
5141
d0c45c85 5142int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
e126ba97
EC		 5143{
5144 struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5145 u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
e126ba97 5146
d0c45c85 5147 return mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
e126ba97 5148}
79b20a6c 5149
350d0e4c
YH		 5150static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5151{
5152 struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5153 struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
5154 struct ib_event event;
5155
5156 if (rwq->ibwq.event_handler) {
5157 event.device = rwq->ibwq.device;
5158 event.element.wq = &rwq->ibwq;
5159 switch (type) {
5160 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5161 event.event = IB_EVENT_WQ_FATAL;
5162 break;
5163 default:
5164 mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5165 return;
5166 }
5167
5168 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5169 }
5170}
5171
03404e8a
MG		 5172static int set_delay_drop(struct mlx5_ib_dev *dev)
5173{
5174 int err = 0;
5175
5176 mutex_lock(&dev->delay_drop.lock);
5177 if (dev->delay_drop.activate)
5178 goto out;
5179
333fbaa0 5180 err = mlx5_core_set_delay_drop(dev, dev->delay_drop.timeout);
03404e8a
MG		 5181 if (err)
5182 goto out;
5183
5184 dev->delay_drop.activate = true;
5185out:
5186 mutex_unlock(&dev->delay_drop.lock);
fe248c3a
MG		 5187
5188 if (!err)
5189 atomic_inc(&dev->delay_drop.rqs_cnt);
03404e8a
MG		 5190 return err;
5191}
5192
79b20a6c
YH		 5193static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5194 struct ib_wq_init_attr *init_attr)
5195{
5196 struct mlx5_ib_dev *dev;
4be6da1e 5197 int has_net_offloads;
79b20a6c 5198 __be64 *rq_pas0;
8256c69b 5199 int ts_format;
79b20a6c
YH		 5200 void *in;
5201 void *rqc;
5202 void *wq;
5203 int inlen;
5204 int err;
5205
5206 dev = to_mdev(pd->device);
5207
8256c69b
MG		 5208 ts_format = get_rq_ts_format(dev, to_mcq(init_attr->cq));
5209 if (ts_format < 0)
5210 return ts_format;
5211
79b20a6c 5212 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
1b9a07ee 5213 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5214 if (!in)
5215 return -ENOMEM;
5216
34d57585 5217 MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
79b20a6c
YH		 5218 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5219 MLX5_SET(rqc, rqc, mem_rq_type,
5220 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
8256c69b 5221 MLX5_SET(rqc, rqc, ts_format, ts_format);
79b20a6c
YH		 5222 MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5223 MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5224 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
5225 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
5226 wq = MLX5_ADDR_OF(rqc, rqc, wq);
ccc87087
NO		 5227 MLX5_SET(wq, wq, wq_type,
5228 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5229 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 5230 if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5231 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5232 mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5233 err = -EOPNOTSUPP;
5234 goto out;
5235 } else {
5236 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5237 }
5238 }
79b20a6c 5239 MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
ccc87087 5240 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
c16339b6
MZ		 5241 /*
5242 * In Firmware number of strides in each WQE is:
5243 * "512 * 2^single_wqe_log_num_of_strides"
5244 * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
5245 * accepted as 0 to 9
5246 */
5247 static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
5248 2, 3, 4, 5, 6, 7, 8, 9 };
ccc87087
NO		 5249 MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5250 MLX5_SET(wq, wq, log_wqe_stride_size,
5251 rwq->single_stride_log_num_of_bytes -
5252 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
c16339b6
MZ		 5253 MLX5_SET(wq, wq, log_wqe_num_of_strides,
5254 fw_map[rwq->log_num_strides -
5255 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
ccc87087 5256 }
79b20a6c
YH		 5257 MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5258 MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5259 MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5260 MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5261 MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5262 MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
4be6da1e 5263 has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
b1f74a84 5264 if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
4be6da1e 5265 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
b1f74a84
NO		 5266 mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5267 err = -EOPNOTSUPP;
5268 goto out;
5269 }
5270 } else {
5271 MLX5_SET(rqc, rqc, vsd, 1);
5272 }
4be6da1e
NO		 5273 if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5274 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5275 mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5276 err = -EOPNOTSUPP;
5277 goto out;
5278 }
5279 MLX5_SET(rqc, rqc, scatter_fcs, 1);
5280 }
03404e8a
MG		 5281 if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5282 if (!(dev->ib_dev.attrs.raw_packet_caps &
5283 IB_RAW_PACKET_CAP_DELAY_DROP)) {
5284 mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5285 err = -EOPNOTSUPP;
5286 goto out;
5287 }
5288 MLX5_SET(rqc, rqc, delay_drop_en, 1);
5289 }
79b20a6c 5290 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
aab8d396 5291 mlx5_ib_populate_pas(rwq->umem, 1UL << rwq->page_shift, rq_pas0, 0);
333fbaa0 5292 err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
03404e8a
MG		 5293 if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5294 err = set_delay_drop(dev);
5295 if (err) {
5296 mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5297 err);
333fbaa0 5298 mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
03404e8a
MG		 5299 } else {
5300 rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5301 }
5302 }
b1f74a84 5303out:
79b20a6c
YH		 5304 kvfree(in);
5305 return err;
5306}
5307
5308static int set_user_rq_size(struct mlx5_ib_dev *dev,
5309 struct ib_wq_init_attr *wq_init_attr,
5310 struct mlx5_ib_create_wq *ucmd,
5311 struct mlx5_ib_rwq *rwq)
5312{
5313 /* Sanity check RQ size before proceeding */
5314 if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5315 return -EINVAL;
5316
5317 if (!ucmd->rq_wqe_count)
5318 return -EINVAL;
5319
5320 rwq->wqe_count = ucmd->rq_wqe_count;
5321 rwq->wqe_shift = ucmd->rq_wqe_shift;
0dfe4522
LR		 5322 if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
5323 return -EINVAL;
5324
79b20a6c
YH		 5325 rwq->log_rq_stride = rwq->wqe_shift;
5326 rwq->log_rq_size = ilog2(rwq->wqe_count);
5327 return 0;
5328}
5329
c16339b6
MZ		 5330static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
5331{
5332 if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5333 (log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5334 return false;
5335
5336 if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
5337 (log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5338 return false;
5339
5340 return true;
5341}
5342
79b20a6c
YH		 5343static int prepare_user_rq(struct ib_pd *pd,
5344 struct ib_wq_init_attr *init_attr,
5345 struct ib_udata *udata,
5346 struct mlx5_ib_rwq *rwq)
5347{
5348 struct mlx5_ib_dev *dev = to_mdev(pd->device);
5349 struct mlx5_ib_create_wq ucmd = {};
5350 int err;
5351 size_t required_cmd_sz;
5352
70c1430f
LR		 5353 required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,
5354 single_stride_log_num_of_bytes);
79b20a6c
YH		 5355 if (udata->inlen < required_cmd_sz) {
5356 mlx5_ib_dbg(dev, "invalid inlen\n");
5357 return -EINVAL;
5358 }
5359
5360 if (udata->inlen > sizeof(ucmd) &&
5361 !ib_is_udata_cleared(udata, sizeof(ucmd),
5362 udata->inlen - sizeof(ucmd))) {
5363 mlx5_ib_dbg(dev, "inlen is not supported\n");
5364 return -EOPNOTSUPP;
5365 }
5366
5367 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5368 mlx5_ib_dbg(dev, "copy failed\n");
5369 return -EFAULT;
5370 }
5371
ccc87087 5372 if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
79b20a6c
YH		 5373 mlx5_ib_dbg(dev, "invalid comp mask\n");
5374 return -EOPNOTSUPP;
ccc87087
NO		 5375 } else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5376 if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5377 mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5378 return -EOPNOTSUPP;
5379 }
5380 if ((ucmd.single_stride_log_num_of_bytes <
5381 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5382 (ucmd.single_stride_log_num_of_bytes >
5383 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5384 mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5385 ucmd.single_stride_log_num_of_bytes,
5386 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5387 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5388 return -EINVAL;
5389 }
c16339b6
MZ		 5390 if (!log_of_strides_valid(dev,
5391 ucmd.single_wqe_log_num_of_strides)) {
5392 mlx5_ib_dbg(
5393 dev,
5394 "Invalid log num strides (%u. Range is %u - %u)\n",
5395 ucmd.single_wqe_log_num_of_strides,
5396 MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
5397 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
5398 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5399 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
ccc87087
NO		 5400 return -EINVAL;
5401 }
5402 rwq->single_stride_log_num_of_bytes =
5403 ucmd.single_stride_log_num_of_bytes;
5404 rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5405 rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5406 rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
79b20a6c
YH		 5407 }
5408
5409 err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
5410 if (err) {
5411 mlx5_ib_dbg(dev, "err %d\n", err);
5412 return err;
5413 }
5414
b0ea0fa5 5415 err = create_user_rq(dev, pd, udata, rwq, &ucmd);
79b20a6c
YH		 5416 if (err) {
5417 mlx5_ib_dbg(dev, "err %d\n", err);
645ba597 5418 return err;
79b20a6c
YH		 5419 }
5420
5421 rwq->user_index = ucmd.user_index;
5422 return 0;
5423}
5424
5425struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5426 struct ib_wq_init_attr *init_attr,
5427 struct ib_udata *udata)
5428{
5429 struct mlx5_ib_dev *dev;
5430 struct mlx5_ib_rwq *rwq;
5431 struct mlx5_ib_create_wq_resp resp = {};
5432 size_t min_resp_len;
5433 int err;
5434
5435 if (!udata)
5436 return ERR_PTR(-ENOSYS);
5437
70c1430f 5438 min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);
79b20a6c
YH		 5439 if (udata->outlen && udata->outlen < min_resp_len)
5440 return ERR_PTR(-EINVAL);
5441
ba80013f
MG		 5442 if (!capable(CAP_SYS_RAWIO) &&
5443 init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP)
5444 return ERR_PTR(-EPERM);
5445
79b20a6c
YH		 5446 dev = to_mdev(pd->device);
5447 switch (init_attr->wq_type) {
5448 case IB_WQT_RQ:
5449 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5450 if (!rwq)
5451 return ERR_PTR(-ENOMEM);
5452 err = prepare_user_rq(pd, init_attr, udata, rwq);
5453 if (err)
5454 goto err;
5455 err = create_rq(rwq, pd, init_attr);
5456 if (err)
5457 goto err_user_rq;
5458 break;
5459 default:
5460 mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5461 init_attr->wq_type);
5462 return ERR_PTR(-EINVAL);
5463 }
5464
350d0e4c 5465 rwq->ibwq.wq_num = rwq->core_qp.qpn;
79b20a6c
YH		 5466 rwq->ibwq.state = IB_WQS_RESET;
5467 if (udata->outlen) {
70c1430f
LR		 5468 resp.response_length = offsetofend(
5469 struct mlx5_ib_create_wq_resp, response_length);
79b20a6c
YH		 5470 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5471 if (err)
5472 goto err_copy;
5473 }
5474
350d0e4c
YH		 5475 rwq->core_qp.event = mlx5_ib_wq_event;
5476 rwq->ibwq.event_handler = init_attr->event_handler;
79b20a6c
YH		 5477 return &rwq->ibwq;
5478
5479err_copy:
333fbaa0 5480 mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
79b20a6c 5481err_user_rq:
bdeacabd 5482 destroy_user_rq(dev, pd, rwq, udata);
79b20a6c
YH		 5483err:
5484 kfree(rwq);
5485 return ERR_PTR(err);
5486}
5487
add53535 5488int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
79b20a6c
YH		 5489{
5490 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5491 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
add53535 5492 int ret;
79b20a6c 5493
add53535
LR		 5494 ret = mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5495 if (ret)
5496 return ret;
bdeacabd 5497 destroy_user_rq(dev, wq->pd, rwq, udata);
79b20a6c 5498 kfree(rwq);
add53535 5499 return 0;
79b20a6c
YH		 5500}
5501
c0a6b5ec
LR		 5502int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
5503 struct ib_rwq_ind_table_init_attr *init_attr,
5504 struct ib_udata *udata)
c5f90929 5505{
c0a6b5ec
LR		 5506 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =
5507 to_mrwq_ind_table(ib_rwq_ind_table);
5508 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table->device);
c5f90929
YH		 5509 int sz = 1 << init_attr->log_ind_tbl_size;
5510 struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5511 size_t min_resp_len;
5512 int inlen;
5513 int err;
5514 int i;
5515 u32 *in;
5516 void *rqtc;
5517
5518 if (udata->inlen > 0 &&
5519 !ib_is_udata_cleared(udata, 0,
5520 udata->inlen))
c0a6b5ec 5521 return -EOPNOTSUPP;
c5f90929 5522
efd7f400
MG		 5523 if (init_attr->log_ind_tbl_size >
5524 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5525 mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5526 init_attr->log_ind_tbl_size,
5527 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
c0a6b5ec 5528 return -EINVAL;
efd7f400
MG		 5529 }
5530
70c1430f
LR		 5531 min_resp_len =
5532 offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);
c5f90929 5533 if (udata->outlen && udata->outlen < min_resp_len)
c0a6b5ec 5534 return -EINVAL;
c5f90929
YH		 5535
5536 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 5537 in = kvzalloc(inlen, GFP_KERNEL);
c0a6b5ec
LR		 5538 if (!in)
5539 return -ENOMEM;
c5f90929
YH		 5540
5541 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5542
5543 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5544 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5545
5546 for (i = 0; i < sz; i++)
5547 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5548
5deba86e
YH		 5549 rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
5550 MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
5551
c5f90929
YH		 5552 err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5553 kvfree(in);
c5f90929 5554 if (err)
c0a6b5ec 5555 return err;
c5f90929
YH		 5556
5557 rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5558 if (udata->outlen) {
70c1430f
LR		 5559 resp.response_length =
5560 offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
5561 response_length);
c5f90929
YH		 5562 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5563 if (err)
5564 goto err_copy;
5565 }
5566
c0a6b5ec 5567 return 0;
c5f90929
YH		 5568
5569err_copy:
5deba86e 5570 mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
c0a6b5ec 5571 return err;
c5f90929
YH		 5572}
5573
5574int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5575{
5576 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5577 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5578
c0a6b5ec 5579 return mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
c5f90929
YH		 5580}
5581
79b20a6c
YH		 5582int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5583 u32 wq_attr_mask, struct ib_udata *udata)
5584{
5585 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5586 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5587 struct mlx5_ib_modify_wq ucmd = {};
5588 size_t required_cmd_sz;
5589 int curr_wq_state;
5590 int wq_state;
5591 int inlen;
5592 int err;
5593 void *rqc;
5594 void *in;
5595
70c1430f 5596 required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);
79b20a6c
YH		 5597 if (udata->inlen < required_cmd_sz)
5598 return -EINVAL;
5599
5600 if (udata->inlen > sizeof(ucmd) &&
5601 !ib_is_udata_cleared(udata, sizeof(ucmd),
5602 udata->inlen - sizeof(ucmd)))
5603 return -EOPNOTSUPP;
5604
5605 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5606 return -EFAULT;
5607
5608 if (ucmd.comp_mask || ucmd.reserved)
5609 return -EOPNOTSUPP;
5610
5611 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 5612 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5613 if (!in)
5614 return -ENOMEM;
5615
5616 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5617
f9744288
LR		 5618 curr_wq_state = wq_attr->curr_wq_state;
5619 wq_state = wq_attr->wq_state;
79b20a6c
YH		 5620 if (curr_wq_state == IB_WQS_ERR)
5621 curr_wq_state = MLX5_RQC_STATE_ERR;
5622 if (wq_state == IB_WQS_ERR)
5623 wq_state = MLX5_RQC_STATE_ERR;
5624 MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
34d57585 5625 MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
79b20a6c
YH		 5626 MLX5_SET(rqc, rqc, state, wq_state);
5627
b1f74a84
NO		 5628 if (wq_attr_mask & IB_WQ_FLAGS) {
5629 if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5630 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5631 MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
c4526fe2 5632 mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
b1f74a84
NO		 5633 err = -EOPNOTSUPP;
5634 goto out;
5635 }
5636 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5637 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5638 MLX5_SET(rqc, rqc, vsd,
5639 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5640 }
b1383aa6
NO		 5641
5642 if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5643 mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5644 err = -EOPNOTSUPP;
5645 goto out;
5646 }
b1f74a84
NO		 5647 }
5648
23a6964e 5649 if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
3e1f000f
PP		 5650 u16 set_id;
5651
5652 set_id = mlx5_ib_get_counters_id(dev, 0);
23a6964e
MD		 5653 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5654 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5655 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
3e1f000f 5656 MLX5_SET(rqc, rqc, counter_set_id, set_id);
23a6964e 5657 } else
5a738b5d
JG		 5658 dev_info_once(
5659 &dev->ib_dev.dev,
5660 "Receive WQ counters are not supported on current FW\n");
23a6964e
MD		 5661 }
5662
e0b4b472 5663 err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in);
79b20a6c
YH		 5664 if (!err)
5665 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5666
b1f74a84
NO		 5667out:
5668 kvfree(in);
79b20a6c
YH		 5669 return err;
5670}
d0e84c0a
YH		 5671
5672struct mlx5_ib_drain_cqe {
5673 struct ib_cqe cqe;
5674 struct completion done;
5675};
5676
5677static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
5678{
5679 struct mlx5_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
5680 struct mlx5_ib_drain_cqe,
5681 cqe);
5682
5683 complete(&cqe->done);
5684}
5685
5686/* This function returns only once the drained WR was completed */
5687static void handle_drain_completion(struct ib_cq *cq,
5688 struct mlx5_ib_drain_cqe *sdrain,
5689 struct mlx5_ib_dev *dev)
5690{
5691 struct mlx5_core_dev *mdev = dev->mdev;
5692
5693 if (cq->poll_ctx == IB_POLL_DIRECT) {
5694 while (wait_for_completion_timeout(&sdrain->done, HZ / 10) <= 0)
5695 ib_process_cq_direct(cq, -1);
5696 return;
5697 }
5698
5699 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5700 struct mlx5_ib_cq *mcq = to_mcq(cq);
5701 bool triggered = false;
5702 unsigned long flags;
5703
5704 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
5705 /* Make sure that the CQ handler won't run if wasn't run yet */
5706 if (!mcq->mcq.reset_notify_added)
5707 mcq->mcq.reset_notify_added = 1;
5708 else
5709 triggered = true;
5710 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
5711
5712 if (triggered) {
5713 /* Wait for any scheduled/running task to be ended */
5714 switch (cq->poll_ctx) {
5715 case IB_POLL_SOFTIRQ:
5716 irq_poll_disable(&cq->iop);
5717 irq_poll_enable(&cq->iop);
5718 break;
5719 case IB_POLL_WORKQUEUE:
5720 cancel_work_sync(&cq->work);
5721 break;
5722 default:
5723 WARN_ON_ONCE(1);
5724 }
5725 }
5726
5727 /* Run the CQ handler - this makes sure that the drain WR will
5728 * be processed if wasn't processed yet.
5729 */
4e0e2ea1 5730 mcq->mcq.comp(&mcq->mcq, NULL);
d0e84c0a
YH		 5731 }
5732
5733 wait_for_completion(&sdrain->done);
5734}
5735
5736void mlx5_ib_drain_sq(struct ib_qp *qp)
5737{
5738 struct ib_cq *cq = qp->send_cq;
5739 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5740 struct mlx5_ib_drain_cqe sdrain;
d34ac5cd 5741 const struct ib_send_wr *bad_swr;
d0e84c0a
YH		 5742 struct ib_rdma_wr swr = {
5743 .wr = {
5744 .next = NULL,
5745 { .wr_cqe = &sdrain.cqe, },
5746 .opcode = IB_WR_RDMA_WRITE,
5747 },
5748 };
5749 int ret;
5750 struct mlx5_ib_dev *dev = to_mdev(qp->device);
5751 struct mlx5_core_dev *mdev = dev->mdev;
5752
5753 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5754 if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5755 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5756 return;
5757 }
5758
5759 sdrain.cqe.done = mlx5_ib_drain_qp_done;
5760 init_completion(&sdrain.done);
5761
029e88fd 5762 ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
d0e84c0a
YH		 5763 if (ret) {
5764 WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5765 return;
5766 }
5767
5768 handle_drain_completion(cq, &sdrain, dev);
5769}
5770
5771void mlx5_ib_drain_rq(struct ib_qp *qp)
5772{
5773 struct ib_cq *cq = qp->recv_cq;
5774 struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5775 struct mlx5_ib_drain_cqe rdrain;
d34ac5cd
BVA		 5776 struct ib_recv_wr rwr = {};
5777 const struct ib_recv_wr *bad_rwr;
d0e84c0a
YH		 5778 int ret;
5779 struct mlx5_ib_dev *dev = to_mdev(qp->device);
5780 struct mlx5_core_dev *mdev = dev->mdev;
5781
5782 ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5783 if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5784 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5785 return;
5786 }
5787
5788 rwr.wr_cqe = &rdrain.cqe;
5789 rdrain.cqe.done = mlx5_ib_drain_qp_done;
5790 init_completion(&rdrain.done);
5791
029e88fd 5792 ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
d0e84c0a
YH		 5793 if (ret) {
5794 WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5795 return;
5796 }
5797
5798 handle_drain_completion(cq, &rdrain, dev);
5799}
d14133dd 5800
30cd9fc5 5801/*
d14133dd
MZ		 5802 * Bind a qp to a counter. If @counter is NULL then bind the qp to
5803 * the default counter
5804 */
5805int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
5806{
10189e8e 5807 struct mlx5_ib_dev *dev = to_mdev(qp->device);
d14133dd
MZ		 5808 struct mlx5_ib_qp *mqp = to_mqp(qp);
5809 int err = 0;
5810
5811 mutex_lock(&mqp->mutex);
5812 if (mqp->state == IB_QPS_RESET) {
5813 qp->counter = counter;
5814 goto out;
5815 }
5816
10189e8e
MZ		 5817 if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
5818 err = -EOPNOTSUPP;
5819 goto out;
5820 }
5821
d14133dd
MZ		 5822 if (mqp->state == IB_QPS_RTS) {
5823 err = __mlx5_ib_qp_set_counter(qp, counter);
5824 if (!err)
5825 qp->counter = counter;
5826
5827 goto out;
5828 }
5829
5830 mqp->counter_pending = 1;
5831 qp->counter = counter;
5832
5833out:
5834 mutex_unlock(&mqp->mutex);
5835 return err;
5836}
312b8f79
MZ		 5837
5838int mlx5_ib_qp_event_init(void)
5839{
5840 mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
5841 if (!mlx5_ib_qp_event_wq)
5842 return -ENOMEM;
5843
5844 return 0;
5845}
5846
5847void mlx5_ib_qp_event_cleanup(void)
5848{
5849 destroy_workqueue(mlx5_ib_qp_event_wq);
5850}
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git