]> git.ipfire.org Git - thirdparty/kernel/stable.git/blame - drivers/infiniband/hw/mlx5/qp.c
projects / thirdparty / kernel / stable.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/mlx5: Packet pacing enhancement
[thirdparty/kernel/stable.git] / drivers / infiniband / hw / mlx5 / qp.c
CommitLineData
e126ba97 1/*
6cf0a15f 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/module.h>
34#include <rdma/ib_umem.h>
2811ba51 35#include <rdma/ib_cache.h>
cfb5e088 36#include <rdma/ib_user_verbs.h>
c2e53b2c 37#include <linux/mlx5/fs.h>
e126ba97 38#include "mlx5_ib.h"
b96c9dde 39#include "ib_rep.h"
e126ba97
EC		 40
41/* not supported currently */
42static int wq_signature;
43
44enum {
45 MLX5_IB_ACK_REQ_FREQ = 8,
46};
47
48enum {
49 MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83,
50 MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f,
51 MLX5_IB_LINK_TYPE_IB = 0,
52 MLX5_IB_LINK_TYPE_ETH = 1
53};
54
55enum {
56 MLX5_IB_SQ_STRIDE = 6,
e126ba97
EC		 57};
58
59static const u32 mlx5_ib_opcode[] = {
60 [IB_WR_SEND] = MLX5_OPCODE_SEND,
f0313965 61 [IB_WR_LSO] = MLX5_OPCODE_LSO,
e126ba97
EC		 62 [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM,
63 [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE,
64 [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM,
65 [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ,
66 [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS,
67 [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA,
68 [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL,
69 [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR,
8a187ee5 70 [IB_WR_REG_MR] = MLX5_OPCODE_UMR,
e126ba97
EC		 71 [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS,
72 [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA,
73 [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR,
74};
75
f0313965
ES		 76struct mlx5_wqe_eth_pad {
77 u8 rsvd0[16];
78};
e126ba97 79
eb49ab0c
AV		 80enum raw_qp_set_mask_map {
81 MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID = 1UL << 0,
7d29f349 82 MLX5_RAW_QP_RATE_LIMIT = 1UL << 1,
eb49ab0c
AV		 83};
84
0680efa2
AV		 85struct mlx5_modify_raw_qp_param {
86 u16 operation;
eb49ab0c
AV		 87
88 u32 set_mask; /* raw_qp_set_mask_map */
7d29f349 89 u32 rate_limit;
eb49ab0c 90 u8 rq_q_ctr_id;
0680efa2
AV		 91};
92
89ea94a7
MG		 93static void get_cqs(enum ib_qp_type qp_type,
94 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
95 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
96
e126ba97
EC		 97static int is_qp0(enum ib_qp_type qp_type)
98{
99 return qp_type == IB_QPT_SMI;
100}
101
e126ba97
EC		 102static int is_sqp(enum ib_qp_type qp_type)
103{
104 return is_qp0(qp_type) || is_qp1(qp_type);
105}
106
107static void *get_wqe(struct mlx5_ib_qp *qp, int offset)
108{
109 return mlx5_buf_offset(&qp->buf, offset);
110}
111
112static void *get_recv_wqe(struct mlx5_ib_qp *qp, int n)
113{
114 return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
115}
116
117void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n)
118{
119 return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE));
120}
121
c1395a2a
HE		 122/**
123 * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space.
124 *
125 * @qp: QP to copy from.
126 * @send: copy from the send queue when non-zero, use the receive queue
127 * otherwise.
128 * @wqe_index: index to start copying from. For send work queues, the
129 * wqe_index is in units of MLX5_SEND_WQE_BB.
130 * For receive work queue, it is the number of work queue
131 * element in the queue.
132 * @buffer: destination buffer.
133 * @length: maximum number of bytes to copy.
134 *
135 * Copies at least a single WQE, but may copy more data.
136 *
137 * Return: the number of bytes copied, or an error code.
138 */
139int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
19098df2 140 void *buffer, u32 length,
141 struct mlx5_ib_qp_base *base)
c1395a2a
HE		 142{
143 struct ib_device *ibdev = qp->ibqp.device;
144 struct mlx5_ib_dev *dev = to_mdev(ibdev);
145 struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq;
146 size_t offset;
147 size_t wq_end;
19098df2 148 struct ib_umem *umem = base->ubuffer.umem;
c1395a2a
HE		 149 u32 first_copy_length;
150 int wqe_length;
151 int ret;
152
153 if (wq->wqe_cnt == 0) {
154 mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n",
155 qp->ibqp.qp_type);
156 return -EINVAL;
157 }
158
159 offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift);
160 wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift);
161
162 if (send && length < sizeof(struct mlx5_wqe_ctrl_seg))
163 return -EINVAL;
164
165 if (offset > umem->length ||
166 (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length))
167 return -EINVAL;
168
169 first_copy_length = min_t(u32, offset + length, wq_end) - offset;
170 ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length);
171 if (ret)
172 return ret;
173
174 if (send) {
175 struct mlx5_wqe_ctrl_seg *ctrl = buffer;
176 int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
177
178 wqe_length = ds * MLX5_WQE_DS_UNITS;
179 } else {
180 wqe_length = 1 << wq->wqe_shift;
181 }
182
183 if (wqe_length <= first_copy_length)
184 return first_copy_length;
185
186 ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset,
187 wqe_length - first_copy_length);
188 if (ret)
189 return ret;
190
191 return wqe_length;
192}
193
e126ba97
EC		 194static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
195{
196 struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
197 struct ib_event event;
198
19098df2 199 if (type == MLX5_EVENT_TYPE_PATH_MIG) {
200 /* This event is only valid for trans_qps */
201 to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
202 }
e126ba97
EC		 203
204 if (ibqp->event_handler) {
205 event.device = ibqp->device;
206 event.element.qp = ibqp;
207 switch (type) {
208 case MLX5_EVENT_TYPE_PATH_MIG:
209 event.event = IB_EVENT_PATH_MIG;
210 break;
211 case MLX5_EVENT_TYPE_COMM_EST:
212 event.event = IB_EVENT_COMM_EST;
213 break;
214 case MLX5_EVENT_TYPE_SQ_DRAINED:
215 event.event = IB_EVENT_SQ_DRAINED;
216 break;
217 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
218 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
219 break;
220 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
221 event.event = IB_EVENT_QP_FATAL;
222 break;
223 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
224 event.event = IB_EVENT_PATH_MIG_ERR;
225 break;
226 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
227 event.event = IB_EVENT_QP_REQ_ERR;
228 break;
229 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
230 event.event = IB_EVENT_QP_ACCESS_ERR;
231 break;
232 default:
233 pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn);
234 return;
235 }
236
237 ibqp->event_handler(&event, ibqp->qp_context);
238 }
239}
240
241static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
242 int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
243{
244 int wqe_size;
245 int wq_size;
246
247 /* Sanity check RQ size before proceeding */
938fe83c 248 if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
e126ba97
EC		 249 return -EINVAL;
250
251 if (!has_rq) {
252 qp->rq.max_gs = 0;
253 qp->rq.wqe_cnt = 0;
254 qp->rq.wqe_shift = 0;
0540d814
NO		 255 cap->max_recv_wr = 0;
256 cap->max_recv_sge = 0;
e126ba97
EC		 257 } else {
258 if (ucmd) {
259 qp->rq.wqe_cnt = ucmd->rq_wqe_count;
260 qp->rq.wqe_shift = ucmd->rq_wqe_shift;
261 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
262 qp->rq.max_post = qp->rq.wqe_cnt;
263 } else {
264 wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0;
265 wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
266 wqe_size = roundup_pow_of_two(wqe_size);
267 wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
268 wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
269 qp->rq.wqe_cnt = wq_size / wqe_size;
938fe83c 270 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
e126ba97
EC		 271 mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
272 wqe_size,
938fe83c
SM		 273 MLX5_CAP_GEN(dev->mdev,
274 max_wqe_sz_rq));
e126ba97
EC		 275 return -EINVAL;
276 }
277 qp->rq.wqe_shift = ilog2(wqe_size);
278 qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig;
279 qp->rq.max_post = qp->rq.wqe_cnt;
280 }
281 }
282
283 return 0;
284}
285
f0313965 286static int sq_overhead(struct ib_qp_init_attr *attr)
e126ba97 287{
618af384 288 int size = 0;
e126ba97 289
f0313965 290 switch (attr->qp_type) {
e126ba97 291 case IB_QPT_XRC_INI:
b125a54b 292 size += sizeof(struct mlx5_wqe_xrc_seg);
e126ba97
EC		 293 /* fall through */
294 case IB_QPT_RC:
295 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 296 max(sizeof(struct mlx5_wqe_atomic_seg) +
297 sizeof(struct mlx5_wqe_raddr_seg),
298 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
299 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 300 break;
301
b125a54b
EC		 302 case IB_QPT_XRC_TGT:
303 return 0;
304
e126ba97 305 case IB_QPT_UC:
b125a54b 306 size += sizeof(struct mlx5_wqe_ctrl_seg) +
75c1657e
LR		 307 max(sizeof(struct mlx5_wqe_raddr_seg),
308 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
309 sizeof(struct mlx5_mkey_seg));
e126ba97
EC		 310 break;
311
312 case IB_QPT_UD:
f0313965
ES		 313 if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
314 size += sizeof(struct mlx5_wqe_eth_pad) +
315 sizeof(struct mlx5_wqe_eth_seg);
316 /* fall through */
e126ba97 317 case IB_QPT_SMI:
d16e91da 318 case MLX5_IB_QPT_HW_GSI:
b125a54b 319 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 320 sizeof(struct mlx5_wqe_datagram_seg);
321 break;
322
323 case MLX5_IB_QPT_REG_UMR:
b125a54b 324 size += sizeof(struct mlx5_wqe_ctrl_seg) +
e126ba97
EC		 325 sizeof(struct mlx5_wqe_umr_ctrl_seg) +
326 sizeof(struct mlx5_mkey_seg);
327 break;
328
329 default:
330 return -EINVAL;
331 }
332
333 return size;
334}
335
336static int calc_send_wqe(struct ib_qp_init_attr *attr)
337{
338 int inl_size = 0;
339 int size;
340
f0313965 341 size = sq_overhead(attr);
e126ba97
EC		 342 if (size < 0)
343 return size;
344
345 if (attr->cap.max_inline_data) {
346 inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
347 attr->cap.max_inline_data;
348 }
349
350 size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
e1e66cc2
SG		 351 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN &&
352 ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
353 return MLX5_SIG_WQE_SIZE;
354 else
355 return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
e126ba97
EC		 356}
357
288c01b7
EC		 358static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
359{
360 int max_sge;
361
362 if (attr->qp_type == IB_QPT_RC)
363 max_sge = (min_t(int, wqe_size, 512) -
364 sizeof(struct mlx5_wqe_ctrl_seg) -
365 sizeof(struct mlx5_wqe_raddr_seg)) /
366 sizeof(struct mlx5_wqe_data_seg);
367 else if (attr->qp_type == IB_QPT_XRC_INI)
368 max_sge = (min_t(int, wqe_size, 512) -
369 sizeof(struct mlx5_wqe_ctrl_seg) -
370 sizeof(struct mlx5_wqe_xrc_seg) -
371 sizeof(struct mlx5_wqe_raddr_seg)) /
372 sizeof(struct mlx5_wqe_data_seg);
373 else
374 max_sge = (wqe_size - sq_overhead(attr)) /
375 sizeof(struct mlx5_wqe_data_seg);
376
377 return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
378 sizeof(struct mlx5_wqe_data_seg));
379}
380
e126ba97
EC		 381static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
382 struct mlx5_ib_qp *qp)
383{
384 int wqe_size;
385 int wq_size;
386
387 if (!attr->cap.max_send_wr)
388 return 0;
389
390 wqe_size = calc_send_wqe(attr);
391 mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
392 if (wqe_size < 0)
393 return wqe_size;
394
938fe83c 395 if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
b125a54b 396 mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
938fe83c 397 wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 398 return -EINVAL;
399 }
400
f0313965
ES		 401 qp->max_inline_data = wqe_size - sq_overhead(attr) -
402 sizeof(struct mlx5_wqe_inline_seg);
e126ba97
EC		 403 attr->cap.max_inline_data = qp->max_inline_data;
404
e1e66cc2
SG		 405 if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
406 qp->signature_en = true;
407
e126ba97
EC		 408 wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
409 qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
938fe83c 410 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
1974ab9d
BVA		 411 mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
412 attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
938fe83c
SM		 413 qp->sq.wqe_cnt,
414 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
b125a54b
EC		 415 return -ENOMEM;
416 }
e126ba97 417 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
288c01b7
EC		 418 qp->sq.max_gs = get_send_sge(attr, wqe_size);
419 if (qp->sq.max_gs < attr->cap.max_send_sge)
420 return -ENOMEM;
421
422 attr->cap.max_send_sge = qp->sq.max_gs;
b125a54b
EC		 423 qp->sq.max_post = wq_size / wqe_size;
424 attr->cap.max_send_wr = qp->sq.max_post;
e126ba97
EC		 425
426 return wq_size;
427}
428
429static int set_user_buf_size(struct mlx5_ib_dev *dev,
430 struct mlx5_ib_qp *qp,
19098df2 431 struct mlx5_ib_create_qp *ucmd,
0fb2ed66 432 struct mlx5_ib_qp_base *base,
433 struct ib_qp_init_attr *attr)
e126ba97
EC		 434{
435 int desc_sz = 1 << qp->sq.wqe_shift;
436
938fe83c 437 if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
e126ba97 438 mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
938fe83c 439 desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
e126ba97
EC		 440 return -EINVAL;
441 }
442
443 if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) {
444 mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n",
445 ucmd->sq_wqe_count, ucmd->sq_wqe_count);
446 return -EINVAL;
447 }
448
449 qp->sq.wqe_cnt = ucmd->sq_wqe_count;
450
938fe83c 451 if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
e126ba97 452 mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
938fe83c
SM		 453 qp->sq.wqe_cnt,
454 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
e126ba97
EC		 455 return -EINVAL;
456 }
457
c2e53b2c
YH		 458 if (attr->qp_type == IB_QPT_RAW_PACKET ||
459 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 460 base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
461 qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
462 } else {
463 base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
464 (qp->sq.wqe_cnt << 6);
465 }
e126ba97
EC		 466
467 return 0;
468}
469
470static int qp_has_rq(struct ib_qp_init_attr *attr)
471{
472 if (attr->qp_type == IB_QPT_XRC_INI ||
473 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
474 attr->qp_type == MLX5_IB_QPT_REG_UMR ||
475 !attr->cap.max_recv_wr)
476 return 0;
477
478 return 1;
479}
480
2f5ff264 481static int first_med_bfreg(void)
c1be5232
EC		 482{
483 return 1;
484}
485
0b80c14f
EC		 486enum {
487 /* this is the first blue flame register in the array of bfregs assigned
488 * to a processes. Since we do not use it for blue flame but rather
489 * regular 64 bit doorbells, we do not need a lock for maintaiing
490 * "odd/even" order
491 */
492 NUM_NON_BLUE_FLAME_BFREGS = 1,
493};
494
b037c29a
EC		 495static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
496{
31a78a5a 497 return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a
EC		 498}
499
500static int num_med_bfreg(struct mlx5_ib_dev *dev,
501 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 502{
503 int n;
504
b037c29a
EC		 505 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
506 NUM_NON_BLUE_FLAME_BFREGS;
c1be5232
EC		 507
508 return n >= 0 ? n : 0;
509}
510
b037c29a
EC		 511static int first_hi_bfreg(struct mlx5_ib_dev *dev,
512 struct mlx5_bfreg_info *bfregi)
c1be5232
EC		 513{
514 int med;
c1be5232 515
b037c29a
EC		 516 med = num_med_bfreg(dev, bfregi);
517 return ++med;
c1be5232
EC		 518}
519
b037c29a
EC		 520static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
521 struct mlx5_bfreg_info *bfregi)
e126ba97 522{
e126ba97
EC		 523 int i;
524
b037c29a
EC		 525 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
526 if (!bfregi->count[i]) {
2f5ff264 527 bfregi->count[i]++;
e126ba97
EC		 528 return i;
529 }
530 }
531
532 return -ENOMEM;
533}
534
b037c29a
EC		 535static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
536 struct mlx5_bfreg_info *bfregi)
e126ba97 537{
2f5ff264 538 int minidx = first_med_bfreg();
e126ba97
EC		 539 int i;
540
b037c29a 541 for (i = first_med_bfreg(); i < first_hi_bfreg(dev, bfregi); i++) {
2f5ff264 542 if (bfregi->count[i] < bfregi->count[minidx])
e126ba97 543 minidx = i;
0b80c14f
EC		 544 if (!bfregi->count[minidx])
545 break;
e126ba97
EC		 546 }
547
2f5ff264 548 bfregi->count[minidx]++;
e126ba97
EC		 549 return minidx;
550}
551
b037c29a
EC		 552static int alloc_bfreg(struct mlx5_ib_dev *dev,
553 struct mlx5_bfreg_info *bfregi,
2f5ff264 554 enum mlx5_ib_latency_class lat)
e126ba97 555{
2f5ff264 556 int bfregn = -EINVAL;
e126ba97 557
2f5ff264 558 mutex_lock(&bfregi->lock);
e126ba97
EC		 559 switch (lat) {
560 case MLX5_IB_LATENCY_CLASS_LOW:
0b80c14f 561 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
2f5ff264
EC		 562 bfregn = 0;
563 bfregi->count[bfregn]++;
e126ba97
EC		 564 break;
565
566 case MLX5_IB_LATENCY_CLASS_MEDIUM:
2f5ff264
EC		 567 if (bfregi->ver < 2)
568 bfregn = -ENOMEM;
78c0f98c 569 else
b037c29a 570 bfregn = alloc_med_class_bfreg(dev, bfregi);
e126ba97
EC		 571 break;
572
573 case MLX5_IB_LATENCY_CLASS_HIGH:
2f5ff264
EC		 574 if (bfregi->ver < 2)
575 bfregn = -ENOMEM;
78c0f98c 576 else
b037c29a 577 bfregn = alloc_high_class_bfreg(dev, bfregi);
e126ba97
EC		 578 break;
579 }
2f5ff264 580 mutex_unlock(&bfregi->lock);
e126ba97 581
2f5ff264 582 return bfregn;
e126ba97
EC		 583}
584
4ed131d0 585void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
e126ba97 586{
2f5ff264 587 mutex_lock(&bfregi->lock);
b037c29a 588 bfregi->count[bfregn]--;
2f5ff264 589 mutex_unlock(&bfregi->lock);
e126ba97
EC		 590}
591
592static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
593{
594 switch (state) {
595 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
596 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;
597 case IB_QPS_RTR: return MLX5_QP_STATE_RTR;
598 case IB_QPS_RTS: return MLX5_QP_STATE_RTS;
599 case IB_QPS_SQD: return MLX5_QP_STATE_SQD;
600 case IB_QPS_SQE: return MLX5_QP_STATE_SQER;
601 case IB_QPS_ERR: return MLX5_QP_STATE_ERR;
602 default: return -1;
603 }
604}
605
606static int to_mlx5_st(enum ib_qp_type type)
607{
608 switch (type) {
609 case IB_QPT_RC: return MLX5_QP_ST_RC;
610 case IB_QPT_UC: return MLX5_QP_ST_UC;
611 case IB_QPT_UD: return MLX5_QP_ST_UD;
612 case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR;
613 case IB_QPT_XRC_INI:
614 case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC;
615 case IB_QPT_SMI: return MLX5_QP_ST_QP0;
d16e91da 616 case MLX5_IB_QPT_HW_GSI: return MLX5_QP_ST_QP1;
c32a4f29 617 case MLX5_IB_QPT_DCI: return MLX5_QP_ST_DCI;
e126ba97 618 case IB_QPT_RAW_IPV6: return MLX5_QP_ST_RAW_IPV6;
e126ba97 619 case IB_QPT_RAW_PACKET:
0fb2ed66 620 case IB_QPT_RAW_ETHERTYPE: return MLX5_QP_ST_RAW_ETHERTYPE;
e126ba97
EC		 621 case IB_QPT_MAX:
622 default: return -EINVAL;
623 }
624}
625
89ea94a7
MG		 626static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
627 struct mlx5_ib_cq *recv_cq);
628static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
629 struct mlx5_ib_cq *recv_cq);
630
b037c29a 631static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
1ee47ab3
YH		 632 struct mlx5_bfreg_info *bfregi, int bfregn,
633 bool dyn_bfreg)
e126ba97 634{
b037c29a
EC		 635 int bfregs_per_sys_page;
636 int index_of_sys_page;
637 int offset;
638
639 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
640 MLX5_NON_FP_BFREGS_PER_UAR;
641 index_of_sys_page = bfregn / bfregs_per_sys_page;
642
1ee47ab3
YH		 643 if (dyn_bfreg) {
644 index_of_sys_page += bfregi->num_static_sys_pages;
645 if (bfregn > bfregi->num_dyn_bfregs ||
646 bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
647 mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
648 return -EINVAL;
649 }
650 }
b037c29a 651
1ee47ab3 652 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
b037c29a 653 return bfregi->sys_pages[index_of_sys_page] + offset;
e126ba97
EC		 654}
655
19098df2 656static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
657 struct ib_pd *pd,
658 unsigned long addr, size_t size,
659 struct ib_umem **umem,
660 int *npages, int *page_shift, int *ncont,
661 u32 *offset)
662{
663 int err;
664
665 *umem = ib_umem_get(pd->uobject->context, addr, size, 0, 0);
666 if (IS_ERR(*umem)) {
667 mlx5_ib_dbg(dev, "umem_get failed\n");
668 return PTR_ERR(*umem);
669 }
670
762f899a 671 mlx5_ib_cont_pages(*umem, addr, 0, npages, page_shift, ncont, NULL);
19098df2 672
673 err = mlx5_ib_get_buf_offset(addr, *page_shift, offset);
674 if (err) {
675 mlx5_ib_warn(dev, "bad offset\n");
676 goto err_umem;
677 }
678
679 mlx5_ib_dbg(dev, "addr 0x%lx, size %zu, npages %d, page_shift %d, ncont %d, offset %d\n",
680 addr, size, *npages, *page_shift, *ncont, *offset);
681
682 return 0;
683
684err_umem:
685 ib_umem_release(*umem);
686 *umem = NULL;
687
688 return err;
689}
690
fe248c3a
MG		 691static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
692 struct mlx5_ib_rwq *rwq)
79b20a6c
YH		 693{
694 struct mlx5_ib_ucontext *context;
695
fe248c3a
MG		 696 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
697 atomic_dec(&dev->delay_drop.rqs_cnt);
698
79b20a6c
YH		 699 context = to_mucontext(pd->uobject->context);
700 mlx5_ib_db_unmap_user(context, &rwq->db);
701 if (rwq->umem)
702 ib_umem_release(rwq->umem);
703}
704
705static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
706 struct mlx5_ib_rwq *rwq,
707 struct mlx5_ib_create_wq *ucmd)
708{
709 struct mlx5_ib_ucontext *context;
710 int page_shift = 0;
711 int npages;
712 u32 offset = 0;
713 int ncont = 0;
714 int err;
715
716 if (!ucmd->buf_addr)
717 return -EINVAL;
718
719 context = to_mucontext(pd->uobject->context);
720 rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,
721 rwq->buf_size, 0, 0);
722 if (IS_ERR(rwq->umem)) {
723 mlx5_ib_dbg(dev, "umem_get failed\n");
724 err = PTR_ERR(rwq->umem);
725 return err;
726 }
727
762f899a 728 mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
79b20a6c
YH		 729 &ncont, NULL);
730 err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
731 &rwq->rq_page_offset);
732 if (err) {
733 mlx5_ib_warn(dev, "bad offset\n");
734 goto err_umem;
735 }
736
737 rwq->rq_num_pas = ncont;
738 rwq->page_shift = page_shift;
739 rwq->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
740 rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
741
742 mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
743 (unsigned long long)ucmd->buf_addr, rwq->buf_size,
744 npages, page_shift, ncont, offset);
745
746 err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);
747 if (err) {
748 mlx5_ib_dbg(dev, "map failed\n");
749 goto err_umem;
750 }
751
752 rwq->create_type = MLX5_WQ_USER;
753 return 0;
754
755err_umem:
756 ib_umem_release(rwq->umem);
757 return err;
758}
759
b037c29a
EC		 760static int adjust_bfregn(struct mlx5_ib_dev *dev,
761 struct mlx5_bfreg_info *bfregi, int bfregn)
762{
763 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
764 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
765}
766
e126ba97
EC		 767static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
768 struct mlx5_ib_qp *qp, struct ib_udata *udata,
0fb2ed66 769 struct ib_qp_init_attr *attr,
09a7d9ec 770 u32 **in,
19098df2 771 struct mlx5_ib_create_qp_resp *resp, int *inlen,
772 struct mlx5_ib_qp_base *base)
e126ba97
EC		 773{
774 struct mlx5_ib_ucontext *context;
775 struct mlx5_ib_create_qp ucmd;
19098df2 776 struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
9e9c47d0 777 int page_shift = 0;
1ee47ab3 778 int uar_index = 0;
e126ba97 779 int npages;
9e9c47d0 780 u32 offset = 0;
2f5ff264 781 int bfregn;
9e9c47d0 782 int ncont = 0;
09a7d9ec
SM		 783 __be64 *pas;
784 void *qpc;
e126ba97
EC		 785 int err;
786
787 err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
788 if (err) {
789 mlx5_ib_dbg(dev, "copy failed\n");
790 return err;
791 }
792
793 context = to_mucontext(pd->uobject->context);
1ee47ab3
YH		 794 if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {
795 uar_index = bfregn_to_uar_index(dev, &context->bfregi,
796 ucmd.bfreg_index, true);
797 if (uar_index < 0)
798 return uar_index;
799
800 bfregn = MLX5_IB_INVALID_BFREG;
801 } else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) {
802 /*
803 * TBD: should come from the verbs when we have the API
804 */
051f2630 805 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
2f5ff264 806 bfregn = MLX5_CROSS_CHANNEL_BFREG;
1ee47ab3 807 }
051f2630 808 else {
b037c29a 809 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
2f5ff264
EC		 810 if (bfregn < 0) {
811 mlx5_ib_dbg(dev, "failed to allocate low latency BFREG\n");
051f2630 812 mlx5_ib_dbg(dev, "reverting to medium latency\n");
b037c29a 813 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_MEDIUM);
2f5ff264
EC		 814 if (bfregn < 0) {
815 mlx5_ib_dbg(dev, "failed to allocate medium latency BFREG\n");
051f2630 816 mlx5_ib_dbg(dev, "reverting to high latency\n");
b037c29a 817 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_LOW);
2f5ff264
EC		 818 if (bfregn < 0) {
819 mlx5_ib_warn(dev, "bfreg allocation failed\n");
820 return bfregn;
051f2630 821 }
c1be5232 822 }
e126ba97
EC		 823 }
824 }
825
2f5ff264 826 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
1ee47ab3
YH		 827 if (bfregn != MLX5_IB_INVALID_BFREG)
828 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
829 false);
e126ba97 830
48fea837
HE		 831 qp->rq.offset = 0;
832 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
833 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
834
0fb2ed66 835 err = set_user_buf_size(dev, qp, &ucmd, base, attr);
e126ba97 836 if (err)
2f5ff264 837 goto err_bfreg;
e126ba97 838
19098df2 839 if (ucmd.buf_addr && ubuffer->buf_size) {
840 ubuffer->buf_addr = ucmd.buf_addr;
841 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr,
842 ubuffer->buf_size,
843 &ubuffer->umem, &npages, &page_shift,
844 &ncont, &offset);
845 if (err)
2f5ff264 846 goto err_bfreg;
9e9c47d0 847 } else {
19098df2 848 ubuffer->umem = NULL;
e126ba97 849 }
e126ba97 850
09a7d9ec
SM		 851 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
852 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1b9a07ee 853 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 854 if (!*in) {
855 err = -ENOMEM;
856 goto err_umem;
857 }
09a7d9ec
SM		 858
859 pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
19098df2 860 if (ubuffer->umem)
09a7d9ec
SM		 861 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
862
863 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
864
865 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
866 MLX5_SET(qpc, qpc, page_offset, offset);
e126ba97 867
09a7d9ec 868 MLX5_SET(qpc, qpc, uar_page, uar_index);
1ee47ab3
YH		 869 if (bfregn != MLX5_IB_INVALID_BFREG)
870 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
871 else
872 resp->bfreg_index = MLX5_IB_INVALID_BFREG;
2f5ff264 873 qp->bfregn = bfregn;
e126ba97
EC		 874
875 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
876 if (err) {
877 mlx5_ib_dbg(dev, "map failed\n");
878 goto err_free;
879 }
880
881 err = ib_copy_to_udata(udata, resp, sizeof(*resp));
882 if (err) {
883 mlx5_ib_dbg(dev, "copy failed\n");
884 goto err_unmap;
885 }
886 qp->create_type = MLX5_QP_USER;
887
888 return 0;
889
890err_unmap:
891 mlx5_ib_db_unmap_user(context, &qp->db);
892
893err_free:
479163f4 894 kvfree(*in);
e126ba97
EC		 895
896err_umem:
19098df2 897 if (ubuffer->umem)
898 ib_umem_release(ubuffer->umem);
e126ba97 899
2f5ff264 900err_bfreg:
1ee47ab3
YH		 901 if (bfregn != MLX5_IB_INVALID_BFREG)
902 mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
e126ba97
EC		 903 return err;
904}
905
b037c29a
EC		 906static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
907 struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base)
e126ba97
EC		 908{
909 struct mlx5_ib_ucontext *context;
910
911 context = to_mucontext(pd->uobject->context);
912 mlx5_ib_db_unmap_user(context, &qp->db);
19098df2 913 if (base->ubuffer.umem)
914 ib_umem_release(base->ubuffer.umem);
1ee47ab3
YH		 915
916 /*
917 * Free only the BFREGs which are handled by the kernel.
918 * BFREGs of UARs allocated dynamically are handled by user.
919 */
920 if (qp->bfregn != MLX5_IB_INVALID_BFREG)
921 mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
e126ba97
EC		 922}
923
924static int create_kernel_qp(struct mlx5_ib_dev *dev,
925 struct ib_qp_init_attr *init_attr,
926 struct mlx5_ib_qp *qp,
09a7d9ec 927 u32 **in, int *inlen,
19098df2 928 struct mlx5_ib_qp_base *base)
e126ba97 929{
e126ba97 930 int uar_index;
09a7d9ec 931 void *qpc;
e126ba97
EC		 932 int err;
933
f0313965
ES		 934 if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
935 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
b11a4f9c 936 IB_QP_CREATE_IPOIB_UD_LSO |
93d576af 937 IB_QP_CREATE_NETIF_QP |
b11a4f9c 938 mlx5_ib_create_qp_sqpn_qp1()))
1a4c3a3d 939 return -EINVAL;
e126ba97
EC		 940
941 if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
5fe9dec0
EC		 942 qp->bf.bfreg = &dev->fp_bfreg;
943 else
944 qp->bf.bfreg = &dev->bfreg;
e126ba97 945
d8030b0d
EC		 946 /* We need to divide by two since each register is comprised of
947 * two buffers of identical size, namely odd and even
948 */
949 qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
5fe9dec0 950 uar_index = qp->bf.bfreg->index;
e126ba97
EC		 951
952 err = calc_sq_size(dev, init_attr, qp);
953 if (err < 0) {
954 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 955 return err;
e126ba97
EC		 956 }
957
958 qp->rq.offset = 0;
959 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
19098df2 960 base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
e126ba97 961
19098df2 962 err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
e126ba97
EC		 963 if (err) {
964 mlx5_ib_dbg(dev, "err %d\n", err);
5fe9dec0 965 return err;
e126ba97
EC		 966 }
967
968 qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
09a7d9ec
SM		 969 *inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
970 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1b9a07ee 971 *in = kvzalloc(*inlen, GFP_KERNEL);
e126ba97
EC		 972 if (!*in) {
973 err = -ENOMEM;
974 goto err_buf;
975 }
09a7d9ec
SM		 976
977 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
978 MLX5_SET(qpc, qpc, uar_page, uar_index);
979 MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
980
e126ba97 981 /* Set "fast registration enabled" for all kernel QPs */
09a7d9ec
SM		 982 MLX5_SET(qpc, qpc, fre, 1);
983 MLX5_SET(qpc, qpc, rlky, 1);
e126ba97 984
b11a4f9c 985 if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
09a7d9ec 986 MLX5_SET(qpc, qpc, deth_sqpn, 1);
b11a4f9c
HE		 987 qp->flags |= MLX5_IB_QP_SQPN_QP1;
988 }
989
09a7d9ec
SM		 990 mlx5_fill_page_array(&qp->buf,
991 (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas));
e126ba97 992
9603b61d 993 err = mlx5_db_alloc(dev->mdev, &qp->db);
e126ba97
EC		 994 if (err) {
995 mlx5_ib_dbg(dev, "err %d\n", err);
996 goto err_free;
997 }
998
b5883008
LD		 999 qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1000 sizeof(*qp->sq.wrid), GFP_KERNEL);
1001 qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1002 sizeof(*qp->sq.wr_data), GFP_KERNEL);
1003 qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1004 sizeof(*qp->rq.wrid), GFP_KERNEL);
1005 qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1006 sizeof(*qp->sq.w_list), GFP_KERNEL);
1007 qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1008 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
e126ba97
EC		 1009
1010 if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1011 !qp->sq.w_list || !qp->sq.wqe_head) {
1012 err = -ENOMEM;
1013 goto err_wrid;
1014 }
1015 qp->create_type = MLX5_QP_KERNEL;
1016
1017 return 0;
1018
1019err_wrid:
b5883008
LD		 1020 kvfree(qp->sq.wqe_head);
1021 kvfree(qp->sq.w_list);
1022 kvfree(qp->sq.wrid);
1023 kvfree(qp->sq.wr_data);
1024 kvfree(qp->rq.wrid);
f4044dac 1025 mlx5_db_free(dev->mdev, &qp->db);
e126ba97
EC		 1026
1027err_free:
479163f4 1028 kvfree(*in);
e126ba97
EC		 1029
1030err_buf:
9603b61d 1031 mlx5_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1032 return err;
1033}
1034
1035static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1036{
b5883008
LD		 1037 kvfree(qp->sq.wqe_head);
1038 kvfree(qp->sq.w_list);
1039 kvfree(qp->sq.wrid);
1040 kvfree(qp->sq.wr_data);
1041 kvfree(qp->rq.wrid);
f4044dac 1042 mlx5_db_free(dev->mdev, &qp->db);
9603b61d 1043 mlx5_buf_free(dev->mdev, &qp->buf);
e126ba97
EC		 1044}
1045
09a7d9ec 1046static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
e126ba97
EC		 1047{
1048 if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
c32a4f29 1049 (attr->qp_type == MLX5_IB_QPT_DCI) ||
e126ba97 1050 (attr->qp_type == IB_QPT_XRC_INI))
09a7d9ec 1051 return MLX5_SRQ_RQ;
e126ba97 1052 else if (!qp->has_rq)
09a7d9ec 1053 return MLX5_ZERO_LEN_RQ;
e126ba97 1054 else
09a7d9ec 1055 return MLX5_NON_ZERO_RQ;
e126ba97
EC		 1056}
1057
1058static int is_connected(enum ib_qp_type qp_type)
1059{
1060 if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC)
1061 return 1;
1062
1063 return 0;
1064}
1065
0fb2ed66 1066static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
c2e53b2c 1067 struct mlx5_ib_qp *qp,
0fb2ed66 1068 struct mlx5_ib_sq *sq, u32 tdn)
1069{
c4f287c4 1070 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
0fb2ed66 1071 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1072
0fb2ed66 1073 MLX5_SET(tisc, tisc, transport_domain, tdn);
c2e53b2c
YH		 1074 if (qp->flags & MLX5_IB_QP_UNDERLAY)
1075 MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1076
0fb2ed66 1077 return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
1078}
1079
1080static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1081 struct mlx5_ib_sq *sq)
1082{
1083 mlx5_core_destroy_tis(dev->mdev, sq->tisn);
1084}
1085
b96c9dde
MB		 1086static void destroy_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
1087 struct mlx5_ib_sq *sq)
1088{
1089 if (sq->flow_rule)
1090 mlx5_del_flow_rules(sq->flow_rule);
1091}
1092
0fb2ed66 1093static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1094 struct mlx5_ib_sq *sq, void *qpin,
1095 struct ib_pd *pd)
1096{
1097 struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1098 __be64 *pas;
1099 void *in;
1100 void *sqc;
1101 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1102 void *wq;
1103 int inlen;
1104 int err;
1105 int page_shift = 0;
1106 int npages;
1107 int ncont = 0;
1108 u32 offset = 0;
1109
1110 err = mlx5_ib_umem_get(dev, pd, ubuffer->buf_addr, ubuffer->buf_size,
1111 &sq->ubuffer.umem, &npages, &page_shift,
1112 &ncont, &offset);
1113 if (err)
1114 return err;
1115
1116 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * ncont;
1b9a07ee 1117 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1118 if (!in) {
1119 err = -ENOMEM;
1120 goto err_umem;
1121 }
1122
1123 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1124 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
795b609c
BW		 1125 if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1126 MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
0fb2ed66 1127 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1128 MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1129 MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1130 MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1131 MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
96dc3fc5
NO		 1132 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1133 MLX5_CAP_ETH(dev->mdev, swp))
1134 MLX5_SET(sqc, sqc, allow_swp, 1);
0fb2ed66 1135
1136 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1137 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1138 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1139 MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1140 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1141 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1142 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1143 MLX5_SET(wq, wq, log_wq_pg_sz, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1144 MLX5_SET(wq, wq, page_offset, offset);
1145
1146 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1147 mlx5_ib_populate_pas(dev, sq->ubuffer.umem, page_shift, pas, 0);
1148
1149 err = mlx5_core_create_sq_tracked(dev->mdev, in, inlen, &sq->base.mqp);
1150
1151 kvfree(in);
1152
1153 if (err)
1154 goto err_umem;
1155
b96c9dde
MB		 1156 err = create_flow_rule_vport_sq(dev, sq);
1157 if (err)
1158 goto err_flow;
1159
0fb2ed66 1160 return 0;
1161
b96c9dde
MB		 1162err_flow:
1163 mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1164
0fb2ed66 1165err_umem:
1166 ib_umem_release(sq->ubuffer.umem);
1167 sq->ubuffer.umem = NULL;
1168
1169 return err;
1170}
1171
1172static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1173 struct mlx5_ib_sq *sq)
1174{
b96c9dde 1175 destroy_flow_rule_vport_sq(dev, sq);
0fb2ed66 1176 mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
1177 ib_umem_release(sq->ubuffer.umem);
1178}
1179
2c292dbb 1180static size_t get_rq_pas_size(void *qpc)
0fb2ed66 1181{
1182 u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
1183 u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
1184 u32 log_rq_size = MLX5_GET(qpc, qpc, log_rq_size);
1185 u32 page_offset = MLX5_GET(qpc, qpc, page_offset);
1186 u32 po_quanta = 1 << (log_page_size - 6);
1187 u32 rq_sz = 1 << (log_rq_size + 4 + log_rq_stride);
1188 u32 page_size = 1 << log_page_size;
1189 u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
1190 u32 rq_num_pas = (rq_sz_po + page_size - 1) / page_size;
1191
1192 return rq_num_pas * sizeof(u64);
1193}
1194
1195static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2c292dbb
BP		 1196 struct mlx5_ib_rq *rq, void *qpin,
1197 size_t qpinlen)
0fb2ed66 1198{
358e42ea 1199 struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
0fb2ed66 1200 __be64 *pas;
1201 __be64 *qp_pas;
1202 void *in;
1203 void *rqc;
1204 void *wq;
1205 void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
2c292dbb
BP		 1206 size_t rq_pas_size = get_rq_pas_size(qpc);
1207 size_t inlen;
0fb2ed66 1208 int err;
2c292dbb
BP		 1209
1210 if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))
1211 return -EINVAL;
0fb2ed66 1212
1213 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
1b9a07ee 1214 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1215 if (!in)
1216 return -ENOMEM;
1217
1218 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
e4cc4fa7
NO		 1219 if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1220 MLX5_SET(rqc, rqc, vsd, 1);
0fb2ed66 1221 MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1222 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1223 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1224 MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1225 MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1226
358e42ea
MD		 1227 if (mqp->flags & MLX5_IB_QP_CAP_SCATTER_FCS)
1228 MLX5_SET(rqc, rqc, scatter_fcs, 1);
1229
0fb2ed66 1230 wq = MLX5_ADDR_OF(rqc, rqc, wq);
1231 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 1232 if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1233 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
0fb2ed66 1234 MLX5_SET(wq, wq, page_offset, MLX5_GET(qpc, qpc, page_offset));
1235 MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1236 MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1237 MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1238 MLX5_SET(wq, wq, log_wq_pg_sz, MLX5_GET(qpc, qpc, log_page_size));
1239 MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1240
1241 pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1242 qp_pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, qpin, pas);
1243 memcpy(pas, qp_pas, rq_pas_size);
1244
1245 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rq->base.mqp);
1246
1247 kvfree(in);
1248
1249 return err;
1250}
1251
1252static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1253 struct mlx5_ib_rq *rq)
1254{
1255 mlx5_core_destroy_rq_tracked(dev->mdev, &rq->base.mqp);
1256}
1257
f95ef6cb
MG		 1258static bool tunnel_offload_supported(struct mlx5_core_dev *dev)
1259{
1260 return (MLX5_CAP_ETH(dev, tunnel_stateless_vxlan) ||
1261 MLX5_CAP_ETH(dev, tunnel_stateless_gre) ||
1262 MLX5_CAP_ETH(dev, tunnel_stateless_geneve_rx));
1263}
1264
0fb2ed66 1265static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
f95ef6cb
MG		 1266 struct mlx5_ib_rq *rq, u32 tdn,
1267 bool tunnel_offload_en)
0fb2ed66 1268{
1269 u32 *in;
1270 void *tirc;
1271 int inlen;
1272 int err;
1273
1274 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1275 in = kvzalloc(inlen, GFP_KERNEL);
0fb2ed66 1276 if (!in)
1277 return -ENOMEM;
1278
1279 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1280 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1281 MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1282 MLX5_SET(tirc, tirc, transport_domain, tdn);
f95ef6cb
MG		 1283 if (tunnel_offload_en)
1284 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
0fb2ed66 1285
ec9c2fb8
MB		 1286 if (dev->rep)
1287 MLX5_SET(tirc, tirc, self_lb_block,
1288 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
1289
0fb2ed66 1290 err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
1291
1292 kvfree(in);
1293
1294 return err;
1295}
1296
1297static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1298 struct mlx5_ib_rq *rq)
1299{
1300 mlx5_core_destroy_tir(dev->mdev, rq->tirn);
1301}
1302
1303static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2c292dbb 1304 u32 *in, size_t inlen,
0fb2ed66 1305 struct ib_pd *pd)
1306{
1307 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1308 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1309 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1310 struct ib_uobject *uobj = pd->uobject;
1311 struct ib_ucontext *ucontext = uobj->context;
1312 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1313 int err;
1314 u32 tdn = mucontext->tdn;
1315
1316 if (qp->sq.wqe_cnt) {
c2e53b2c 1317 err = create_raw_packet_qp_tis(dev, qp, sq, tdn);
0fb2ed66 1318 if (err)
1319 return err;
1320
1321 err = create_raw_packet_qp_sq(dev, sq, in, pd);
1322 if (err)
1323 goto err_destroy_tis;
1324
1325 sq->base.container_mibqp = qp;
1d31e9c0 1326 sq->base.mqp.event = mlx5_ib_qp_event;
0fb2ed66 1327 }
1328
1329 if (qp->rq.wqe_cnt) {
358e42ea
MD		 1330 rq->base.container_mibqp = qp;
1331
e4cc4fa7
NO		 1332 if (qp->flags & MLX5_IB_QP_CVLAN_STRIPPING)
1333 rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
b1383aa6
NO		 1334 if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING)
1335 rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
2c292dbb 1336 err = create_raw_packet_qp_rq(dev, rq, in, inlen);
0fb2ed66 1337 if (err)
1338 goto err_destroy_sq;
1339
0fb2ed66 1340
f95ef6cb
MG		 1341 err = create_raw_packet_qp_tir(dev, rq, tdn,
1342 qp->tunnel_offload_en);
0fb2ed66 1343 if (err)
1344 goto err_destroy_rq;
1345 }
1346
1347 qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1348 rq->base.mqp.qpn;
1349
1350 return 0;
1351
1352err_destroy_rq:
1353 destroy_raw_packet_qp_rq(dev, rq);
1354err_destroy_sq:
1355 if (!qp->sq.wqe_cnt)
1356 return err;
1357 destroy_raw_packet_qp_sq(dev, sq);
1358err_destroy_tis:
1359 destroy_raw_packet_qp_tis(dev, sq);
1360
1361 return err;
1362}
1363
1364static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1365 struct mlx5_ib_qp *qp)
1366{
1367 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1368 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1369 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1370
1371 if (qp->rq.wqe_cnt) {
1372 destroy_raw_packet_qp_tir(dev, rq);
1373 destroy_raw_packet_qp_rq(dev, rq);
1374 }
1375
1376 if (qp->sq.wqe_cnt) {
1377 destroy_raw_packet_qp_sq(dev, sq);
1378 destroy_raw_packet_qp_tis(dev, sq);
1379 }
1380}
1381
1382static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1383 struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1384{
1385 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1386 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1387
1388 sq->sq = &qp->sq;
1389 rq->rq = &qp->rq;
1390 sq->doorbell = &qp->db;
1391 rq->doorbell = &qp->db;
1392}
1393
28d61370
YH		 1394static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1395{
1396 mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn);
1397}
1398
1399static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1400 struct ib_pd *pd,
1401 struct ib_qp_init_attr *init_attr,
1402 struct ib_udata *udata)
1403{
1404 struct ib_uobject *uobj = pd->uobject;
1405 struct ib_ucontext *ucontext = uobj->context;
1406 struct mlx5_ib_ucontext *mucontext = to_mucontext(ucontext);
1407 struct mlx5_ib_create_qp_resp resp = {};
1408 int inlen;
1409 int err;
1410 u32 *in;
1411 void *tirc;
1412 void *hfso;
1413 u32 selected_fields = 0;
1414 size_t min_resp_len;
1415 u32 tdn = mucontext->tdn;
1416 struct mlx5_ib_create_qp_rss ucmd = {};
1417 size_t required_cmd_sz;
1418
1419 if (init_attr->qp_type != IB_QPT_RAW_PACKET)
1420 return -EOPNOTSUPP;
1421
1422 if (init_attr->create_flags || init_attr->send_cq)
1423 return -EINVAL;
1424
2f5ff264 1425 min_resp_len = offsetof(typeof(resp), bfreg_index) + sizeof(resp.bfreg_index);
28d61370
YH		 1426 if (udata->outlen < min_resp_len)
1427 return -EINVAL;
1428
f95ef6cb 1429 required_cmd_sz = offsetof(typeof(ucmd), flags) + sizeof(ucmd.flags);
28d61370
YH		 1430 if (udata->inlen < required_cmd_sz) {
1431 mlx5_ib_dbg(dev, "invalid inlen\n");
1432 return -EINVAL;
1433 }
1434
1435 if (udata->inlen > sizeof(ucmd) &&
1436 !ib_is_udata_cleared(udata, sizeof(ucmd),
1437 udata->inlen - sizeof(ucmd))) {
1438 mlx5_ib_dbg(dev, "inlen is not supported\n");
1439 return -EOPNOTSUPP;
1440 }
1441
1442 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
1443 mlx5_ib_dbg(dev, "copy failed\n");
1444 return -EFAULT;
1445 }
1446
1447 if (ucmd.comp_mask) {
1448 mlx5_ib_dbg(dev, "invalid comp mask\n");
1449 return -EOPNOTSUPP;
1450 }
1451
f95ef6cb
MG		 1452 if (ucmd.flags & ~MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1453 mlx5_ib_dbg(dev, "invalid flags\n");
1454 return -EOPNOTSUPP;
1455 }
1456
1457 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS &&
1458 !tunnel_offload_supported(dev->mdev)) {
1459 mlx5_ib_dbg(dev, "tunnel offloads isn't supported\n");
28d61370
YH		 1460 return -EOPNOTSUPP;
1461 }
1462
309fa347
MG		 1463 if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1464 !(ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1465 mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1466 return -EOPNOTSUPP;
1467 }
1468
28d61370
YH		 1469 err = ib_copy_to_udata(udata, &resp, min_resp_len);
1470 if (err) {
1471 mlx5_ib_dbg(dev, "copy failed\n");
1472 return -EINVAL;
1473 }
1474
1475 inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1b9a07ee 1476 in = kvzalloc(inlen, GFP_KERNEL);
28d61370
YH		 1477 if (!in)
1478 return -ENOMEM;
1479
1480 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1481 MLX5_SET(tirc, tirc, disp_type,
1482 MLX5_TIRC_DISP_TYPE_INDIRECT);
1483 MLX5_SET(tirc, tirc, indirect_table,
1484 init_attr->rwq_ind_tbl->ind_tbl_num);
1485 MLX5_SET(tirc, tirc, transport_domain, tdn);
1486
1487 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
f95ef6cb
MG		 1488
1489 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1490 MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1491
309fa347
MG		 1492 if (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1493 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1494 else
1495 hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1496
28d61370
YH		 1497 switch (ucmd.rx_hash_function) {
1498 case MLX5_RX_HASH_FUNC_TOEPLITZ:
1499 {
1500 void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1501 size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1502
1503 if (len != ucmd.rx_key_len) {
1504 err = -EINVAL;
1505 goto err;
1506 }
1507
1508 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1509 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
1510 memcpy(rss_key, ucmd.rx_hash_key, len);
1511 break;
1512 }
1513 default:
1514 err = -EOPNOTSUPP;
1515 goto err;
1516 }
1517
1518 if (!ucmd.rx_hash_fields_mask) {
1519 /* special case when this TIR serves as steering entry without hashing */
1520 if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1521 goto create_tir;
1522 err = -EINVAL;
1523 goto err;
1524 }
1525
1526 if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1527 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1528 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1529 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1530 err = -EINVAL;
1531 goto err;
1532 }
1533
1534 /* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1535 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1536 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1537 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1538 MLX5_L3_PROT_TYPE_IPV4);
1539 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1540 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1541 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1542 MLX5_L3_PROT_TYPE_IPV6);
1543
1544 if (((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1545 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP)) &&
1546 ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1547 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))) {
1548 err = -EINVAL;
1549 goto err;
1550 }
1551
1552 /* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1553 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1554 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1555 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1556 MLX5_L4_PROT_TYPE_TCP);
1557 else if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1558 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1559 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1560 MLX5_L4_PROT_TYPE_UDP);
1561
1562 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1563 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1564 selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1565
1566 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1567 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1568 selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1569
1570 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1571 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1572 selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1573
1574 if ((ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1575 (ucmd.rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1576 selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1577
1578 MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1579
1580create_tir:
ec9c2fb8
MB		 1581 if (dev->rep)
1582 MLX5_SET(tirc, tirc, self_lb_block,
1583 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST_);
1584
28d61370
YH		 1585 err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn);
1586
1587 if (err)
1588 goto err;
1589
1590 kvfree(in);
1591 /* qpn is reserved for that QP */
1592 qp->trans_qp.base.mqp.qpn = 0;
d9f88e5a 1593 qp->flags |= MLX5_IB_QP_RSS;
28d61370
YH		 1594 return 0;
1595
1596err:
1597 kvfree(in);
1598 return err;
1599}
1600
e126ba97
EC		 1601static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1602 struct ib_qp_init_attr *init_attr,
1603 struct ib_udata *udata, struct mlx5_ib_qp *qp)
1604{
1605 struct mlx5_ib_resources *devr = &dev->devr;
09a7d9ec 1606 int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
938fe83c 1607 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97 1608 struct mlx5_ib_create_qp_resp resp;
89ea94a7
MG		 1609 struct mlx5_ib_cq *send_cq;
1610 struct mlx5_ib_cq *recv_cq;
1611 unsigned long flags;
cfb5e088 1612 u32 uidx = MLX5_IB_DEFAULT_UIDX;
09a7d9ec
SM		 1613 struct mlx5_ib_create_qp ucmd;
1614 struct mlx5_ib_qp_base *base;
e7b169f3 1615 int mlx5_st;
cfb5e088 1616 void *qpc;
09a7d9ec
SM		 1617 u32 *in;
1618 int err;
e126ba97
EC		 1619
1620 mutex_init(&qp->mutex);
1621 spin_lock_init(&qp->sq.lock);
1622 spin_lock_init(&qp->rq.lock);
1623
e7b169f3
NO		 1624 mlx5_st = to_mlx5_st(init_attr->qp_type);
1625 if (mlx5_st < 0)
1626 return -EINVAL;
1627
28d61370
YH		 1628 if (init_attr->rwq_ind_tbl) {
1629 if (!udata)
1630 return -ENOSYS;
1631
1632 err = create_rss_raw_qp_tir(dev, qp, pd, init_attr, udata);
1633 return err;
1634 }
1635
f360d88a 1636 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
938fe83c 1637 if (!MLX5_CAP_GEN(mdev, block_lb_mc)) {
f360d88a
EC		 1638 mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
1639 return -EINVAL;
1640 } else {
1641 qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
1642 }
1643 }
1644
051f2630
LR		 1645 if (init_attr->create_flags &
1646 (IB_QP_CREATE_CROSS_CHANNEL |
1647 IB_QP_CREATE_MANAGED_SEND |
1648 IB_QP_CREATE_MANAGED_RECV)) {
1649 if (!MLX5_CAP_GEN(mdev, cd)) {
1650 mlx5_ib_dbg(dev, "cross-channel isn't supported\n");
1651 return -EINVAL;
1652 }
1653 if (init_attr->create_flags & IB_QP_CREATE_CROSS_CHANNEL)
1654 qp->flags |= MLX5_IB_QP_CROSS_CHANNEL;
1655 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_SEND)
1656 qp->flags |= MLX5_IB_QP_MANAGED_SEND;
1657 if (init_attr->create_flags & IB_QP_CREATE_MANAGED_RECV)
1658 qp->flags |= MLX5_IB_QP_MANAGED_RECV;
1659 }
f0313965
ES		 1660
1661 if (init_attr->qp_type == IB_QPT_UD &&
1662 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO))
1663 if (!MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1664 mlx5_ib_dbg(dev, "ipoib UD lso qp isn't supported\n");
1665 return -EOPNOTSUPP;
1666 }
1667
358e42ea
MD		 1668 if (init_attr->create_flags & IB_QP_CREATE_SCATTER_FCS) {
1669 if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1670 mlx5_ib_dbg(dev, "Scatter FCS is supported only for Raw Packet QPs");
1671 return -EOPNOTSUPP;
1672 }
1673 if (!MLX5_CAP_GEN(dev->mdev, eth_net_offloads) ||
1674 !MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
1675 mlx5_ib_dbg(dev, "Scatter FCS isn't supported\n");
1676 return -EOPNOTSUPP;
1677 }
1678 qp->flags |= MLX5_IB_QP_CAP_SCATTER_FCS;
1679 }
1680
e126ba97
EC		 1681 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1682 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
1683
e4cc4fa7
NO		 1684 if (init_attr->create_flags & IB_QP_CREATE_CVLAN_STRIPPING) {
1685 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1686 MLX5_CAP_ETH(dev->mdev, vlan_cap)) ||
1687 (init_attr->qp_type != IB_QPT_RAW_PACKET))
1688 return -EOPNOTSUPP;
1689 qp->flags |= MLX5_IB_QP_CVLAN_STRIPPING;
1690 }
1691
e126ba97
EC		 1692 if (pd && pd->uobject) {
1693 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
1694 mlx5_ib_dbg(dev, "copy failed\n");
1695 return -EFAULT;
1696 }
1697
cfb5e088
HA		 1698 err = get_qp_user_index(to_mucontext(pd->uobject->context),
1699 &ucmd, udata->inlen, &uidx);
1700 if (err)
1701 return err;
1702
e126ba97
EC		 1703 qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE);
1704 qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE);
f95ef6cb
MG		 1705 if (ucmd.flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS) {
1706 if (init_attr->qp_type != IB_QPT_RAW_PACKET ||
1707 !tunnel_offload_supported(mdev)) {
1708 mlx5_ib_dbg(dev, "Tunnel offload isn't supported\n");
1709 return -EOPNOTSUPP;
1710 }
1711 qp->tunnel_offload_en = true;
1712 }
c2e53b2c
YH		 1713
1714 if (init_attr->create_flags & IB_QP_CREATE_SOURCE_QPN) {
1715 if (init_attr->qp_type != IB_QPT_UD ||
1716 (MLX5_CAP_GEN(dev->mdev, port_type) !=
1717 MLX5_CAP_PORT_TYPE_IB) ||
1718 !mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS)) {
1719 mlx5_ib_dbg(dev, "Source QP option isn't supported\n");
1720 return -EOPNOTSUPP;
1721 }
1722
1723 qp->flags |= MLX5_IB_QP_UNDERLAY;
1724 qp->underlay_qpn = init_attr->source_qpn;
1725 }
e126ba97
EC		 1726 } else {
1727 qp->wq_sig = !!wq_signature;
1728 }
1729
c2e53b2c
YH		 1730 base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
1731 qp->flags & MLX5_IB_QP_UNDERLAY) ?
1732 &qp->raw_packet_qp.rq.base :
1733 &qp->trans_qp.base;
1734
e126ba97
EC		 1735 qp->has_rq = qp_has_rq(init_attr);
1736 err = set_rq_size(dev, &init_attr->cap, qp->has_rq,
1737 qp, (pd && pd->uobject) ? &ucmd : NULL);
1738 if (err) {
1739 mlx5_ib_dbg(dev, "err %d\n", err);
1740 return err;
1741 }
1742
1743 if (pd) {
1744 if (pd->uobject) {
938fe83c
SM		 1745 __u32 max_wqes =
1746 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
e126ba97
EC		 1747 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count);
1748 if (ucmd.rq_wqe_shift != qp->rq.wqe_shift ||
1749 ucmd.rq_wqe_count != qp->rq.wqe_cnt) {
1750 mlx5_ib_dbg(dev, "invalid rq params\n");
1751 return -EINVAL;
1752 }
938fe83c 1753 if (ucmd.sq_wqe_count > max_wqes) {
e126ba97 1754 mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n",
938fe83c 1755 ucmd.sq_wqe_count, max_wqes);
e126ba97
EC		 1756 return -EINVAL;
1757 }
b11a4f9c
HE		 1758 if (init_attr->create_flags &
1759 mlx5_ib_create_qp_sqpn_qp1()) {
1760 mlx5_ib_dbg(dev, "user-space is not allowed to create UD QPs spoofing as QP1\n");
1761 return -EINVAL;
1762 }
0fb2ed66 1763 err = create_user_qp(dev, pd, qp, udata, init_attr, &in,
1764 &resp, &inlen, base);
e126ba97
EC		 1765 if (err)
1766 mlx5_ib_dbg(dev, "err %d\n", err);
1767 } else {
19098df2 1768 err = create_kernel_qp(dev, init_attr, qp, &in, &inlen,
1769 base);
e126ba97
EC		 1770 if (err)
1771 mlx5_ib_dbg(dev, "err %d\n", err);
e126ba97
EC		 1772 }
1773
1774 if (err)
1775 return err;
1776 } else {
1b9a07ee 1777 in = kvzalloc(inlen, GFP_KERNEL);
e126ba97
EC		 1778 if (!in)
1779 return -ENOMEM;
1780
1781 qp->create_type = MLX5_QP_EMPTY;
1782 }
1783
1784 if (is_sqp(init_attr->qp_type))
1785 qp->port = init_attr->port_num;
1786
09a7d9ec
SM		 1787 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1788
e7b169f3 1789 MLX5_SET(qpc, qpc, st, mlx5_st);
09a7d9ec 1790 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
e126ba97
EC		 1791
1792 if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
09a7d9ec 1793 MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
e126ba97 1794 else
09a7d9ec
SM		 1795 MLX5_SET(qpc, qpc, latency_sensitive, 1);
1796
e126ba97
EC		 1797
1798 if (qp->wq_sig)
09a7d9ec 1799 MLX5_SET(qpc, qpc, wq_signature, 1);
e126ba97 1800
f360d88a 1801 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
09a7d9ec 1802 MLX5_SET(qpc, qpc, block_lb_mc, 1);
f360d88a 1803
051f2630 1804 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
09a7d9ec 1805 MLX5_SET(qpc, qpc, cd_master, 1);
051f2630 1806 if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
09a7d9ec 1807 MLX5_SET(qpc, qpc, cd_slave_send, 1);
051f2630 1808 if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
09a7d9ec 1809 MLX5_SET(qpc, qpc, cd_slave_receive, 1);
051f2630 1810
e126ba97
EC		 1811 if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
1812 int rcqe_sz;
1813 int scqe_sz;
1814
1815 rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq);
1816 scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq);
1817
1818 if (rcqe_sz == 128)
09a7d9ec 1819 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
e126ba97 1820 else
09a7d9ec 1821 MLX5_SET(qpc, qpc, cs_res, MLX5_RES_SCAT_DATA32_CQE);
e126ba97
EC		 1822
1823 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) {
1824 if (scqe_sz == 128)
09a7d9ec 1825 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
e126ba97 1826 else
09a7d9ec 1827 MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
e126ba97
EC		 1828 }
1829 }
1830
1831 if (qp->rq.wqe_cnt) {
09a7d9ec
SM		 1832 MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
1833 MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
e126ba97
EC		 1834 }
1835
09a7d9ec 1836 MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
e126ba97 1837
3fd3307e 1838 if (qp->sq.wqe_cnt) {
09a7d9ec 1839 MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
3fd3307e 1840 } else {
09a7d9ec 1841 MLX5_SET(qpc, qpc, no_sq, 1);
3fd3307e
AK		 1842 if (init_attr->srq &&
1843 init_attr->srq->srq_type == IB_SRQT_TM)
1844 MLX5_SET(qpc, qpc, offload_type,
1845 MLX5_QPC_OFFLOAD_TYPE_RNDV);
1846 }
e126ba97
EC		 1847
1848 /* Set default resources */
1849 switch (init_attr->qp_type) {
1850 case IB_QPT_XRC_TGT:
09a7d9ec
SM		 1851 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1852 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
1853 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
1854 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(init_attr->xrcd)->xrcdn);
e126ba97
EC		 1855 break;
1856 case IB_QPT_XRC_INI:
09a7d9ec
SM		 1857 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
1858 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1859 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
e126ba97
EC		 1860 break;
1861 default:
1862 if (init_attr->srq) {
09a7d9ec
SM		 1863 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x0)->xrcdn);
1864 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
e126ba97 1865 } else {
09a7d9ec
SM		 1866 MLX5_SET(qpc, qpc, xrcd, to_mxrcd(devr->x1)->xrcdn);
1867 MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
e126ba97
EC		 1868 }
1869 }
1870
1871 if (init_attr->send_cq)
09a7d9ec 1872 MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
e126ba97
EC		 1873
1874 if (init_attr->recv_cq)
09a7d9ec 1875 MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
e126ba97 1876
09a7d9ec 1877 MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
e126ba97 1878
09a7d9ec
SM		 1879 /* 0xffffff means we ask to work with cqe version 0 */
1880 if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
cfb5e088 1881 MLX5_SET(qpc, qpc, user_index, uidx);
09a7d9ec 1882
f0313965
ES		 1883 /* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
1884 if (init_attr->qp_type == IB_QPT_UD &&
1885 (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)) {
f0313965
ES		 1886 MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
1887 qp->flags |= MLX5_IB_QP_LSO;
1888 }
cfb5e088 1889
b1383aa6
NO		 1890 if (init_attr->create_flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
1891 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
1892 mlx5_ib_dbg(dev, "scatter end padding is not supported\n");
1893 err = -EOPNOTSUPP;
1894 goto err;
1895 } else if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
1896 MLX5_SET(qpc, qpc, end_padding_mode,
1897 MLX5_WQ_END_PAD_MODE_ALIGN);
1898 } else {
1899 qp->flags |= MLX5_IB_QP_PCI_WRITE_END_PADDING;
1900 }
1901 }
1902
2c292dbb
BP		 1903 if (inlen < 0) {
1904 err = -EINVAL;
1905 goto err;
1906 }
1907
c2e53b2c
YH		 1908 if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
1909 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 1910 qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
1911 raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
2c292dbb 1912 err = create_raw_packet_qp(dev, qp, in, inlen, pd);
0fb2ed66 1913 } else {
1914 err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
1915 }
1916
e126ba97
EC		 1917 if (err) {
1918 mlx5_ib_dbg(dev, "create qp failed\n");
1919 goto err_create;
1920 }
1921
479163f4 1922 kvfree(in);
e126ba97 1923
19098df2 1924 base->container_mibqp = qp;
1925 base->mqp.event = mlx5_ib_qp_event;
e126ba97 1926
89ea94a7
MG		 1927 get_cqs(init_attr->qp_type, init_attr->send_cq, init_attr->recv_cq,
1928 &send_cq, &recv_cq);
1929 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
1930 mlx5_ib_lock_cqs(send_cq, recv_cq);
1931 /* Maintain device to QPs access, needed for further handling via reset
1932 * flow
1933 */
1934 list_add_tail(&qp->qps_list, &dev->qp_list);
1935 /* Maintain CQ to QPs access, needed for further handling via reset flow
1936 */
1937 if (send_cq)
1938 list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
1939 if (recv_cq)
1940 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
1941 mlx5_ib_unlock_cqs(send_cq, recv_cq);
1942 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1943
e126ba97
EC		 1944 return 0;
1945
1946err_create:
1947 if (qp->create_type == MLX5_QP_USER)
b037c29a 1948 destroy_qp_user(dev, pd, qp, base);
e126ba97
EC		 1949 else if (qp->create_type == MLX5_QP_KERNEL)
1950 destroy_qp_kernel(dev, qp);
1951
b1383aa6 1952err:
479163f4 1953 kvfree(in);
e126ba97
EC		 1954 return err;
1955}
1956
1957static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
1958 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1959{
1960 if (send_cq) {
1961 if (recv_cq) {
1962 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
89ea94a7 1963 spin_lock(&send_cq->lock);
e126ba97
EC		 1964 spin_lock_nested(&recv_cq->lock,
1965 SINGLE_DEPTH_NESTING);
1966 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
89ea94a7 1967 spin_lock(&send_cq->lock);
e126ba97
EC		 1968 __acquire(&recv_cq->lock);
1969 } else {
89ea94a7 1970 spin_lock(&recv_cq->lock);
e126ba97
EC		 1971 spin_lock_nested(&send_cq->lock,
1972 SINGLE_DEPTH_NESTING);
1973 }
1974 } else {
89ea94a7 1975 spin_lock(&send_cq->lock);
6a4f139a 1976 __acquire(&recv_cq->lock);
e126ba97
EC		 1977 }
1978 } else if (recv_cq) {
89ea94a7 1979 spin_lock(&recv_cq->lock);
6a4f139a
EC		 1980 __acquire(&send_cq->lock);
1981 } else {
1982 __acquire(&send_cq->lock);
1983 __acquire(&recv_cq->lock);
e126ba97
EC		 1984 }
1985}
1986
1987static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
1988 __releases(&send_cq->lock) __releases(&recv_cq->lock)
1989{
1990 if (send_cq) {
1991 if (recv_cq) {
1992 if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
1993 spin_unlock(&recv_cq->lock);
89ea94a7 1994 spin_unlock(&send_cq->lock);
e126ba97
EC		 1995 } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
1996 __release(&recv_cq->lock);
89ea94a7 1997 spin_unlock(&send_cq->lock);
e126ba97
EC		 1998 } else {
1999 spin_unlock(&send_cq->lock);
89ea94a7 2000 spin_unlock(&recv_cq->lock);
e126ba97
EC		 2001 }
2002 } else {
6a4f139a 2003 __release(&recv_cq->lock);
89ea94a7 2004 spin_unlock(&send_cq->lock);
e126ba97
EC		 2005 }
2006 } else if (recv_cq) {
6a4f139a 2007 __release(&send_cq->lock);
89ea94a7 2008 spin_unlock(&recv_cq->lock);
6a4f139a
EC		 2009 } else {
2010 __release(&recv_cq->lock);
2011 __release(&send_cq->lock);
e126ba97
EC		 2012 }
2013}
2014
2015static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp)
2016{
2017 return to_mpd(qp->ibqp.pd);
2018}
2019
89ea94a7
MG		 2020static void get_cqs(enum ib_qp_type qp_type,
2021 struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
e126ba97
EC		 2022 struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2023{
89ea94a7 2024 switch (qp_type) {
e126ba97
EC		 2025 case IB_QPT_XRC_TGT:
2026 *send_cq = NULL;
2027 *recv_cq = NULL;
2028 break;
2029 case MLX5_IB_QPT_REG_UMR:
2030 case IB_QPT_XRC_INI:
89ea94a7 2031 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
e126ba97
EC		 2032 *recv_cq = NULL;
2033 break;
2034
2035 case IB_QPT_SMI:
d16e91da 2036 case MLX5_IB_QPT_HW_GSI:
e126ba97
EC		 2037 case IB_QPT_RC:
2038 case IB_QPT_UC:
2039 case IB_QPT_UD:
2040 case IB_QPT_RAW_IPV6:
2041 case IB_QPT_RAW_ETHERTYPE:
0fb2ed66 2042 case IB_QPT_RAW_PACKET:
89ea94a7
MG		 2043 *send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2044 *recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
e126ba97
EC		 2045 break;
2046
e126ba97
EC		 2047 case IB_QPT_MAX:
2048 default:
2049 *send_cq = NULL;
2050 *recv_cq = NULL;
2051 break;
2052 }
2053}
2054
ad5f8e96 2055static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2056 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2057 u8 lag_tx_affinity);
ad5f8e96 2058
e126ba97
EC		 2059static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
2060{
2061 struct mlx5_ib_cq *send_cq, *recv_cq;
c2e53b2c 2062 struct mlx5_ib_qp_base *base;
89ea94a7 2063 unsigned long flags;
e126ba97
EC		 2064 int err;
2065
28d61370
YH		 2066 if (qp->ibqp.rwq_ind_tbl) {
2067 destroy_rss_raw_qp_tir(dev, qp);
2068 return;
2069 }
2070
c2e53b2c
YH		 2071 base = (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2072 qp->flags & MLX5_IB_QP_UNDERLAY) ?
0fb2ed66 2073 &qp->raw_packet_qp.rq.base :
2074 &qp->trans_qp.base;
2075
6aec21f6 2076 if (qp->state != IB_QPS_RESET) {
c2e53b2c
YH		 2077 if (qp->ibqp.qp_type != IB_QPT_RAW_PACKET &&
2078 !(qp->flags & MLX5_IB_QP_UNDERLAY)) {
ad5f8e96 2079 err = mlx5_core_qp_modify(dev->mdev,
1a412fb1
SM		 2080 MLX5_CMD_OP_2RST_QP, 0,
2081 NULL, &base->mqp);
ad5f8e96 2082 } else {
0680efa2
AV		 2083 struct mlx5_modify_raw_qp_param raw_qp_param = {
2084 .operation = MLX5_CMD_OP_2RST_QP
2085 };
2086
13eab21f 2087 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
ad5f8e96 2088 }
2089 if (err)
427c1e7b 2090 mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
19098df2 2091 base->mqp.qpn);
6aec21f6 2092 }
e126ba97 2093
89ea94a7
MG		 2094 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
2095 &send_cq, &recv_cq);
2096
2097 spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2098 mlx5_ib_lock_cqs(send_cq, recv_cq);
2099 /* del from lists under both locks above to protect reset flow paths */
2100 list_del(&qp->qps_list);
2101 if (send_cq)
2102 list_del(&qp->cq_send_list);
2103
2104 if (recv_cq)
2105 list_del(&qp->cq_recv_list);
e126ba97
EC		 2106
2107 if (qp->create_type == MLX5_QP_KERNEL) {
19098df2 2108 __mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 2109 qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2110 if (send_cq != recv_cq)
19098df2 2111 __mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2112 NULL);
e126ba97 2113 }
89ea94a7
MG		 2114 mlx5_ib_unlock_cqs(send_cq, recv_cq);
2115 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
e126ba97 2116
c2e53b2c
YH		 2117 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
2118 qp->flags & MLX5_IB_QP_UNDERLAY) {
0fb2ed66 2119 destroy_raw_packet_qp(dev, qp);
2120 } else {
2121 err = mlx5_core_destroy_qp(dev->mdev, &base->mqp);
2122 if (err)
2123 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2124 base->mqp.qpn);
2125 }
e126ba97 2126
e126ba97
EC		 2127 if (qp->create_type == MLX5_QP_KERNEL)
2128 destroy_qp_kernel(dev, qp);
2129 else if (qp->create_type == MLX5_QP_USER)
b037c29a 2130 destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
e126ba97
EC		 2131}
2132
2133static const char *ib_qp_type_str(enum ib_qp_type type)
2134{
2135 switch (type) {
2136 case IB_QPT_SMI:
2137 return "IB_QPT_SMI";
2138 case IB_QPT_GSI:
2139 return "IB_QPT_GSI";
2140 case IB_QPT_RC:
2141 return "IB_QPT_RC";
2142 case IB_QPT_UC:
2143 return "IB_QPT_UC";
2144 case IB_QPT_UD:
2145 return "IB_QPT_UD";
2146 case IB_QPT_RAW_IPV6:
2147 return "IB_QPT_RAW_IPV6";
2148 case IB_QPT_RAW_ETHERTYPE:
2149 return "IB_QPT_RAW_ETHERTYPE";
2150 case IB_QPT_XRC_INI:
2151 return "IB_QPT_XRC_INI";
2152 case IB_QPT_XRC_TGT:
2153 return "IB_QPT_XRC_TGT";
2154 case IB_QPT_RAW_PACKET:
2155 return "IB_QPT_RAW_PACKET";
2156 case MLX5_IB_QPT_REG_UMR:
2157 return "MLX5_IB_QPT_REG_UMR";
b4aaa1f0
MS		 2158 case IB_QPT_DRIVER:
2159 return "IB_QPT_DRIVER";
e126ba97
EC		 2160 case IB_QPT_MAX:
2161 default:
2162 return "Invalid QP type";
2163 }
2164}
2165
b4aaa1f0
MS		 2166static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd,
2167 struct ib_qp_init_attr *attr,
2168 struct mlx5_ib_create_qp *ucmd)
2169{
b4aaa1f0
MS		 2170 struct mlx5_ib_qp *qp;
2171 int err = 0;
2172 u32 uidx = MLX5_IB_DEFAULT_UIDX;
2173 void *dctc;
2174
2175 if (!attr->srq || !attr->recv_cq)
2176 return ERR_PTR(-EINVAL);
2177
b4aaa1f0
MS		 2178 err = get_qp_user_index(to_mucontext(pd->uobject->context),
2179 ucmd, sizeof(*ucmd), &uidx);
2180 if (err)
2181 return ERR_PTR(err);
2182
2183 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2184 if (!qp)
2185 return ERR_PTR(-ENOMEM);
2186
2187 qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2188 if (!qp->dct.in) {
2189 err = -ENOMEM;
2190 goto err_free;
2191 }
2192
2193 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
776a3906 2194 qp->qp_sub_type = MLX5_IB_QPT_DCT;
b4aaa1f0
MS		 2195 MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2196 MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2197 MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2198 MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2199 MLX5_SET(dctc, dctc, user_index, uidx);
2200
2201 qp->state = IB_QPS_RESET;
2202
2203 return &qp->ibqp;
2204err_free:
2205 kfree(qp);
2206 return ERR_PTR(err);
2207}
2208
2209static int set_mlx_qp_type(struct mlx5_ib_dev *dev,
2210 struct ib_qp_init_attr *init_attr,
2211 struct mlx5_ib_create_qp *ucmd,
2212 struct ib_udata *udata)
2213{
2214 enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI };
2215 int err;
2216
2217 if (!udata)
2218 return -EINVAL;
2219
2220 if (udata->inlen < sizeof(*ucmd)) {
2221 mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");
2222 return -EINVAL;
2223 }
2224 err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));
2225 if (err)
2226 return err;
2227
2228 if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {
2229 init_attr->qp_type = MLX5_IB_QPT_DCI;
2230 } else {
2231 if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {
2232 init_attr->qp_type = MLX5_IB_QPT_DCT;
2233 } else {
2234 mlx5_ib_dbg(dev, "Invalid QP flags\n");
2235 return -EINVAL;
2236 }
2237 }
2238
2239 if (!MLX5_CAP_GEN(dev->mdev, dct)) {
2240 mlx5_ib_dbg(dev, "DC transport is not supported\n");
2241 return -EOPNOTSUPP;
2242 }
2243
2244 return 0;
2245}
2246
e126ba97 2247struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
b4aaa1f0 2248 struct ib_qp_init_attr *verbs_init_attr,
e126ba97
EC		 2249 struct ib_udata *udata)
2250{
2251 struct mlx5_ib_dev *dev;
2252 struct mlx5_ib_qp *qp;
2253 u16 xrcdn = 0;
2254 int err;
b4aaa1f0
MS		 2255 struct ib_qp_init_attr mlx_init_attr;
2256 struct ib_qp_init_attr *init_attr = verbs_init_attr;
e126ba97
EC		 2257
2258 if (pd) {
2259 dev = to_mdev(pd->device);
0fb2ed66 2260
2261 if (init_attr->qp_type == IB_QPT_RAW_PACKET) {
2262 if (!pd->uobject) {
2263 mlx5_ib_dbg(dev, "Raw Packet QP is not supported for kernel consumers\n");
2264 return ERR_PTR(-EINVAL);
2265 } else if (!to_mucontext(pd->uobject->context)->cqe_version) {
2266 mlx5_ib_dbg(dev, "Raw Packet QP is only supported for CQE version > 0\n");
2267 return ERR_PTR(-EINVAL);
2268 }
2269 }
09f16cf5
MD		 2270 } else {
2271 /* being cautious here */
2272 if (init_attr->qp_type != IB_QPT_XRC_TGT &&
2273 init_attr->qp_type != MLX5_IB_QPT_REG_UMR) {
2274 pr_warn("%s: no PD for transport %s\n", __func__,
2275 ib_qp_type_str(init_attr->qp_type));
2276 return ERR_PTR(-EINVAL);
2277 }
2278 dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
e126ba97
EC		 2279 }
2280
b4aaa1f0
MS		 2281 if (init_attr->qp_type == IB_QPT_DRIVER) {
2282 struct mlx5_ib_create_qp ucmd;
2283
2284 init_attr = &mlx_init_attr;
2285 memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));
2286 err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);
2287 if (err)
2288 return ERR_PTR(err);
c32a4f29
MS		 2289
2290 if (init_attr->qp_type == MLX5_IB_QPT_DCI) {
2291 if (init_attr->cap.max_recv_wr ||
2292 init_attr->cap.max_recv_sge) {
2293 mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");
2294 return ERR_PTR(-EINVAL);
2295 }
776a3906
MS		 2296 } else {
2297 return mlx5_ib_create_dct(pd, init_attr, &ucmd);
c32a4f29 2298 }
b4aaa1f0
MS		 2299 }
2300
e126ba97
EC		 2301 switch (init_attr->qp_type) {
2302 case IB_QPT_XRC_TGT:
2303 case IB_QPT_XRC_INI:
938fe83c 2304 if (!MLX5_CAP_GEN(dev->mdev, xrc)) {
e126ba97
EC		 2305 mlx5_ib_dbg(dev, "XRC not supported\n");
2306 return ERR_PTR(-ENOSYS);
2307 }
2308 init_attr->recv_cq = NULL;
2309 if (init_attr->qp_type == IB_QPT_XRC_TGT) {
2310 xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn;
2311 init_attr->send_cq = NULL;
2312 }
2313
2314 /* fall through */
0fb2ed66 2315 case IB_QPT_RAW_PACKET:
e126ba97
EC		 2316 case IB_QPT_RC:
2317 case IB_QPT_UC:
2318 case IB_QPT_UD:
2319 case IB_QPT_SMI:
d16e91da 2320 case MLX5_IB_QPT_HW_GSI:
e126ba97 2321 case MLX5_IB_QPT_REG_UMR:
c32a4f29 2322 case MLX5_IB_QPT_DCI:
e126ba97
EC		 2323 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2324 if (!qp)
2325 return ERR_PTR(-ENOMEM);
2326
2327 err = create_qp_common(dev, pd, init_attr, udata, qp);
2328 if (err) {
2329 mlx5_ib_dbg(dev, "create_qp_common failed\n");
2330 kfree(qp);
2331 return ERR_PTR(err);
2332 }
2333
2334 if (is_qp0(init_attr->qp_type))
2335 qp->ibqp.qp_num = 0;
2336 else if (is_qp1(init_attr->qp_type))
2337 qp->ibqp.qp_num = 1;
2338 else
19098df2 2339 qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
e126ba97
EC		 2340
2341 mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n",
19098df2 2342 qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
a1ab8402
EC		 2343 init_attr->recv_cq ? to_mcq(init_attr->recv_cq)->mcq.cqn : -1,
2344 init_attr->send_cq ? to_mcq(init_attr->send_cq)->mcq.cqn : -1);
e126ba97 2345
19098df2 2346 qp->trans_qp.xrcdn = xrcdn;
e126ba97
EC		 2347
2348 break;
2349
d16e91da
HE		 2350 case IB_QPT_GSI:
2351 return mlx5_ib_gsi_create_qp(pd, init_attr);
2352
e126ba97
EC		 2353 case IB_QPT_RAW_IPV6:
2354 case IB_QPT_RAW_ETHERTYPE:
e126ba97
EC		 2355 case IB_QPT_MAX:
2356 default:
2357 mlx5_ib_dbg(dev, "unsupported qp type %d\n",
2358 init_attr->qp_type);
2359 /* Don't support raw QPs */
2360 return ERR_PTR(-EINVAL);
2361 }
2362
b4aaa1f0
MS		 2363 if (verbs_init_attr->qp_type == IB_QPT_DRIVER)
2364 qp->qp_sub_type = init_attr->qp_type;
2365
e126ba97
EC		 2366 return &qp->ibqp;
2367}
2368
776a3906
MS		 2369static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
2370{
2371 struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
2372
2373 if (mqp->state == IB_QPS_RTR) {
2374 int err;
2375
2376 err = mlx5_core_destroy_dct(dev->mdev, &mqp->dct.mdct);
2377 if (err) {
2378 mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
2379 return err;
2380 }
2381 }
2382
2383 kfree(mqp->dct.in);
2384 kfree(mqp);
2385 return 0;
2386}
2387
e126ba97
EC		 2388int mlx5_ib_destroy_qp(struct ib_qp *qp)
2389{
2390 struct mlx5_ib_dev *dev = to_mdev(qp->device);
2391 struct mlx5_ib_qp *mqp = to_mqp(qp);
2392
d16e91da
HE		 2393 if (unlikely(qp->qp_type == IB_QPT_GSI))
2394 return mlx5_ib_gsi_destroy_qp(qp);
2395
776a3906
MS		 2396 if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
2397 return mlx5_ib_destroy_dct(mqp);
2398
e126ba97
EC		 2399 destroy_qp_common(dev, mqp);
2400
2401 kfree(mqp);
2402
2403 return 0;
2404}
2405
2406static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr,
2407 int attr_mask)
2408{
2409 u32 hw_access_flags = 0;
2410 u8 dest_rd_atomic;
2411 u32 access_flags;
2412
2413 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2414 dest_rd_atomic = attr->max_dest_rd_atomic;
2415 else
19098df2 2416 dest_rd_atomic = qp->trans_qp.resp_depth;
e126ba97
EC		 2417
2418 if (attr_mask & IB_QP_ACCESS_FLAGS)
2419 access_flags = attr->qp_access_flags;
2420 else
19098df2 2421 access_flags = qp->trans_qp.atomic_rd_en;
e126ba97
EC		 2422
2423 if (!dest_rd_atomic)
2424 access_flags &= IB_ACCESS_REMOTE_WRITE;
2425
2426 if (access_flags & IB_ACCESS_REMOTE_READ)
2427 hw_access_flags |= MLX5_QP_BIT_RRE;
2428 if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
2429 hw_access_flags |= (MLX5_QP_BIT_RAE | MLX5_ATOMIC_MODE_CX);
2430 if (access_flags & IB_ACCESS_REMOTE_WRITE)
2431 hw_access_flags |= MLX5_QP_BIT_RWE;
2432
2433 return cpu_to_be32(hw_access_flags);
2434}
2435
2436enum {
2437 MLX5_PATH_FLAG_FL = 1 << 0,
2438 MLX5_PATH_FLAG_FREE_AR = 1 << 1,
2439 MLX5_PATH_FLAG_COUNTER = 1 << 2,
2440};
2441
2442static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
2443{
2444 if (rate == IB_RATE_PORT_CURRENT) {
2445 return 0;
2446 } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) {
2447 return -EINVAL;
2448 } else {
2449 while (rate != IB_RATE_2_5_GBPS &&
2450 !(1 << (rate + MLX5_STAT_RATE_OFFSET) &
938fe83c 2451 MLX5_CAP_GEN(dev->mdev, stat_rate_support)))
e126ba97
EC		 2452 --rate;
2453 }
2454
2455 return rate + MLX5_STAT_RATE_OFFSET;
2456}
2457
75850d0b 2458static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
2459 struct mlx5_ib_sq *sq, u8 sl)
2460{
2461 void *in;
2462 void *tisc;
2463 int inlen;
2464 int err;
2465
2466 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 2467 in = kvzalloc(inlen, GFP_KERNEL);
75850d0b 2468 if (!in)
2469 return -ENOMEM;
2470
2471 MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
2472
2473 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2474 MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
2475
2476 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
2477
2478 kvfree(in);
2479
2480 return err;
2481}
2482
13eab21f
AH		 2483static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
2484 struct mlx5_ib_sq *sq, u8 tx_affinity)
2485{
2486 void *in;
2487 void *tisc;
2488 int inlen;
2489 int err;
2490
2491 inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
1b9a07ee 2492 in = kvzalloc(inlen, GFP_KERNEL);
13eab21f
AH		 2493 if (!in)
2494 return -ENOMEM;
2495
2496 MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
2497
2498 tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
2499 MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
2500
2501 err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);
2502
2503 kvfree(in);
2504
2505 return err;
2506}
2507
75850d0b 2508static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
90898850 2509 const struct rdma_ah_attr *ah,
e126ba97 2510 struct mlx5_qp_path *path, u8 port, int attr_mask,
f879ee8d
AS		 2511 u32 path_flags, const struct ib_qp_attr *attr,
2512 bool alt)
e126ba97 2513{
d8966fcd 2514 const struct ib_global_route *grh = rdma_ah_read_grh(ah);
e126ba97 2515 int err;
ed88451e 2516 enum ib_gid_type gid_type;
d8966fcd
DC		 2517 u8 ah_flags = rdma_ah_get_ah_flags(ah);
2518 u8 sl = rdma_ah_get_sl(ah);
e126ba97 2519
e126ba97 2520 if (attr_mask & IB_QP_PKEY_INDEX)
f879ee8d
AS		 2521 path->pkey_index = cpu_to_be16(alt ? attr->alt_pkey_index :
2522 attr->pkey_index);
e126ba97 2523
d8966fcd
DC		 2524 if (ah_flags & IB_AH_GRH) {
2525 if (grh->sgid_index >=
938fe83c 2526 dev->mdev->port_caps[port - 1].gid_table_len) {
f4f01b54 2527 pr_err("sgid_index (%u) too large. max is %d\n",
d8966fcd 2528 grh->sgid_index,
938fe83c 2529 dev->mdev->port_caps[port - 1].gid_table_len);
f83b4263
EC		 2530 return -EINVAL;
2531 }
2811ba51 2532 }
44c58487
DC		 2533
2534 if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
d8966fcd 2535 if (!(ah_flags & IB_AH_GRH))
2811ba51 2536 return -EINVAL;
d8966fcd 2537 err = mlx5_get_roce_gid_type(dev, port, grh->sgid_index,
ed88451e
MD		 2538 &gid_type);
2539 if (err)
2540 return err;
44c58487 2541 memcpy(path->rmac, ah->roce.dmac, sizeof(ah->roce.dmac));
2b621851
MD		 2542 if (qp->ibqp.qp_type == IB_QPT_RC ||
2543 qp->ibqp.qp_type == IB_QPT_UC ||
2544 qp->ibqp.qp_type == IB_QPT_XRC_INI ||
2545 qp->ibqp.qp_type == IB_QPT_XRC_TGT)
2546 path->udp_sport = mlx5_get_roce_udp_sport(dev, port,
2547 grh->sgid_index);
d8966fcd 2548 path->dci_cfi_prio_sl = (sl & 0x7) << 4;
ed88451e 2549 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
d8966fcd 2550 path->ecn_dscp = (grh->traffic_class >> 2) & 0x3f;
2811ba51 2551 } else {
d3ae2bde
NO		 2552 path->fl_free_ar = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0;
2553 path->fl_free_ar |=
2554 (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x40 : 0;
d8966fcd
DC		 2555 path->rlid = cpu_to_be16(rdma_ah_get_dlid(ah));
2556 path->grh_mlid = rdma_ah_get_path_bits(ah) & 0x7f;
2557 if (ah_flags & IB_AH_GRH)
2811ba51 2558 path->grh_mlid |= 1 << 7;
d8966fcd 2559 path->dci_cfi_prio_sl = sl & 0xf;
2811ba51
AS		 2560 }
2561
d8966fcd
DC		 2562 if (ah_flags & IB_AH_GRH) {
2563 path->mgid_index = grh->sgid_index;
2564 path->hop_limit = grh->hop_limit;
e126ba97 2565 path->tclass_flowlabel =
d8966fcd
DC		 2566 cpu_to_be32((grh->traffic_class << 20) |
2567 (grh->flow_label));
2568 memcpy(path->rgid, grh->dgid.raw, 16);
e126ba97
EC		 2569 }
2570
d8966fcd 2571 err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
e126ba97
EC		 2572 if (err < 0)
2573 return err;
2574 path->static_rate = err;
2575 path->port = port;
2576
e126ba97 2577 if (attr_mask & IB_QP_TIMEOUT)
f879ee8d 2578 path->ackto_lt = (alt ? attr->alt_timeout : attr->timeout) << 3;
e126ba97 2579
75850d0b 2580 if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
2581 return modify_raw_packet_eth_prio(dev->mdev,
2582 &qp->raw_packet_qp.sq,
d8966fcd 2583 sl & 0xf);
75850d0b 2584
e126ba97
EC		 2585 return 0;
2586}
2587
2588static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
2589 [MLX5_QP_STATE_INIT] = {
2590 [MLX5_QP_STATE_INIT] = {
2591 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
2592 MLX5_QP_OPTPAR_RAE |
2593 MLX5_QP_OPTPAR_RWE |
2594 MLX5_QP_OPTPAR_PKEY_INDEX |
2595 MLX5_QP_OPTPAR_PRI_PORT,
2596 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
2597 MLX5_QP_OPTPAR_PKEY_INDEX |
2598 MLX5_QP_OPTPAR_PRI_PORT,
2599 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
2600 MLX5_QP_OPTPAR_Q_KEY |
2601 MLX5_QP_OPTPAR_PRI_PORT,
2602 },
2603 [MLX5_QP_STATE_RTR] = {
2604 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2605 MLX5_QP_OPTPAR_RRE |
2606 MLX5_QP_OPTPAR_RAE |
2607 MLX5_QP_OPTPAR_RWE |
2608 MLX5_QP_OPTPAR_PKEY_INDEX,
2609 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2610 MLX5_QP_OPTPAR_RWE |
2611 MLX5_QP_OPTPAR_PKEY_INDEX,
2612 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX |
2613 MLX5_QP_OPTPAR_Q_KEY,
2614 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX |
2615 MLX5_QP_OPTPAR_Q_KEY,
a4774e90
EC		 2616 [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2617 MLX5_QP_OPTPAR_RRE |
2618 MLX5_QP_OPTPAR_RAE |
2619 MLX5_QP_OPTPAR_RWE |
2620 MLX5_QP_OPTPAR_PKEY_INDEX,
e126ba97
EC		 2621 },
2622 },
2623 [MLX5_QP_STATE_RTR] = {
2624 [MLX5_QP_STATE_RTS] = {
2625 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2626 MLX5_QP_OPTPAR_RRE |
2627 MLX5_QP_OPTPAR_RAE |
2628 MLX5_QP_OPTPAR_RWE |
2629 MLX5_QP_OPTPAR_PM_STATE |
2630 MLX5_QP_OPTPAR_RNR_TIMEOUT,
2631 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
2632 MLX5_QP_OPTPAR_RWE |
2633 MLX5_QP_OPTPAR_PM_STATE,
2634 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
2635 },
2636 },
2637 [MLX5_QP_STATE_RTS] = {
2638 [MLX5_QP_STATE_RTS] = {
2639 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE |
2640 MLX5_QP_OPTPAR_RAE |
2641 MLX5_QP_OPTPAR_RWE |
2642 MLX5_QP_OPTPAR_RNR_TIMEOUT |
c2a3431e
EC		 2643 MLX5_QP_OPTPAR_PM_STATE |
2644 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97 2645 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE |
c2a3431e
EC		 2646 MLX5_QP_OPTPAR_PM_STATE |
2647 MLX5_QP_OPTPAR_ALT_ADDR_PATH,
e126ba97
EC		 2648 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY |
2649 MLX5_QP_OPTPAR_SRQN |
2650 MLX5_QP_OPTPAR_CQN_RCV,
2651 },
2652 },
2653 [MLX5_QP_STATE_SQER] = {
2654 [MLX5_QP_STATE_RTS] = {
2655 [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
2656 [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
75959f56 2657 [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
a4774e90
EC		 2658 [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT |
2659 MLX5_QP_OPTPAR_RWE |
2660 MLX5_QP_OPTPAR_RAE |
2661 MLX5_QP_OPTPAR_RRE,
e126ba97
EC		 2662 },
2663 },
2664};
2665
2666static int ib_nr_to_mlx5_nr(int ib_mask)
2667{
2668 switch (ib_mask) {
2669 case IB_QP_STATE:
2670 return 0;
2671 case IB_QP_CUR_STATE:
2672 return 0;
2673 case IB_QP_EN_SQD_ASYNC_NOTIFY:
2674 return 0;
2675 case IB_QP_ACCESS_FLAGS:
2676 return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
2677 MLX5_QP_OPTPAR_RAE;
2678 case IB_QP_PKEY_INDEX:
2679 return MLX5_QP_OPTPAR_PKEY_INDEX;
2680 case IB_QP_PORT:
2681 return MLX5_QP_OPTPAR_PRI_PORT;
2682 case IB_QP_QKEY:
2683 return MLX5_QP_OPTPAR_Q_KEY;
2684 case IB_QP_AV:
2685 return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
2686 MLX5_QP_OPTPAR_PRI_PORT;
2687 case IB_QP_PATH_MTU:
2688 return 0;
2689 case IB_QP_TIMEOUT:
2690 return MLX5_QP_OPTPAR_ACK_TIMEOUT;
2691 case IB_QP_RETRY_CNT:
2692 return MLX5_QP_OPTPAR_RETRY_COUNT;
2693 case IB_QP_RNR_RETRY:
2694 return MLX5_QP_OPTPAR_RNR_RETRY;
2695 case IB_QP_RQ_PSN:
2696 return 0;
2697 case IB_QP_MAX_QP_RD_ATOMIC:
2698 return MLX5_QP_OPTPAR_SRA_MAX;
2699 case IB_QP_ALT_PATH:
2700 return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
2701 case IB_QP_MIN_RNR_TIMER:
2702 return MLX5_QP_OPTPAR_RNR_TIMEOUT;
2703 case IB_QP_SQ_PSN:
2704 return 0;
2705 case IB_QP_MAX_DEST_RD_ATOMIC:
2706 return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
2707 MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
2708 case IB_QP_PATH_MIG_STATE:
2709 return MLX5_QP_OPTPAR_PM_STATE;
2710 case IB_QP_CAP:
2711 return 0;
2712 case IB_QP_DEST_QPN:
2713 return 0;
2714 }
2715 return 0;
2716}
2717
2718static int ib_mask_to_mlx5_opt(int ib_mask)
2719{
2720 int result = 0;
2721 int i;
2722
2723 for (i = 0; i < 8 * sizeof(int); i++) {
2724 if ((1 << i) & ib_mask)
2725 result |= ib_nr_to_mlx5_nr(1 << i);
2726 }
2727
2728 return result;
2729}
2730
eb49ab0c
AV		 2731static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
2732 struct mlx5_ib_rq *rq, int new_state,
2733 const struct mlx5_modify_raw_qp_param *raw_qp_param)
ad5f8e96 2734{
2735 void *in;
2736 void *rqc;
2737 int inlen;
2738 int err;
2739
2740 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 2741 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 2742 if (!in)
2743 return -ENOMEM;
2744
2745 MLX5_SET(modify_rq_in, in, rq_state, rq->state);
2746
2747 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
2748 MLX5_SET(rqc, rqc, state, new_state);
2749
eb49ab0c
AV		 2750 if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
2751 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
2752 MLX5_SET64(modify_rq_in, in, modify_bitmask,
23a6964e 2753 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
eb49ab0c
AV		 2754 MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
2755 } else
2756 pr_info_once("%s: RAW PACKET QP counters are not supported on current FW\n",
2757 dev->ib_dev.name);
2758 }
2759
2760 err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in, inlen);
ad5f8e96 2761 if (err)
2762 goto out;
2763
2764 rq->state = new_state;
2765
2766out:
2767 kvfree(in);
2768 return err;
2769}
2770
2771static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
7d29f349
BW		 2772 struct mlx5_ib_sq *sq,
2773 int new_state,
2774 const struct mlx5_modify_raw_qp_param *raw_qp_param)
ad5f8e96 2775{
7d29f349
BW		 2776 struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
2777 u32 old_rate = ibqp->rate_limit;
2778 u32 new_rate = old_rate;
2779 u16 rl_index = 0;
ad5f8e96 2780 void *in;
2781 void *sqc;
2782 int inlen;
2783 int err;
2784
2785 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 2786 in = kvzalloc(inlen, GFP_KERNEL);
ad5f8e96 2787 if (!in)
2788 return -ENOMEM;
2789
2790 MLX5_SET(modify_sq_in, in, sq_state, sq->state);
2791
2792 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
2793 MLX5_SET(sqc, sqc, state, new_state);
2794
7d29f349
BW		 2795 if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
2796 if (new_state != MLX5_SQC_STATE_RDY)
2797 pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
2798 __func__);
2799 else
2800 new_rate = raw_qp_param->rate_limit;
2801 }
2802
2803 if (old_rate != new_rate) {
2804 if (new_rate) {
2805 err = mlx5_rl_add_rate(dev, new_rate, &rl_index);
2806 if (err) {
2807 pr_err("Failed configuring rate %u: %d\n",
2808 new_rate, err);
2809 goto out;
2810 }
2811 }
2812
2813 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
2814 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
2815 }
2816
ad5f8e96 2817 err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in, inlen);
7d29f349
BW		 2818 if (err) {
2819 /* Remove new rate from table if failed */
2820 if (new_rate &&
2821 old_rate != new_rate)
2822 mlx5_rl_remove_rate(dev, new_rate);
ad5f8e96 2823 goto out;
7d29f349
BW		 2824 }
2825
2826 /* Only remove the old rate after new rate was set */
2827 if ((old_rate &&
2828 (old_rate != new_rate)) ||
2829 (new_state != MLX5_SQC_STATE_RDY))
2830 mlx5_rl_remove_rate(dev, old_rate);
ad5f8e96 2831
7d29f349 2832 ibqp->rate_limit = new_rate;
ad5f8e96 2833 sq->state = new_state;
2834
2835out:
2836 kvfree(in);
2837 return err;
2838}
2839
2840static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
13eab21f
AH		 2841 const struct mlx5_modify_raw_qp_param *raw_qp_param,
2842 u8 tx_affinity)
ad5f8e96 2843{
2844 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
2845 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
2846 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
7d29f349
BW		 2847 int modify_rq = !!qp->rq.wqe_cnt;
2848 int modify_sq = !!qp->sq.wqe_cnt;
ad5f8e96 2849 int rq_state;
2850 int sq_state;
2851 int err;
2852
0680efa2 2853 switch (raw_qp_param->operation) {
ad5f8e96 2854 case MLX5_CMD_OP_RST2INIT_QP:
2855 rq_state = MLX5_RQC_STATE_RDY;
2856 sq_state = MLX5_SQC_STATE_RDY;
2857 break;
2858 case MLX5_CMD_OP_2ERR_QP:
2859 rq_state = MLX5_RQC_STATE_ERR;
2860 sq_state = MLX5_SQC_STATE_ERR;
2861 break;
2862 case MLX5_CMD_OP_2RST_QP:
2863 rq_state = MLX5_RQC_STATE_RST;
2864 sq_state = MLX5_SQC_STATE_RST;
2865 break;
ad5f8e96 2866 case MLX5_CMD_OP_RTR2RTS_QP:
2867 case MLX5_CMD_OP_RTS2RTS_QP:
7d29f349
BW		 2868 if (raw_qp_param->set_mask ==
2869 MLX5_RAW_QP_RATE_LIMIT) {
2870 modify_rq = 0;
2871 sq_state = sq->state;
2872 } else {
2873 return raw_qp_param->set_mask ? -EINVAL : 0;
2874 }
2875 break;
2876 case MLX5_CMD_OP_INIT2INIT_QP:
2877 case MLX5_CMD_OP_INIT2RTR_QP:
eb49ab0c
AV		 2878 if (raw_qp_param->set_mask)
2879 return -EINVAL;
2880 else
2881 return 0;
ad5f8e96 2882 default:
2883 WARN_ON(1);
2884 return -EINVAL;
2885 }
2886
7d29f349
BW		 2887 if (modify_rq) {
2888 err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
ad5f8e96 2889 if (err)
2890 return err;
2891 }
2892
7d29f349 2893 if (modify_sq) {
13eab21f
AH		 2894 if (tx_affinity) {
2895 err = modify_raw_packet_tx_affinity(dev->mdev, sq,
2896 tx_affinity);
2897 if (err)
2898 return err;
2899 }
2900
7d29f349 2901 return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state, raw_qp_param);
13eab21f 2902 }
ad5f8e96 2903
2904 return 0;
2905}
2906
e126ba97
EC		 2907static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
2908 const struct ib_qp_attr *attr, int attr_mask,
2909 enum ib_qp_state cur_state, enum ib_qp_state new_state)
2910{
427c1e7b 2911 static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
2912 [MLX5_QP_STATE_RST] = {
2913 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2914 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2915 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_RST2INIT_QP,
2916 },
2917 [MLX5_QP_STATE_INIT] = {
2918 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2919 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2920 [MLX5_QP_STATE_INIT] = MLX5_CMD_OP_INIT2INIT_QP,
2921 [MLX5_QP_STATE_RTR] = MLX5_CMD_OP_INIT2RTR_QP,
2922 },
2923 [MLX5_QP_STATE_RTR] = {
2924 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2925 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2926 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTR2RTS_QP,
2927 },
2928 [MLX5_QP_STATE_RTS] = {
2929 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2930 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2931 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_RTS2RTS_QP,
2932 },
2933 [MLX5_QP_STATE_SQD] = {
2934 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2935 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2936 },
2937 [MLX5_QP_STATE_SQER] = {
2938 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2939 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2940 [MLX5_QP_STATE_RTS] = MLX5_CMD_OP_SQERR2RTS_QP,
2941 },
2942 [MLX5_QP_STATE_ERR] = {
2943 [MLX5_QP_STATE_RST] = MLX5_CMD_OP_2RST_QP,
2944 [MLX5_QP_STATE_ERR] = MLX5_CMD_OP_2ERR_QP,
2945 }
2946 };
2947
e126ba97
EC		 2948 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2949 struct mlx5_ib_qp *qp = to_mqp(ibqp);
19098df2 2950 struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
e126ba97
EC		 2951 struct mlx5_ib_cq *send_cq, *recv_cq;
2952 struct mlx5_qp_context *context;
e126ba97 2953 struct mlx5_ib_pd *pd;
eb49ab0c 2954 struct mlx5_ib_port *mibport = NULL;
e126ba97
EC		 2955 enum mlx5_qp_state mlx5_cur, mlx5_new;
2956 enum mlx5_qp_optpar optpar;
e126ba97
EC		 2957 int mlx5_st;
2958 int err;
427c1e7b 2959 u16 op;
13eab21f 2960 u8 tx_affinity = 0;
e126ba97 2961
55de9a77
LR		 2962 mlx5_st = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
2963 qp->qp_sub_type : ibqp->qp_type);
2964 if (mlx5_st < 0)
2965 return -EINVAL;
2966
1a412fb1
SM		 2967 context = kzalloc(sizeof(*context), GFP_KERNEL);
2968 if (!context)
e126ba97
EC		 2969 return -ENOMEM;
2970
55de9a77 2971 context->flags = cpu_to_be32(mlx5_st << 16);
e126ba97
EC		 2972
2973 if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
2974 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2975 } else {
2976 switch (attr->path_mig_state) {
2977 case IB_MIG_MIGRATED:
2978 context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
2979 break;
2980 case IB_MIG_REARM:
2981 context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11);
2982 break;
2983 case IB_MIG_ARMED:
2984 context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11);
2985 break;
2986 }
2987 }
2988
13eab21f
AH		 2989 if ((cur_state == IB_QPS_RESET) && (new_state == IB_QPS_INIT)) {
2990 if ((ibqp->qp_type == IB_QPT_RC) ||
2991 (ibqp->qp_type == IB_QPT_UD &&
2992 !(qp->flags & MLX5_IB_QP_SQPN_QP1)) ||
2993 (ibqp->qp_type == IB_QPT_UC) ||
2994 (ibqp->qp_type == IB_QPT_RAW_PACKET) ||
2995 (ibqp->qp_type == IB_QPT_XRC_INI) ||
2996 (ibqp->qp_type == IB_QPT_XRC_TGT)) {
2997 if (mlx5_lag_is_active(dev->mdev)) {
7fd8aefb 2998 u8 p = mlx5_core_native_port_num(dev->mdev);
13eab21f 2999 tx_affinity = (unsigned int)atomic_add_return(1,
7fd8aefb 3000 &dev->roce[p].next_port) %
13eab21f
AH		 3001 MLX5_MAX_PORTS + 1;
3002 context->flags |= cpu_to_be32(tx_affinity << 24);
3003 }
3004 }
3005 }
3006
d16e91da 3007 if (is_sqp(ibqp->qp_type)) {
e126ba97 3008 context->mtu_msgmax = (IB_MTU_256 << 5) | 8;
c2e53b2c
YH		 3009 } else if ((ibqp->qp_type == IB_QPT_UD &&
3010 !(qp->flags & MLX5_IB_QP_UNDERLAY)) ||
e126ba97
EC		 3011 ibqp->qp_type == MLX5_IB_QPT_REG_UMR) {
3012 context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
3013 } else if (attr_mask & IB_QP_PATH_MTU) {
3014 if (attr->path_mtu < IB_MTU_256 ||
3015 attr->path_mtu > IB_MTU_4096) {
3016 mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
3017 err = -EINVAL;
3018 goto out;
3019 }
938fe83c
SM		 3020 context->mtu_msgmax = (attr->path_mtu << 5) |
3021 (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg);
e126ba97
EC		 3022 }
3023
3024 if (attr_mask & IB_QP_DEST_QPN)
3025 context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num);
3026
3027 if (attr_mask & IB_QP_PKEY_INDEX)
d3ae2bde 3028 context->pri_path.pkey_index = cpu_to_be16(attr->pkey_index);
e126ba97
EC		 3029
3030 /* todo implement counter_index functionality */
3031
3032 if (is_sqp(ibqp->qp_type))
3033 context->pri_path.port = qp->port;
3034
3035 if (attr_mask & IB_QP_PORT)
3036 context->pri_path.port = attr->port_num;
3037
3038 if (attr_mask & IB_QP_AV) {
75850d0b 3039 err = mlx5_set_path(dev, qp, &attr->ah_attr, &context->pri_path,
e126ba97 3040 attr_mask & IB_QP_PORT ? attr->port_num : qp->port,
f879ee8d 3041 attr_mask, 0, attr, false);
e126ba97
EC		 3042 if (err)
3043 goto out;
3044 }
3045
3046 if (attr_mask & IB_QP_TIMEOUT)
3047 context->pri_path.ackto_lt |= attr->timeout << 3;
3048
3049 if (attr_mask & IB_QP_ALT_PATH) {
75850d0b 3050 err = mlx5_set_path(dev, qp, &attr->alt_ah_attr,
3051 &context->alt_path,
f879ee8d
AS		 3052 attr->alt_port_num,
3053 attr_mask | IB_QP_PKEY_INDEX | IB_QP_TIMEOUT,
3054 0, attr, true);
e126ba97
EC		 3055 if (err)
3056 goto out;
3057 }
3058
3059 pd = get_pd(qp);
89ea94a7
MG		 3060 get_cqs(qp->ibqp.qp_type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
3061 &send_cq, &recv_cq);
e126ba97
EC		 3062
3063 context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
3064 context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0;
3065 context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0;
3066 context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28);
3067
3068 if (attr_mask & IB_QP_RNR_RETRY)
3069 context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
3070
3071 if (attr_mask & IB_QP_RETRY_CNT)
3072 context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
3073
3074 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3075 if (attr->max_rd_atomic)
3076 context->params1 |=
3077 cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
3078 }
3079
3080 if (attr_mask & IB_QP_SQ_PSN)
3081 context->next_send_psn = cpu_to_be32(attr->sq_psn);
3082
3083 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3084 if (attr->max_dest_rd_atomic)
3085 context->params2 |=
3086 cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
3087 }
3088
3089 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
3090 context->params2 |= to_mlx5_access_flags(qp, attr, attr_mask);
3091
3092 if (attr_mask & IB_QP_MIN_RNR_TIMER)
3093 context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
3094
3095 if (attr_mask & IB_QP_RQ_PSN)
3096 context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
3097
3098 if (attr_mask & IB_QP_QKEY)
3099 context->qkey = cpu_to_be32(attr->qkey);
3100
3101 if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3102 context->db_rec_addr = cpu_to_be64(qp->db.dma);
3103
0837e86a
MB		 3104 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3105 u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
3106 qp->port) - 1;
c2e53b2c
YH		 3107
3108 /* Underlay port should be used - index 0 function per port */
3109 if (qp->flags & MLX5_IB_QP_UNDERLAY)
3110 port_num = 0;
3111
eb49ab0c 3112 mibport = &dev->port[port_num];
0837e86a 3113 context->qp_counter_set_usr_page |=
e1f24a79 3114 cpu_to_be32((u32)(mibport->cnts.set_id) << 24);
0837e86a
MB		 3115 }
3116
e126ba97
EC		 3117 if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
3118 context->sq_crq_size |= cpu_to_be16(1 << 4);
3119
b11a4f9c
HE		 3120 if (qp->flags & MLX5_IB_QP_SQPN_QP1)
3121 context->deth_sqpn = cpu_to_be32(1);
e126ba97
EC		 3122
3123 mlx5_cur = to_mlx5_state(cur_state);
3124 mlx5_new = to_mlx5_state(new_state);
e126ba97 3125
427c1e7b 3126 if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
5d414b17
DC		 3127 !optab[mlx5_cur][mlx5_new]) {
3128 err = -EINVAL;
427c1e7b 3129 goto out;
5d414b17 3130 }
427c1e7b 3131
3132 op = optab[mlx5_cur][mlx5_new];
e126ba97
EC		 3133 optpar = ib_mask_to_mlx5_opt(attr_mask);
3134 optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
ad5f8e96 3135
c2e53b2c
YH		 3136 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
3137 qp->flags & MLX5_IB_QP_UNDERLAY) {
0680efa2
AV		 3138 struct mlx5_modify_raw_qp_param raw_qp_param = {};
3139
3140 raw_qp_param.operation = op;
eb49ab0c 3141 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
e1f24a79 3142 raw_qp_param.rq_q_ctr_id = mibport->cnts.set_id;
eb49ab0c
AV		 3143 raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
3144 }
7d29f349
BW		 3145
3146 if (attr_mask & IB_QP_RATE_LIMIT) {
3147 raw_qp_param.rate_limit = attr->rate_limit;
3148 raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
3149 }
3150
13eab21f 3151 err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
0680efa2 3152 } else {
1a412fb1 3153 err = mlx5_core_qp_modify(dev->mdev, op, optpar, context,
ad5f8e96 3154 &base->mqp);
0680efa2
AV		 3155 }
3156
e126ba97
EC		 3157 if (err)
3158 goto out;
3159
3160 qp->state = new_state;
3161
3162 if (attr_mask & IB_QP_ACCESS_FLAGS)
19098df2 3163 qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
e126ba97 3164 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
19098df2 3165 qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
e126ba97
EC		 3166 if (attr_mask & IB_QP_PORT)
3167 qp->port = attr->port_num;
3168 if (attr_mask & IB_QP_ALT_PATH)
19098df2 3169 qp->trans_qp.alt_port = attr->alt_port_num;
e126ba97
EC		 3170
3171 /*
3172 * If we moved a kernel QP to RESET, clean up all old CQ
3173 * entries and reinitialize the QP.
3174 */
75a45982
LR		 3175 if (new_state == IB_QPS_RESET &&
3176 !ibqp->uobject && ibqp->qp_type != IB_QPT_XRC_TGT) {
19098df2 3177 mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
e126ba97
EC		 3178 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
3179 if (send_cq != recv_cq)
19098df2 3180 mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
e126ba97
EC		 3181
3182 qp->rq.head = 0;
3183 qp->rq.tail = 0;
3184 qp->sq.head = 0;
3185 qp->sq.tail = 0;
3186 qp->sq.cur_post = 0;
3187 qp->sq.last_poll = 0;
3188 qp->db.db[MLX5_RCV_DBR] = 0;
3189 qp->db.db[MLX5_SND_DBR] = 0;
3190 }
3191
3192out:
1a412fb1 3193 kfree(context);
e126ba97
EC		 3194 return err;
3195}
3196
c32a4f29
MS		 3197static inline bool is_valid_mask(int mask, int req, int opt)
3198{
3199 if ((mask & req) != req)
3200 return false;
3201
3202 if (mask & ~(req | opt))
3203 return false;
3204
3205 return true;
3206}
3207
3208/* check valid transition for driver QP types
3209 * for now the only QP type that this function supports is DCI
3210 */
3211static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
3212 enum ib_qp_attr_mask attr_mask)
3213{
3214 int req = IB_QP_STATE;
3215 int opt = 0;
3216
3217 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3218 req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
3219 return is_valid_mask(attr_mask, req, opt);
3220 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3221 opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
3222 return is_valid_mask(attr_mask, req, opt);
3223 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3224 req |= IB_QP_PATH_MTU;
3225 opt = IB_QP_PKEY_INDEX;
3226 return is_valid_mask(attr_mask, req, opt);
3227 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3228 req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
3229 IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
3230 opt = IB_QP_MIN_RNR_TIMER;
3231 return is_valid_mask(attr_mask, req, opt);
3232 } else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
3233 opt = IB_QP_MIN_RNR_TIMER;
3234 return is_valid_mask(attr_mask, req, opt);
3235 } else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
3236 return is_valid_mask(attr_mask, req, opt);
3237 }
3238 return false;
3239}
3240
776a3906
MS		 3241/* mlx5_ib_modify_dct: modify a DCT QP
3242 * valid transitions are:
3243 * RESET to INIT: must set access_flags, pkey_index and port
3244 * INIT to RTR : must set min_rnr_timer, tclass, flow_label,
3245 * mtu, gid_index and hop_limit
3246 * Other transitions and attributes are illegal
3247 */
3248static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3249 int attr_mask, struct ib_udata *udata)
3250{
3251 struct mlx5_ib_qp *qp = to_mqp(ibqp);
3252 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3253 enum ib_qp_state cur_state, new_state;
3254 int err = 0;
3255 int required = IB_QP_STATE;
3256 void *dctc;
3257
3258 if (!(attr_mask & IB_QP_STATE))
3259 return -EINVAL;
3260
3261 cur_state = qp->state;
3262 new_state = attr->qp_state;
3263
3264 dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
3265 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3266 required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
3267 if (!is_valid_mask(attr_mask, required, 0))
3268 return -EINVAL;
3269
3270 if (attr->port_num == 0 ||
3271 attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
3272 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
3273 attr->port_num, dev->num_ports);
3274 return -EINVAL;
3275 }
3276 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
3277 MLX5_SET(dctc, dctc, rre, 1);
3278 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
3279 MLX5_SET(dctc, dctc, rwe, 1);
3280 if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3281 if (!mlx5_ib_dc_atomic_is_supported(dev))
3282 return -EOPNOTSUPP;
3283 MLX5_SET(dctc, dctc, rae, 1);
3284 MLX5_SET(dctc, dctc, atomic_mode, MLX5_ATOMIC_MODE_DCT_CX);
3285 }
3286 MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
3287 MLX5_SET(dctc, dctc, port, attr->port_num);
3288 MLX5_SET(dctc, dctc, counter_set_id, dev->port[attr->port_num - 1].cnts.set_id);
3289
3290 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3291 struct mlx5_ib_modify_qp_resp resp = {};
3292 u32 min_resp_len = offsetof(typeof(resp), dctn) +
3293 sizeof(resp.dctn);
3294
3295 if (udata->outlen < min_resp_len)
3296 return -EINVAL;
3297 resp.response_length = min_resp_len;
3298
3299 required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
3300 if (!is_valid_mask(attr_mask, required, 0))
3301 return -EINVAL;
3302 MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
3303 MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
3304 MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
3305 MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
3306 MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
3307 MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
3308
3309 err = mlx5_core_create_dct(dev->mdev, &qp->dct.mdct, qp->dct.in,
3310 MLX5_ST_SZ_BYTES(create_dct_in));
3311 if (err)
3312 return err;
3313 resp.dctn = qp->dct.mdct.mqp.qpn;
3314 err = ib_copy_to_udata(udata, &resp, resp.response_length);
3315 if (err) {
3316 mlx5_core_destroy_dct(dev->mdev, &qp->dct.mdct);
3317 return err;
3318 }
3319 } else {
3320 mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
3321 return -EINVAL;
3322 }
3323 if (err)
3324 qp->state = IB_QPS_ERR;
3325 else
3326 qp->state = new_state;
3327 return err;
3328}
3329
e126ba97
EC		 3330int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
3331 int attr_mask, struct ib_udata *udata)
3332{
3333 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3334 struct mlx5_ib_qp *qp = to_mqp(ibqp);
d16e91da 3335 enum ib_qp_type qp_type;
e126ba97
EC		 3336 enum ib_qp_state cur_state, new_state;
3337 int err = -EINVAL;
3338 int port;
2811ba51 3339 enum rdma_link_layer ll = IB_LINK_LAYER_UNSPECIFIED;
e126ba97 3340
28d61370
YH		 3341 if (ibqp->rwq_ind_tbl)
3342 return -ENOSYS;
3343
d16e91da
HE		 3344 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
3345 return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
3346
c32a4f29
MS		 3347 if (ibqp->qp_type == IB_QPT_DRIVER)
3348 qp_type = qp->qp_sub_type;
3349 else
3350 qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
3351 IB_QPT_GSI : ibqp->qp_type;
3352
776a3906
MS		 3353 if (qp_type == MLX5_IB_QPT_DCT)
3354 return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
d16e91da 3355
e126ba97
EC		 3356 mutex_lock(&qp->mutex);
3357
3358 cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
3359 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
3360
2811ba51
AS		 3361 if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
3362 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
3363 ll = dev->ib_dev.get_link_layer(&dev->ib_dev, port);
3364 }
3365
c2e53b2c
YH		 3366 if (qp->flags & MLX5_IB_QP_UNDERLAY) {
3367 if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
3368 mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
3369 attr_mask);
3370 goto out;
3371 }
3372 } else if (qp_type != MLX5_IB_QPT_REG_UMR &&
c32a4f29
MS		 3373 qp_type != MLX5_IB_QPT_DCI &&
3374 !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
158abf86
HE		 3375 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
3376 cur_state, new_state, ibqp->qp_type, attr_mask);
e126ba97 3377 goto out;
c32a4f29
MS		 3378 } else if (qp_type == MLX5_IB_QPT_DCI &&
3379 !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
3380 mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
3381 cur_state, new_state, qp_type, attr_mask);
3382 goto out;
158abf86 3383 }
e126ba97
EC		 3384
3385 if ((attr_mask & IB_QP_PORT) &&
938fe83c 3386 (attr->port_num == 0 ||
508562d6 3387 attr->port_num > dev->num_ports)) {
158abf86
HE		 3388 mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
3389 attr->port_num, dev->num_ports);
e126ba97 3390 goto out;
158abf86 3391 }
e126ba97
EC		 3392
3393 if (attr_mask & IB_QP_PKEY_INDEX) {
3394 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
938fe83c 3395 if (attr->pkey_index >=
158abf86
HE		 3396 dev->mdev->port_caps[port - 1].pkey_table_len) {
3397 mlx5_ib_dbg(dev, "invalid pkey index %d\n",
3398 attr->pkey_index);
e126ba97 3399 goto out;
158abf86 3400 }
e126ba97
EC		 3401 }
3402
3403 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
938fe83c 3404 attr->max_rd_atomic >
158abf86
HE		 3405 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) {
3406 mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
3407 attr->max_rd_atomic);
e126ba97 3408 goto out;
158abf86 3409 }
e126ba97
EC		 3410
3411 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
938fe83c 3412 attr->max_dest_rd_atomic >
158abf86
HE		 3413 (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) {
3414 mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
3415 attr->max_dest_rd_atomic);
e126ba97 3416 goto out;
158abf86 3417 }
e126ba97
EC		 3418
3419 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
3420 err = 0;
3421 goto out;
3422 }
3423
3424 err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
3425
3426out:
3427 mutex_unlock(&qp->mutex);
3428 return err;
3429}
3430
3431static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
3432{
3433 struct mlx5_ib_cq *cq;
3434 unsigned cur;
3435
3436 cur = wq->head - wq->tail;
3437 if (likely(cur + nreq < wq->max_post))
3438 return 0;
3439
3440 cq = to_mcq(ib_cq);
3441 spin_lock(&cq->lock);
3442 cur = wq->head - wq->tail;
3443 spin_unlock(&cq->lock);
3444
3445 return cur + nreq >= wq->max_post;
3446}
3447
3448static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg,
3449 u64 remote_addr, u32 rkey)
3450{
3451 rseg->raddr = cpu_to_be64(remote_addr);
3452 rseg->rkey = cpu_to_be32(rkey);
3453 rseg->reserved = 0;
3454}
3455
f0313965
ES		 3456static void *set_eth_seg(struct mlx5_wqe_eth_seg *eseg,
3457 struct ib_send_wr *wr, void *qend,
3458 struct mlx5_ib_qp *qp, int *size)
3459{
3460 void *seg = eseg;
3461
3462 memset(eseg, 0, sizeof(struct mlx5_wqe_eth_seg));
3463
3464 if (wr->send_flags & IB_SEND_IP_CSUM)
3465 eseg->cs_flags = MLX5_ETH_WQE_L3_CSUM |
3466 MLX5_ETH_WQE_L4_CSUM;
3467
3468 seg += sizeof(struct mlx5_wqe_eth_seg);
3469 *size += sizeof(struct mlx5_wqe_eth_seg) / 16;
3470
3471 if (wr->opcode == IB_WR_LSO) {
3472 struct ib_ud_wr *ud_wr = container_of(wr, struct ib_ud_wr, wr);
2b31f7ae 3473 int size_of_inl_hdr_start = sizeof(eseg->inline_hdr.start);
f0313965
ES		 3474 u64 left, leftlen, copysz;
3475 void *pdata = ud_wr->header;
3476
3477 left = ud_wr->hlen;
3478 eseg->mss = cpu_to_be16(ud_wr->mss);
2b31f7ae 3479 eseg->inline_hdr.sz = cpu_to_be16(left);
f0313965
ES		 3480
3481 /*
3482 * check if there is space till the end of queue, if yes,
3483 * copy all in one shot, otherwise copy till the end of queue,
3484 * rollback and than the copy the left
3485 */
2b31f7ae 3486 leftlen = qend - (void *)eseg->inline_hdr.start;
f0313965
ES		 3487 copysz = min_t(u64, leftlen, left);
3488
3489 memcpy(seg - size_of_inl_hdr_start, pdata, copysz);
3490
3491 if (likely(copysz > size_of_inl_hdr_start)) {
3492 seg += ALIGN(copysz - size_of_inl_hdr_start, 16);
3493 *size += ALIGN(copysz - size_of_inl_hdr_start, 16) / 16;
3494 }
3495
3496 if (unlikely(copysz < left)) { /* the last wqe in the queue */
3497 seg = mlx5_get_send_wqe(qp, 0);
3498 left -= copysz;
3499 pdata += copysz;
3500 memcpy(seg, pdata, left);
3501 seg += ALIGN(left, 16);
3502 *size += ALIGN(left, 16) / 16;
3503 }
3504 }
3505
3506 return seg;
3507}
3508
e126ba97
EC		 3509static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
3510 struct ib_send_wr *wr)
3511{
e622f2f4
CH		 3512 memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
3513 dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
3514 dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
e126ba97
EC		 3515}
3516
3517static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
3518{
3519 dseg->byte_count = cpu_to_be32(sg->length);
3520 dseg->lkey = cpu_to_be32(sg->lkey);
3521 dseg->addr = cpu_to_be64(sg->addr);
3522}
3523
31616255 3524static u64 get_xlt_octo(u64 bytes)
e126ba97 3525{
31616255
AK		 3526 return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /
3527 MLX5_IB_UMR_OCTOWORD;
e126ba97
EC		 3528}
3529
3530static __be64 frwr_mkey_mask(void)
3531{
3532 u64 result;
3533
3534 result = MLX5_MKEY_MASK_LEN |
3535 MLX5_MKEY_MASK_PAGE_SIZE |
3536 MLX5_MKEY_MASK_START_ADDR |
3537 MLX5_MKEY_MASK_EN_RINVAL |
3538 MLX5_MKEY_MASK_KEY |
3539 MLX5_MKEY_MASK_LR |
3540 MLX5_MKEY_MASK_LW |
3541 MLX5_MKEY_MASK_RR |
3542 MLX5_MKEY_MASK_RW |
3543 MLX5_MKEY_MASK_A |
3544 MLX5_MKEY_MASK_SMALL_FENCE |
3545 MLX5_MKEY_MASK_FREE;
3546
3547 return cpu_to_be64(result);
3548}
3549
e6631814
SG		 3550static __be64 sig_mkey_mask(void)
3551{
3552 u64 result;
3553
3554 result = MLX5_MKEY_MASK_LEN |
3555 MLX5_MKEY_MASK_PAGE_SIZE |
3556 MLX5_MKEY_MASK_START_ADDR |
d5436ba0 3557 MLX5_MKEY_MASK_EN_SIGERR |
e6631814
SG		 3558 MLX5_MKEY_MASK_EN_RINVAL |
3559 MLX5_MKEY_MASK_KEY |
3560 MLX5_MKEY_MASK_LR |
3561 MLX5_MKEY_MASK_LW |
3562 MLX5_MKEY_MASK_RR |
3563 MLX5_MKEY_MASK_RW |
3564 MLX5_MKEY_MASK_SMALL_FENCE |
3565 MLX5_MKEY_MASK_FREE |
3566 MLX5_MKEY_MASK_BSF_EN;
3567
3568 return cpu_to_be64(result);
3569}
3570
8a187ee5 3571static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr,
31616255 3572 struct mlx5_ib_mr *mr)
8a187ee5 3573{
31616255 3574 int size = mr->ndescs * mr->desc_size;
8a187ee5
SG		 3575
3576 memset(umr, 0, sizeof(*umr));
b005d316 3577
8a187ee5 3578 umr->flags = MLX5_UMR_CHECK_NOT_FREE;
31616255 3579 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
8a187ee5
SG		 3580 umr->mkey_mask = frwr_mkey_mask();
3581}
3582
dd01e66a 3583static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr)
e126ba97
EC		 3584{
3585 memset(umr, 0, sizeof(*umr));
dd01e66a 3586 umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
2d221588 3587 umr->flags = MLX5_UMR_INLINE;
e126ba97
EC		 3588}
3589
31616255 3590static __be64 get_umr_enable_mr_mask(void)
968e78dd
HE		 3591{
3592 u64 result;
3593
31616255 3594 result = MLX5_MKEY_MASK_KEY |
968e78dd
HE		 3595 MLX5_MKEY_MASK_FREE;
3596
968e78dd
HE		 3597 return cpu_to_be64(result);
3598}
3599
31616255 3600static __be64 get_umr_disable_mr_mask(void)
968e78dd
HE		 3601{
3602 u64 result;
3603
3604 result = MLX5_MKEY_MASK_FREE;
3605
3606 return cpu_to_be64(result);
3607}
3608
56e11d62
NO		 3609static __be64 get_umr_update_translation_mask(void)
3610{
3611 u64 result;
3612
3613 result = MLX5_MKEY_MASK_LEN |
3614 MLX5_MKEY_MASK_PAGE_SIZE |
31616255 3615 MLX5_MKEY_MASK_START_ADDR;
56e11d62
NO		 3616
3617 return cpu_to_be64(result);
3618}
3619
31616255 3620static __be64 get_umr_update_access_mask(int atomic)
56e11d62
NO		 3621{
3622 u64 result;
3623
31616255
AK		 3624 result = MLX5_MKEY_MASK_LR |
3625 MLX5_MKEY_MASK_LW |
56e11d62 3626 MLX5_MKEY_MASK_RR |
31616255
AK		 3627 MLX5_MKEY_MASK_RW;
3628
3629 if (atomic)
3630 result |= MLX5_MKEY_MASK_A;
56e11d62
NO		 3631
3632 return cpu_to_be64(result);
3633}
3634
3635static __be64 get_umr_update_pd_mask(void)
3636{
3637 u64 result;
3638
31616255 3639 result = MLX5_MKEY_MASK_PD;
56e11d62
NO		 3640
3641 return cpu_to_be64(result);
3642}
3643
e126ba97 3644static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
578e7264 3645 struct ib_send_wr *wr, int atomic)
e126ba97 3646{
e622f2f4 3647 struct mlx5_umr_wr *umrwr = umr_wr(wr);
e126ba97
EC		 3648
3649 memset(umr, 0, sizeof(*umr));
3650
968e78dd
HE		 3651 if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)
3652 umr->flags = MLX5_UMR_CHECK_FREE; /* fail if free */
3653 else
3654 umr->flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */
3655
31616255
AK		 3656 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));
3657 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {
3658 u64 offset = get_xlt_octo(umrwr->offset);
3659
3660 umr->xlt_offset = cpu_to_be16(offset & 0xffff);
3661 umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);
3662 umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
e126ba97 3663 }
31616255
AK		 3664 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)
3665 umr->mkey_mask |= get_umr_update_translation_mask();
3666 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {
3667 umr->mkey_mask |= get_umr_update_access_mask(atomic);
3668 umr->mkey_mask |= get_umr_update_pd_mask();
3669 }
3670 if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)
3671 umr->mkey_mask |= get_umr_enable_mr_mask();
3672 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
3673 umr->mkey_mask |= get_umr_disable_mr_mask();
e126ba97
EC		 3674
3675 if (!wr->num_sge)
968e78dd 3676 umr->flags |= MLX5_UMR_INLINE;
e126ba97
EC		 3677}
3678
3679static u8 get_umr_flags(int acc)
3680{
3681 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
3682 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
3683 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
3684 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
2ac45934 3685 MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN;
e126ba97
EC		 3686}
3687
8a187ee5
SG		 3688static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg,
3689 struct mlx5_ib_mr *mr,
3690 u32 key, int access)
3691{
3692 int ndescs = ALIGN(mr->ndescs, 8) >> 1;
3693
3694 memset(seg, 0, sizeof(*seg));
b005d316 3695
ec22eb53 3696 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_MTT)
b005d316 3697 seg->log2_page_size = ilog2(mr->ibmr.page_size);
ec22eb53 3698 else if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
b005d316
SG		 3699 /* KLMs take twice the size of MTTs */
3700 ndescs *= 2;
3701
3702 seg->flags = get_umr_flags(access) | mr->access_mode;
8a187ee5
SG		 3703 seg->qpn_mkey7_0 = cpu_to_be32((key & 0xff) | 0xffffff00);
3704 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
3705 seg->start_addr = cpu_to_be64(mr->ibmr.iova);
3706 seg->len = cpu_to_be64(mr->ibmr.length);
3707 seg->xlt_oct_size = cpu_to_be32(ndescs);
8a187ee5
SG		 3708}
3709
dd01e66a 3710static void set_linv_mkey_seg(struct mlx5_mkey_seg *seg)
e126ba97
EC		 3711{
3712 memset(seg, 0, sizeof(*seg));
dd01e66a 3713 seg->status = MLX5_MKEY_STATUS_FREE;
e126ba97
EC		 3714}
3715
3716static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr)
3717{
e622f2f4 3718 struct mlx5_umr_wr *umrwr = umr_wr(wr);
968e78dd 3719
e126ba97 3720 memset(seg, 0, sizeof(*seg));
31616255 3721 if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)
968e78dd 3722 seg->status = MLX5_MKEY_STATUS_FREE;
e126ba97 3723
968e78dd 3724 seg->flags = convert_access(umrwr->access_flags);
31616255
AK		 3725 if (umrwr->pd)
3726 seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);
3727 if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION &&
3728 !umrwr->length)
3729 seg->flags_pd |= cpu_to_be32(MLX5_MKEY_LEN64);
3730
3731 seg->start_addr = cpu_to_be64(umrwr->virt_addr);
968e78dd
HE		 3732 seg->len = cpu_to_be64(umrwr->length);
3733 seg->log2_page_size = umrwr->page_shift;
746b5583 3734 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |
968e78dd 3735 mlx5_mkey_variant(umrwr->mkey));
e126ba97
EC		 3736}
3737
8a187ee5
SG		 3738static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg,
3739 struct mlx5_ib_mr *mr,
3740 struct mlx5_ib_pd *pd)
3741{
3742 int bcount = mr->desc_size * mr->ndescs;
3743
3744 dseg->addr = cpu_to_be64(mr->desc_map);
3745 dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64));
3746 dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey);
3747}
3748
e126ba97
EC		 3749static __be32 send_ieth(struct ib_send_wr *wr)
3750{
3751 switch (wr->opcode) {
3752 case IB_WR_SEND_WITH_IMM:
3753 case IB_WR_RDMA_WRITE_WITH_IMM:
3754 return wr->ex.imm_data;
3755
3756 case IB_WR_SEND_WITH_INV:
3757 return cpu_to_be32(wr->ex.invalidate_rkey);
3758
3759 default:
3760 return 0;
3761 }
3762}
3763
3764static u8 calc_sig(void *wqe, int size)
3765{
3766 u8 *p = wqe;
3767 u8 res = 0;
3768 int i;
3769
3770 for (i = 0; i < size; i++)
3771 res ^= p[i];
3772
3773 return ~res;
3774}
3775
3776static u8 wq_sig(void *wqe)
3777{
3778 return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4);
3779}
3780
3781static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr,
3782 void *wqe, int *sz)
3783{
3784 struct mlx5_wqe_inline_seg *seg;
3785 void *qend = qp->sq.qend;
3786 void *addr;
3787 int inl = 0;
3788 int copy;
3789 int len;
3790 int i;
3791
3792 seg = wqe;
3793 wqe += sizeof(*seg);
3794 for (i = 0; i < wr->num_sge; i++) {
3795 addr = (void *)(unsigned long)(wr->sg_list[i].addr);
3796 len = wr->sg_list[i].length;
3797 inl += len;
3798
3799 if (unlikely(inl > qp->max_inline_data))
3800 return -ENOMEM;
3801
3802 if (unlikely(wqe + len > qend)) {
3803 copy = qend - wqe;
3804 memcpy(wqe, addr, copy);
3805 addr += copy;
3806 len -= copy;
3807 wqe = mlx5_get_send_wqe(qp, 0);
3808 }
3809 memcpy(wqe, addr, len);
3810 wqe += len;
3811 }
3812
3813 seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);
3814
3815 *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;
3816
3817 return 0;
3818}
3819
e6631814
SG		 3820static u16 prot_field_size(enum ib_signature_type type)
3821{
3822 switch (type) {
3823 case IB_SIG_TYPE_T10_DIF:
3824 return MLX5_DIF_SIZE;
3825 default:
3826 return 0;
3827 }
3828}
3829
3830static u8 bs_selector(int block_size)
3831{
3832 switch (block_size) {
3833 case 512: return 0x1;
3834 case 520: return 0x2;
3835 case 4096: return 0x3;
3836 case 4160: return 0x4;
3837 case 1073741824: return 0x5;
3838 default: return 0;
3839 }
3840}
3841
78eda2bb
SG		 3842static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
3843 struct mlx5_bsf_inl *inl)
e6631814 3844{
142537f4
SG		 3845 /* Valid inline section and allow BSF refresh */
3846 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
3847 MLX5_BSF_REFRESH_DIF);
3848 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
3849 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
78eda2bb
SG		 3850 /* repeating block */
3851 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
3852 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
3853 MLX5_DIF_CRC : MLX5_DIF_IPCS;
e6631814 3854
78eda2bb
SG		 3855 if (domain->sig.dif.ref_remap)
3856 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
e6631814 3857
78eda2bb
SG		 3858 if (domain->sig.dif.app_escape) {
3859 if (domain->sig.dif.ref_escape)
3860 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
3861 else
3862 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
e6631814
SG		 3863 }
3864
78eda2bb
SG		 3865 inl->dif_app_bitmask_check =
3866 cpu_to_be16(domain->sig.dif.apptag_check_mask);
e6631814
SG		 3867}
3868
3869static int mlx5_set_bsf(struct ib_mr *sig_mr,
3870 struct ib_sig_attrs *sig_attrs,
3871 struct mlx5_bsf *bsf, u32 data_size)
3872{
3873 struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig;
3874 struct mlx5_bsf_basic *basic = &bsf->basic;
3875 struct ib_sig_domain *mem = &sig_attrs->mem;
3876 struct ib_sig_domain *wire = &sig_attrs->wire;
e6631814 3877
c7f44fbd 3878 memset(bsf, 0, sizeof(*bsf));
78eda2bb
SG		 3879
3880 /* Basic + Extended + Inline */
3881 basic->bsf_size_sbs = 1 << 7;
3882 /* Input domain check byte mask */
3883 basic->check_byte_mask = sig_attrs->check_mask;
3884 basic->raw_data_size = cpu_to_be32(data_size);
3885
3886 /* Memory domain */
e6631814 3887 switch (sig_attrs->mem.sig_type) {
78eda2bb
SG		 3888 case IB_SIG_TYPE_NONE:
3889 break;
e6631814 3890 case IB_SIG_TYPE_T10_DIF:
78eda2bb
SG		 3891 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
3892 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
3893 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
3894 break;
3895 default:
3896 return -EINVAL;
3897 }
e6631814 3898
78eda2bb
SG		 3899 /* Wire domain */
3900 switch (sig_attrs->wire.sig_type) {
3901 case IB_SIG_TYPE_NONE:
3902 break;
3903 case IB_SIG_TYPE_T10_DIF:
e6631814 3904 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
78eda2bb 3905 mem->sig_type == wire->sig_type) {
e6631814 3906 /* Same block structure */
142537f4 3907 basic->bsf_size_sbs |= 1 << 4;
e6631814 3908 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
fd22f78c 3909 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
c7f44fbd 3910 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
fd22f78c 3911 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
c7f44fbd 3912 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
fd22f78c 3913 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
e6631814
SG		 3914 } else
3915 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
3916
142537f4 3917 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
78eda2bb 3918 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
e6631814 3919 break;
e6631814
SG		 3920 default:
3921 return -EINVAL;
3922 }
3923
3924 return 0;
3925}
3926
e622f2f4
CH		 3927static int set_sig_data_segment(struct ib_sig_handover_wr *wr,
3928 struct mlx5_ib_qp *qp, void **seg, int *size)
e6631814 3929{
e622f2f4
CH		 3930 struct ib_sig_attrs *sig_attrs = wr->sig_attrs;
3931 struct ib_mr *sig_mr = wr->sig_mr;
e6631814 3932 struct mlx5_bsf *bsf;
e622f2f4
CH		 3933 u32 data_len = wr->wr.sg_list->length;
3934 u32 data_key = wr->wr.sg_list->lkey;
3935 u64 data_va = wr->wr.sg_list->addr;
e6631814
SG		 3936 int ret;
3937 int wqe_size;
3938
e622f2f4
CH		 3939 if (!wr->prot ||
3940 (data_key == wr->prot->lkey &&
3941 data_va == wr->prot->addr &&
3942 data_len == wr->prot->length)) {
e6631814
SG		 3943 /**
3944 * Source domain doesn't contain signature information
5c273b16 3945 * or data and protection are interleaved in memory.
e6631814
SG		 3946 * So need construct:
3947 * ------------------
3948 * | data_klm |
3949 * ------------------
3950 * | BSF |
3951 * ------------------
3952 **/
3953 struct mlx5_klm *data_klm = *seg;
3954
3955 data_klm->bcount = cpu_to_be32(data_len);
3956 data_klm->key = cpu_to_be32(data_key);
3957 data_klm->va = cpu_to_be64(data_va);
3958 wqe_size = ALIGN(sizeof(*data_klm), 64);
3959 } else {
3960 /**
3961 * Source domain contains signature information
3962 * So need construct a strided block format:
3963 * ---------------------------
3964 * | stride_block_ctrl |
3965 * ---------------------------
3966 * | data_klm |
3967 * ---------------------------
3968 * | prot_klm |
3969 * ---------------------------
3970 * | BSF |
3971 * ---------------------------
3972 **/
3973 struct mlx5_stride_block_ctrl_seg *sblock_ctrl;
3974 struct mlx5_stride_block_entry *data_sentry;
3975 struct mlx5_stride_block_entry *prot_sentry;
e622f2f4
CH		 3976 u32 prot_key = wr->prot->lkey;
3977 u64 prot_va = wr->prot->addr;
e6631814
SG		 3978 u16 block_size = sig_attrs->mem.sig.dif.pi_interval;
3979 int prot_size;
3980
3981 sblock_ctrl = *seg;
3982 data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl);
3983 prot_sentry = (void *)data_sentry + sizeof(*data_sentry);
3984
3985 prot_size = prot_field_size(sig_attrs->mem.sig_type);
3986 if (!prot_size) {
3987 pr_err("Bad block size given: %u\n", block_size);
3988 return -EINVAL;
3989 }
3990 sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size +
3991 prot_size);
3992 sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP);
3993 sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size);
3994 sblock_ctrl->num_entries = cpu_to_be16(2);
3995
3996 data_sentry->bcount = cpu_to_be16(block_size);
3997 data_sentry->key = cpu_to_be32(data_key);
3998 data_sentry->va = cpu_to_be64(data_va);
5c273b16
SG		 3999 data_sentry->stride = cpu_to_be16(block_size);
4000
e6631814
SG		 4001 prot_sentry->bcount = cpu_to_be16(prot_size);
4002 prot_sentry->key = cpu_to_be32(prot_key);
5c273b16
SG		 4003 prot_sentry->va = cpu_to_be64(prot_va);
4004 prot_sentry->stride = cpu_to_be16(prot_size);
e6631814 4005
e6631814
SG		 4006 wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) +
4007 sizeof(*prot_sentry), 64);
4008 }
4009
4010 *seg += wqe_size;
4011 *size += wqe_size / 16;
4012 if (unlikely((*seg == qp->sq.qend)))
4013 *seg = mlx5_get_send_wqe(qp, 0);
4014
4015 bsf = *seg;
4016 ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len);
4017 if (ret)
4018 return -EINVAL;
4019
4020 *seg += sizeof(*bsf);
4021 *size += sizeof(*bsf) / 16;
4022 if (unlikely((*seg == qp->sq.qend)))
4023 *seg = mlx5_get_send_wqe(qp, 0);
4024
4025 return 0;
4026}
4027
4028static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,
31616255 4029 struct ib_sig_handover_wr *wr, u32 size,
e6631814
SG		 4030 u32 length, u32 pdn)
4031{
e622f2f4 4032 struct ib_mr *sig_mr = wr->sig_mr;
e6631814 4033 u32 sig_key = sig_mr->rkey;
d5436ba0 4034 u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1;
e6631814
SG		 4035
4036 memset(seg, 0, sizeof(*seg));
4037
e622f2f4 4038 seg->flags = get_umr_flags(wr->access_flags) |
ec22eb53 4039 MLX5_MKC_ACCESS_MODE_KLMS;
e6631814 4040 seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);
d5436ba0 4041 seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |
e6631814
SG		 4042 MLX5_MKEY_BSF_EN | pdn);
4043 seg->len = cpu_to_be64(length);
31616255 4044 seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));
e6631814
SG		 4045 seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);
4046}
4047
4048static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
31616255 4049 u32 size)
e6631814
SG		 4050{
4051 memset(umr, 0, sizeof(*umr));
4052
4053 umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
31616255 4054 umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
e6631814
SG		 4055 umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
4056 umr->mkey_mask = sig_mkey_mask();
4057}
4058
4059
e622f2f4 4060static int set_sig_umr_wr(struct ib_send_wr *send_wr, struct mlx5_ib_qp *qp,
e6631814
SG		 4061 void **seg, int *size)
4062{
e622f2f4
CH		 4063 struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr);
4064 struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr);
e6631814 4065 u32 pdn = get_pd(qp)->pdn;
31616255 4066 u32 xlt_size;
e6631814
SG		 4067 int region_len, ret;
4068
e622f2f4
CH		 4069 if (unlikely(wr->wr.num_sge != 1) ||
4070 unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) ||
d5436ba0
SG		 4071 unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) ||
4072 unlikely(!sig_mr->sig->sig_status_checked))
e6631814
SG		 4073 return -EINVAL;
4074
4075 /* length of the protected region, data + protection */
e622f2f4
CH		 4076 region_len = wr->wr.sg_list->length;
4077 if (wr->prot &&
4078 (wr->prot->lkey != wr->wr.sg_list->lkey ||
4079 wr->prot->addr != wr->wr.sg_list->addr ||
4080 wr->prot->length != wr->wr.sg_list->length))
4081 region_len += wr->prot->length;
e6631814
SG		 4082
4083 /**
4084 * KLM octoword size - if protection was provided
4085 * then we use strided block format (3 octowords),
4086 * else we use single KLM (1 octoword)
4087 **/
31616255 4088 xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm);
e6631814 4089
31616255 4090 set_sig_umr_segment(*seg, xlt_size);
e6631814
SG		 4091 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4092 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4093 if (unlikely((*seg == qp->sq.qend)))
4094 *seg = mlx5_get_send_wqe(qp, 0);
4095
31616255 4096 set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn);
e6631814
SG		 4097 *seg += sizeof(struct mlx5_mkey_seg);
4098 *size += sizeof(struct mlx5_mkey_seg) / 16;
4099 if (unlikely((*seg == qp->sq.qend)))
4100 *seg = mlx5_get_send_wqe(qp, 0);
4101
4102 ret = set_sig_data_segment(wr, qp, seg, size);
4103 if (ret)
4104 return ret;
4105
d5436ba0 4106 sig_mr->sig->sig_status_checked = false;
e6631814
SG		 4107 return 0;
4108}
4109
4110static int set_psv_wr(struct ib_sig_domain *domain,
4111 u32 psv_idx, void **seg, int *size)
4112{
4113 struct mlx5_seg_set_psv *psv_seg = *seg;
4114
4115 memset(psv_seg, 0, sizeof(*psv_seg));
4116 psv_seg->psv_num = cpu_to_be32(psv_idx);
4117 switch (domain->sig_type) {
78eda2bb
SG		 4118 case IB_SIG_TYPE_NONE:
4119 break;
e6631814
SG		 4120 case IB_SIG_TYPE_T10_DIF:
4121 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
4122 domain->sig.dif.app_tag);
4123 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
e6631814 4124 break;
e6631814 4125 default:
12bbf1ea
LR		 4126 pr_err("Bad signature type (%d) is given.\n",
4127 domain->sig_type);
4128 return -EINVAL;
e6631814
SG		 4129 }
4130
78eda2bb
SG		 4131 *seg += sizeof(*psv_seg);
4132 *size += sizeof(*psv_seg) / 16;
4133
e6631814
SG		 4134 return 0;
4135}
4136
8a187ee5
SG		 4137static int set_reg_wr(struct mlx5_ib_qp *qp,
4138 struct ib_reg_wr *wr,
4139 void **seg, int *size)
4140{
4141 struct mlx5_ib_mr *mr = to_mmr(wr->mr);
4142 struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd);
4143
4144 if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) {
4145 mlx5_ib_warn(to_mdev(qp->ibqp.device),
4146 "Invalid IB_SEND_INLINE send flag\n");
4147 return -EINVAL;
4148 }
4149
4150 set_reg_umr_seg(*seg, mr);
4151 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4152 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4153 if (unlikely((*seg == qp->sq.qend)))
4154 *seg = mlx5_get_send_wqe(qp, 0);
4155
4156 set_reg_mkey_seg(*seg, mr, wr->key, wr->access);
4157 *seg += sizeof(struct mlx5_mkey_seg);
4158 *size += sizeof(struct mlx5_mkey_seg) / 16;
4159 if (unlikely((*seg == qp->sq.qend)))
4160 *seg = mlx5_get_send_wqe(qp, 0);
4161
4162 set_reg_data_seg(*seg, mr, pd);
4163 *seg += sizeof(struct mlx5_wqe_data_seg);
4164 *size += (sizeof(struct mlx5_wqe_data_seg) / 16);
4165
4166 return 0;
4167}
4168
dd01e66a 4169static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size)
e126ba97 4170{
dd01e66a 4171 set_linv_umr_seg(*seg);
e126ba97
EC		 4172 *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4173 *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4174 if (unlikely((*seg == qp->sq.qend)))
4175 *seg = mlx5_get_send_wqe(qp, 0);
dd01e66a 4176 set_linv_mkey_seg(*seg);
e126ba97
EC		 4177 *seg += sizeof(struct mlx5_mkey_seg);
4178 *size += sizeof(struct mlx5_mkey_seg) / 16;
4179 if (unlikely((*seg == qp->sq.qend)))
4180 *seg = mlx5_get_send_wqe(qp, 0);
e126ba97
EC		 4181}
4182
4183static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
4184{
4185 __be32 *p = NULL;
4186 int tidx = idx;
4187 int i, j;
4188
4189 pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx));
4190 for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
4191 if ((i & 0xf) == 0) {
4192 void *buf = mlx5_get_send_wqe(qp, tidx);
4193 tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1);
4194 p = buf;
4195 j = 0;
4196 }
4197 pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
4198 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
4199 be32_to_cpu(p[j + 3]));
4200 }
4201}
4202
6e5eadac
SG		 4203static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
4204 struct mlx5_wqe_ctrl_seg **ctrl,
6a4f139a 4205 struct ib_send_wr *wr, unsigned *idx,
6e5eadac
SG		 4206 int *size, int nreq)
4207{
b2a232d2
LR		 4208 if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
4209 return -ENOMEM;
6e5eadac
SG		 4210
4211 *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
4212 *seg = mlx5_get_send_wqe(qp, *idx);
4213 *ctrl = *seg;
4214 *(uint32_t *)(*seg + 8) = 0;
4215 (*ctrl)->imm = send_ieth(wr);
4216 (*ctrl)->fm_ce_se = qp->sq_signal_bits |
4217 (wr->send_flags & IB_SEND_SIGNALED ?
4218 MLX5_WQE_CTRL_CQ_UPDATE : 0) |
4219 (wr->send_flags & IB_SEND_SOLICITED ?
4220 MLX5_WQE_CTRL_SOLICITED : 0);
4221
4222 *seg += sizeof(**ctrl);
4223 *size = sizeof(**ctrl) / 16;
4224
b2a232d2 4225 return 0;
6e5eadac
SG		 4226}
4227
4228static void finish_wqe(struct mlx5_ib_qp *qp,
4229 struct mlx5_wqe_ctrl_seg *ctrl,
4230 u8 size, unsigned idx, u64 wr_id,
6e8484c5 4231 int nreq, u8 fence, u32 mlx5_opcode)
6e5eadac
SG		 4232{
4233 u8 opmod = 0;
4234
4235 ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) |
4236 mlx5_opcode | ((u32)opmod << 24));
19098df2 4237 ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
6e5eadac 4238 ctrl->fm_ce_se |= fence;
6e5eadac
SG		 4239 if (unlikely(qp->wq_sig))
4240 ctrl->signature = wq_sig(ctrl);
4241
4242 qp->sq.wrid[idx] = wr_id;
4243 qp->sq.w_list[idx].opcode = mlx5_opcode;
4244 qp->sq.wqe_head[idx] = qp->sq.head + nreq;
4245 qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB);
4246 qp->sq.w_list[idx].next = qp->sq.cur_post;
4247}
4248
4249
e126ba97
EC		 4250int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
4251 struct ib_send_wr **bad_wr)
4252{
4253 struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */
4254 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
89ea94a7 4255 struct mlx5_core_dev *mdev = dev->mdev;
d16e91da 4256 struct mlx5_ib_qp *qp;
e6631814 4257 struct mlx5_ib_mr *mr;
e126ba97
EC		 4258 struct mlx5_wqe_data_seg *dpseg;
4259 struct mlx5_wqe_xrc_seg *xrc;
d16e91da 4260 struct mlx5_bf *bf;
e126ba97 4261 int uninitialized_var(size);
d16e91da 4262 void *qend;
e126ba97 4263 unsigned long flags;
e126ba97
EC		 4264 unsigned idx;
4265 int err = 0;
e126ba97
EC		 4266 int num_sge;
4267 void *seg;
4268 int nreq;
4269 int i;
4270 u8 next_fence = 0;
e126ba97
EC		 4271 u8 fence;
4272
d16e91da
HE		 4273 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4274 return mlx5_ib_gsi_post_send(ibqp, wr, bad_wr);
4275
4276 qp = to_mqp(ibqp);
5fe9dec0 4277 bf = &qp->bf;
d16e91da
HE		 4278 qend = qp->sq.qend;
4279
e126ba97
EC		 4280 spin_lock_irqsave(&qp->sq.lock, flags);
4281
89ea94a7
MG		 4282 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
4283 err = -EIO;
4284 *bad_wr = wr;
4285 nreq = 0;
4286 goto out;
4287 }
4288
e126ba97 4289 for (nreq = 0; wr; nreq++, wr = wr->next) {
a8f731eb 4290 if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) {
e126ba97
EC		 4291 mlx5_ib_warn(dev, "\n");
4292 err = -EINVAL;
4293 *bad_wr = wr;
4294 goto out;
4295 }
4296
6e5eadac
SG		 4297 num_sge = wr->num_sge;
4298 if (unlikely(num_sge > qp->sq.max_gs)) {
e126ba97 4299 mlx5_ib_warn(dev, "\n");
24be409b 4300 err = -EINVAL;
e126ba97
EC		 4301 *bad_wr = wr;
4302 goto out;
4303 }
4304
6e5eadac
SG		 4305 err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
4306 if (err) {
e126ba97
EC		 4307 mlx5_ib_warn(dev, "\n");
4308 err = -ENOMEM;
4309 *bad_wr = wr;
4310 goto out;
4311 }
4312
6e8484c5
MG		 4313 if (wr->opcode == IB_WR_LOCAL_INV ||
4314 wr->opcode == IB_WR_REG_MR) {
4315 fence = dev->umr_fence;
4316 next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
4317 } else if (wr->send_flags & IB_SEND_FENCE) {
4318 if (qp->next_fence)
4319 fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
4320 else
4321 fence = MLX5_FENCE_MODE_FENCE;
4322 } else {
4323 fence = qp->next_fence;
4324 }
4325
e126ba97
EC		 4326 switch (ibqp->qp_type) {
4327 case IB_QPT_XRC_INI:
4328 xrc = seg;
e126ba97
EC		 4329 seg += sizeof(*xrc);
4330 size += sizeof(*xrc) / 16;
4331 /* fall through */
4332 case IB_QPT_RC:
4333 switch (wr->opcode) {
4334 case IB_WR_RDMA_READ:
4335 case IB_WR_RDMA_WRITE:
4336 case IB_WR_RDMA_WRITE_WITH_IMM:
e622f2f4
CH		 4337 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4338 rdma_wr(wr)->rkey);
f241e749 4339 seg += sizeof(struct mlx5_wqe_raddr_seg);
e126ba97
EC		 4340 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4341 break;
4342
4343 case IB_WR_ATOMIC_CMP_AND_SWP:
4344 case IB_WR_ATOMIC_FETCH_AND_ADD:
e126ba97 4345 case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
81bea28f
EC		 4346 mlx5_ib_warn(dev, "Atomic operations are not supported yet\n");
4347 err = -ENOSYS;
4348 *bad_wr = wr;
4349 goto out;
e126ba97
EC		 4350
4351 case IB_WR_LOCAL_INV:
e126ba97
EC		 4352 qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
4353 ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
dd01e66a 4354 set_linv_wr(qp, &seg, &size);
e126ba97
EC		 4355 num_sge = 0;
4356 break;
4357
8a187ee5 4358 case IB_WR_REG_MR:
8a187ee5
SG		 4359 qp->sq.wr_data[idx] = IB_WR_REG_MR;
4360 ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
4361 err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
4362 if (err) {
4363 *bad_wr = wr;
4364 goto out;
4365 }
4366 num_sge = 0;
4367 break;
4368
e6631814
SG		 4369 case IB_WR_REG_SIG_MR:
4370 qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR;
e622f2f4 4371 mr = to_mmr(sig_handover_wr(wr)->sig_mr);
e6631814
SG		 4372
4373 ctrl->imm = cpu_to_be32(mr->ibmr.rkey);
4374 err = set_sig_umr_wr(wr, qp, &seg, &size);
4375 if (err) {
4376 mlx5_ib_warn(dev, "\n");
4377 *bad_wr = wr;
4378 goto out;
4379 }
4380
6e8484c5
MG		 4381 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4382 fence, MLX5_OPCODE_UMR);
e6631814
SG		 4383 /*
4384 * SET_PSV WQEs are not signaled and solicited
4385 * on error
4386 */
4387 wr->send_flags &= ~IB_SEND_SIGNALED;
4388 wr->send_flags |= IB_SEND_SOLICITED;
4389 err = begin_wqe(qp, &seg, &ctrl, wr,
4390 &idx, &size, nreq);
4391 if (err) {
4392 mlx5_ib_warn(dev, "\n");
4393 err = -ENOMEM;
4394 *bad_wr = wr;
4395 goto out;
4396 }
4397
e622f2f4 4398 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem,
e6631814
SG		 4399 mr->sig->psv_memory.psv_idx, &seg,
4400 &size);
4401 if (err) {
4402 mlx5_ib_warn(dev, "\n");
4403 *bad_wr = wr;
4404 goto out;
4405 }
4406
6e8484c5
MG		 4407 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4408 fence, MLX5_OPCODE_SET_PSV);
e6631814
SG		 4409 err = begin_wqe(qp, &seg, &ctrl, wr,
4410 &idx, &size, nreq);
4411 if (err) {
4412 mlx5_ib_warn(dev, "\n");
4413 err = -ENOMEM;
4414 *bad_wr = wr;
4415 goto out;
4416 }
4417
e622f2f4 4418 err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
e6631814
SG		 4419 mr->sig->psv_wire.psv_idx, &seg,
4420 &size);
4421 if (err) {
4422 mlx5_ib_warn(dev, "\n");
4423 *bad_wr = wr;
4424 goto out;
4425 }
4426
6e8484c5
MG		 4427 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
4428 fence, MLX5_OPCODE_SET_PSV);
4429 qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
e6631814
SG		 4430 num_sge = 0;
4431 goto skip_psv;
4432
e126ba97
EC		 4433 default:
4434 break;
4435 }
4436 break;
4437
4438 case IB_QPT_UC:
4439 switch (wr->opcode) {
4440 case IB_WR_RDMA_WRITE:
4441 case IB_WR_RDMA_WRITE_WITH_IMM:
e622f2f4
CH		 4442 set_raddr_seg(seg, rdma_wr(wr)->remote_addr,
4443 rdma_wr(wr)->rkey);
e126ba97
EC		 4444 seg += sizeof(struct mlx5_wqe_raddr_seg);
4445 size += sizeof(struct mlx5_wqe_raddr_seg) / 16;
4446 break;
4447
4448 default:
4449 break;
4450 }
4451 break;
4452
e126ba97 4453 case IB_QPT_SMI:
1e0e50b6
MG		 4454 if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
4455 mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
4456 err = -EPERM;
4457 *bad_wr = wr;
4458 goto out;
4459 }
f6b1ee34 4460 /* fall through */
d16e91da 4461 case MLX5_IB_QPT_HW_GSI:
e126ba97 4462 set_datagram_seg(seg, wr);
f241e749 4463 seg += sizeof(struct mlx5_wqe_datagram_seg);
e126ba97
EC		 4464 size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4465 if (unlikely((seg == qend)))
4466 seg = mlx5_get_send_wqe(qp, 0);
4467 break;
f0313965
ES		 4468 case IB_QPT_UD:
4469 set_datagram_seg(seg, wr);
4470 seg += sizeof(struct mlx5_wqe_datagram_seg);
4471 size += sizeof(struct mlx5_wqe_datagram_seg) / 16;
4472
4473 if (unlikely((seg == qend)))
4474 seg = mlx5_get_send_wqe(qp, 0);
4475
4476 /* handle qp that supports ud offload */
4477 if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO) {
4478 struct mlx5_wqe_eth_pad *pad;
e126ba97 4479
f0313965
ES		 4480 pad = seg;
4481 memset(pad, 0, sizeof(struct mlx5_wqe_eth_pad));
4482 seg += sizeof(struct mlx5_wqe_eth_pad);
4483 size += sizeof(struct mlx5_wqe_eth_pad) / 16;
4484
4485 seg = set_eth_seg(seg, wr, qend, qp, &size);
4486
4487 if (unlikely((seg == qend)))
4488 seg = mlx5_get_send_wqe(qp, 0);
4489 }
4490 break;
e126ba97
EC		 4491 case MLX5_IB_QPT_REG_UMR:
4492 if (wr->opcode != MLX5_IB_WR_UMR) {
4493 err = -EINVAL;
4494 mlx5_ib_warn(dev, "bad opcode\n");
4495 goto out;
4496 }
4497 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
e622f2f4 4498 ctrl->imm = cpu_to_be32(umr_wr(wr)->mkey);
578e7264 4499 set_reg_umr_segment(seg, wr, !!(MLX5_CAP_GEN(mdev, atomic)));
e126ba97
EC		 4500 seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);
4501 size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16;
4502 if (unlikely((seg == qend)))
4503 seg = mlx5_get_send_wqe(qp, 0);
4504 set_reg_mkey_segment(seg, wr);
4505 seg += sizeof(struct mlx5_mkey_seg);
4506 size += sizeof(struct mlx5_mkey_seg) / 16;
4507 if (unlikely((seg == qend)))
4508 seg = mlx5_get_send_wqe(qp, 0);
4509 break;
4510
4511 default:
4512 break;
4513 }
4514
4515 if (wr->send_flags & IB_SEND_INLINE && num_sge) {
4516 int uninitialized_var(sz);
4517
4518 err = set_data_inl_seg(qp, wr, seg, &sz);
4519 if (unlikely(err)) {
4520 mlx5_ib_warn(dev, "\n");
4521 *bad_wr = wr;
4522 goto out;
4523 }
e126ba97
EC		 4524 size += sz;
4525 } else {
4526 dpseg = seg;
4527 for (i = 0; i < num_sge; i++) {
4528 if (unlikely(dpseg == qend)) {
4529 seg = mlx5_get_send_wqe(qp, 0);
4530 dpseg = seg;
4531 }
4532 if (likely(wr->sg_list[i].length)) {
4533 set_data_ptr_seg(dpseg, wr->sg_list + i);
4534 size += sizeof(struct mlx5_wqe_data_seg) / 16;
4535 dpseg++;
4536 }
4537 }
4538 }
4539
6e8484c5
MG		 4540 qp->next_fence = next_fence;
4541 finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,
6e5eadac 4542 mlx5_ib_opcode[wr->opcode]);
e6631814 4543skip_psv:
e126ba97
EC		 4544 if (0)
4545 dump_wqe(qp, idx, size);
4546 }
4547
4548out:
4549 if (likely(nreq)) {
4550 qp->sq.head += nreq;
4551
4552 /* Make sure that descriptors are written before
4553 * updating doorbell record and ringing the doorbell
4554 */
4555 wmb();
4556
4557 qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post);
4558
ada388f7
EC		 4559 /* Make sure doorbell record is visible to the HCA before
4560 * we hit doorbell */
4561 wmb();
4562
5fe9dec0
EC		 4563 /* currently we support only regular doorbells */
4564 mlx5_write64((__be32 *)ctrl, bf->bfreg->map + bf->offset, NULL);
4565 /* Make sure doorbells don't leak out of SQ spinlock
4566 * and reach the HCA out of order.
4567 */
4568 mmiowb();
e126ba97 4569 bf->offset ^= bf->buf_size;
e126ba97
EC		 4570 }
4571
4572 spin_unlock_irqrestore(&qp->sq.lock, flags);
4573
4574 return err;
4575}
4576
4577static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size)
4578{
4579 sig->signature = calc_sig(sig, size);
4580}
4581
4582int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
4583 struct ib_recv_wr **bad_wr)
4584{
4585 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4586 struct mlx5_wqe_data_seg *scat;
4587 struct mlx5_rwqe_sig *sig;
89ea94a7
MG		 4588 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4589 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97
EC		 4590 unsigned long flags;
4591 int err = 0;
4592 int nreq;
4593 int ind;
4594 int i;
4595
d16e91da
HE		 4596 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
4597 return mlx5_ib_gsi_post_recv(ibqp, wr, bad_wr);
4598
e126ba97
EC		 4599 spin_lock_irqsave(&qp->rq.lock, flags);
4600
89ea94a7
MG		 4601 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
4602 err = -EIO;
4603 *bad_wr = wr;
4604 nreq = 0;
4605 goto out;
4606 }
4607
e126ba97
EC		 4608 ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
4609
4610 for (nreq = 0; wr; nreq++, wr = wr->next) {
4611 if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
4612 err = -ENOMEM;
4613 *bad_wr = wr;
4614 goto out;
4615 }
4616
4617 if (unlikely(wr->num_sge > qp->rq.max_gs)) {
4618 err = -EINVAL;
4619 *bad_wr = wr;
4620 goto out;
4621 }
4622
4623 scat = get_recv_wqe(qp, ind);
4624 if (qp->wq_sig)
4625 scat++;
4626
4627 for (i = 0; i < wr->num_sge; i++)
4628 set_data_ptr_seg(scat + i, wr->sg_list + i);
4629
4630 if (i < qp->rq.max_gs) {
4631 scat[i].byte_count = 0;
4632 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
4633 scat[i].addr = 0;
4634 }
4635
4636 if (qp->wq_sig) {
4637 sig = (struct mlx5_rwqe_sig *)scat;
4638 set_sig_seg(sig, (qp->rq.max_gs + 1) << 2);
4639 }
4640
4641 qp->rq.wrid[ind] = wr->wr_id;
4642
4643 ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
4644 }
4645
4646out:
4647 if (likely(nreq)) {
4648 qp->rq.head += nreq;
4649
4650 /* Make sure that descriptors are written before
4651 * doorbell record.
4652 */
4653 wmb();
4654
4655 *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
4656 }
4657
4658 spin_unlock_irqrestore(&qp->rq.lock, flags);
4659
4660 return err;
4661}
4662
4663static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4664{
4665 switch (mlx5_state) {
4666 case MLX5_QP_STATE_RST: return IB_QPS_RESET;
4667 case MLX5_QP_STATE_INIT: return IB_QPS_INIT;
4668 case MLX5_QP_STATE_RTR: return IB_QPS_RTR;
4669 case MLX5_QP_STATE_RTS: return IB_QPS_RTS;
4670 case MLX5_QP_STATE_SQ_DRAINING:
4671 case MLX5_QP_STATE_SQD: return IB_QPS_SQD;
4672 case MLX5_QP_STATE_SQER: return IB_QPS_SQE;
4673 case MLX5_QP_STATE_ERR: return IB_QPS_ERR;
4674 default: return -1;
4675 }
4676}
4677
4678static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4679{
4680 switch (mlx5_mig_state) {
4681 case MLX5_QP_PM_ARMED: return IB_MIG_ARMED;
4682 case MLX5_QP_PM_REARM: return IB_MIG_REARM;
4683 case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED;
4684 default: return -1;
4685 }
4686}
4687
4688static int to_ib_qp_access_flags(int mlx5_flags)
4689{
4690 int ib_flags = 0;
4691
4692 if (mlx5_flags & MLX5_QP_BIT_RRE)
4693 ib_flags |= IB_ACCESS_REMOTE_READ;
4694 if (mlx5_flags & MLX5_QP_BIT_RWE)
4695 ib_flags |= IB_ACCESS_REMOTE_WRITE;
4696 if (mlx5_flags & MLX5_QP_BIT_RAE)
4697 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
4698
4699 return ib_flags;
4700}
4701
38349389 4702static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
d8966fcd 4703 struct rdma_ah_attr *ah_attr,
38349389 4704 struct mlx5_qp_path *path)
e126ba97 4705{
e126ba97 4706
d8966fcd 4707 memset(ah_attr, 0, sizeof(*ah_attr));
e126ba97 4708
e7996a9a 4709 if (!path->port || path->port > ibdev->num_ports)
e126ba97
EC		 4710 return;
4711
ae59c3f0
LR		 4712 ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
4713
d8966fcd
DC		 4714 rdma_ah_set_port_num(ah_attr, path->port);
4715 rdma_ah_set_sl(ah_attr, path->dci_cfi_prio_sl & 0xf);
4716
4717 rdma_ah_set_dlid(ah_attr, be16_to_cpu(path->rlid));
4718 rdma_ah_set_path_bits(ah_attr, path->grh_mlid & 0x7f);
4719 rdma_ah_set_static_rate(ah_attr,
4720 path->static_rate ? path->static_rate - 5 : 0);
4721 if (path->grh_mlid & (1 << 7)) {
4722 u32 tc_fl = be32_to_cpu(path->tclass_flowlabel);
4723
4724 rdma_ah_set_grh(ah_attr, NULL,
4725 tc_fl & 0xfffff,
4726 path->mgid_index,
4727 path->hop_limit,
4728 (tc_fl >> 20) & 0xff);
4729 rdma_ah_set_dgid_raw(ah_attr, path->rgid);
e126ba97
EC		 4730 }
4731}
4732
6d2f89df 4733static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4734 struct mlx5_ib_sq *sq,
4735 u8 *sq_state)
4736{
4737 void *out;
4738 void *sqc;
4739 int inlen;
4740 int err;
4741
4742 inlen = MLX5_ST_SZ_BYTES(query_sq_out);
1b9a07ee 4743 out = kvzalloc(inlen, GFP_KERNEL);
6d2f89df 4744 if (!out)
4745 return -ENOMEM;
4746
4747 err = mlx5_core_query_sq(dev->mdev, sq->base.mqp.qpn, out);
4748 if (err)
4749 goto out;
4750
4751 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
4752 *sq_state = MLX5_GET(sqc, sqc, state);
4753 sq->state = *sq_state;
4754
4755out:
4756 kvfree(out);
4757 return err;
4758}
4759
4760static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4761 struct mlx5_ib_rq *rq,
4762 u8 *rq_state)
4763{
4764 void *out;
4765 void *rqc;
4766 int inlen;
4767 int err;
4768
4769 inlen = MLX5_ST_SZ_BYTES(query_rq_out);
1b9a07ee 4770 out = kvzalloc(inlen, GFP_KERNEL);
6d2f89df 4771 if (!out)
4772 return -ENOMEM;
4773
4774 err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4775 if (err)
4776 goto out;
4777
4778 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4779 *rq_state = MLX5_GET(rqc, rqc, state);
4780 rq->state = *rq_state;
4781
4782out:
4783 kvfree(out);
4784 return err;
4785}
4786
4787static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4788 struct mlx5_ib_qp *qp, u8 *qp_state)
4789{
4790 static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4791 [MLX5_RQC_STATE_RST] = {
4792 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4793 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4794 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE_BAD,
4795 [MLX5_SQ_STATE_NA] = IB_QPS_RESET,
4796 },
4797 [MLX5_RQC_STATE_RDY] = {
4798 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4799 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4800 [MLX5_SQC_STATE_ERR] = IB_QPS_SQE,
4801 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE,
4802 },
4803 [MLX5_RQC_STATE_ERR] = {
4804 [MLX5_SQC_STATE_RST] = MLX5_QP_STATE_BAD,
4805 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE_BAD,
4806 [MLX5_SQC_STATE_ERR] = IB_QPS_ERR,
4807 [MLX5_SQ_STATE_NA] = IB_QPS_ERR,
4808 },
4809 [MLX5_RQ_STATE_NA] = {
4810 [MLX5_SQC_STATE_RST] = IB_QPS_RESET,
4811 [MLX5_SQC_STATE_RDY] = MLX5_QP_STATE,
4812 [MLX5_SQC_STATE_ERR] = MLX5_QP_STATE,
4813 [MLX5_SQ_STATE_NA] = MLX5_QP_STATE_BAD,
4814 },
4815 };
4816
4817 *qp_state = sqrq_trans[rq_state][sq_state];
4818
4819 if (*qp_state == MLX5_QP_STATE_BAD) {
4820 WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4821 qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4822 qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4823 return -EINVAL;
4824 }
4825
4826 if (*qp_state == MLX5_QP_STATE)
4827 *qp_state = qp->state;
4828
4829 return 0;
4830}
4831
4832static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4833 struct mlx5_ib_qp *qp,
4834 u8 *raw_packet_qp_state)
4835{
4836 struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4837 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4838 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4839 int err;
4840 u8 sq_state = MLX5_SQ_STATE_NA;
4841 u8 rq_state = MLX5_RQ_STATE_NA;
4842
4843 if (qp->sq.wqe_cnt) {
4844 err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4845 if (err)
4846 return err;
4847 }
4848
4849 if (qp->rq.wqe_cnt) {
4850 err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4851 if (err)
4852 return err;
4853 }
4854
4855 return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4856 raw_packet_qp_state);
4857}
4858
4859static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4860 struct ib_qp_attr *qp_attr)
e126ba97 4861{
09a7d9ec 4862 int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
e126ba97
EC		 4863 struct mlx5_qp_context *context;
4864 int mlx5_state;
09a7d9ec 4865 u32 *outb;
e126ba97
EC		 4866 int err = 0;
4867
09a7d9ec 4868 outb = kzalloc(outlen, GFP_KERNEL);
6d2f89df 4869 if (!outb)
4870 return -ENOMEM;
4871
19098df2 4872 err = mlx5_core_qp_query(dev->mdev, &qp->trans_qp.base.mqp, outb,
09a7d9ec 4873 outlen);
e126ba97 4874 if (err)
6d2f89df 4875 goto out;
e126ba97 4876
09a7d9ec
SM		 4877 /* FIXME: use MLX5_GET rather than mlx5_qp_context manual struct */
4878 context = (struct mlx5_qp_context *)MLX5_ADDR_OF(query_qp_out, outb, qpc);
4879
e126ba97
EC		 4880 mlx5_state = be32_to_cpu(context->flags) >> 28;
4881
4882 qp->state = to_ib_qp_state(mlx5_state);
e126ba97
EC		 4883 qp_attr->path_mtu = context->mtu_msgmax >> 5;
4884 qp_attr->path_mig_state =
4885 to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3);
4886 qp_attr->qkey = be32_to_cpu(context->qkey);
4887 qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff;
4888 qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff;
4889 qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff;
4890 qp_attr->qp_access_flags =
4891 to_ib_qp_access_flags(be32_to_cpu(context->params2));
4892
4893 if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
38349389
DC		 4894 to_rdma_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
4895 to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
d3ae2bde
NO		 4896 qp_attr->alt_pkey_index =
4897 be16_to_cpu(context->alt_path.pkey_index);
d8966fcd
DC		 4898 qp_attr->alt_port_num =
4899 rdma_ah_get_port_num(&qp_attr->alt_ah_attr);
e126ba97
EC		 4900 }
4901
d3ae2bde 4902 qp_attr->pkey_index = be16_to_cpu(context->pri_path.pkey_index);
e126ba97
EC		 4903 qp_attr->port_num = context->pri_path.port;
4904
4905 /* qp_attr->en_sqd_async_notify is only applicable in modify qp */
4906 qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING;
4907
4908 qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7);
4909
4910 qp_attr->max_dest_rd_atomic =
4911 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
4912 qp_attr->min_rnr_timer =
4913 (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
4914 qp_attr->timeout = context->pri_path.ackto_lt >> 3;
4915 qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7;
4916 qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7;
4917 qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3;
6d2f89df 4918
4919out:
4920 kfree(outb);
4921 return err;
4922}
4923
776a3906
MS		 4924static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
4925 struct ib_qp_attr *qp_attr, int qp_attr_mask,
4926 struct ib_qp_init_attr *qp_init_attr)
4927{
4928 struct mlx5_core_dct *dct = &mqp->dct.mdct;
4929 u32 *out;
4930 u32 access_flags = 0;
4931 int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
4932 void *dctc;
4933 int err;
4934 int supported_mask = IB_QP_STATE |
4935 IB_QP_ACCESS_FLAGS |
4936 IB_QP_PORT |
4937 IB_QP_MIN_RNR_TIMER |
4938 IB_QP_AV |
4939 IB_QP_PATH_MTU |
4940 IB_QP_PKEY_INDEX;
4941
4942 if (qp_attr_mask & ~supported_mask)
4943 return -EINVAL;
4944 if (mqp->state != IB_QPS_RTR)
4945 return -EINVAL;
4946
4947 out = kzalloc(outlen, GFP_KERNEL);
4948 if (!out)
4949 return -ENOMEM;
4950
4951 err = mlx5_core_dct_query(dev->mdev, dct, out, outlen);
4952 if (err)
4953 goto out;
4954
4955 dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
4956
4957 if (qp_attr_mask & IB_QP_STATE)
4958 qp_attr->qp_state = IB_QPS_RTR;
4959
4960 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
4961 if (MLX5_GET(dctc, dctc, rre))
4962 access_flags |= IB_ACCESS_REMOTE_READ;
4963 if (MLX5_GET(dctc, dctc, rwe))
4964 access_flags |= IB_ACCESS_REMOTE_WRITE;
4965 if (MLX5_GET(dctc, dctc, rae))
4966 access_flags |= IB_ACCESS_REMOTE_ATOMIC;
4967 qp_attr->qp_access_flags = access_flags;
4968 }
4969
4970 if (qp_attr_mask & IB_QP_PORT)
4971 qp_attr->port_num = MLX5_GET(dctc, dctc, port);
4972 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
4973 qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
4974 if (qp_attr_mask & IB_QP_AV) {
4975 qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
4976 qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
4977 qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
4978 qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
4979 }
4980 if (qp_attr_mask & IB_QP_PATH_MTU)
4981 qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
4982 if (qp_attr_mask & IB_QP_PKEY_INDEX)
4983 qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
4984out:
4985 kfree(out);
4986 return err;
4987}
4988
6d2f89df 4989int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
4990 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
4991{
4992 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4993 struct mlx5_ib_qp *qp = to_mqp(ibqp);
4994 int err = 0;
4995 u8 raw_packet_qp_state;
4996
28d61370
YH		 4997 if (ibqp->rwq_ind_tbl)
4998 return -ENOSYS;
4999
d16e91da
HE		 5000 if (unlikely(ibqp->qp_type == IB_QPT_GSI))
5001 return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
5002 qp_init_attr);
5003
c2e53b2c
YH		 5004 /* Not all of output fields are applicable, make sure to zero them */
5005 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5006 memset(qp_attr, 0, sizeof(*qp_attr));
5007
776a3906
MS		 5008 if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT))
5009 return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5010 qp_attr_mask, qp_init_attr);
5011
6d2f89df 5012 mutex_lock(&qp->mutex);
5013
c2e53b2c
YH		 5014 if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
5015 qp->flags & MLX5_IB_QP_UNDERLAY) {
6d2f89df 5016 err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5017 if (err)
5018 goto out;
5019 qp->state = raw_packet_qp_state;
5020 qp_attr->port_num = 1;
5021 } else {
5022 err = query_qp_attr(dev, qp, qp_attr);
5023 if (err)
5024 goto out;
5025 }
5026
5027 qp_attr->qp_state = qp->state;
e126ba97
EC		 5028 qp_attr->cur_qp_state = qp_attr->qp_state;
5029 qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt;
5030 qp_attr->cap.max_recv_sge = qp->rq.max_gs;
5031
5032 if (!ibqp->uobject) {
0540d814 5033 qp_attr->cap.max_send_wr = qp->sq.max_post;
e126ba97 5034 qp_attr->cap.max_send_sge = qp->sq.max_gs;
0540d814 5035 qp_init_attr->qp_context = ibqp->qp_context;
e126ba97
EC		 5036 } else {
5037 qp_attr->cap.max_send_wr = 0;
5038 qp_attr->cap.max_send_sge = 0;
5039 }
5040
0540d814
NO		 5041 qp_init_attr->qp_type = ibqp->qp_type;
5042 qp_init_attr->recv_cq = ibqp->recv_cq;
5043 qp_init_attr->send_cq = ibqp->send_cq;
5044 qp_init_attr->srq = ibqp->srq;
5045 qp_attr->cap.max_inline_data = qp->max_inline_data;
e126ba97
EC		 5046
5047 qp_init_attr->cap = qp_attr->cap;
5048
5049 qp_init_attr->create_flags = 0;
5050 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
5051 qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
5052
051f2630
LR		 5053 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
5054 qp_init_attr->create_flags |= IB_QP_CREATE_CROSS_CHANNEL;
5055 if (qp->flags & MLX5_IB_QP_MANAGED_SEND)
5056 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_SEND;
5057 if (qp->flags & MLX5_IB_QP_MANAGED_RECV)
5058 qp_init_attr->create_flags |= IB_QP_CREATE_MANAGED_RECV;
b11a4f9c
HE		 5059 if (qp->flags & MLX5_IB_QP_SQPN_QP1)
5060 qp_init_attr->create_flags |= mlx5_ib_create_qp_sqpn_qp1();
051f2630 5061
e126ba97
EC		 5062 qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5063 IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5064
e126ba97
EC		 5065out:
5066 mutex_unlock(&qp->mutex);
5067 return err;
5068}
5069
5070struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
5071 struct ib_ucontext *context,
5072 struct ib_udata *udata)
5073{
5074 struct mlx5_ib_dev *dev = to_mdev(ibdev);
5075 struct mlx5_ib_xrcd *xrcd;
5076 int err;
5077
938fe83c 5078 if (!MLX5_CAP_GEN(dev->mdev, xrc))
e126ba97
EC		 5079 return ERR_PTR(-ENOSYS);
5080
5081 xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL);
5082 if (!xrcd)
5083 return ERR_PTR(-ENOMEM);
5084
9603b61d 5085 err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn);
e126ba97
EC		 5086 if (err) {
5087 kfree(xrcd);
5088 return ERR_PTR(-ENOMEM);
5089 }
5090
5091 return &xrcd->ibxrcd;
5092}
5093
5094int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
5095{
5096 struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5097 u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
5098 int err;
5099
9603b61d 5100 err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
b081808a 5101 if (err)
e126ba97 5102 mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
e126ba97
EC		 5103
5104 kfree(xrcd);
e126ba97
EC		 5105 return 0;
5106}
79b20a6c 5107
350d0e4c
YH		 5108static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5109{
5110 struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5111 struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
5112 struct ib_event event;
5113
5114 if (rwq->ibwq.event_handler) {
5115 event.device = rwq->ibwq.device;
5116 event.element.wq = &rwq->ibwq;
5117 switch (type) {
5118 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5119 event.event = IB_EVENT_WQ_FATAL;
5120 break;
5121 default:
5122 mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5123 return;
5124 }
5125
5126 rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5127 }
5128}
5129
03404e8a
MG		 5130static int set_delay_drop(struct mlx5_ib_dev *dev)
5131{
5132 int err = 0;
5133
5134 mutex_lock(&dev->delay_drop.lock);
5135 if (dev->delay_drop.activate)
5136 goto out;
5137
5138 err = mlx5_core_set_delay_drop(dev->mdev, dev->delay_drop.timeout);
5139 if (err)
5140 goto out;
5141
5142 dev->delay_drop.activate = true;
5143out:
5144 mutex_unlock(&dev->delay_drop.lock);
fe248c3a
MG		 5145
5146 if (!err)
5147 atomic_inc(&dev->delay_drop.rqs_cnt);
03404e8a
MG		 5148 return err;
5149}
5150
79b20a6c
YH		 5151static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5152 struct ib_wq_init_attr *init_attr)
5153{
5154 struct mlx5_ib_dev *dev;
4be6da1e 5155 int has_net_offloads;
79b20a6c
YH		 5156 __be64 *rq_pas0;
5157 void *in;
5158 void *rqc;
5159 void *wq;
5160 int inlen;
5161 int err;
5162
5163 dev = to_mdev(pd->device);
5164
5165 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
1b9a07ee 5166 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5167 if (!in)
5168 return -ENOMEM;
5169
5170 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5171 MLX5_SET(rqc, rqc, mem_rq_type,
5172 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
5173 MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5174 MLX5_SET(rqc, rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5175 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
5176 MLX5_SET(rqc, rqc, flush_in_error_en, 1);
5177 wq = MLX5_ADDR_OF(rqc, rqc, wq);
ccc87087
NO		 5178 MLX5_SET(wq, wq, wq_type,
5179 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5180 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
b1383aa6
NO		 5181 if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5182 if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5183 mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5184 err = -EOPNOTSUPP;
5185 goto out;
5186 } else {
5187 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5188 }
5189 }
79b20a6c 5190 MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
ccc87087
NO		 5191 if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
5192 MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5193 MLX5_SET(wq, wq, log_wqe_stride_size,
5194 rwq->single_stride_log_num_of_bytes -
5195 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5196 MLX5_SET(wq, wq, log_wqe_num_of_strides, rwq->log_num_strides -
5197 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES);
5198 }
79b20a6c
YH		 5199 MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5200 MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5201 MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5202 MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5203 MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5204 MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
4be6da1e 5205 has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
b1f74a84 5206 if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
4be6da1e 5207 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
b1f74a84
NO		 5208 mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5209 err = -EOPNOTSUPP;
5210 goto out;
5211 }
5212 } else {
5213 MLX5_SET(rqc, rqc, vsd, 1);
5214 }
4be6da1e
NO		 5215 if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5216 if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5217 mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5218 err = -EOPNOTSUPP;
5219 goto out;
5220 }
5221 MLX5_SET(rqc, rqc, scatter_fcs, 1);
5222 }
03404e8a
MG		 5223 if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5224 if (!(dev->ib_dev.attrs.raw_packet_caps &
5225 IB_RAW_PACKET_CAP_DELAY_DROP)) {
5226 mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5227 err = -EOPNOTSUPP;
5228 goto out;
5229 }
5230 MLX5_SET(rqc, rqc, delay_drop_en, 1);
5231 }
79b20a6c
YH		 5232 rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5233 mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);
350d0e4c 5234 err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);
03404e8a
MG		 5235 if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5236 err = set_delay_drop(dev);
5237 if (err) {
5238 mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5239 err);
5240 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
5241 } else {
5242 rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5243 }
5244 }
b1f74a84 5245out:
79b20a6c
YH		 5246 kvfree(in);
5247 return err;
5248}
5249
5250static int set_user_rq_size(struct mlx5_ib_dev *dev,
5251 struct ib_wq_init_attr *wq_init_attr,
5252 struct mlx5_ib_create_wq *ucmd,
5253 struct mlx5_ib_rwq *rwq)
5254{
5255 /* Sanity check RQ size before proceeding */
5256 if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5257 return -EINVAL;
5258
5259 if (!ucmd->rq_wqe_count)
5260 return -EINVAL;
5261
5262 rwq->wqe_count = ucmd->rq_wqe_count;
5263 rwq->wqe_shift = ucmd->rq_wqe_shift;
5264 rwq->buf_size = (rwq->wqe_count << rwq->wqe_shift);
5265 rwq->log_rq_stride = rwq->wqe_shift;
5266 rwq->log_rq_size = ilog2(rwq->wqe_count);
5267 return 0;
5268}
5269
5270static int prepare_user_rq(struct ib_pd *pd,
5271 struct ib_wq_init_attr *init_attr,
5272 struct ib_udata *udata,
5273 struct mlx5_ib_rwq *rwq)
5274{
5275 struct mlx5_ib_dev *dev = to_mdev(pd->device);
5276 struct mlx5_ib_create_wq ucmd = {};
5277 int err;
5278 size_t required_cmd_sz;
5279
ccc87087
NO		 5280 required_cmd_sz = offsetof(typeof(ucmd), single_stride_log_num_of_bytes)
5281 + sizeof(ucmd.single_stride_log_num_of_bytes);
79b20a6c
YH		 5282 if (udata->inlen < required_cmd_sz) {
5283 mlx5_ib_dbg(dev, "invalid inlen\n");
5284 return -EINVAL;
5285 }
5286
5287 if (udata->inlen > sizeof(ucmd) &&
5288 !ib_is_udata_cleared(udata, sizeof(ucmd),
5289 udata->inlen - sizeof(ucmd))) {
5290 mlx5_ib_dbg(dev, "inlen is not supported\n");
5291 return -EOPNOTSUPP;
5292 }
5293
5294 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5295 mlx5_ib_dbg(dev, "copy failed\n");
5296 return -EFAULT;
5297 }
5298
ccc87087 5299 if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
79b20a6c
YH		 5300 mlx5_ib_dbg(dev, "invalid comp mask\n");
5301 return -EOPNOTSUPP;
ccc87087
NO		 5302 } else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5303 if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5304 mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5305 return -EOPNOTSUPP;
5306 }
5307 if ((ucmd.single_stride_log_num_of_bytes <
5308 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5309 (ucmd.single_stride_log_num_of_bytes >
5310 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5311 mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5312 ucmd.single_stride_log_num_of_bytes,
5313 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5314 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5315 return -EINVAL;
5316 }
5317 if ((ucmd.single_wqe_log_num_of_strides >
5318 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5319 (ucmd.single_wqe_log_num_of_strides <
5320 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES)) {
5321 mlx5_ib_dbg(dev, "Invalid log num strides (%u. Range is %u - %u)\n",
5322 ucmd.single_wqe_log_num_of_strides,
5323 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5324 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5325 return -EINVAL;
5326 }
5327 rwq->single_stride_log_num_of_bytes =
5328 ucmd.single_stride_log_num_of_bytes;
5329 rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5330 rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5331 rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
79b20a6c
YH		 5332 }
5333
5334 err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
5335 if (err) {
5336 mlx5_ib_dbg(dev, "err %d\n", err);
5337 return err;
5338 }
5339
5340 err = create_user_rq(dev, pd, rwq, &ucmd);
5341 if (err) {
5342 mlx5_ib_dbg(dev, "err %d\n", err);
5343 if (err)
5344 return err;
5345 }
5346
5347 rwq->user_index = ucmd.user_index;
5348 return 0;
5349}
5350
5351struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5352 struct ib_wq_init_attr *init_attr,
5353 struct ib_udata *udata)
5354{
5355 struct mlx5_ib_dev *dev;
5356 struct mlx5_ib_rwq *rwq;
5357 struct mlx5_ib_create_wq_resp resp = {};
5358 size_t min_resp_len;
5359 int err;
5360
5361 if (!udata)
5362 return ERR_PTR(-ENOSYS);
5363
5364 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5365 if (udata->outlen && udata->outlen < min_resp_len)
5366 return ERR_PTR(-EINVAL);
5367
5368 dev = to_mdev(pd->device);
5369 switch (init_attr->wq_type) {
5370 case IB_WQT_RQ:
5371 rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5372 if (!rwq)
5373 return ERR_PTR(-ENOMEM);
5374 err = prepare_user_rq(pd, init_attr, udata, rwq);
5375 if (err)
5376 goto err;
5377 err = create_rq(rwq, pd, init_attr);
5378 if (err)
5379 goto err_user_rq;
5380 break;
5381 default:
5382 mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5383 init_attr->wq_type);
5384 return ERR_PTR(-EINVAL);
5385 }
5386
350d0e4c 5387 rwq->ibwq.wq_num = rwq->core_qp.qpn;
79b20a6c
YH		 5388 rwq->ibwq.state = IB_WQS_RESET;
5389 if (udata->outlen) {
5390 resp.response_length = offsetof(typeof(resp), response_length) +
5391 sizeof(resp.response_length);
5392 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5393 if (err)
5394 goto err_copy;
5395 }
5396
350d0e4c
YH		 5397 rwq->core_qp.event = mlx5_ib_wq_event;
5398 rwq->ibwq.event_handler = init_attr->event_handler;
79b20a6c
YH		 5399 return &rwq->ibwq;
5400
5401err_copy:
350d0e4c 5402 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
79b20a6c 5403err_user_rq:
fe248c3a 5404 destroy_user_rq(dev, pd, rwq);
79b20a6c
YH		 5405err:
5406 kfree(rwq);
5407 return ERR_PTR(err);
5408}
5409
5410int mlx5_ib_destroy_wq(struct ib_wq *wq)
5411{
5412 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5413 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5414
350d0e4c 5415 mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
fe248c3a 5416 destroy_user_rq(dev, wq->pd, rwq);
79b20a6c
YH		 5417 kfree(rwq);
5418
5419 return 0;
5420}
5421
c5f90929
YH		 5422struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
5423 struct ib_rwq_ind_table_init_attr *init_attr,
5424 struct ib_udata *udata)
5425{
5426 struct mlx5_ib_dev *dev = to_mdev(device);
5427 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl;
5428 int sz = 1 << init_attr->log_ind_tbl_size;
5429 struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5430 size_t min_resp_len;
5431 int inlen;
5432 int err;
5433 int i;
5434 u32 *in;
5435 void *rqtc;
5436
5437 if (udata->inlen > 0 &&
5438 !ib_is_udata_cleared(udata, 0,
5439 udata->inlen))
5440 return ERR_PTR(-EOPNOTSUPP);
5441
efd7f400
MG		 5442 if (init_attr->log_ind_tbl_size >
5443 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5444 mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5445 init_attr->log_ind_tbl_size,
5446 MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5447 return ERR_PTR(-EINVAL);
5448 }
5449
c5f90929
YH		 5450 min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
5451 if (udata->outlen && udata->outlen < min_resp_len)
5452 return ERR_PTR(-EINVAL);
5453
5454 rwq_ind_tbl = kzalloc(sizeof(*rwq_ind_tbl), GFP_KERNEL);
5455 if (!rwq_ind_tbl)
5456 return ERR_PTR(-ENOMEM);
5457
5458 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
1b9a07ee 5459 in = kvzalloc(inlen, GFP_KERNEL);
c5f90929
YH		 5460 if (!in) {
5461 err = -ENOMEM;
5462 goto err;
5463 }
5464
5465 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5466
5467 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5468 MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5469
5470 for (i = 0; i < sz; i++)
5471 MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5472
5473 err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5474 kvfree(in);
5475
5476 if (err)
5477 goto err;
5478
5479 rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5480 if (udata->outlen) {
5481 resp.response_length = offsetof(typeof(resp), response_length) +
5482 sizeof(resp.response_length);
5483 err = ib_copy_to_udata(udata, &resp, resp.response_length);
5484 if (err)
5485 goto err_copy;
5486 }
5487
5488 return &rwq_ind_tbl->ib_rwq_ind_tbl;
5489
5490err_copy:
5491 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5492err:
5493 kfree(rwq_ind_tbl);
5494 return ERR_PTR(err);
5495}
5496
5497int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5498{
5499 struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5500 struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5501
5502 mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
5503
5504 kfree(rwq_ind_tbl);
5505 return 0;
5506}
5507
79b20a6c
YH		 5508int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5509 u32 wq_attr_mask, struct ib_udata *udata)
5510{
5511 struct mlx5_ib_dev *dev = to_mdev(wq->device);
5512 struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5513 struct mlx5_ib_modify_wq ucmd = {};
5514 size_t required_cmd_sz;
5515 int curr_wq_state;
5516 int wq_state;
5517 int inlen;
5518 int err;
5519 void *rqc;
5520 void *in;
5521
5522 required_cmd_sz = offsetof(typeof(ucmd), reserved) + sizeof(ucmd.reserved);
5523 if (udata->inlen < required_cmd_sz)
5524 return -EINVAL;
5525
5526 if (udata->inlen > sizeof(ucmd) &&
5527 !ib_is_udata_cleared(udata, sizeof(ucmd),
5528 udata->inlen - sizeof(ucmd)))
5529 return -EOPNOTSUPP;
5530
5531 if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5532 return -EFAULT;
5533
5534 if (ucmd.comp_mask || ucmd.reserved)
5535 return -EOPNOTSUPP;
5536
5537 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 5538 in = kvzalloc(inlen, GFP_KERNEL);
79b20a6c
YH		 5539 if (!in)
5540 return -ENOMEM;
5541
5542 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5543
5544 curr_wq_state = (wq_attr_mask & IB_WQ_CUR_STATE) ?
5545 wq_attr->curr_wq_state : wq->state;
5546 wq_state = (wq_attr_mask & IB_WQ_STATE) ?
5547 wq_attr->wq_state : curr_wq_state;
5548 if (curr_wq_state == IB_WQS_ERR)
5549 curr_wq_state = MLX5_RQC_STATE_ERR;
5550 if (wq_state == IB_WQS_ERR)
5551 wq_state = MLX5_RQC_STATE_ERR;
5552 MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
5553 MLX5_SET(rqc, rqc, state, wq_state);
5554
b1f74a84
NO		 5555 if (wq_attr_mask & IB_WQ_FLAGS) {
5556 if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5557 if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5558 MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5559 mlx5_ib_dbg(dev, "VLAN offloads are not "
5560 "supported\n");
5561 err = -EOPNOTSUPP;
5562 goto out;
5563 }
5564 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5565 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5566 MLX5_SET(rqc, rqc, vsd,
5567 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5568 }
b1383aa6
NO		 5569
5570 if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5571 mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5572 err = -EOPNOTSUPP;
5573 goto out;
5574 }
b1f74a84
NO		 5575 }
5576
23a6964e
MD		 5577 if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
5578 if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5579 MLX5_SET64(modify_rq_in, in, modify_bitmask,
5580 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
e1f24a79
PP		 5581 MLX5_SET(rqc, rqc, counter_set_id,
5582 dev->port->cnts.set_id);
23a6964e
MD		 5583 } else
5584 pr_info_once("%s: Receive WQ counters are not supported on current FW\n",
5585 dev->ib_dev.name);
5586 }
5587
350d0e4c 5588 err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in, inlen);
79b20a6c
YH		 5589 if (!err)
5590 rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5591
b1f74a84
NO		 5592out:
5593 kvfree(in);
79b20a6c
YH		 5594 return err;
5595}
Mirror https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git