2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
37 static int db_delay_usecs
= 1;
38 module_param(db_delay_usecs
, int, 0644);
39 MODULE_PARM_DESC(db_delay_usecs
, "Usecs to delay awaiting db fifo to drain");
41 static int ocqp_support
= 1;
42 module_param(ocqp_support
, int, 0644);
43 MODULE_PARM_DESC(ocqp_support
, "Support on-chip SQs (default=1)");
45 int db_fc_threshold
= 1000;
46 module_param(db_fc_threshold
, int, 0644);
47 MODULE_PARM_DESC(db_fc_threshold
,
48 "QP count/threshold that triggers"
49 " automatic db flow control mode (default = 1000)");
51 int db_coalescing_threshold
;
52 module_param(db_coalescing_threshold
, int, 0644);
53 MODULE_PARM_DESC(db_coalescing_threshold
,
54 "QP count/threshold that triggers"
55 " disabling db coalescing (default = 0)");
57 static int max_fr_immd
= T4_MAX_FR_IMMD
;
58 module_param(max_fr_immd
, int, 0644);
59 MODULE_PARM_DESC(max_fr_immd
, "fastreg threshold for using DSGL instead of immedate");
61 static int alloc_ird(struct c4iw_dev
*dev
, u32 ird
)
65 spin_lock_irq(&dev
->lock
);
66 if (ird
<= dev
->avail_ird
)
67 dev
->avail_ird
-= ird
;
70 spin_unlock_irq(&dev
->lock
);
73 dev_warn(&dev
->rdev
.lldi
.pdev
->dev
,
74 "device IRD resources exhausted\n");
79 static void free_ird(struct c4iw_dev
*dev
, int ird
)
81 spin_lock_irq(&dev
->lock
);
82 dev
->avail_ird
+= ird
;
83 spin_unlock_irq(&dev
->lock
);
86 static void set_state(struct c4iw_qp
*qhp
, enum c4iw_qp_state state
)
89 spin_lock_irqsave(&qhp
->lock
, flag
);
90 qhp
->attr
.state
= state
;
91 spin_unlock_irqrestore(&qhp
->lock
, flag
);
94 static void dealloc_oc_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
)
96 c4iw_ocqp_pool_free(rdev
, sq
->dma_addr
, sq
->memsize
);
99 static void dealloc_host_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
)
101 dma_free_coherent(&(rdev
->lldi
.pdev
->dev
), sq
->memsize
, sq
->queue
,
102 dma_unmap_addr(sq
, mapping
));
105 static void dealloc_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
)
107 if (t4_sq_onchip(sq
))
108 dealloc_oc_sq(rdev
, sq
);
110 dealloc_host_sq(rdev
, sq
);
113 static int alloc_oc_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
)
115 if (!ocqp_support
|| !ocqp_supported(&rdev
->lldi
))
117 sq
->dma_addr
= c4iw_ocqp_pool_alloc(rdev
, sq
->memsize
);
120 sq
->phys_addr
= rdev
->oc_mw_pa
+ sq
->dma_addr
-
121 rdev
->lldi
.vr
->ocq
.start
;
122 sq
->queue
= (__force
union t4_wr
*)(rdev
->oc_mw_kva
+ sq
->dma_addr
-
123 rdev
->lldi
.vr
->ocq
.start
);
124 sq
->flags
|= T4_SQ_ONCHIP
;
128 static int alloc_host_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
)
130 sq
->queue
= dma_alloc_coherent(&(rdev
->lldi
.pdev
->dev
), sq
->memsize
,
131 &(sq
->dma_addr
), GFP_KERNEL
);
134 sq
->phys_addr
= virt_to_phys(sq
->queue
);
135 dma_unmap_addr_set(sq
, mapping
, sq
->dma_addr
);
139 static int alloc_sq(struct c4iw_rdev
*rdev
, struct t4_sq
*sq
, int user
)
143 ret
= alloc_oc_sq(rdev
, sq
);
145 ret
= alloc_host_sq(rdev
, sq
);
149 static int destroy_qp(struct c4iw_rdev
*rdev
, struct t4_wq
*wq
,
150 struct c4iw_dev_ucontext
*uctx
, int has_rq
)
153 * uP clears EQ contexts when the connection exits rdma mode,
154 * so no need to post a RESET WR for these EQs.
156 dealloc_sq(rdev
, &wq
->sq
);
158 c4iw_put_qpid(rdev
, wq
->sq
.qid
, uctx
);
161 dma_free_coherent(&rdev
->lldi
.pdev
->dev
,
162 wq
->rq
.memsize
, wq
->rq
.queue
,
163 dma_unmap_addr(&wq
->rq
, mapping
));
164 c4iw_rqtpool_free(rdev
, wq
->rq
.rqt_hwaddr
, wq
->rq
.rqt_size
);
166 c4iw_put_qpid(rdev
, wq
->rq
.qid
, uctx
);
172 * Determine the BAR2 virtual address and qid. If pbar2_pa is not NULL,
173 * then this is a user mapping so compute the page-aligned physical address
176 void __iomem
*c4iw_bar2_addrs(struct c4iw_rdev
*rdev
, unsigned int qid
,
177 enum cxgb4_bar2_qtype qtype
,
178 unsigned int *pbar2_qid
, u64
*pbar2_pa
)
183 ret
= cxgb4_bar2_sge_qregs(rdev
->lldi
.ports
[0], qid
, qtype
,
185 &bar2_qoffset
, pbar2_qid
);
190 *pbar2_pa
= (rdev
->bar2_pa
+ bar2_qoffset
) & PAGE_MASK
;
192 if (is_t4(rdev
->lldi
.adapter_type
))
195 return rdev
->bar2_kva
+ bar2_qoffset
;
198 static int create_qp(struct c4iw_rdev
*rdev
, struct t4_wq
*wq
,
199 struct t4_cq
*rcq
, struct t4_cq
*scq
,
200 struct c4iw_dev_ucontext
*uctx
,
201 struct c4iw_wr_wait
*wr_waitp
,
204 int user
= (uctx
!= &rdev
->uctx
);
205 struct fw_ri_res_wr
*res_wr
;
206 struct fw_ri_res
*res
;
212 wq
->sq
.qid
= c4iw_get_qpid(rdev
, uctx
);
217 wq
->rq
.qid
= c4iw_get_qpid(rdev
, uctx
);
225 wq
->sq
.sw_sq
= kcalloc(wq
->sq
.size
, sizeof(*wq
->sq
.sw_sq
),
229 goto free_rq_qid
;//FIXME
233 wq
->rq
.sw_rq
= kcalloc(wq
->rq
.size
,
234 sizeof(*wq
->rq
.sw_rq
),
245 * RQT must be a power of 2 and at least 16 deep.
248 roundup_pow_of_two(max_t(u16
, wq
->rq
.size
, 16));
249 wq
->rq
.rqt_hwaddr
= c4iw_rqtpool_alloc(rdev
, wq
->rq
.rqt_size
);
250 if (!wq
->rq
.rqt_hwaddr
) {
256 ret
= alloc_sq(rdev
, &wq
->sq
, user
);
259 memset(wq
->sq
.queue
, 0, wq
->sq
.memsize
);
260 dma_unmap_addr_set(&wq
->sq
, mapping
, wq
->sq
.dma_addr
);
263 wq
->rq
.queue
= dma_alloc_coherent(&rdev
->lldi
.pdev
->dev
,
271 pr_debug("sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx\n",
273 (unsigned long long)virt_to_phys(wq
->sq
.queue
),
275 (unsigned long long)virt_to_phys(wq
->rq
.queue
));
276 memset(wq
->rq
.queue
, 0, wq
->rq
.memsize
);
277 dma_unmap_addr_set(&wq
->rq
, mapping
, wq
->rq
.dma_addr
);
280 wq
->db
= rdev
->lldi
.db_reg
;
282 wq
->sq
.bar2_va
= c4iw_bar2_addrs(rdev
, wq
->sq
.qid
,
283 CXGB4_BAR2_QTYPE_EGRESS
,
285 user
? &wq
->sq
.bar2_pa
: NULL
);
287 wq
->rq
.bar2_va
= c4iw_bar2_addrs(rdev
, wq
->rq
.qid
,
288 CXGB4_BAR2_QTYPE_EGRESS
,
290 user
? &wq
->rq
.bar2_pa
: NULL
);
293 * User mode must have bar2 access.
295 if (user
&& (!wq
->sq
.bar2_pa
|| (need_rq
&& !wq
->rq
.bar2_pa
))) {
296 pr_warn("%s: sqid %u or rqid %u not in BAR2 range\n",
297 pci_name(rdev
->lldi
.pdev
), wq
->sq
.qid
, wq
->rq
.qid
);
304 /* build fw_ri_res_wr */
305 wr_len
= sizeof *res_wr
+ 2 * sizeof *res
;
307 wr_len
+= sizeof(*res
);
308 skb
= alloc_skb(wr_len
, GFP_KERNEL
);
313 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
315 res_wr
= __skb_put_zero(skb
, wr_len
);
316 res_wr
->op_nres
= cpu_to_be32(
317 FW_WR_OP_V(FW_RI_RES_WR
) |
318 FW_RI_RES_WR_NRES_V(need_rq
? 2 : 1) |
320 res_wr
->len16_pkd
= cpu_to_be32(DIV_ROUND_UP(wr_len
, 16));
321 res_wr
->cookie
= (uintptr_t)wr_waitp
;
323 res
->u
.sqrq
.restype
= FW_RI_RES_TYPE_SQ
;
324 res
->u
.sqrq
.op
= FW_RI_RES_OP_WRITE
;
327 * eqsize is the number of 64B entries plus the status page size.
329 eqsize
= wq
->sq
.size
* T4_SQ_NUM_SLOTS
+
330 rdev
->hw_queue
.t4_eq_status_entries
;
332 res
->u
.sqrq
.fetchszm_to_iqid
= cpu_to_be32(
333 FW_RI_RES_WR_HOSTFCMODE_V(0) | /* no host cidx updates */
334 FW_RI_RES_WR_CPRIO_V(0) | /* don't keep in chip cache */
335 FW_RI_RES_WR_PCIECHN_V(0) | /* set by uP at ri_init time */
336 (t4_sq_onchip(&wq
->sq
) ? FW_RI_RES_WR_ONCHIP_F
: 0) |
337 FW_RI_RES_WR_IQID_V(scq
->cqid
));
338 res
->u
.sqrq
.dcaen_to_eqsize
= cpu_to_be32(
339 FW_RI_RES_WR_DCAEN_V(0) |
340 FW_RI_RES_WR_DCACPU_V(0) |
341 FW_RI_RES_WR_FBMIN_V(2) |
342 (t4_sq_onchip(&wq
->sq
) ? FW_RI_RES_WR_FBMAX_V(2) :
343 FW_RI_RES_WR_FBMAX_V(3)) |
344 FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
345 FW_RI_RES_WR_CIDXFTHRESH_V(0) |
346 FW_RI_RES_WR_EQSIZE_V(eqsize
));
347 res
->u
.sqrq
.eqid
= cpu_to_be32(wq
->sq
.qid
);
348 res
->u
.sqrq
.eqaddr
= cpu_to_be64(wq
->sq
.dma_addr
);
352 res
->u
.sqrq
.restype
= FW_RI_RES_TYPE_RQ
;
353 res
->u
.sqrq
.op
= FW_RI_RES_OP_WRITE
;
356 * eqsize is the number of 64B entries plus the status page size
358 eqsize
= wq
->rq
.size
* T4_RQ_NUM_SLOTS
+
359 rdev
->hw_queue
.t4_eq_status_entries
;
360 res
->u
.sqrq
.fetchszm_to_iqid
=
361 /* no host cidx updates */
362 cpu_to_be32(FW_RI_RES_WR_HOSTFCMODE_V(0) |
363 /* don't keep in chip cache */
364 FW_RI_RES_WR_CPRIO_V(0) |
365 /* set by uP at ri_init time */
366 FW_RI_RES_WR_PCIECHN_V(0) |
367 FW_RI_RES_WR_IQID_V(rcq
->cqid
));
368 res
->u
.sqrq
.dcaen_to_eqsize
=
369 cpu_to_be32(FW_RI_RES_WR_DCAEN_V(0) |
370 FW_RI_RES_WR_DCACPU_V(0) |
371 FW_RI_RES_WR_FBMIN_V(2) |
372 FW_RI_RES_WR_FBMAX_V(3) |
373 FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
374 FW_RI_RES_WR_CIDXFTHRESH_V(0) |
375 FW_RI_RES_WR_EQSIZE_V(eqsize
));
376 res
->u
.sqrq
.eqid
= cpu_to_be32(wq
->rq
.qid
);
377 res
->u
.sqrq
.eqaddr
= cpu_to_be64(wq
->rq
.dma_addr
);
380 c4iw_init_wr_wait(wr_waitp
);
381 ret
= c4iw_ref_send_wait(rdev
, skb
, wr_waitp
, 0, wq
->sq
.qid
, __func__
);
385 pr_debug("sqid 0x%x rqid 0x%x kdb 0x%p sq_bar2_addr %p rq_bar2_addr %p\n",
386 wq
->sq
.qid
, wq
->rq
.qid
, wq
->db
,
387 wq
->sq
.bar2_va
, wq
->rq
.bar2_va
);
392 dma_free_coherent(&rdev
->lldi
.pdev
->dev
,
393 wq
->rq
.memsize
, wq
->rq
.queue
,
394 dma_unmap_addr(&wq
->rq
, mapping
));
396 dealloc_sq(rdev
, &wq
->sq
);
399 c4iw_rqtpool_free(rdev
, wq
->rq
.rqt_hwaddr
, wq
->rq
.rqt_size
);
407 c4iw_put_qpid(rdev
, wq
->rq
.qid
, uctx
);
409 c4iw_put_qpid(rdev
, wq
->sq
.qid
, uctx
);
413 static int build_immd(struct t4_sq
*sq
, struct fw_ri_immd
*immdp
,
414 const struct ib_send_wr
*wr
, int max
, u32
*plenp
)
421 dstp
= (u8
*)immdp
->data
;
422 for (i
= 0; i
< wr
->num_sge
; i
++) {
423 if ((plen
+ wr
->sg_list
[i
].length
) > max
)
425 srcp
= (u8
*)(unsigned long)wr
->sg_list
[i
].addr
;
426 plen
+= wr
->sg_list
[i
].length
;
427 rem
= wr
->sg_list
[i
].length
;
429 if (dstp
== (u8
*)&sq
->queue
[sq
->size
])
430 dstp
= (u8
*)sq
->queue
;
431 if (rem
<= (u8
*)&sq
->queue
[sq
->size
] - dstp
)
434 len
= (u8
*)&sq
->queue
[sq
->size
] - dstp
;
435 memcpy(dstp
, srcp
, len
);
441 len
= roundup(plen
+ sizeof *immdp
, 16) - (plen
+ sizeof *immdp
);
443 memset(dstp
, 0, len
);
444 immdp
->op
= FW_RI_DATA_IMMD
;
447 immdp
->immdlen
= cpu_to_be32(plen
);
452 static int build_isgl(__be64
*queue_start
, __be64
*queue_end
,
453 struct fw_ri_isgl
*isglp
, struct ib_sge
*sg_list
,
454 int num_sge
, u32
*plenp
)
461 if ((__be64
*)isglp
== queue_end
)
462 isglp
= (struct fw_ri_isgl
*)queue_start
;
464 flitp
= (__be64
*)isglp
->sge
;
466 for (i
= 0; i
< num_sge
; i
++) {
467 if ((plen
+ sg_list
[i
].length
) < plen
)
469 plen
+= sg_list
[i
].length
;
470 *flitp
= cpu_to_be64(((u64
)sg_list
[i
].lkey
<< 32) |
472 if (++flitp
== queue_end
)
474 *flitp
= cpu_to_be64(sg_list
[i
].addr
);
475 if (++flitp
== queue_end
)
478 *flitp
= (__force __be64
)0;
479 isglp
->op
= FW_RI_DATA_ISGL
;
481 isglp
->nsge
= cpu_to_be16(num_sge
);
488 static int build_rdma_send(struct t4_sq
*sq
, union t4_wr
*wqe
,
489 const struct ib_send_wr
*wr
, u8
*len16
)
495 if (wr
->num_sge
> T4_MAX_SEND_SGE
)
497 switch (wr
->opcode
) {
499 if (wr
->send_flags
& IB_SEND_SOLICITED
)
500 wqe
->send
.sendop_pkd
= cpu_to_be32(
501 FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE
));
503 wqe
->send
.sendop_pkd
= cpu_to_be32(
504 FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND
));
505 wqe
->send
.stag_inv
= 0;
507 case IB_WR_SEND_WITH_INV
:
508 if (wr
->send_flags
& IB_SEND_SOLICITED
)
509 wqe
->send
.sendop_pkd
= cpu_to_be32(
510 FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE_INV
));
512 wqe
->send
.sendop_pkd
= cpu_to_be32(
513 FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_INV
));
514 wqe
->send
.stag_inv
= cpu_to_be32(wr
->ex
.invalidate_rkey
);
525 if (wr
->send_flags
& IB_SEND_INLINE
) {
526 ret
= build_immd(sq
, wqe
->send
.u
.immd_src
, wr
,
527 T4_MAX_SEND_INLINE
, &plen
);
530 size
= sizeof wqe
->send
+ sizeof(struct fw_ri_immd
) +
533 ret
= build_isgl((__be64
*)sq
->queue
,
534 (__be64
*)&sq
->queue
[sq
->size
],
535 wqe
->send
.u
.isgl_src
,
536 wr
->sg_list
, wr
->num_sge
, &plen
);
539 size
= sizeof wqe
->send
+ sizeof(struct fw_ri_isgl
) +
540 wr
->num_sge
* sizeof(struct fw_ri_sge
);
543 wqe
->send
.u
.immd_src
[0].op
= FW_RI_DATA_IMMD
;
544 wqe
->send
.u
.immd_src
[0].r1
= 0;
545 wqe
->send
.u
.immd_src
[0].r2
= 0;
546 wqe
->send
.u
.immd_src
[0].immdlen
= 0;
547 size
= sizeof wqe
->send
+ sizeof(struct fw_ri_immd
);
550 *len16
= DIV_ROUND_UP(size
, 16);
551 wqe
->send
.plen
= cpu_to_be32(plen
);
555 static int build_rdma_write(struct t4_sq
*sq
, union t4_wr
*wqe
,
556 const struct ib_send_wr
*wr
, u8
*len16
)
562 if (wr
->num_sge
> T4_MAX_SEND_SGE
)
566 * iWARP protocol supports 64 bit immediate data but rdma api
567 * limits it to 32bit.
569 if (wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
)
570 wqe
->write
.iw_imm_data
.ib_imm_data
.imm_data32
= wr
->ex
.imm_data
;
572 wqe
->write
.iw_imm_data
.ib_imm_data
.imm_data32
= 0;
573 wqe
->write
.stag_sink
= cpu_to_be32(rdma_wr(wr
)->rkey
);
574 wqe
->write
.to_sink
= cpu_to_be64(rdma_wr(wr
)->remote_addr
);
576 if (wr
->send_flags
& IB_SEND_INLINE
) {
577 ret
= build_immd(sq
, wqe
->write
.u
.immd_src
, wr
,
578 T4_MAX_WRITE_INLINE
, &plen
);
581 size
= sizeof wqe
->write
+ sizeof(struct fw_ri_immd
) +
584 ret
= build_isgl((__be64
*)sq
->queue
,
585 (__be64
*)&sq
->queue
[sq
->size
],
586 wqe
->write
.u
.isgl_src
,
587 wr
->sg_list
, wr
->num_sge
, &plen
);
590 size
= sizeof wqe
->write
+ sizeof(struct fw_ri_isgl
) +
591 wr
->num_sge
* sizeof(struct fw_ri_sge
);
594 wqe
->write
.u
.immd_src
[0].op
= FW_RI_DATA_IMMD
;
595 wqe
->write
.u
.immd_src
[0].r1
= 0;
596 wqe
->write
.u
.immd_src
[0].r2
= 0;
597 wqe
->write
.u
.immd_src
[0].immdlen
= 0;
598 size
= sizeof wqe
->write
+ sizeof(struct fw_ri_immd
);
601 *len16
= DIV_ROUND_UP(size
, 16);
602 wqe
->write
.plen
= cpu_to_be32(plen
);
606 static void build_immd_cmpl(struct t4_sq
*sq
, struct fw_ri_immd_cmpl
*immdp
,
607 struct ib_send_wr
*wr
)
609 memcpy((u8
*)immdp
->data
, (u8
*)(uintptr_t)wr
->sg_list
->addr
, 16);
610 memset(immdp
->r1
, 0, 6);
611 immdp
->op
= FW_RI_DATA_IMMD
;
615 static void build_rdma_write_cmpl(struct t4_sq
*sq
,
616 struct fw_ri_rdma_write_cmpl_wr
*wcwr
,
617 const struct ib_send_wr
*wr
, u8
*len16
)
623 * This code assumes the struct fields preceding the write isgl
624 * fit in one 64B WR slot. This is because the WQE is built
625 * directly in the dma queue, and wrapping is only handled
626 * by the code buildling sgls. IE the "fixed part" of the wr
627 * structs must all fit in 64B. The WQE build code should probably be
628 * redesigned to avoid this restriction, but for now just add
629 * the BUILD_BUG_ON() to catch if this WQE struct gets too big.
631 BUILD_BUG_ON(offsetof(struct fw_ri_rdma_write_cmpl_wr
, u
) > 64);
633 wcwr
->stag_sink
= cpu_to_be32(rdma_wr(wr
)->rkey
);
634 wcwr
->to_sink
= cpu_to_be64(rdma_wr(wr
)->remote_addr
);
635 wcwr
->stag_inv
= cpu_to_be32(wr
->next
->ex
.invalidate_rkey
);
640 if (wr
->next
->send_flags
& IB_SEND_INLINE
)
641 build_immd_cmpl(sq
, &wcwr
->u_cmpl
.immd_src
, wr
->next
);
643 build_isgl((__be64
*)sq
->queue
, (__be64
*)&sq
->queue
[sq
->size
],
644 &wcwr
->u_cmpl
.isgl_src
, wr
->next
->sg_list
, 1, NULL
);
647 build_isgl((__be64
*)sq
->queue
, (__be64
*)&sq
->queue
[sq
->size
],
648 wcwr
->u
.isgl_src
, wr
->sg_list
, wr
->num_sge
, &plen
);
650 size
= sizeof(*wcwr
) + sizeof(struct fw_ri_isgl
) +
651 wr
->num_sge
* sizeof(struct fw_ri_sge
);
652 wcwr
->plen
= cpu_to_be32(plen
);
653 *len16
= DIV_ROUND_UP(size
, 16);
656 static int build_rdma_read(union t4_wr
*wqe
, const struct ib_send_wr
*wr
,
661 if (wr
->num_sge
&& wr
->sg_list
[0].length
) {
662 wqe
->read
.stag_src
= cpu_to_be32(rdma_wr(wr
)->rkey
);
663 wqe
->read
.to_src_hi
= cpu_to_be32((u32
)(rdma_wr(wr
)->remote_addr
665 wqe
->read
.to_src_lo
= cpu_to_be32((u32
)rdma_wr(wr
)->remote_addr
);
666 wqe
->read
.stag_sink
= cpu_to_be32(wr
->sg_list
[0].lkey
);
667 wqe
->read
.plen
= cpu_to_be32(wr
->sg_list
[0].length
);
668 wqe
->read
.to_sink_hi
= cpu_to_be32((u32
)(wr
->sg_list
[0].addr
670 wqe
->read
.to_sink_lo
= cpu_to_be32((u32
)(wr
->sg_list
[0].addr
));
672 wqe
->read
.stag_src
= cpu_to_be32(2);
673 wqe
->read
.to_src_hi
= 0;
674 wqe
->read
.to_src_lo
= 0;
675 wqe
->read
.stag_sink
= cpu_to_be32(2);
677 wqe
->read
.to_sink_hi
= 0;
678 wqe
->read
.to_sink_lo
= 0;
682 *len16
= DIV_ROUND_UP(sizeof wqe
->read
, 16);
686 static void post_write_cmpl(struct c4iw_qp
*qhp
, const struct ib_send_wr
*wr
)
688 bool send_signaled
= (wr
->next
->send_flags
& IB_SEND_SIGNALED
) ||
690 bool write_signaled
= (wr
->send_flags
& IB_SEND_SIGNALED
) ||
692 struct t4_swsqe
*swsqe
;
699 * The sw_sq entries still look like a WRITE and a SEND and consume
700 * 2 slots. The FW WR, however, will be a single uber-WR.
702 wqe
= (union t4_wr
*)((u8
*)qhp
->wq
.sq
.queue
+
703 qhp
->wq
.sq
.wq_pidx
* T4_EQ_ENTRY_SIZE
);
704 build_rdma_write_cmpl(&qhp
->wq
.sq
, &wqe
->write_cmpl
, wr
, &len16
);
707 swsqe
= &qhp
->wq
.sq
.sw_sq
[qhp
->wq
.sq
.pidx
];
708 swsqe
->opcode
= FW_RI_RDMA_WRITE
;
709 swsqe
->idx
= qhp
->wq
.sq
.pidx
;
711 swsqe
->signaled
= write_signaled
;
713 swsqe
->wr_id
= wr
->wr_id
;
716 cxgb4_read_sge_timestamp(qhp
->rhp
->rdev
.lldi
.ports
[0]);
717 swsqe
->host_time
= ktime_get();
720 write_wrid
= qhp
->wq
.sq
.pidx
;
722 /* just bump the sw_sq */
724 if (++qhp
->wq
.sq
.pidx
== qhp
->wq
.sq
.size
)
727 /* SEND_WITH_INV swsqe */
728 swsqe
= &qhp
->wq
.sq
.sw_sq
[qhp
->wq
.sq
.pidx
];
729 swsqe
->opcode
= FW_RI_SEND_WITH_INV
;
730 swsqe
->idx
= qhp
->wq
.sq
.pidx
;
732 swsqe
->signaled
= send_signaled
;
734 swsqe
->wr_id
= wr
->next
->wr_id
;
737 cxgb4_read_sge_timestamp(qhp
->rhp
->rdev
.lldi
.ports
[0]);
738 swsqe
->host_time
= ktime_get();
741 wqe
->write_cmpl
.flags_send
= send_signaled
? FW_RI_COMPLETION_FLAG
: 0;
742 wqe
->write_cmpl
.wrid_send
= qhp
->wq
.sq
.pidx
;
744 init_wr_hdr(wqe
, write_wrid
, FW_RI_RDMA_WRITE_CMPL_WR
,
745 write_signaled
? FW_RI_COMPLETION_FLAG
: 0, len16
);
746 t4_sq_produce(&qhp
->wq
, len16
);
747 idx
= DIV_ROUND_UP(len16
* 16, T4_EQ_ENTRY_SIZE
);
749 t4_ring_sq_db(&qhp
->wq
, idx
, wqe
);
752 static int build_rdma_recv(struct c4iw_qp
*qhp
, union t4_recv_wr
*wqe
,
753 const struct ib_recv_wr
*wr
, u8
*len16
)
757 ret
= build_isgl((__be64
*)qhp
->wq
.rq
.queue
,
758 (__be64
*)&qhp
->wq
.rq
.queue
[qhp
->wq
.rq
.size
],
759 &wqe
->recv
.isgl
, wr
->sg_list
, wr
->num_sge
, NULL
);
762 *len16
= DIV_ROUND_UP(sizeof wqe
->recv
+
763 wr
->num_sge
* sizeof(struct fw_ri_sge
), 16);
767 static int build_srq_recv(union t4_recv_wr
*wqe
, const struct ib_recv_wr
*wr
,
772 ret
= build_isgl((__be64
*)wqe
, (__be64
*)(wqe
+ 1),
773 &wqe
->recv
.isgl
, wr
->sg_list
, wr
->num_sge
, NULL
);
776 *len16
= DIV_ROUND_UP(sizeof(wqe
->recv
) +
777 wr
->num_sge
* sizeof(struct fw_ri_sge
), 16);
781 static void build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr
*fr
,
782 const struct ib_reg_wr
*wr
, struct c4iw_mr
*mhp
,
785 __be64
*p
= (__be64
*)fr
->pbl
;
787 fr
->r2
= cpu_to_be32(0);
788 fr
->stag
= cpu_to_be32(mhp
->ibmr
.rkey
);
790 fr
->tpte
.valid_to_pdid
= cpu_to_be32(FW_RI_TPTE_VALID_F
|
791 FW_RI_TPTE_STAGKEY_V((mhp
->ibmr
.rkey
& FW_RI_TPTE_STAGKEY_M
)) |
792 FW_RI_TPTE_STAGSTATE_V(1) |
793 FW_RI_TPTE_STAGTYPE_V(FW_RI_STAG_NSMR
) |
794 FW_RI_TPTE_PDID_V(mhp
->attr
.pdid
));
795 fr
->tpte
.locread_to_qpid
= cpu_to_be32(
796 FW_RI_TPTE_PERM_V(c4iw_ib_to_tpt_access(wr
->access
)) |
797 FW_RI_TPTE_ADDRTYPE_V(FW_RI_VA_BASED_TO
) |
798 FW_RI_TPTE_PS_V(ilog2(wr
->mr
->page_size
) - 12));
799 fr
->tpte
.nosnoop_pbladdr
= cpu_to_be32(FW_RI_TPTE_PBLADDR_V(
800 PBL_OFF(&mhp
->rhp
->rdev
, mhp
->attr
.pbl_addr
)>>3));
801 fr
->tpte
.dca_mwbcnt_pstag
= cpu_to_be32(0);
802 fr
->tpte
.len_hi
= cpu_to_be32(0);
803 fr
->tpte
.len_lo
= cpu_to_be32(mhp
->ibmr
.length
);
804 fr
->tpte
.va_hi
= cpu_to_be32(mhp
->ibmr
.iova
>> 32);
805 fr
->tpte
.va_lo_fbo
= cpu_to_be32(mhp
->ibmr
.iova
& 0xffffffff);
807 p
[0] = cpu_to_be64((u64
)mhp
->mpl
[0]);
808 p
[1] = cpu_to_be64((u64
)mhp
->mpl
[1]);
810 *len16
= DIV_ROUND_UP(sizeof(*fr
), 16);
813 static int build_memreg(struct t4_sq
*sq
, union t4_wr
*wqe
,
814 const struct ib_reg_wr
*wr
, struct c4iw_mr
*mhp
,
815 u8
*len16
, bool dsgl_supported
)
817 struct fw_ri_immd
*imdp
;
820 int pbllen
= roundup(mhp
->mpl_len
* sizeof(u64
), 32);
823 if (mhp
->mpl_len
> t4_max_fr_depth(dsgl_supported
&& use_dsgl
))
826 wqe
->fr
.qpbinde_to_dcacpu
= 0;
827 wqe
->fr
.pgsz_shift
= ilog2(wr
->mr
->page_size
) - 12;
828 wqe
->fr
.addr_type
= FW_RI_VA_BASED_TO
;
829 wqe
->fr
.mem_perms
= c4iw_ib_to_tpt_access(wr
->access
);
831 wqe
->fr
.len_lo
= cpu_to_be32(mhp
->ibmr
.length
);
832 wqe
->fr
.stag
= cpu_to_be32(wr
->key
);
833 wqe
->fr
.va_hi
= cpu_to_be32(mhp
->ibmr
.iova
>> 32);
834 wqe
->fr
.va_lo_fbo
= cpu_to_be32(mhp
->ibmr
.iova
&
837 if (dsgl_supported
&& use_dsgl
&& (pbllen
> max_fr_immd
)) {
838 struct fw_ri_dsgl
*sglp
;
840 for (i
= 0; i
< mhp
->mpl_len
; i
++)
841 mhp
->mpl
[i
] = (__force u64
)cpu_to_be64((u64
)mhp
->mpl
[i
]);
843 sglp
= (struct fw_ri_dsgl
*)(&wqe
->fr
+ 1);
844 sglp
->op
= FW_RI_DATA_DSGL
;
846 sglp
->nsge
= cpu_to_be16(1);
847 sglp
->addr0
= cpu_to_be64(mhp
->mpl_addr
);
848 sglp
->len0
= cpu_to_be32(pbllen
);
850 *len16
= DIV_ROUND_UP(sizeof(wqe
->fr
) + sizeof(*sglp
), 16);
852 imdp
= (struct fw_ri_immd
*)(&wqe
->fr
+ 1);
853 imdp
->op
= FW_RI_DATA_IMMD
;
856 imdp
->immdlen
= cpu_to_be32(pbllen
);
857 p
= (__be64
*)(imdp
+ 1);
859 for (i
= 0; i
< mhp
->mpl_len
; i
++) {
860 *p
= cpu_to_be64((u64
)mhp
->mpl
[i
]);
862 if (++p
== (__be64
*)&sq
->queue
[sq
->size
])
863 p
= (__be64
*)sq
->queue
;
868 if (++p
== (__be64
*)&sq
->queue
[sq
->size
])
869 p
= (__be64
*)sq
->queue
;
871 *len16
= DIV_ROUND_UP(sizeof(wqe
->fr
) + sizeof(*imdp
)
877 static int build_inv_stag(union t4_wr
*wqe
, const struct ib_send_wr
*wr
,
880 wqe
->inv
.stag_inv
= cpu_to_be32(wr
->ex
.invalidate_rkey
);
882 *len16
= DIV_ROUND_UP(sizeof wqe
->inv
, 16);
886 static void free_qp_work(struct work_struct
*work
)
888 struct c4iw_ucontext
*ucontext
;
890 struct c4iw_dev
*rhp
;
892 qhp
= container_of(work
, struct c4iw_qp
, free_work
);
893 ucontext
= qhp
->ucontext
;
896 pr_debug("qhp %p ucontext %p\n", qhp
, ucontext
);
897 destroy_qp(&rhp
->rdev
, &qhp
->wq
,
898 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
, !qhp
->srq
);
901 c4iw_put_ucontext(ucontext
);
902 c4iw_put_wr_wait(qhp
->wr_waitp
);
906 static void queue_qp_free(struct kref
*kref
)
910 qhp
= container_of(kref
, struct c4iw_qp
, kref
);
911 pr_debug("qhp %p\n", qhp
);
912 queue_work(qhp
->rhp
->rdev
.free_workq
, &qhp
->free_work
);
915 void c4iw_qp_add_ref(struct ib_qp
*qp
)
917 pr_debug("ib_qp %p\n", qp
);
918 kref_get(&to_c4iw_qp(qp
)->kref
);
921 void c4iw_qp_rem_ref(struct ib_qp
*qp
)
923 pr_debug("ib_qp %p\n", qp
);
924 kref_put(&to_c4iw_qp(qp
)->kref
, queue_qp_free
);
927 static void add_to_fc_list(struct list_head
*head
, struct list_head
*entry
)
929 if (list_empty(entry
))
930 list_add_tail(entry
, head
);
933 static int ring_kernel_sq_db(struct c4iw_qp
*qhp
, u16 inc
)
937 spin_lock_irqsave(&qhp
->rhp
->lock
, flags
);
938 spin_lock(&qhp
->lock
);
939 if (qhp
->rhp
->db_state
== NORMAL
)
940 t4_ring_sq_db(&qhp
->wq
, inc
, NULL
);
942 add_to_fc_list(&qhp
->rhp
->db_fc_list
, &qhp
->db_fc_entry
);
943 qhp
->wq
.sq
.wq_pidx_inc
+= inc
;
945 spin_unlock(&qhp
->lock
);
946 spin_unlock_irqrestore(&qhp
->rhp
->lock
, flags
);
950 static int ring_kernel_rq_db(struct c4iw_qp
*qhp
, u16 inc
)
954 spin_lock_irqsave(&qhp
->rhp
->lock
, flags
);
955 spin_lock(&qhp
->lock
);
956 if (qhp
->rhp
->db_state
== NORMAL
)
957 t4_ring_rq_db(&qhp
->wq
, inc
, NULL
);
959 add_to_fc_list(&qhp
->rhp
->db_fc_list
, &qhp
->db_fc_entry
);
960 qhp
->wq
.rq
.wq_pidx_inc
+= inc
;
962 spin_unlock(&qhp
->lock
);
963 spin_unlock_irqrestore(&qhp
->rhp
->lock
, flags
);
967 static int ib_to_fw_opcode(int ib_opcode
)
972 case IB_WR_SEND_WITH_INV
:
973 opcode
= FW_RI_SEND_WITH_INV
;
978 case IB_WR_RDMA_WRITE
:
979 opcode
= FW_RI_RDMA_WRITE
;
981 case IB_WR_RDMA_WRITE_WITH_IMM
:
982 opcode
= FW_RI_WRITE_IMMEDIATE
;
984 case IB_WR_RDMA_READ
:
985 case IB_WR_RDMA_READ_WITH_INV
:
986 opcode
= FW_RI_READ_REQ
;
989 opcode
= FW_RI_FAST_REGISTER
;
991 case IB_WR_LOCAL_INV
:
992 opcode
= FW_RI_LOCAL_INV
;
1000 static int complete_sq_drain_wr(struct c4iw_qp
*qhp
,
1001 const struct ib_send_wr
*wr
)
1003 struct t4_cqe cqe
= {};
1004 struct c4iw_cq
*schp
;
1009 schp
= to_c4iw_cq(qhp
->ibqp
.send_cq
);
1012 opcode
= ib_to_fw_opcode(wr
->opcode
);
1016 cqe
.u
.drain_cookie
= wr
->wr_id
;
1017 cqe
.header
= cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH
) |
1018 CQE_OPCODE_V(opcode
) |
1022 CQE_QPID_V(qhp
->wq
.sq
.qid
));
1024 spin_lock_irqsave(&schp
->lock
, flag
);
1025 cqe
.bits_type_ts
= cpu_to_be64(CQE_GENBIT_V((u64
)cq
->gen
));
1026 cq
->sw_queue
[cq
->sw_pidx
] = cqe
;
1027 t4_swcq_produce(cq
);
1028 spin_unlock_irqrestore(&schp
->lock
, flag
);
1030 if (t4_clear_cq_armed(&schp
->cq
)) {
1031 spin_lock_irqsave(&schp
->comp_handler_lock
, flag
);
1032 (*schp
->ibcq
.comp_handler
)(&schp
->ibcq
,
1033 schp
->ibcq
.cq_context
);
1034 spin_unlock_irqrestore(&schp
->comp_handler_lock
, flag
);
1039 static int complete_sq_drain_wrs(struct c4iw_qp
*qhp
,
1040 const struct ib_send_wr
*wr
,
1041 const struct ib_send_wr
**bad_wr
)
1046 ret
= complete_sq_drain_wr(qhp
, wr
);
1056 static void complete_rq_drain_wr(struct c4iw_qp
*qhp
,
1057 const struct ib_recv_wr
*wr
)
1059 struct t4_cqe cqe
= {};
1060 struct c4iw_cq
*rchp
;
1064 rchp
= to_c4iw_cq(qhp
->ibqp
.recv_cq
);
1067 cqe
.u
.drain_cookie
= wr
->wr_id
;
1068 cqe
.header
= cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH
) |
1069 CQE_OPCODE_V(FW_RI_SEND
) |
1073 CQE_QPID_V(qhp
->wq
.sq
.qid
));
1075 spin_lock_irqsave(&rchp
->lock
, flag
);
1076 cqe
.bits_type_ts
= cpu_to_be64(CQE_GENBIT_V((u64
)cq
->gen
));
1077 cq
->sw_queue
[cq
->sw_pidx
] = cqe
;
1078 t4_swcq_produce(cq
);
1079 spin_unlock_irqrestore(&rchp
->lock
, flag
);
1081 if (t4_clear_cq_armed(&rchp
->cq
)) {
1082 spin_lock_irqsave(&rchp
->comp_handler_lock
, flag
);
1083 (*rchp
->ibcq
.comp_handler
)(&rchp
->ibcq
,
1084 rchp
->ibcq
.cq_context
);
1085 spin_unlock_irqrestore(&rchp
->comp_handler_lock
, flag
);
1089 static void complete_rq_drain_wrs(struct c4iw_qp
*qhp
,
1090 const struct ib_recv_wr
*wr
)
1093 complete_rq_drain_wr(qhp
, wr
);
1098 int c4iw_post_send(struct ib_qp
*ibqp
, const struct ib_send_wr
*wr
,
1099 const struct ib_send_wr
**bad_wr
)
1103 enum fw_wr_opcodes fw_opcode
= 0;
1104 enum fw_ri_wr_flags fw_flags
;
1105 struct c4iw_qp
*qhp
;
1106 struct c4iw_dev
*rhp
;
1107 union t4_wr
*wqe
= NULL
;
1109 struct t4_swsqe
*swsqe
;
1113 qhp
= to_c4iw_qp(ibqp
);
1115 spin_lock_irqsave(&qhp
->lock
, flag
);
1118 * If the qp has been flushed, then just insert a special
1121 if (qhp
->wq
.flushed
) {
1122 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1123 err
= complete_sq_drain_wrs(qhp
, wr
, bad_wr
);
1126 num_wrs
= t4_sq_avail(&qhp
->wq
);
1128 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1134 * Fastpath for NVMe-oF target WRITE + SEND_WITH_INV wr chain which is
1135 * the response for small NVMEe-oF READ requests. If the chain is
1136 * exactly a WRITE->SEND_WITH_INV and the sgl depths and lengths
1137 * meet the requirements of the fw_ri_write_cmpl_wr work request,
1138 * then build and post the write_cmpl WR. If any of the tests
1139 * below are not true, then we continue on with the tradtional WRITE
1142 if (qhp
->rhp
->rdev
.lldi
.write_cmpl_support
&&
1143 CHELSIO_CHIP_VERSION(qhp
->rhp
->rdev
.lldi
.adapter_type
) >=
1145 wr
&& wr
->next
&& !wr
->next
->next
&&
1146 wr
->opcode
== IB_WR_RDMA_WRITE
&&
1147 wr
->sg_list
[0].length
&& wr
->num_sge
<= T4_WRITE_CMPL_MAX_SGL
&&
1148 wr
->next
->opcode
== IB_WR_SEND_WITH_INV
&&
1149 wr
->next
->sg_list
[0].length
== T4_WRITE_CMPL_MAX_CQE
&&
1150 wr
->next
->num_sge
== 1 && num_wrs
>= 2) {
1151 post_write_cmpl(qhp
, wr
);
1152 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1162 wqe
= (union t4_wr
*)((u8
*)qhp
->wq
.sq
.queue
+
1163 qhp
->wq
.sq
.wq_pidx
* T4_EQ_ENTRY_SIZE
);
1166 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1167 fw_flags
|= FW_RI_SOLICITED_EVENT_FLAG
;
1168 if (wr
->send_flags
& IB_SEND_SIGNALED
|| qhp
->sq_sig_all
)
1169 fw_flags
|= FW_RI_COMPLETION_FLAG
;
1170 swsqe
= &qhp
->wq
.sq
.sw_sq
[qhp
->wq
.sq
.pidx
];
1171 switch (wr
->opcode
) {
1172 case IB_WR_SEND_WITH_INV
:
1174 if (wr
->send_flags
& IB_SEND_FENCE
)
1175 fw_flags
|= FW_RI_READ_FENCE_FLAG
;
1176 fw_opcode
= FW_RI_SEND_WR
;
1177 if (wr
->opcode
== IB_WR_SEND
)
1178 swsqe
->opcode
= FW_RI_SEND
;
1180 swsqe
->opcode
= FW_RI_SEND_WITH_INV
;
1181 err
= build_rdma_send(&qhp
->wq
.sq
, wqe
, wr
, &len16
);
1183 case IB_WR_RDMA_WRITE_WITH_IMM
:
1184 if (unlikely(!rhp
->rdev
.lldi
.write_w_imm_support
)) {
1188 fw_flags
|= FW_RI_RDMA_WRITE_WITH_IMMEDIATE
;
1190 case IB_WR_RDMA_WRITE
:
1191 fw_opcode
= FW_RI_RDMA_WRITE_WR
;
1192 swsqe
->opcode
= FW_RI_RDMA_WRITE
;
1193 err
= build_rdma_write(&qhp
->wq
.sq
, wqe
, wr
, &len16
);
1195 case IB_WR_RDMA_READ
:
1196 case IB_WR_RDMA_READ_WITH_INV
:
1197 fw_opcode
= FW_RI_RDMA_READ_WR
;
1198 swsqe
->opcode
= FW_RI_READ_REQ
;
1199 if (wr
->opcode
== IB_WR_RDMA_READ_WITH_INV
) {
1200 c4iw_invalidate_mr(rhp
, wr
->sg_list
[0].lkey
);
1201 fw_flags
= FW_RI_RDMA_READ_INVALIDATE
;
1205 err
= build_rdma_read(wqe
, wr
, &len16
);
1208 swsqe
->read_len
= wr
->sg_list
[0].length
;
1209 if (!qhp
->wq
.sq
.oldest_read
)
1210 qhp
->wq
.sq
.oldest_read
= swsqe
;
1212 case IB_WR_REG_MR
: {
1213 struct c4iw_mr
*mhp
= to_c4iw_mr(reg_wr(wr
)->mr
);
1215 swsqe
->opcode
= FW_RI_FAST_REGISTER
;
1216 if (rhp
->rdev
.lldi
.fr_nsmr_tpte_wr_support
&&
1217 !mhp
->attr
.state
&& mhp
->mpl_len
<= 2) {
1218 fw_opcode
= FW_RI_FR_NSMR_TPTE_WR
;
1219 build_tpte_memreg(&wqe
->fr_tpte
, reg_wr(wr
),
1222 fw_opcode
= FW_RI_FR_NSMR_WR
;
1223 err
= build_memreg(&qhp
->wq
.sq
, wqe
, reg_wr(wr
),
1225 rhp
->rdev
.lldi
.ulptx_memwrite_dsgl
);
1229 mhp
->attr
.state
= 1;
1232 case IB_WR_LOCAL_INV
:
1233 if (wr
->send_flags
& IB_SEND_FENCE
)
1234 fw_flags
|= FW_RI_LOCAL_FENCE_FLAG
;
1235 fw_opcode
= FW_RI_INV_LSTAG_WR
;
1236 swsqe
->opcode
= FW_RI_LOCAL_INV
;
1237 err
= build_inv_stag(wqe
, wr
, &len16
);
1238 c4iw_invalidate_mr(rhp
, wr
->ex
.invalidate_rkey
);
1241 pr_warn("%s post of type=%d TBD!\n", __func__
,
1249 swsqe
->idx
= qhp
->wq
.sq
.pidx
;
1250 swsqe
->complete
= 0;
1251 swsqe
->signaled
= (wr
->send_flags
& IB_SEND_SIGNALED
) ||
1254 swsqe
->wr_id
= wr
->wr_id
;
1256 swsqe
->sge_ts
= cxgb4_read_sge_timestamp(
1257 rhp
->rdev
.lldi
.ports
[0]);
1258 swsqe
->host_time
= ktime_get();
1261 init_wr_hdr(wqe
, qhp
->wq
.sq
.pidx
, fw_opcode
, fw_flags
, len16
);
1263 pr_debug("cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u\n",
1264 (unsigned long long)wr
->wr_id
, qhp
->wq
.sq
.pidx
,
1265 swsqe
->opcode
, swsqe
->read_len
);
1268 t4_sq_produce(&qhp
->wq
, len16
);
1269 idx
+= DIV_ROUND_UP(len16
*16, T4_EQ_ENTRY_SIZE
);
1271 if (!rhp
->rdev
.status_page
->db_off
) {
1272 t4_ring_sq_db(&qhp
->wq
, idx
, wqe
);
1273 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1275 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1276 ring_kernel_sq_db(qhp
, idx
);
1281 int c4iw_post_receive(struct ib_qp
*ibqp
, const struct ib_recv_wr
*wr
,
1282 const struct ib_recv_wr
**bad_wr
)
1285 struct c4iw_qp
*qhp
;
1286 union t4_recv_wr
*wqe
= NULL
;
1292 qhp
= to_c4iw_qp(ibqp
);
1293 spin_lock_irqsave(&qhp
->lock
, flag
);
1296 * If the qp has been flushed, then just insert a special
1299 if (qhp
->wq
.flushed
) {
1300 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1301 complete_rq_drain_wrs(qhp
, wr
);
1304 num_wrs
= t4_rq_avail(&qhp
->wq
);
1306 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1311 if (wr
->num_sge
> T4_MAX_RECV_SGE
) {
1316 wqe
= (union t4_recv_wr
*)((u8
*)qhp
->wq
.rq
.queue
+
1317 qhp
->wq
.rq
.wq_pidx
*
1320 err
= build_rdma_recv(qhp
, wqe
, wr
, &len16
);
1328 qhp
->wq
.rq
.sw_rq
[qhp
->wq
.rq
.pidx
].wr_id
= wr
->wr_id
;
1330 qhp
->wq
.rq
.sw_rq
[qhp
->wq
.rq
.pidx
].sge_ts
=
1331 cxgb4_read_sge_timestamp(
1332 qhp
->rhp
->rdev
.lldi
.ports
[0]);
1333 qhp
->wq
.rq
.sw_rq
[qhp
->wq
.rq
.pidx
].host_time
=
1337 wqe
->recv
.opcode
= FW_RI_RECV_WR
;
1339 wqe
->recv
.wrid
= qhp
->wq
.rq
.pidx
;
1340 wqe
->recv
.r2
[0] = 0;
1341 wqe
->recv
.r2
[1] = 0;
1342 wqe
->recv
.r2
[2] = 0;
1343 wqe
->recv
.len16
= len16
;
1344 pr_debug("cookie 0x%llx pidx %u\n",
1345 (unsigned long long)wr
->wr_id
, qhp
->wq
.rq
.pidx
);
1346 t4_rq_produce(&qhp
->wq
, len16
);
1347 idx
+= DIV_ROUND_UP(len16
*16, T4_EQ_ENTRY_SIZE
);
1351 if (!qhp
->rhp
->rdev
.status_page
->db_off
) {
1352 t4_ring_rq_db(&qhp
->wq
, idx
, wqe
);
1353 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1355 spin_unlock_irqrestore(&qhp
->lock
, flag
);
1356 ring_kernel_rq_db(qhp
, idx
);
1361 static void defer_srq_wr(struct t4_srq
*srq
, union t4_recv_wr
*wqe
,
1362 u64 wr_id
, u8 len16
)
1364 struct t4_srq_pending_wr
*pwr
= &srq
->pending_wrs
[srq
->pending_pidx
];
1366 pr_debug("%s cidx %u pidx %u wq_pidx %u in_use %u ooo_count %u wr_id 0x%llx pending_cidx %u pending_pidx %u pending_in_use %u\n",
1367 __func__
, srq
->cidx
, srq
->pidx
, srq
->wq_pidx
,
1368 srq
->in_use
, srq
->ooo_count
,
1369 (unsigned long long)wr_id
, srq
->pending_cidx
,
1370 srq
->pending_pidx
, srq
->pending_in_use
);
1373 memcpy(&pwr
->wqe
, wqe
, len16
* 16);
1374 t4_srq_produce_pending_wr(srq
);
1377 int c4iw_post_srq_recv(struct ib_srq
*ibsrq
, const struct ib_recv_wr
*wr
,
1378 const struct ib_recv_wr
**bad_wr
)
1380 union t4_recv_wr
*wqe
, lwqe
;
1381 struct c4iw_srq
*srq
;
1388 srq
= to_c4iw_srq(ibsrq
);
1389 spin_lock_irqsave(&srq
->lock
, flag
);
1390 num_wrs
= t4_srq_avail(&srq
->wq
);
1392 spin_unlock_irqrestore(&srq
->lock
, flag
);
1396 if (wr
->num_sge
> T4_MAX_RECV_SGE
) {
1403 err
= build_srq_recv(wqe
, wr
, &len16
);
1411 wqe
->recv
.opcode
= FW_RI_RECV_WR
;
1413 wqe
->recv
.wrid
= srq
->wq
.pidx
;
1414 wqe
->recv
.r2
[0] = 0;
1415 wqe
->recv
.r2
[1] = 0;
1416 wqe
->recv
.r2
[2] = 0;
1417 wqe
->recv
.len16
= len16
;
1419 if (srq
->wq
.ooo_count
||
1420 srq
->wq
.pending_in_use
||
1421 srq
->wq
.sw_rq
[srq
->wq
.pidx
].valid
) {
1422 defer_srq_wr(&srq
->wq
, wqe
, wr
->wr_id
, len16
);
1424 srq
->wq
.sw_rq
[srq
->wq
.pidx
].wr_id
= wr
->wr_id
;
1425 srq
->wq
.sw_rq
[srq
->wq
.pidx
].valid
= 1;
1426 c4iw_copy_wr_to_srq(&srq
->wq
, wqe
, len16
);
1427 pr_debug("%s cidx %u pidx %u wq_pidx %u in_use %u wr_id 0x%llx\n",
1428 __func__
, srq
->wq
.cidx
,
1429 srq
->wq
.pidx
, srq
->wq
.wq_pidx
,
1431 (unsigned long long)wr
->wr_id
);
1432 t4_srq_produce(&srq
->wq
, len16
);
1433 idx
+= DIV_ROUND_UP(len16
* 16, T4_EQ_ENTRY_SIZE
);
1439 t4_ring_srq_db(&srq
->wq
, idx
, len16
, wqe
);
1440 spin_unlock_irqrestore(&srq
->lock
, flag
);
1444 static inline void build_term_codes(struct t4_cqe
*err_cqe
, u8
*layer_type
,
1454 *layer_type
= LAYER_RDMAP
|DDP_LOCAL_CATA
;
1459 status
= CQE_STATUS(err_cqe
);
1460 opcode
= CQE_OPCODE(err_cqe
);
1461 rqtype
= RQ_TYPE(err_cqe
);
1462 send_inv
= (opcode
== FW_RI_SEND_WITH_INV
) ||
1463 (opcode
== FW_RI_SEND_WITH_SE_INV
);
1464 tagged
= (opcode
== FW_RI_RDMA_WRITE
) ||
1465 (rqtype
&& (opcode
== FW_RI_READ_RESP
));
1470 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_OP
;
1471 *ecode
= RDMAP_CANT_INV_STAG
;
1473 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1474 *ecode
= RDMAP_INV_STAG
;
1478 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1479 if ((opcode
== FW_RI_SEND_WITH_INV
) ||
1480 (opcode
== FW_RI_SEND_WITH_SE_INV
))
1481 *ecode
= RDMAP_CANT_INV_STAG
;
1483 *ecode
= RDMAP_STAG_NOT_ASSOC
;
1486 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1487 *ecode
= RDMAP_STAG_NOT_ASSOC
;
1490 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1491 *ecode
= RDMAP_ACC_VIOL
;
1494 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1495 *ecode
= RDMAP_TO_WRAP
;
1499 *layer_type
= LAYER_DDP
|DDP_TAGGED_ERR
;
1500 *ecode
= DDPT_BASE_BOUNDS
;
1502 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_PROT
;
1503 *ecode
= RDMAP_BASE_BOUNDS
;
1506 case T4_ERR_INVALIDATE_SHARED_MR
:
1507 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND
:
1508 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_OP
;
1509 *ecode
= RDMAP_CANT_INV_STAG
;
1512 case T4_ERR_ECC_PSTAG
:
1513 case T4_ERR_INTERNAL_ERR
:
1514 *layer_type
= LAYER_RDMAP
|RDMAP_LOCAL_CATA
;
1517 case T4_ERR_OUT_OF_RQE
:
1518 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1519 *ecode
= DDPU_INV_MSN_NOBUF
;
1521 case T4_ERR_PBL_ADDR_BOUND
:
1522 *layer_type
= LAYER_DDP
|DDP_TAGGED_ERR
;
1523 *ecode
= DDPT_BASE_BOUNDS
;
1526 *layer_type
= LAYER_MPA
|DDP_LLP
;
1527 *ecode
= MPA_CRC_ERR
;
1530 *layer_type
= LAYER_MPA
|DDP_LLP
;
1531 *ecode
= MPA_MARKER_ERR
;
1533 case T4_ERR_PDU_LEN_ERR
:
1534 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1535 *ecode
= DDPU_MSG_TOOBIG
;
1537 case T4_ERR_DDP_VERSION
:
1539 *layer_type
= LAYER_DDP
|DDP_TAGGED_ERR
;
1540 *ecode
= DDPT_INV_VERS
;
1542 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1543 *ecode
= DDPU_INV_VERS
;
1546 case T4_ERR_RDMA_VERSION
:
1547 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_OP
;
1548 *ecode
= RDMAP_INV_VERS
;
1551 *layer_type
= LAYER_RDMAP
|RDMAP_REMOTE_OP
;
1552 *ecode
= RDMAP_INV_OPCODE
;
1554 case T4_ERR_DDP_QUEUE_NUM
:
1555 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1556 *ecode
= DDPU_INV_QN
;
1559 case T4_ERR_MSN_GAP
:
1560 case T4_ERR_MSN_RANGE
:
1561 case T4_ERR_IRD_OVERFLOW
:
1562 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1563 *ecode
= DDPU_INV_MSN_RANGE
;
1566 *layer_type
= LAYER_DDP
|DDP_LOCAL_CATA
;
1570 *layer_type
= LAYER_DDP
|DDP_UNTAGGED_ERR
;
1571 *ecode
= DDPU_INV_MO
;
1574 *layer_type
= LAYER_RDMAP
|DDP_LOCAL_CATA
;
1580 static void post_terminate(struct c4iw_qp
*qhp
, struct t4_cqe
*err_cqe
,
1583 struct fw_ri_wr
*wqe
;
1584 struct sk_buff
*skb
;
1585 struct terminate_message
*term
;
1587 pr_debug("qhp %p qid 0x%x tid %u\n", qhp
, qhp
->wq
.sq
.qid
,
1590 skb
= skb_dequeue(&qhp
->ep
->com
.ep_skb_list
);
1594 set_wr_txq(skb
, CPL_PRIORITY_DATA
, qhp
->ep
->txq_idx
);
1596 wqe
= __skb_put_zero(skb
, sizeof(*wqe
));
1597 wqe
->op_compl
= cpu_to_be32(FW_WR_OP_V(FW_RI_INIT_WR
));
1598 wqe
->flowid_len16
= cpu_to_be32(
1599 FW_WR_FLOWID_V(qhp
->ep
->hwtid
) |
1600 FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe
), 16)));
1602 wqe
->u
.terminate
.type
= FW_RI_TYPE_TERMINATE
;
1603 wqe
->u
.terminate
.immdlen
= cpu_to_be32(sizeof *term
);
1604 term
= (struct terminate_message
*)wqe
->u
.terminate
.termmsg
;
1605 if (qhp
->attr
.layer_etype
== (LAYER_MPA
|DDP_LLP
)) {
1606 term
->layer_etype
= qhp
->attr
.layer_etype
;
1607 term
->ecode
= qhp
->attr
.ecode
;
1609 build_term_codes(err_cqe
, &term
->layer_etype
, &term
->ecode
);
1610 c4iw_ofld_send(&qhp
->rhp
->rdev
, skb
);
1614 * Assumes qhp lock is held.
1616 static void __flush_qp(struct c4iw_qp
*qhp
, struct c4iw_cq
*rchp
,
1617 struct c4iw_cq
*schp
)
1620 int rq_flushed
= 0, sq_flushed
;
1623 pr_debug("qhp %p rchp %p schp %p\n", qhp
, rchp
, schp
);
1625 /* locking hierarchy: cqs lock first, then qp lock. */
1626 spin_lock_irqsave(&rchp
->lock
, flag
);
1628 spin_lock(&schp
->lock
);
1629 spin_lock(&qhp
->lock
);
1631 if (qhp
->wq
.flushed
) {
1632 spin_unlock(&qhp
->lock
);
1634 spin_unlock(&schp
->lock
);
1635 spin_unlock_irqrestore(&rchp
->lock
, flag
);
1638 qhp
->wq
.flushed
= 1;
1639 t4_set_wq_in_error(&qhp
->wq
, 0);
1641 c4iw_flush_hw_cq(rchp
, qhp
);
1643 c4iw_count_rcqes(&rchp
->cq
, &qhp
->wq
, &count
);
1644 rq_flushed
= c4iw_flush_rq(&qhp
->wq
, &rchp
->cq
, count
);
1648 c4iw_flush_hw_cq(schp
, qhp
);
1649 sq_flushed
= c4iw_flush_sq(qhp
);
1651 spin_unlock(&qhp
->lock
);
1653 spin_unlock(&schp
->lock
);
1654 spin_unlock_irqrestore(&rchp
->lock
, flag
);
1657 if ((rq_flushed
|| sq_flushed
) &&
1658 t4_clear_cq_armed(&rchp
->cq
)) {
1659 spin_lock_irqsave(&rchp
->comp_handler_lock
, flag
);
1660 (*rchp
->ibcq
.comp_handler
)(&rchp
->ibcq
,
1661 rchp
->ibcq
.cq_context
);
1662 spin_unlock_irqrestore(&rchp
->comp_handler_lock
, flag
);
1665 if (rq_flushed
&& t4_clear_cq_armed(&rchp
->cq
)) {
1666 spin_lock_irqsave(&rchp
->comp_handler_lock
, flag
);
1667 (*rchp
->ibcq
.comp_handler
)(&rchp
->ibcq
,
1668 rchp
->ibcq
.cq_context
);
1669 spin_unlock_irqrestore(&rchp
->comp_handler_lock
, flag
);
1671 if (sq_flushed
&& t4_clear_cq_armed(&schp
->cq
)) {
1672 spin_lock_irqsave(&schp
->comp_handler_lock
, flag
);
1673 (*schp
->ibcq
.comp_handler
)(&schp
->ibcq
,
1674 schp
->ibcq
.cq_context
);
1675 spin_unlock_irqrestore(&schp
->comp_handler_lock
, flag
);
1680 static void flush_qp(struct c4iw_qp
*qhp
)
1682 struct c4iw_cq
*rchp
, *schp
;
1685 rchp
= to_c4iw_cq(qhp
->ibqp
.recv_cq
);
1686 schp
= to_c4iw_cq(qhp
->ibqp
.send_cq
);
1688 if (qhp
->ibqp
.uobject
) {
1690 /* for user qps, qhp->wq.flushed is protected by qhp->mutex */
1691 if (qhp
->wq
.flushed
)
1694 qhp
->wq
.flushed
= 1;
1695 t4_set_wq_in_error(&qhp
->wq
, 0);
1696 t4_set_cq_in_error(&rchp
->cq
);
1697 spin_lock_irqsave(&rchp
->comp_handler_lock
, flag
);
1698 (*rchp
->ibcq
.comp_handler
)(&rchp
->ibcq
, rchp
->ibcq
.cq_context
);
1699 spin_unlock_irqrestore(&rchp
->comp_handler_lock
, flag
);
1701 t4_set_cq_in_error(&schp
->cq
);
1702 spin_lock_irqsave(&schp
->comp_handler_lock
, flag
);
1703 (*schp
->ibcq
.comp_handler
)(&schp
->ibcq
,
1704 schp
->ibcq
.cq_context
);
1705 spin_unlock_irqrestore(&schp
->comp_handler_lock
, flag
);
1709 __flush_qp(qhp
, rchp
, schp
);
1712 static int rdma_fini(struct c4iw_dev
*rhp
, struct c4iw_qp
*qhp
,
1715 struct fw_ri_wr
*wqe
;
1717 struct sk_buff
*skb
;
1719 pr_debug("qhp %p qid 0x%x tid %u\n", qhp
, qhp
->wq
.sq
.qid
, ep
->hwtid
);
1721 skb
= skb_dequeue(&ep
->com
.ep_skb_list
);
1725 set_wr_txq(skb
, CPL_PRIORITY_DATA
, ep
->txq_idx
);
1727 wqe
= __skb_put_zero(skb
, sizeof(*wqe
));
1728 wqe
->op_compl
= cpu_to_be32(
1729 FW_WR_OP_V(FW_RI_INIT_WR
) |
1731 wqe
->flowid_len16
= cpu_to_be32(
1732 FW_WR_FLOWID_V(ep
->hwtid
) |
1733 FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe
), 16)));
1734 wqe
->cookie
= (uintptr_t)ep
->com
.wr_waitp
;
1736 wqe
->u
.fini
.type
= FW_RI_TYPE_FINI
;
1738 ret
= c4iw_ref_send_wait(&rhp
->rdev
, skb
, ep
->com
.wr_waitp
,
1739 qhp
->ep
->hwtid
, qhp
->wq
.sq
.qid
, __func__
);
1741 pr_debug("ret %d\n", ret
);
1745 static void build_rtr_msg(u8 p2p_type
, struct fw_ri_init
*init
)
1747 pr_debug("p2p_type = %d\n", p2p_type
);
1748 memset(&init
->u
, 0, sizeof init
->u
);
1750 case FW_RI_INIT_P2PTYPE_RDMA_WRITE
:
1751 init
->u
.write
.opcode
= FW_RI_RDMA_WRITE_WR
;
1752 init
->u
.write
.stag_sink
= cpu_to_be32(1);
1753 init
->u
.write
.to_sink
= cpu_to_be64(1);
1754 init
->u
.write
.u
.immd_src
[0].op
= FW_RI_DATA_IMMD
;
1755 init
->u
.write
.len16
= DIV_ROUND_UP(sizeof init
->u
.write
+
1756 sizeof(struct fw_ri_immd
),
1759 case FW_RI_INIT_P2PTYPE_READ_REQ
:
1760 init
->u
.write
.opcode
= FW_RI_RDMA_READ_WR
;
1761 init
->u
.read
.stag_src
= cpu_to_be32(1);
1762 init
->u
.read
.to_src_lo
= cpu_to_be32(1);
1763 init
->u
.read
.stag_sink
= cpu_to_be32(1);
1764 init
->u
.read
.to_sink_lo
= cpu_to_be32(1);
1765 init
->u
.read
.len16
= DIV_ROUND_UP(sizeof init
->u
.read
, 16);
1770 static int rdma_init(struct c4iw_dev
*rhp
, struct c4iw_qp
*qhp
)
1772 struct fw_ri_wr
*wqe
;
1774 struct sk_buff
*skb
;
1776 pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp
,
1777 qhp
->wq
.sq
.qid
, qhp
->ep
->hwtid
, qhp
->ep
->ird
, qhp
->ep
->ord
);
1779 skb
= alloc_skb(sizeof *wqe
, GFP_KERNEL
);
1784 ret
= alloc_ird(rhp
, qhp
->attr
.max_ird
);
1786 qhp
->attr
.max_ird
= 0;
1790 set_wr_txq(skb
, CPL_PRIORITY_DATA
, qhp
->ep
->txq_idx
);
1792 wqe
= __skb_put_zero(skb
, sizeof(*wqe
));
1793 wqe
->op_compl
= cpu_to_be32(
1794 FW_WR_OP_V(FW_RI_INIT_WR
) |
1796 wqe
->flowid_len16
= cpu_to_be32(
1797 FW_WR_FLOWID_V(qhp
->ep
->hwtid
) |
1798 FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe
), 16)));
1800 wqe
->cookie
= (uintptr_t)qhp
->ep
->com
.wr_waitp
;
1802 wqe
->u
.init
.type
= FW_RI_TYPE_INIT
;
1803 wqe
->u
.init
.mpareqbit_p2ptype
=
1804 FW_RI_WR_MPAREQBIT_V(qhp
->attr
.mpa_attr
.initiator
) |
1805 FW_RI_WR_P2PTYPE_V(qhp
->attr
.mpa_attr
.p2p_type
);
1806 wqe
->u
.init
.mpa_attrs
= FW_RI_MPA_IETF_ENABLE
;
1807 if (qhp
->attr
.mpa_attr
.recv_marker_enabled
)
1808 wqe
->u
.init
.mpa_attrs
|= FW_RI_MPA_RX_MARKER_ENABLE
;
1809 if (qhp
->attr
.mpa_attr
.xmit_marker_enabled
)
1810 wqe
->u
.init
.mpa_attrs
|= FW_RI_MPA_TX_MARKER_ENABLE
;
1811 if (qhp
->attr
.mpa_attr
.crc_enabled
)
1812 wqe
->u
.init
.mpa_attrs
|= FW_RI_MPA_CRC_ENABLE
;
1814 wqe
->u
.init
.qp_caps
= FW_RI_QP_RDMA_READ_ENABLE
|
1815 FW_RI_QP_RDMA_WRITE_ENABLE
|
1816 FW_RI_QP_BIND_ENABLE
;
1817 if (!qhp
->ibqp
.uobject
)
1818 wqe
->u
.init
.qp_caps
|= FW_RI_QP_FAST_REGISTER_ENABLE
|
1819 FW_RI_QP_STAG0_ENABLE
;
1820 wqe
->u
.init
.nrqe
= cpu_to_be16(t4_rqes_posted(&qhp
->wq
));
1821 wqe
->u
.init
.pdid
= cpu_to_be32(qhp
->attr
.pd
);
1822 wqe
->u
.init
.qpid
= cpu_to_be32(qhp
->wq
.sq
.qid
);
1823 wqe
->u
.init
.sq_eqid
= cpu_to_be32(qhp
->wq
.sq
.qid
);
1825 wqe
->u
.init
.rq_eqid
= cpu_to_be32(FW_RI_INIT_RQEQID_SRQ
|
1828 wqe
->u
.init
.rq_eqid
= cpu_to_be32(qhp
->wq
.rq
.qid
);
1829 wqe
->u
.init
.hwrqsize
= cpu_to_be32(qhp
->wq
.rq
.rqt_size
);
1830 wqe
->u
.init
.hwrqaddr
= cpu_to_be32(qhp
->wq
.rq
.rqt_hwaddr
-
1831 rhp
->rdev
.lldi
.vr
->rq
.start
);
1833 wqe
->u
.init
.scqid
= cpu_to_be32(qhp
->attr
.scq
);
1834 wqe
->u
.init
.rcqid
= cpu_to_be32(qhp
->attr
.rcq
);
1835 wqe
->u
.init
.ord_max
= cpu_to_be32(qhp
->attr
.max_ord
);
1836 wqe
->u
.init
.ird_max
= cpu_to_be32(qhp
->attr
.max_ird
);
1837 wqe
->u
.init
.iss
= cpu_to_be32(qhp
->ep
->snd_seq
);
1838 wqe
->u
.init
.irs
= cpu_to_be32(qhp
->ep
->rcv_seq
);
1839 if (qhp
->attr
.mpa_attr
.initiator
)
1840 build_rtr_msg(qhp
->attr
.mpa_attr
.p2p_type
, &wqe
->u
.init
);
1842 ret
= c4iw_ref_send_wait(&rhp
->rdev
, skb
, qhp
->ep
->com
.wr_waitp
,
1843 qhp
->ep
->hwtid
, qhp
->wq
.sq
.qid
, __func__
);
1847 free_ird(rhp
, qhp
->attr
.max_ird
);
1849 pr_debug("ret %d\n", ret
);
1853 int c4iw_modify_qp(struct c4iw_dev
*rhp
, struct c4iw_qp
*qhp
,
1854 enum c4iw_qp_attr_mask mask
,
1855 struct c4iw_qp_attributes
*attrs
,
1859 struct c4iw_qp_attributes newattr
= qhp
->attr
;
1864 struct c4iw_ep
*ep
= NULL
;
1866 pr_debug("qhp %p sqid 0x%x rqid 0x%x ep %p state %d -> %d\n",
1867 qhp
, qhp
->wq
.sq
.qid
, qhp
->wq
.rq
.qid
, qhp
->ep
, qhp
->attr
.state
,
1868 (mask
& C4IW_QP_ATTR_NEXT_STATE
) ? attrs
->next_state
: -1);
1870 mutex_lock(&qhp
->mutex
);
1872 /* Process attr changes if in IDLE */
1873 if (mask
& C4IW_QP_ATTR_VALID_MODIFY
) {
1874 if (qhp
->attr
.state
!= C4IW_QP_STATE_IDLE
) {
1878 if (mask
& C4IW_QP_ATTR_ENABLE_RDMA_READ
)
1879 newattr
.enable_rdma_read
= attrs
->enable_rdma_read
;
1880 if (mask
& C4IW_QP_ATTR_ENABLE_RDMA_WRITE
)
1881 newattr
.enable_rdma_write
= attrs
->enable_rdma_write
;
1882 if (mask
& C4IW_QP_ATTR_ENABLE_RDMA_BIND
)
1883 newattr
.enable_bind
= attrs
->enable_bind
;
1884 if (mask
& C4IW_QP_ATTR_MAX_ORD
) {
1885 if (attrs
->max_ord
> c4iw_max_read_depth
) {
1889 newattr
.max_ord
= attrs
->max_ord
;
1891 if (mask
& C4IW_QP_ATTR_MAX_IRD
) {
1892 if (attrs
->max_ird
> cur_max_read_depth(rhp
)) {
1896 newattr
.max_ird
= attrs
->max_ird
;
1898 qhp
->attr
= newattr
;
1901 if (mask
& C4IW_QP_ATTR_SQ_DB
) {
1902 ret
= ring_kernel_sq_db(qhp
, attrs
->sq_db_inc
);
1905 if (mask
& C4IW_QP_ATTR_RQ_DB
) {
1906 ret
= ring_kernel_rq_db(qhp
, attrs
->rq_db_inc
);
1910 if (!(mask
& C4IW_QP_ATTR_NEXT_STATE
))
1912 if (qhp
->attr
.state
== attrs
->next_state
)
1915 switch (qhp
->attr
.state
) {
1916 case C4IW_QP_STATE_IDLE
:
1917 switch (attrs
->next_state
) {
1918 case C4IW_QP_STATE_RTS
:
1919 if (!(mask
& C4IW_QP_ATTR_LLP_STREAM_HANDLE
)) {
1923 if (!(mask
& C4IW_QP_ATTR_MPA_ATTR
)) {
1927 qhp
->attr
.mpa_attr
= attrs
->mpa_attr
;
1928 qhp
->attr
.llp_stream_handle
= attrs
->llp_stream_handle
;
1929 qhp
->ep
= qhp
->attr
.llp_stream_handle
;
1930 set_state(qhp
, C4IW_QP_STATE_RTS
);
1933 * Ref the endpoint here and deref when we
1934 * disassociate the endpoint from the QP. This
1935 * happens in CLOSING->IDLE transition or *->ERROR
1938 c4iw_get_ep(&qhp
->ep
->com
);
1939 ret
= rdma_init(rhp
, qhp
);
1943 case C4IW_QP_STATE_ERROR
:
1944 set_state(qhp
, C4IW_QP_STATE_ERROR
);
1952 case C4IW_QP_STATE_RTS
:
1953 switch (attrs
->next_state
) {
1954 case C4IW_QP_STATE_CLOSING
:
1955 t4_set_wq_in_error(&qhp
->wq
, 0);
1956 set_state(qhp
, C4IW_QP_STATE_CLOSING
);
1961 c4iw_get_ep(&qhp
->ep
->com
);
1963 ret
= rdma_fini(rhp
, qhp
, ep
);
1967 case C4IW_QP_STATE_TERMINATE
:
1968 t4_set_wq_in_error(&qhp
->wq
, 0);
1969 set_state(qhp
, C4IW_QP_STATE_TERMINATE
);
1970 qhp
->attr
.layer_etype
= attrs
->layer_etype
;
1971 qhp
->attr
.ecode
= attrs
->ecode
;
1974 c4iw_get_ep(&qhp
->ep
->com
);
1978 terminate
= qhp
->attr
.send_term
;
1979 ret
= rdma_fini(rhp
, qhp
, ep
);
1984 case C4IW_QP_STATE_ERROR
:
1985 t4_set_wq_in_error(&qhp
->wq
, 0);
1986 set_state(qhp
, C4IW_QP_STATE_ERROR
);
1991 c4iw_get_ep(&qhp
->ep
->com
);
2000 case C4IW_QP_STATE_CLOSING
:
2003 * Allow kernel users to move to ERROR for qp draining.
2005 if (!internal
&& (qhp
->ibqp
.uobject
|| attrs
->next_state
!=
2006 C4IW_QP_STATE_ERROR
)) {
2010 switch (attrs
->next_state
) {
2011 case C4IW_QP_STATE_IDLE
:
2013 set_state(qhp
, C4IW_QP_STATE_IDLE
);
2014 qhp
->attr
.llp_stream_handle
= NULL
;
2015 c4iw_put_ep(&qhp
->ep
->com
);
2017 wake_up(&qhp
->wait
);
2019 case C4IW_QP_STATE_ERROR
:
2026 case C4IW_QP_STATE_ERROR
:
2027 if (attrs
->next_state
!= C4IW_QP_STATE_IDLE
) {
2031 if (!t4_sq_empty(&qhp
->wq
) || !t4_rq_empty(&qhp
->wq
)) {
2035 set_state(qhp
, C4IW_QP_STATE_IDLE
);
2037 case C4IW_QP_STATE_TERMINATE
:
2045 pr_err("%s in a bad state %d\n", __func__
, qhp
->attr
.state
);
2052 pr_debug("disassociating ep %p qpid 0x%x\n", qhp
->ep
,
2055 /* disassociate the LLP connection */
2056 qhp
->attr
.llp_stream_handle
= NULL
;
2060 set_state(qhp
, C4IW_QP_STATE_ERROR
);
2064 wake_up(&qhp
->wait
);
2066 mutex_unlock(&qhp
->mutex
);
2069 post_terminate(qhp
, NULL
, internal
? GFP_ATOMIC
: GFP_KERNEL
);
2072 * If disconnect is 1, then we need to initiate a disconnect
2073 * on the EP. This can be a normal close (RTS->CLOSING) or
2074 * an abnormal close (RTS/CLOSING->ERROR).
2077 c4iw_ep_disconnect(ep
, abort
, internal
? GFP_ATOMIC
:
2079 c4iw_put_ep(&ep
->com
);
2083 * If free is 1, then we've disassociated the EP from the QP
2084 * and we need to dereference the EP.
2087 c4iw_put_ep(&ep
->com
);
2088 pr_debug("exit state %d\n", qhp
->attr
.state
);
2092 int c4iw_destroy_qp(struct ib_qp
*ib_qp
)
2094 struct c4iw_dev
*rhp
;
2095 struct c4iw_qp
*qhp
;
2096 struct c4iw_qp_attributes attrs
;
2098 qhp
= to_c4iw_qp(ib_qp
);
2101 attrs
.next_state
= C4IW_QP_STATE_ERROR
;
2102 if (qhp
->attr
.state
== C4IW_QP_STATE_TERMINATE
)
2103 c4iw_modify_qp(rhp
, qhp
, C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 1);
2105 c4iw_modify_qp(rhp
, qhp
, C4IW_QP_ATTR_NEXT_STATE
, &attrs
, 0);
2106 wait_event(qhp
->wait
, !qhp
->ep
);
2108 remove_handle(rhp
, &rhp
->qpidr
, qhp
->wq
.sq
.qid
);
2110 spin_lock_irq(&rhp
->lock
);
2111 if (!list_empty(&qhp
->db_fc_entry
))
2112 list_del_init(&qhp
->db_fc_entry
);
2113 spin_unlock_irq(&rhp
->lock
);
2114 free_ird(rhp
, qhp
->attr
.max_ird
);
2116 c4iw_qp_rem_ref(ib_qp
);
2118 pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp
, qhp
->wq
.sq
.qid
);
2122 struct ib_qp
*c4iw_create_qp(struct ib_pd
*pd
, struct ib_qp_init_attr
*attrs
,
2123 struct ib_udata
*udata
)
2125 struct c4iw_dev
*rhp
;
2126 struct c4iw_qp
*qhp
;
2127 struct c4iw_pd
*php
;
2128 struct c4iw_cq
*schp
;
2129 struct c4iw_cq
*rchp
;
2130 struct c4iw_create_qp_resp uresp
;
2131 unsigned int sqsize
, rqsize
= 0;
2132 struct c4iw_ucontext
*ucontext
;
2134 struct c4iw_mm_entry
*sq_key_mm
, *rq_key_mm
= NULL
, *sq_db_key_mm
;
2135 struct c4iw_mm_entry
*rq_db_key_mm
= NULL
, *ma_sync_key_mm
= NULL
;
2137 pr_debug("ib_pd %p\n", pd
);
2139 if (attrs
->qp_type
!= IB_QPT_RC
)
2140 return ERR_PTR(-EINVAL
);
2142 php
= to_c4iw_pd(pd
);
2144 schp
= get_chp(rhp
, ((struct c4iw_cq
*)attrs
->send_cq
)->cq
.cqid
);
2145 rchp
= get_chp(rhp
, ((struct c4iw_cq
*)attrs
->recv_cq
)->cq
.cqid
);
2147 return ERR_PTR(-EINVAL
);
2149 if (attrs
->cap
.max_inline_data
> T4_MAX_SEND_INLINE
)
2150 return ERR_PTR(-EINVAL
);
2153 if (attrs
->cap
.max_recv_wr
> rhp
->rdev
.hw_queue
.t4_max_rq_size
)
2154 return ERR_PTR(-E2BIG
);
2155 rqsize
= attrs
->cap
.max_recv_wr
+ 1;
2160 if (attrs
->cap
.max_send_wr
> rhp
->rdev
.hw_queue
.t4_max_sq_size
)
2161 return ERR_PTR(-E2BIG
);
2162 sqsize
= attrs
->cap
.max_send_wr
+ 1;
2166 ucontext
= udata
? to_c4iw_ucontext(pd
->uobject
->context
) : NULL
;
2168 qhp
= kzalloc(sizeof(*qhp
), GFP_KERNEL
);
2170 return ERR_PTR(-ENOMEM
);
2172 qhp
->wr_waitp
= c4iw_alloc_wr_wait(GFP_KERNEL
);
2173 if (!qhp
->wr_waitp
) {
2178 qhp
->wq
.sq
.size
= sqsize
;
2179 qhp
->wq
.sq
.memsize
=
2180 (sqsize
+ rhp
->rdev
.hw_queue
.t4_eq_status_entries
) *
2181 sizeof(*qhp
->wq
.sq
.queue
) + 16 * sizeof(__be64
);
2182 qhp
->wq
.sq
.flush_cidx
= -1;
2184 qhp
->wq
.rq
.size
= rqsize
;
2185 qhp
->wq
.rq
.memsize
=
2186 (rqsize
+ rhp
->rdev
.hw_queue
.t4_eq_status_entries
) *
2187 sizeof(*qhp
->wq
.rq
.queue
);
2191 qhp
->wq
.sq
.memsize
= roundup(qhp
->wq
.sq
.memsize
, PAGE_SIZE
);
2193 qhp
->wq
.rq
.memsize
=
2194 roundup(qhp
->wq
.rq
.memsize
, PAGE_SIZE
);
2197 ret
= create_qp(&rhp
->rdev
, &qhp
->wq
, &schp
->cq
, &rchp
->cq
,
2198 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
,
2199 qhp
->wr_waitp
, !attrs
->srq
);
2201 goto err_free_wr_wait
;
2203 attrs
->cap
.max_recv_wr
= rqsize
- 1;
2204 attrs
->cap
.max_send_wr
= sqsize
- 1;
2205 attrs
->cap
.max_inline_data
= T4_MAX_SEND_INLINE
;
2208 qhp
->attr
.pd
= php
->pdid
;
2209 qhp
->attr
.scq
= ((struct c4iw_cq
*) attrs
->send_cq
)->cq
.cqid
;
2210 qhp
->attr
.rcq
= ((struct c4iw_cq
*) attrs
->recv_cq
)->cq
.cqid
;
2211 qhp
->attr
.sq_num_entries
= attrs
->cap
.max_send_wr
;
2212 qhp
->attr
.sq_max_sges
= attrs
->cap
.max_send_sge
;
2213 qhp
->attr
.sq_max_sges_rdma_write
= attrs
->cap
.max_send_sge
;
2215 qhp
->attr
.rq_num_entries
= attrs
->cap
.max_recv_wr
;
2216 qhp
->attr
.rq_max_sges
= attrs
->cap
.max_recv_sge
;
2218 qhp
->attr
.state
= C4IW_QP_STATE_IDLE
;
2219 qhp
->attr
.next_state
= C4IW_QP_STATE_IDLE
;
2220 qhp
->attr
.enable_rdma_read
= 1;
2221 qhp
->attr
.enable_rdma_write
= 1;
2222 qhp
->attr
.enable_bind
= 1;
2223 qhp
->attr
.max_ord
= 0;
2224 qhp
->attr
.max_ird
= 0;
2225 qhp
->sq_sig_all
= attrs
->sq_sig_type
== IB_SIGNAL_ALL_WR
;
2226 spin_lock_init(&qhp
->lock
);
2227 mutex_init(&qhp
->mutex
);
2228 init_waitqueue_head(&qhp
->wait
);
2229 kref_init(&qhp
->kref
);
2230 INIT_WORK(&qhp
->free_work
, free_qp_work
);
2232 ret
= insert_handle(rhp
, &rhp
->qpidr
, qhp
, qhp
->wq
.sq
.qid
);
2234 goto err_destroy_qp
;
2236 if (udata
&& ucontext
) {
2237 sq_key_mm
= kmalloc(sizeof(*sq_key_mm
), GFP_KERNEL
);
2240 goto err_remove_handle
;
2243 rq_key_mm
= kmalloc(sizeof(*rq_key_mm
), GFP_KERNEL
);
2246 goto err_free_sq_key
;
2249 sq_db_key_mm
= kmalloc(sizeof(*sq_db_key_mm
), GFP_KERNEL
);
2250 if (!sq_db_key_mm
) {
2252 goto err_free_rq_key
;
2256 kmalloc(sizeof(*rq_db_key_mm
), GFP_KERNEL
);
2257 if (!rq_db_key_mm
) {
2259 goto err_free_sq_db_key
;
2262 memset(&uresp
, 0, sizeof(uresp
));
2263 if (t4_sq_onchip(&qhp
->wq
.sq
)) {
2264 ma_sync_key_mm
= kmalloc(sizeof(*ma_sync_key_mm
),
2266 if (!ma_sync_key_mm
) {
2268 goto err_free_rq_db_key
;
2270 uresp
.flags
= C4IW_QPF_ONCHIP
;
2272 if (rhp
->rdev
.lldi
.write_w_imm_support
)
2273 uresp
.flags
|= C4IW_QPF_WRITE_W_IMM
;
2274 uresp
.qid_mask
= rhp
->rdev
.qpmask
;
2275 uresp
.sqid
= qhp
->wq
.sq
.qid
;
2276 uresp
.sq_size
= qhp
->wq
.sq
.size
;
2277 uresp
.sq_memsize
= qhp
->wq
.sq
.memsize
;
2279 uresp
.rqid
= qhp
->wq
.rq
.qid
;
2280 uresp
.rq_size
= qhp
->wq
.rq
.size
;
2281 uresp
.rq_memsize
= qhp
->wq
.rq
.memsize
;
2283 spin_lock(&ucontext
->mmap_lock
);
2284 if (ma_sync_key_mm
) {
2285 uresp
.ma_sync_key
= ucontext
->key
;
2286 ucontext
->key
+= PAGE_SIZE
;
2288 uresp
.sq_key
= ucontext
->key
;
2289 ucontext
->key
+= PAGE_SIZE
;
2291 uresp
.rq_key
= ucontext
->key
;
2292 ucontext
->key
+= PAGE_SIZE
;
2294 uresp
.sq_db_gts_key
= ucontext
->key
;
2295 ucontext
->key
+= PAGE_SIZE
;
2297 uresp
.rq_db_gts_key
= ucontext
->key
;
2298 ucontext
->key
+= PAGE_SIZE
;
2300 spin_unlock(&ucontext
->mmap_lock
);
2301 ret
= ib_copy_to_udata(udata
, &uresp
, sizeof uresp
);
2303 goto err_free_ma_sync_key
;
2304 sq_key_mm
->key
= uresp
.sq_key
;
2305 sq_key_mm
->addr
= qhp
->wq
.sq
.phys_addr
;
2306 sq_key_mm
->len
= PAGE_ALIGN(qhp
->wq
.sq
.memsize
);
2307 insert_mmap(ucontext
, sq_key_mm
);
2309 rq_key_mm
->key
= uresp
.rq_key
;
2310 rq_key_mm
->addr
= virt_to_phys(qhp
->wq
.rq
.queue
);
2311 rq_key_mm
->len
= PAGE_ALIGN(qhp
->wq
.rq
.memsize
);
2312 insert_mmap(ucontext
, rq_key_mm
);
2314 sq_db_key_mm
->key
= uresp
.sq_db_gts_key
;
2315 sq_db_key_mm
->addr
= (u64
)(unsigned long)qhp
->wq
.sq
.bar2_pa
;
2316 sq_db_key_mm
->len
= PAGE_SIZE
;
2317 insert_mmap(ucontext
, sq_db_key_mm
);
2319 rq_db_key_mm
->key
= uresp
.rq_db_gts_key
;
2320 rq_db_key_mm
->addr
=
2321 (u64
)(unsigned long)qhp
->wq
.rq
.bar2_pa
;
2322 rq_db_key_mm
->len
= PAGE_SIZE
;
2323 insert_mmap(ucontext
, rq_db_key_mm
);
2325 if (ma_sync_key_mm
) {
2326 ma_sync_key_mm
->key
= uresp
.ma_sync_key
;
2327 ma_sync_key_mm
->addr
=
2328 (pci_resource_start(rhp
->rdev
.lldi
.pdev
, 0) +
2329 PCIE_MA_SYNC_A
) & PAGE_MASK
;
2330 ma_sync_key_mm
->len
= PAGE_SIZE
;
2331 insert_mmap(ucontext
, ma_sync_key_mm
);
2334 c4iw_get_ucontext(ucontext
);
2335 qhp
->ucontext
= ucontext
;
2339 &qhp
->wq
.rq
.queue
[qhp
->wq
.rq
.size
].status
.qp_err
;
2342 &qhp
->wq
.sq
.queue
[qhp
->wq
.sq
.size
].status
.qp_err
;
2344 &qhp
->wq
.sq
.queue
[qhp
->wq
.sq
.size
].status
.srqidx
;
2347 qhp
->ibqp
.qp_num
= qhp
->wq
.sq
.qid
;
2349 qhp
->srq
= to_c4iw_srq(attrs
->srq
);
2350 INIT_LIST_HEAD(&qhp
->db_fc_entry
);
2351 pr_debug("sq id %u size %u memsize %zu num_entries %u rq id %u size %u memsize %zu num_entries %u\n",
2352 qhp
->wq
.sq
.qid
, qhp
->wq
.sq
.size
, qhp
->wq
.sq
.memsize
,
2353 attrs
->cap
.max_send_wr
, qhp
->wq
.rq
.qid
, qhp
->wq
.rq
.size
,
2354 qhp
->wq
.rq
.memsize
, attrs
->cap
.max_recv_wr
);
2356 err_free_ma_sync_key
:
2357 kfree(ma_sync_key_mm
);
2360 kfree(rq_db_key_mm
);
2362 kfree(sq_db_key_mm
);
2369 remove_handle(rhp
, &rhp
->qpidr
, qhp
->wq
.sq
.qid
);
2371 destroy_qp(&rhp
->rdev
, &qhp
->wq
,
2372 ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
, !attrs
->srq
);
2374 c4iw_put_wr_wait(qhp
->wr_waitp
);
2377 return ERR_PTR(ret
);
2380 int c4iw_ib_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2381 int attr_mask
, struct ib_udata
*udata
)
2383 struct c4iw_dev
*rhp
;
2384 struct c4iw_qp
*qhp
;
2385 enum c4iw_qp_attr_mask mask
= 0;
2386 struct c4iw_qp_attributes attrs
;
2388 pr_debug("ib_qp %p\n", ibqp
);
2390 /* iwarp does not support the RTR state */
2391 if ((attr_mask
& IB_QP_STATE
) && (attr
->qp_state
== IB_QPS_RTR
))
2392 attr_mask
&= ~IB_QP_STATE
;
2394 /* Make sure we still have something left to do */
2398 memset(&attrs
, 0, sizeof attrs
);
2399 qhp
= to_c4iw_qp(ibqp
);
2402 attrs
.next_state
= c4iw_convert_state(attr
->qp_state
);
2403 attrs
.enable_rdma_read
= (attr
->qp_access_flags
&
2404 IB_ACCESS_REMOTE_READ
) ? 1 : 0;
2405 attrs
.enable_rdma_write
= (attr
->qp_access_flags
&
2406 IB_ACCESS_REMOTE_WRITE
) ? 1 : 0;
2407 attrs
.enable_bind
= (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) ? 1 : 0;
2410 mask
|= (attr_mask
& IB_QP_STATE
) ? C4IW_QP_ATTR_NEXT_STATE
: 0;
2411 mask
|= (attr_mask
& IB_QP_ACCESS_FLAGS
) ?
2412 (C4IW_QP_ATTR_ENABLE_RDMA_READ
|
2413 C4IW_QP_ATTR_ENABLE_RDMA_WRITE
|
2414 C4IW_QP_ATTR_ENABLE_RDMA_BIND
) : 0;
2417 * Use SQ_PSN and RQ_PSN to pass in IDX_INC values for
2418 * ringing the queue db when we're in DB_FULL mode.
2419 * Only allow this on T4 devices.
2421 attrs
.sq_db_inc
= attr
->sq_psn
;
2422 attrs
.rq_db_inc
= attr
->rq_psn
;
2423 mask
|= (attr_mask
& IB_QP_SQ_PSN
) ? C4IW_QP_ATTR_SQ_DB
: 0;
2424 mask
|= (attr_mask
& IB_QP_RQ_PSN
) ? C4IW_QP_ATTR_RQ_DB
: 0;
2425 if (!is_t4(to_c4iw_qp(ibqp
)->rhp
->rdev
.lldi
.adapter_type
) &&
2426 (mask
& (C4IW_QP_ATTR_SQ_DB
|C4IW_QP_ATTR_RQ_DB
)))
2429 return c4iw_modify_qp(rhp
, qhp
, mask
, &attrs
, 0);
2432 struct ib_qp
*c4iw_get_qp(struct ib_device
*dev
, int qpn
)
2434 pr_debug("ib_dev %p qpn 0x%x\n", dev
, qpn
);
2435 return (struct ib_qp
*)get_qhp(to_c4iw_dev(dev
), qpn
);
2438 void c4iw_dispatch_srq_limit_reached_event(struct c4iw_srq
*srq
)
2440 struct ib_event event
= {};
2442 event
.device
= &srq
->rhp
->ibdev
;
2443 event
.element
.srq
= &srq
->ibsrq
;
2444 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
2445 ib_dispatch_event(&event
);
2448 int c4iw_modify_srq(struct ib_srq
*ib_srq
, struct ib_srq_attr
*attr
,
2449 enum ib_srq_attr_mask srq_attr_mask
,
2450 struct ib_udata
*udata
)
2452 struct c4iw_srq
*srq
= to_c4iw_srq(ib_srq
);
2456 * XXX 0 mask == a SW interrupt for srq_limit reached...
2458 if (udata
&& !srq_attr_mask
) {
2459 c4iw_dispatch_srq_limit_reached_event(srq
);
2463 /* no support for this yet */
2464 if (srq_attr_mask
& IB_SRQ_MAX_WR
) {
2469 if (!udata
&& (srq_attr_mask
& IB_SRQ_LIMIT
)) {
2471 srq
->srq_limit
= attr
->srq_limit
;
2477 int c4iw_ib_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2478 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
2480 struct c4iw_qp
*qhp
= to_c4iw_qp(ibqp
);
2482 memset(attr
, 0, sizeof *attr
);
2483 memset(init_attr
, 0, sizeof *init_attr
);
2484 attr
->qp_state
= to_ib_qp_state(qhp
->attr
.state
);
2485 init_attr
->cap
.max_send_wr
= qhp
->attr
.sq_num_entries
;
2486 init_attr
->cap
.max_recv_wr
= qhp
->attr
.rq_num_entries
;
2487 init_attr
->cap
.max_send_sge
= qhp
->attr
.sq_max_sges
;
2488 init_attr
->cap
.max_recv_sge
= qhp
->attr
.sq_max_sges
;
2489 init_attr
->cap
.max_inline_data
= T4_MAX_SEND_INLINE
;
2490 init_attr
->sq_sig_type
= qhp
->sq_sig_all
? IB_SIGNAL_ALL_WR
: 0;
2494 static void free_srq_queue(struct c4iw_srq
*srq
, struct c4iw_dev_ucontext
*uctx
,
2495 struct c4iw_wr_wait
*wr_waitp
)
2497 struct c4iw_rdev
*rdev
= &srq
->rhp
->rdev
;
2498 struct sk_buff
*skb
= srq
->destroy_skb
;
2499 struct t4_srq
*wq
= &srq
->wq
;
2500 struct fw_ri_res_wr
*res_wr
;
2501 struct fw_ri_res
*res
;
2504 wr_len
= sizeof(*res_wr
) + sizeof(*res
);
2505 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
2507 res_wr
= (struct fw_ri_res_wr
*)__skb_put(skb
, wr_len
);
2508 memset(res_wr
, 0, wr_len
);
2509 res_wr
->op_nres
= cpu_to_be32(FW_WR_OP_V(FW_RI_RES_WR
) |
2510 FW_RI_RES_WR_NRES_V(1) |
2512 res_wr
->len16_pkd
= cpu_to_be32(DIV_ROUND_UP(wr_len
, 16));
2513 res_wr
->cookie
= (uintptr_t)wr_waitp
;
2515 res
->u
.srq
.restype
= FW_RI_RES_TYPE_SRQ
;
2516 res
->u
.srq
.op
= FW_RI_RES_OP_RESET
;
2517 res
->u
.srq
.srqid
= cpu_to_be32(srq
->idx
);
2518 res
->u
.srq
.eqid
= cpu_to_be32(wq
->qid
);
2520 c4iw_init_wr_wait(wr_waitp
);
2521 c4iw_ref_send_wait(rdev
, skb
, wr_waitp
, 0, 0, __func__
);
2523 dma_free_coherent(&rdev
->lldi
.pdev
->dev
,
2524 wq
->memsize
, wq
->queue
,
2525 dma_unmap_addr(wq
, mapping
));
2526 c4iw_rqtpool_free(rdev
, wq
->rqt_hwaddr
, wq
->rqt_size
);
2528 c4iw_put_qpid(rdev
, wq
->qid
, uctx
);
2531 static int alloc_srq_queue(struct c4iw_srq
*srq
, struct c4iw_dev_ucontext
*uctx
,
2532 struct c4iw_wr_wait
*wr_waitp
)
2534 struct c4iw_rdev
*rdev
= &srq
->rhp
->rdev
;
2535 int user
= (uctx
!= &rdev
->uctx
);
2536 struct t4_srq
*wq
= &srq
->wq
;
2537 struct fw_ri_res_wr
*res_wr
;
2538 struct fw_ri_res
*res
;
2539 struct sk_buff
*skb
;
2544 wq
->qid
= c4iw_get_qpid(rdev
, uctx
);
2549 wq
->sw_rq
= kcalloc(wq
->size
, sizeof(*wq
->sw_rq
),
2553 wq
->pending_wrs
= kcalloc(srq
->wq
.size
,
2554 sizeof(*srq
->wq
.pending_wrs
),
2556 if (!wq
->pending_wrs
)
2557 goto err_free_sw_rq
;
2560 wq
->rqt_size
= wq
->size
;
2561 wq
->rqt_hwaddr
= c4iw_rqtpool_alloc(rdev
, wq
->rqt_size
);
2562 if (!wq
->rqt_hwaddr
)
2563 goto err_free_pending_wrs
;
2564 wq
->rqt_abs_idx
= (wq
->rqt_hwaddr
- rdev
->lldi
.vr
->rq
.start
) >>
2567 wq
->queue
= dma_zalloc_coherent(&rdev
->lldi
.pdev
->dev
,
2568 wq
->memsize
, &wq
->dma_addr
,
2571 goto err_free_rqtpool
;
2573 dma_unmap_addr_set(wq
, mapping
, wq
->dma_addr
);
2575 wq
->bar2_va
= c4iw_bar2_addrs(rdev
, wq
->qid
, CXGB4_BAR2_QTYPE_EGRESS
,
2577 user
? &wq
->bar2_pa
: NULL
);
2580 * User mode must have bar2 access.
2583 if (user
&& !wq
->bar2_va
) {
2584 pr_warn(MOD
"%s: srqid %u not in BAR2 range.\n",
2585 pci_name(rdev
->lldi
.pdev
), wq
->qid
);
2587 goto err_free_queue
;
2590 /* build fw_ri_res_wr */
2591 wr_len
= sizeof(*res_wr
) + sizeof(*res
);
2593 skb
= alloc_skb(wr_len
, GFP_KERNEL
| __GFP_NOFAIL
);
2595 goto err_free_queue
;
2596 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, 0);
2598 res_wr
= (struct fw_ri_res_wr
*)__skb_put(skb
, wr_len
);
2599 memset(res_wr
, 0, wr_len
);
2600 res_wr
->op_nres
= cpu_to_be32(FW_WR_OP_V(FW_RI_RES_WR
) |
2601 FW_RI_RES_WR_NRES_V(1) |
2603 res_wr
->len16_pkd
= cpu_to_be32(DIV_ROUND_UP(wr_len
, 16));
2604 res_wr
->cookie
= (uintptr_t)wr_waitp
;
2606 res
->u
.srq
.restype
= FW_RI_RES_TYPE_SRQ
;
2607 res
->u
.srq
.op
= FW_RI_RES_OP_WRITE
;
2610 * eqsize is the number of 64B entries plus the status page size.
2612 eqsize
= wq
->size
* T4_RQ_NUM_SLOTS
+
2613 rdev
->hw_queue
.t4_eq_status_entries
;
2614 res
->u
.srq
.eqid
= cpu_to_be32(wq
->qid
);
2615 res
->u
.srq
.fetchszm_to_iqid
=
2616 /* no host cidx updates */
2617 cpu_to_be32(FW_RI_RES_WR_HOSTFCMODE_V(0) |
2618 FW_RI_RES_WR_CPRIO_V(0) | /* don't keep in chip cache */
2619 FW_RI_RES_WR_PCIECHN_V(0) | /* set by uP at ri_init time */
2620 FW_RI_RES_WR_FETCHRO_V(0)); /* relaxed_ordering */
2621 res
->u
.srq
.dcaen_to_eqsize
=
2622 cpu_to_be32(FW_RI_RES_WR_DCAEN_V(0) |
2623 FW_RI_RES_WR_DCACPU_V(0) |
2624 FW_RI_RES_WR_FBMIN_V(2) |
2625 FW_RI_RES_WR_FBMAX_V(3) |
2626 FW_RI_RES_WR_CIDXFTHRESHO_V(0) |
2627 FW_RI_RES_WR_CIDXFTHRESH_V(0) |
2628 FW_RI_RES_WR_EQSIZE_V(eqsize
));
2629 res
->u
.srq
.eqaddr
= cpu_to_be64(wq
->dma_addr
);
2630 res
->u
.srq
.srqid
= cpu_to_be32(srq
->idx
);
2631 res
->u
.srq
.pdid
= cpu_to_be32(srq
->pdid
);
2632 res
->u
.srq
.hwsrqsize
= cpu_to_be32(wq
->rqt_size
);
2633 res
->u
.srq
.hwsrqaddr
= cpu_to_be32(wq
->rqt_hwaddr
-
2634 rdev
->lldi
.vr
->rq
.start
);
2636 c4iw_init_wr_wait(wr_waitp
);
2638 ret
= c4iw_ref_send_wait(rdev
, skb
, wr_waitp
, 0, wq
->qid
, __func__
);
2640 goto err_free_queue
;
2642 pr_debug("%s srq %u eqid %u pdid %u queue va %p pa 0x%llx\n"
2643 " bar2_addr %p rqt addr 0x%x size %d\n",
2644 __func__
, srq
->idx
, wq
->qid
, srq
->pdid
, wq
->queue
,
2645 (u64
)virt_to_phys(wq
->queue
), wq
->bar2_va
,
2646 wq
->rqt_hwaddr
, wq
->rqt_size
);
2650 dma_free_coherent(&rdev
->lldi
.pdev
->dev
,
2651 wq
->memsize
, wq
->queue
,
2652 dma_unmap_addr(wq
, mapping
));
2654 c4iw_rqtpool_free(rdev
, wq
->rqt_hwaddr
, wq
->rqt_size
);
2655 err_free_pending_wrs
:
2657 kfree(wq
->pending_wrs
);
2662 c4iw_put_qpid(rdev
, wq
->qid
, uctx
);
2667 void c4iw_copy_wr_to_srq(struct t4_srq
*srq
, union t4_recv_wr
*wqe
, u8 len16
)
2672 dst
= (u64
*)((u8
*)srq
->queue
+ srq
->wq_pidx
* T4_EQ_ENTRY_SIZE
);
2675 if (dst
>= (u64
*)&srq
->queue
[srq
->size
])
2676 dst
= (u64
*)srq
->queue
;
2678 if (dst
>= (u64
*)&srq
->queue
[srq
->size
])
2679 dst
= (u64
*)srq
->queue
;
2684 struct ib_srq
*c4iw_create_srq(struct ib_pd
*pd
, struct ib_srq_init_attr
*attrs
,
2685 struct ib_udata
*udata
)
2687 struct c4iw_dev
*rhp
;
2688 struct c4iw_srq
*srq
;
2689 struct c4iw_pd
*php
;
2690 struct c4iw_create_srq_resp uresp
;
2691 struct c4iw_ucontext
*ucontext
;
2692 struct c4iw_mm_entry
*srq_key_mm
, *srq_db_key_mm
;
2697 pr_debug("%s ib_pd %p\n", __func__
, pd
);
2699 php
= to_c4iw_pd(pd
);
2702 if (!rhp
->rdev
.lldi
.vr
->srq
.size
)
2703 return ERR_PTR(-EINVAL
);
2704 if (attrs
->attr
.max_wr
> rhp
->rdev
.hw_queue
.t4_max_rq_size
)
2705 return ERR_PTR(-E2BIG
);
2706 if (attrs
->attr
.max_sge
> T4_MAX_RECV_SGE
)
2707 return ERR_PTR(-E2BIG
);
2710 * SRQ RQT and RQ must be a power of 2 and at least 16 deep.
2712 rqsize
= attrs
->attr
.max_wr
+ 1;
2713 rqsize
= roundup_pow_of_two(max_t(u16
, rqsize
, 16));
2715 ucontext
= udata
? to_c4iw_ucontext(pd
->uobject
->context
) : NULL
;
2717 srq
= kzalloc(sizeof(*srq
), GFP_KERNEL
);
2719 return ERR_PTR(-ENOMEM
);
2721 srq
->wr_waitp
= c4iw_alloc_wr_wait(GFP_KERNEL
);
2722 if (!srq
->wr_waitp
) {
2727 srq
->idx
= c4iw_alloc_srq_idx(&rhp
->rdev
);
2730 goto err_free_wr_wait
;
2733 wr_len
= sizeof(struct fw_ri_res_wr
) + sizeof(struct fw_ri_res
);
2734 srq
->destroy_skb
= alloc_skb(wr_len
, GFP_KERNEL
);
2735 if (!srq
->destroy_skb
) {
2737 goto err_free_srq_idx
;
2741 srq
->pdid
= php
->pdid
;
2743 srq
->wq
.size
= rqsize
;
2745 (rqsize
+ rhp
->rdev
.hw_queue
.t4_eq_status_entries
) *
2746 sizeof(*srq
->wq
.queue
);
2748 srq
->wq
.memsize
= roundup(srq
->wq
.memsize
, PAGE_SIZE
);
2750 ret
= alloc_srq_queue(srq
, ucontext
? &ucontext
->uctx
:
2751 &rhp
->rdev
.uctx
, srq
->wr_waitp
);
2754 attrs
->attr
.max_wr
= rqsize
- 1;
2756 if (CHELSIO_CHIP_VERSION(rhp
->rdev
.lldi
.adapter_type
) > CHELSIO_T6
)
2757 srq
->flags
= T4_SRQ_LIMIT_SUPPORT
;
2759 ret
= insert_handle(rhp
, &rhp
->qpidr
, srq
, srq
->wq
.qid
);
2761 goto err_free_queue
;
2764 srq_key_mm
= kmalloc(sizeof(*srq_key_mm
), GFP_KERNEL
);
2767 goto err_remove_handle
;
2769 srq_db_key_mm
= kmalloc(sizeof(*srq_db_key_mm
), GFP_KERNEL
);
2770 if (!srq_db_key_mm
) {
2772 goto err_free_srq_key_mm
;
2774 memset(&uresp
, 0, sizeof(uresp
));
2775 uresp
.flags
= srq
->flags
;
2776 uresp
.qid_mask
= rhp
->rdev
.qpmask
;
2777 uresp
.srqid
= srq
->wq
.qid
;
2778 uresp
.srq_size
= srq
->wq
.size
;
2779 uresp
.srq_memsize
= srq
->wq
.memsize
;
2780 uresp
.rqt_abs_idx
= srq
->wq
.rqt_abs_idx
;
2781 spin_lock(&ucontext
->mmap_lock
);
2782 uresp
.srq_key
= ucontext
->key
;
2783 ucontext
->key
+= PAGE_SIZE
;
2784 uresp
.srq_db_gts_key
= ucontext
->key
;
2785 ucontext
->key
+= PAGE_SIZE
;
2786 spin_unlock(&ucontext
->mmap_lock
);
2787 ret
= ib_copy_to_udata(udata
, &uresp
, sizeof(uresp
));
2789 goto err_free_srq_db_key_mm
;
2790 srq_key_mm
->key
= uresp
.srq_key
;
2791 srq_key_mm
->addr
= virt_to_phys(srq
->wq
.queue
);
2792 srq_key_mm
->len
= PAGE_ALIGN(srq
->wq
.memsize
);
2793 insert_mmap(ucontext
, srq_key_mm
);
2794 srq_db_key_mm
->key
= uresp
.srq_db_gts_key
;
2795 srq_db_key_mm
->addr
= (u64
)(unsigned long)srq
->wq
.bar2_pa
;
2796 srq_db_key_mm
->len
= PAGE_SIZE
;
2797 insert_mmap(ucontext
, srq_db_key_mm
);
2800 pr_debug("%s srq qid %u idx %u size %u memsize %lu num_entries %u\n",
2801 __func__
, srq
->wq
.qid
, srq
->idx
, srq
->wq
.size
,
2802 (unsigned long)srq
->wq
.memsize
, attrs
->attr
.max_wr
);
2804 spin_lock_init(&srq
->lock
);
2806 err_free_srq_db_key_mm
:
2807 kfree(srq_db_key_mm
);
2808 err_free_srq_key_mm
:
2811 remove_handle(rhp
, &rhp
->qpidr
, srq
->wq
.qid
);
2813 free_srq_queue(srq
, ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
,
2816 kfree_skb(srq
->destroy_skb
);
2818 c4iw_free_srq_idx(&rhp
->rdev
, srq
->idx
);
2820 c4iw_put_wr_wait(srq
->wr_waitp
);
2823 return ERR_PTR(ret
);
2826 int c4iw_destroy_srq(struct ib_srq
*ibsrq
)
2828 struct c4iw_dev
*rhp
;
2829 struct c4iw_srq
*srq
;
2830 struct c4iw_ucontext
*ucontext
;
2832 srq
= to_c4iw_srq(ibsrq
);
2835 pr_debug("%s id %d\n", __func__
, srq
->wq
.qid
);
2837 remove_handle(rhp
, &rhp
->qpidr
, srq
->wq
.qid
);
2838 ucontext
= ibsrq
->uobject
?
2839 to_c4iw_ucontext(ibsrq
->uobject
->context
) : NULL
;
2840 free_srq_queue(srq
, ucontext
? &ucontext
->uctx
: &rhp
->rdev
.uctx
,
2842 c4iw_free_srq_idx(&rhp
->rdev
, srq
->idx
);
2843 c4iw_put_wr_wait(srq
->wr_waitp
);