2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved.
3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <rdma/ib_mad.h>
36 #include <rdma/ib_user_verbs.h>
38 #include <linux/module.h>
39 #include <linux/utsname.h>
40 #include <linux/rculist.h>
42 #include <linux/random.h>
43 #include <linux/vmalloc.h>
46 #include "qib_common.h"
48 static unsigned int ib_qib_qp_table_size
= 256;
49 module_param_named(qp_table_size
, ib_qib_qp_table_size
, uint
, S_IRUGO
);
50 MODULE_PARM_DESC(qp_table_size
, "QP table size");
52 unsigned int ib_qib_lkey_table_size
= 16;
53 module_param_named(lkey_table_size
, ib_qib_lkey_table_size
, uint
,
55 MODULE_PARM_DESC(lkey_table_size
,
56 "LKEY table size in bits (2^n, 1 <= n <= 23)");
58 static unsigned int ib_qib_max_pds
= 0xFFFF;
59 module_param_named(max_pds
, ib_qib_max_pds
, uint
, S_IRUGO
);
60 MODULE_PARM_DESC(max_pds
,
61 "Maximum number of protection domains to support");
63 static unsigned int ib_qib_max_ahs
= 0xFFFF;
64 module_param_named(max_ahs
, ib_qib_max_ahs
, uint
, S_IRUGO
);
65 MODULE_PARM_DESC(max_ahs
, "Maximum number of address handles to support");
67 unsigned int ib_qib_max_cqes
= 0x2FFFF;
68 module_param_named(max_cqes
, ib_qib_max_cqes
, uint
, S_IRUGO
);
69 MODULE_PARM_DESC(max_cqes
,
70 "Maximum number of completion queue entries to support");
72 unsigned int ib_qib_max_cqs
= 0x1FFFF;
73 module_param_named(max_cqs
, ib_qib_max_cqs
, uint
, S_IRUGO
);
74 MODULE_PARM_DESC(max_cqs
, "Maximum number of completion queues to support");
76 unsigned int ib_qib_max_qp_wrs
= 0x3FFF;
77 module_param_named(max_qp_wrs
, ib_qib_max_qp_wrs
, uint
, S_IRUGO
);
78 MODULE_PARM_DESC(max_qp_wrs
, "Maximum number of QP WRs to support");
80 unsigned int ib_qib_max_qps
= 16384;
81 module_param_named(max_qps
, ib_qib_max_qps
, uint
, S_IRUGO
);
82 MODULE_PARM_DESC(max_qps
, "Maximum number of QPs to support");
84 unsigned int ib_qib_max_sges
= 0x60;
85 module_param_named(max_sges
, ib_qib_max_sges
, uint
, S_IRUGO
);
86 MODULE_PARM_DESC(max_sges
, "Maximum number of SGEs to support");
88 unsigned int ib_qib_max_mcast_grps
= 16384;
89 module_param_named(max_mcast_grps
, ib_qib_max_mcast_grps
, uint
, S_IRUGO
);
90 MODULE_PARM_DESC(max_mcast_grps
,
91 "Maximum number of multicast groups to support");
93 unsigned int ib_qib_max_mcast_qp_attached
= 16;
94 module_param_named(max_mcast_qp_attached
, ib_qib_max_mcast_qp_attached
,
96 MODULE_PARM_DESC(max_mcast_qp_attached
,
97 "Maximum number of attached QPs to support");
99 unsigned int ib_qib_max_srqs
= 1024;
100 module_param_named(max_srqs
, ib_qib_max_srqs
, uint
, S_IRUGO
);
101 MODULE_PARM_DESC(max_srqs
, "Maximum number of SRQs to support");
103 unsigned int ib_qib_max_srq_sges
= 128;
104 module_param_named(max_srq_sges
, ib_qib_max_srq_sges
, uint
, S_IRUGO
);
105 MODULE_PARM_DESC(max_srq_sges
, "Maximum number of SRQ SGEs to support");
107 unsigned int ib_qib_max_srq_wrs
= 0x1FFFF;
108 module_param_named(max_srq_wrs
, ib_qib_max_srq_wrs
, uint
, S_IRUGO
);
109 MODULE_PARM_DESC(max_srq_wrs
, "Maximum number of SRQ WRs support");
111 static unsigned int ib_qib_disable_sma
;
112 module_param_named(disable_sma
, ib_qib_disable_sma
, uint
, S_IWUSR
| S_IRUGO
);
113 MODULE_PARM_DESC(disable_sma
, "Disable the SMA");
116 * Note that it is OK to post send work requests in the SQE and ERR
117 * states; qib_do_send() will process them and generate error
118 * completions as per IB 1.2 C10-96.
120 const int ib_qib_state_ops
[IB_QPS_ERR
+ 1] = {
122 [IB_QPS_INIT
] = QIB_POST_RECV_OK
,
123 [IB_QPS_RTR
] = QIB_POST_RECV_OK
| QIB_PROCESS_RECV_OK
,
124 [IB_QPS_RTS
] = QIB_POST_RECV_OK
| QIB_PROCESS_RECV_OK
|
125 QIB_POST_SEND_OK
| QIB_PROCESS_SEND_OK
|
126 QIB_PROCESS_NEXT_SEND_OK
,
127 [IB_QPS_SQD
] = QIB_POST_RECV_OK
| QIB_PROCESS_RECV_OK
|
128 QIB_POST_SEND_OK
| QIB_PROCESS_SEND_OK
,
129 [IB_QPS_SQE
] = QIB_POST_RECV_OK
| QIB_PROCESS_RECV_OK
|
130 QIB_POST_SEND_OK
| QIB_FLUSH_SEND
,
131 [IB_QPS_ERR
] = QIB_POST_RECV_OK
| QIB_FLUSH_RECV
|
132 QIB_POST_SEND_OK
| QIB_FLUSH_SEND
,
135 struct qib_ucontext
{
136 struct ib_ucontext ibucontext
;
139 static inline struct qib_ucontext
*to_iucontext(struct ib_ucontext
142 return container_of(ibucontext
, struct qib_ucontext
, ibucontext
);
146 * Translate ib_wr_opcode into ib_wc_opcode.
148 const enum ib_wc_opcode ib_qib_wc_opcode
[] = {
149 [IB_WR_RDMA_WRITE
] = IB_WC_RDMA_WRITE
,
150 [IB_WR_RDMA_WRITE_WITH_IMM
] = IB_WC_RDMA_WRITE
,
151 [IB_WR_SEND
] = IB_WC_SEND
,
152 [IB_WR_SEND_WITH_IMM
] = IB_WC_SEND
,
153 [IB_WR_RDMA_READ
] = IB_WC_RDMA_READ
,
154 [IB_WR_ATOMIC_CMP_AND_SWP
] = IB_WC_COMP_SWAP
,
155 [IB_WR_ATOMIC_FETCH_AND_ADD
] = IB_WC_FETCH_ADD
161 __be64 ib_qib_sys_image_guid
;
164 * qib_copy_sge - copy data to SGE memory
166 * @data: the data to copy
167 * @length: the length of the data
169 void qib_copy_sge(struct qib_sge_state
*ss
, void *data
, u32 length
, int release
)
171 struct qib_sge
*sge
= &ss
->sge
;
174 u32 len
= sge
->length
;
178 if (len
> sge
->sge_length
)
179 len
= sge
->sge_length
;
181 memcpy(sge
->vaddr
, data
, len
);
184 sge
->sge_length
-= len
;
185 if (sge
->sge_length
== 0) {
189 *sge
= *ss
->sg_list
++;
190 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
191 if (++sge
->n
>= QIB_SEGSZ
) {
192 if (++sge
->m
>= sge
->mr
->mapsz
)
197 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
199 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
207 * qib_skip_sge - skip over SGE memory - XXX almost dup of prev func
209 * @length: the number of bytes to skip
211 void qib_skip_sge(struct qib_sge_state
*ss
, u32 length
, int release
)
213 struct qib_sge
*sge
= &ss
->sge
;
216 u32 len
= sge
->length
;
220 if (len
> sge
->sge_length
)
221 len
= sge
->sge_length
;
225 sge
->sge_length
-= len
;
226 if (sge
->sge_length
== 0) {
230 *sge
= *ss
->sg_list
++;
231 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
232 if (++sge
->n
>= QIB_SEGSZ
) {
233 if (++sge
->m
>= sge
->mr
->mapsz
)
238 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
240 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
247 * Count the number of DMA descriptors needed to send length bytes of data.
248 * Don't modify the qib_sge_state to get the count.
249 * Return zero if any of the segments is not aligned.
251 static u32
qib_count_sge(struct qib_sge_state
*ss
, u32 length
)
253 struct qib_sge
*sg_list
= ss
->sg_list
;
254 struct qib_sge sge
= ss
->sge
;
255 u8 num_sge
= ss
->num_sge
;
256 u32 ndesc
= 1; /* count the header */
259 u32 len
= sge
.length
;
263 if (len
> sge
.sge_length
)
264 len
= sge
.sge_length
;
266 if (((long) sge
.vaddr
& (sizeof(u32
) - 1)) ||
267 (len
!= length
&& (len
& (sizeof(u32
) - 1)))) {
274 sge
.sge_length
-= len
;
275 if (sge
.sge_length
== 0) {
278 } else if (sge
.length
== 0 && sge
.mr
->lkey
) {
279 if (++sge
.n
>= QIB_SEGSZ
) {
280 if (++sge
.m
>= sge
.mr
->mapsz
)
285 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].vaddr
;
287 sge
.mr
->map
[sge
.m
]->segs
[sge
.n
].length
;
295 * Copy from the SGEs to the data buffer.
297 static void qib_copy_from_sge(void *data
, struct qib_sge_state
*ss
, u32 length
)
299 struct qib_sge
*sge
= &ss
->sge
;
302 u32 len
= sge
->length
;
306 if (len
> sge
->sge_length
)
307 len
= sge
->sge_length
;
309 memcpy(data
, sge
->vaddr
, len
);
312 sge
->sge_length
-= len
;
313 if (sge
->sge_length
== 0) {
315 *sge
= *ss
->sg_list
++;
316 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
317 if (++sge
->n
>= QIB_SEGSZ
) {
318 if (++sge
->m
>= sge
->mr
->mapsz
)
323 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
325 sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
333 * qib_post_one_send - post one RC, UC, or UD send work request
334 * @qp: the QP to post on
335 * @wr: the work request to send
337 static int qib_post_one_send(struct qib_qp
*qp
, struct ib_send_wr
*wr
,
340 struct qib_swqe
*wqe
;
347 struct qib_lkey_table
*rkt
;
350 spin_lock_irqsave(&qp
->s_lock
, flags
);
352 /* Check that state is OK to post send. */
353 if (unlikely(!(ib_qib_state_ops
[qp
->state
] & QIB_POST_SEND_OK
)))
356 /* IB spec says that num_sge == 0 is OK. */
357 if (wr
->num_sge
> qp
->s_max_sge
)
361 * Don't allow RDMA reads or atomic operations on UC or
362 * undefined operations.
363 * Make sure buffer is large enough to hold the result for atomics.
365 if (wr
->opcode
== IB_WR_FAST_REG_MR
) {
366 if (qib_fast_reg_mr(qp
, wr
))
368 } else if (qp
->ibqp
.qp_type
== IB_QPT_UC
) {
369 if ((unsigned) wr
->opcode
>= IB_WR_RDMA_READ
)
371 } else if (qp
->ibqp
.qp_type
!= IB_QPT_RC
) {
372 /* Check IB_QPT_SMI, IB_QPT_GSI, IB_QPT_UD opcode */
373 if (wr
->opcode
!= IB_WR_SEND
&&
374 wr
->opcode
!= IB_WR_SEND_WITH_IMM
)
376 /* Check UD destination address PD */
377 if (qp
->ibqp
.pd
!= wr
->wr
.ud
.ah
->pd
)
379 } else if ((unsigned) wr
->opcode
> IB_WR_ATOMIC_FETCH_AND_ADD
)
381 else if (wr
->opcode
>= IB_WR_ATOMIC_CMP_AND_SWP
&&
383 wr
->sg_list
[0].length
< sizeof(u64
) ||
384 wr
->sg_list
[0].addr
& (sizeof(u64
) - 1)))
386 else if (wr
->opcode
>= IB_WR_RDMA_READ
&& !qp
->s_max_rd_atomic
)
389 next
= qp
->s_head
+ 1;
390 if (next
>= qp
->s_size
)
392 if (next
== qp
->s_last
) {
397 rkt
= &to_idev(qp
->ibqp
.device
)->lk_table
;
398 pd
= to_ipd(qp
->ibqp
.pd
);
399 wqe
= get_swqe_ptr(qp
, qp
->s_head
);
404 acc
= wr
->opcode
>= IB_WR_RDMA_READ
?
405 IB_ACCESS_LOCAL_WRITE
: 0;
406 for (i
= 0; i
< wr
->num_sge
; i
++) {
407 u32 length
= wr
->sg_list
[i
].length
;
412 ok
= qib_lkey_ok(rkt
, pd
, &wqe
->sg_list
[j
],
413 &wr
->sg_list
[i
], acc
);
415 goto bail_inval_free
;
416 wqe
->length
+= length
;
421 if (qp
->ibqp
.qp_type
== IB_QPT_UC
||
422 qp
->ibqp
.qp_type
== IB_QPT_RC
) {
423 if (wqe
->length
> 0x80000000U
)
424 goto bail_inval_free
;
425 } else if (wqe
->length
> (dd_from_ibdev(qp
->ibqp
.device
)->pport
+
426 qp
->port_num
- 1)->ibmtu
)
427 goto bail_inval_free
;
429 atomic_inc(&to_iah(wr
->wr
.ud
.ah
)->refcount
);
430 wqe
->ssn
= qp
->s_ssn
++;
438 struct qib_sge
*sge
= &wqe
->sg_list
[--j
];
445 if (!ret
&& !wr
->next
&&
447 dd_from_ibdev(qp
->ibqp
.device
)->pport
+ qp
->port_num
- 1)) {
448 qib_schedule_send(qp
);
451 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
456 * qib_post_send - post a send on a QP
457 * @ibqp: the QP to post the send on
458 * @wr: the list of work requests to post
459 * @bad_wr: the first bad WR is put here
461 * This may be called from interrupt context.
463 static int qib_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
464 struct ib_send_wr
**bad_wr
)
466 struct qib_qp
*qp
= to_iqp(ibqp
);
470 for (; wr
; wr
= wr
->next
) {
471 err
= qib_post_one_send(qp
, wr
, &scheduled
);
478 /* Try to do the send work in the caller's context. */
480 qib_do_send(&qp
->s_work
);
487 * qib_post_receive - post a receive on a QP
488 * @ibqp: the QP to post the receive on
489 * @wr: the WR to post
490 * @bad_wr: the first bad WR is put here
492 * This may be called from interrupt context.
494 static int qib_post_receive(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
495 struct ib_recv_wr
**bad_wr
)
497 struct qib_qp
*qp
= to_iqp(ibqp
);
498 struct qib_rwq
*wq
= qp
->r_rq
.wq
;
502 /* Check that state is OK to post receive. */
503 if (!(ib_qib_state_ops
[qp
->state
] & QIB_POST_RECV_OK
) || !wq
) {
509 for (; wr
; wr
= wr
->next
) {
510 struct qib_rwqe
*wqe
;
514 if ((unsigned) wr
->num_sge
> qp
->r_rq
.max_sge
) {
520 spin_lock_irqsave(&qp
->r_rq
.lock
, flags
);
522 if (next
>= qp
->r_rq
.size
)
524 if (next
== wq
->tail
) {
525 spin_unlock_irqrestore(&qp
->r_rq
.lock
, flags
);
531 wqe
= get_rwqe_ptr(&qp
->r_rq
, wq
->head
);
532 wqe
->wr_id
= wr
->wr_id
;
533 wqe
->num_sge
= wr
->num_sge
;
534 for (i
= 0; i
< wr
->num_sge
; i
++)
535 wqe
->sg_list
[i
] = wr
->sg_list
[i
];
536 /* Make sure queue entry is written before the head index. */
539 spin_unlock_irqrestore(&qp
->r_rq
.lock
, flags
);
548 * qib_qp_rcv - processing an incoming packet on a QP
549 * @rcd: the context pointer
550 * @hdr: the packet header
551 * @has_grh: true if the packet has a GRH
552 * @data: the packet data
553 * @tlen: the packet length
554 * @qp: the QP the packet came on
556 * This is called from qib_ib_rcv() to process an incoming packet
558 * Called at interrupt level.
560 static void qib_qp_rcv(struct qib_ctxtdata
*rcd
, struct qib_ib_header
*hdr
,
561 int has_grh
, void *data
, u32 tlen
, struct qib_qp
*qp
)
563 struct qib_ibport
*ibp
= &rcd
->ppd
->ibport_data
;
565 spin_lock(&qp
->r_lock
);
567 /* Check for valid receive state. */
568 if (!(ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_RECV_OK
)) {
573 switch (qp
->ibqp
.qp_type
) {
576 if (ib_qib_disable_sma
)
580 qib_ud_rcv(ibp
, hdr
, has_grh
, data
, tlen
, qp
);
584 qib_rc_rcv(rcd
, hdr
, has_grh
, data
, tlen
, qp
);
588 qib_uc_rcv(ibp
, hdr
, has_grh
, data
, tlen
, qp
);
596 spin_unlock(&qp
->r_lock
);
600 * qib_ib_rcv - process an incoming packet
601 * @rcd: the context pointer
602 * @rhdr: the header of the packet
603 * @data: the packet payload
604 * @tlen: the packet length
606 * This is called from qib_kreceive() to process an incoming packet at
607 * interrupt level. Tlen is the length of the header + data + CRC in bytes.
609 void qib_ib_rcv(struct qib_ctxtdata
*rcd
, void *rhdr
, void *data
, u32 tlen
)
611 struct qib_pportdata
*ppd
= rcd
->ppd
;
612 struct qib_ibport
*ibp
= &ppd
->ibport_data
;
613 struct qib_ib_header
*hdr
= rhdr
;
614 struct qib_other_headers
*ohdr
;
621 /* 24 == LRH+BTH+CRC */
622 if (unlikely(tlen
< 24))
625 /* Check for a valid destination LID (see ch. 7.11.1). */
626 lid
= be16_to_cpu(hdr
->lrh
[1]);
627 if (lid
< QIB_MULTICAST_LID_BASE
) {
628 lid
&= ~((1 << ppd
->lmc
) - 1);
629 if (unlikely(lid
!= ppd
->lid
))
634 lnh
= be16_to_cpu(hdr
->lrh
[0]) & 3;
635 if (lnh
== QIB_LRH_BTH
)
637 else if (lnh
== QIB_LRH_GRH
) {
640 ohdr
= &hdr
->u
.l
.oth
;
641 if (hdr
->u
.l
.grh
.next_hdr
!= IB_GRH_NEXT_HDR
)
643 vtf
= be32_to_cpu(hdr
->u
.l
.grh
.version_tclass_flow
);
644 if ((vtf
>> IB_GRH_VERSION_SHIFT
) != IB_GRH_VERSION
)
649 opcode
= (be32_to_cpu(ohdr
->bth
[0]) >> 24) & 0x7f;
650 #ifdef CONFIG_DEBUG_FS
651 rcd
->opstats
->stats
[opcode
].n_bytes
+= tlen
;
652 rcd
->opstats
->stats
[opcode
].n_packets
++;
655 /* Get the destination QP number. */
656 qp_num
= be32_to_cpu(ohdr
->bth
[1]) & QIB_QPN_MASK
;
657 if (qp_num
== QIB_MULTICAST_QPN
) {
658 struct qib_mcast
*mcast
;
659 struct qib_mcast_qp
*p
;
661 if (lnh
!= QIB_LRH_GRH
)
663 mcast
= qib_mcast_find(ibp
, &hdr
->u
.l
.grh
.dgid
);
666 this_cpu_inc(ibp
->pmastats
->n_multicast_rcv
);
667 list_for_each_entry_rcu(p
, &mcast
->qp_list
, list
)
668 qib_qp_rcv(rcd
, hdr
, 1, data
, tlen
, p
->qp
);
670 * Notify qib_multicast_detach() if it is waiting for us
673 if (atomic_dec_return(&mcast
->refcount
) <= 1)
674 wake_up(&mcast
->wait
);
676 if (rcd
->lookaside_qp
) {
677 if (rcd
->lookaside_qpn
!= qp_num
) {
678 if (atomic_dec_and_test(
679 &rcd
->lookaside_qp
->refcount
))
681 &rcd
->lookaside_qp
->wait
);
682 rcd
->lookaside_qp
= NULL
;
685 if (!rcd
->lookaside_qp
) {
686 qp
= qib_lookup_qpn(ibp
, qp_num
);
689 rcd
->lookaside_qp
= qp
;
690 rcd
->lookaside_qpn
= qp_num
;
692 qp
= rcd
->lookaside_qp
;
693 this_cpu_inc(ibp
->pmastats
->n_unicast_rcv
);
694 qib_qp_rcv(rcd
, hdr
, lnh
== QIB_LRH_GRH
, data
, tlen
, qp
);
703 * This is called from a timer to check for QPs
704 * which need kernel memory in order to send a packet.
706 static void mem_timer(unsigned long data
)
708 struct qib_ibdev
*dev
= (struct qib_ibdev
*) data
;
709 struct list_head
*list
= &dev
->memwait
;
710 struct qib_qp
*qp
= NULL
;
713 spin_lock_irqsave(&dev
->pending_lock
, flags
);
714 if (!list_empty(list
)) {
715 qp
= list_entry(list
->next
, struct qib_qp
, iowait
);
716 list_del_init(&qp
->iowait
);
717 atomic_inc(&qp
->refcount
);
718 if (!list_empty(list
))
719 mod_timer(&dev
->mem_timer
, jiffies
+ 1);
721 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
724 spin_lock_irqsave(&qp
->s_lock
, flags
);
725 if (qp
->s_flags
& QIB_S_WAIT_KMEM
) {
726 qp
->s_flags
&= ~QIB_S_WAIT_KMEM
;
727 qib_schedule_send(qp
);
729 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
730 if (atomic_dec_and_test(&qp
->refcount
))
735 static void update_sge(struct qib_sge_state
*ss
, u32 length
)
737 struct qib_sge
*sge
= &ss
->sge
;
739 sge
->vaddr
+= length
;
740 sge
->length
-= length
;
741 sge
->sge_length
-= length
;
742 if (sge
->sge_length
== 0) {
744 *sge
= *ss
->sg_list
++;
745 } else if (sge
->length
== 0 && sge
->mr
->lkey
) {
746 if (++sge
->n
>= QIB_SEGSZ
) {
747 if (++sge
->m
>= sge
->mr
->mapsz
)
751 sge
->vaddr
= sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].vaddr
;
752 sge
->length
= sge
->mr
->map
[sge
->m
]->segs
[sge
->n
].length
;
756 #ifdef __LITTLE_ENDIAN
757 static inline u32
get_upper_bits(u32 data
, u32 shift
)
759 return data
>> shift
;
762 static inline u32
set_upper_bits(u32 data
, u32 shift
)
764 return data
<< shift
;
767 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
769 data
<<= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
770 data
>>= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
774 static inline u32
get_upper_bits(u32 data
, u32 shift
)
776 return data
<< shift
;
779 static inline u32
set_upper_bits(u32 data
, u32 shift
)
781 return data
>> shift
;
784 static inline u32
clear_upper_bytes(u32 data
, u32 n
, u32 off
)
786 data
>>= ((sizeof(u32
) - n
) * BITS_PER_BYTE
);
787 data
<<= ((sizeof(u32
) - n
- off
) * BITS_PER_BYTE
);
792 static void copy_io(u32 __iomem
*piobuf
, struct qib_sge_state
*ss
,
793 u32 length
, unsigned flush_wc
)
800 u32 len
= ss
->sge
.length
;
805 if (len
> ss
->sge
.sge_length
)
806 len
= ss
->sge
.sge_length
;
808 /* If the source address is not aligned, try to align it. */
809 off
= (unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1);
811 u32
*addr
= (u32
*)((unsigned long)ss
->sge
.vaddr
&
813 u32 v
= get_upper_bits(*addr
, off
* BITS_PER_BYTE
);
816 y
= sizeof(u32
) - off
;
819 if (len
+ extra
>= sizeof(u32
)) {
820 data
|= set_upper_bits(v
, extra
*
822 len
= sizeof(u32
) - extra
;
827 __raw_writel(data
, piobuf
);
832 /* Clear unused upper bytes */
833 data
|= clear_upper_bytes(v
, len
, extra
);
841 /* Source address is aligned. */
842 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
843 int shift
= extra
* BITS_PER_BYTE
;
844 int ushift
= 32 - shift
;
847 while (l
>= sizeof(u32
)) {
850 data
|= set_upper_bits(v
, shift
);
851 __raw_writel(data
, piobuf
);
852 data
= get_upper_bits(v
, ushift
);
858 * We still have 'extra' number of bytes leftover.
863 if (l
+ extra
>= sizeof(u32
)) {
864 data
|= set_upper_bits(v
, shift
);
865 len
-= l
+ extra
- sizeof(u32
);
870 __raw_writel(data
, piobuf
);
875 /* Clear unused upper bytes */
876 data
|= clear_upper_bytes(v
, l
, extra
);
883 } else if (len
== length
) {
887 } else if (len
== length
) {
891 * Need to round up for the last dword in the
895 qib_pio_copy(piobuf
, ss
->sge
.vaddr
, w
- 1);
897 last
= ((u32
*) ss
->sge
.vaddr
)[w
- 1];
902 qib_pio_copy(piobuf
, ss
->sge
.vaddr
, w
);
905 extra
= len
& (sizeof(u32
) - 1);
907 u32 v
= ((u32
*) ss
->sge
.vaddr
)[w
];
909 /* Clear unused upper bytes */
910 data
= clear_upper_bytes(v
, extra
, 0);
916 /* Update address before sending packet. */
917 update_sge(ss
, length
);
919 /* must flush early everything before trigger word */
921 __raw_writel(last
, piobuf
);
922 /* be sure trigger word is written */
925 __raw_writel(last
, piobuf
);
928 static noinline
struct qib_verbs_txreq
*__get_txreq(struct qib_ibdev
*dev
,
931 struct qib_verbs_txreq
*tx
;
934 spin_lock_irqsave(&qp
->s_lock
, flags
);
935 spin_lock(&dev
->pending_lock
);
937 if (!list_empty(&dev
->txreq_free
)) {
938 struct list_head
*l
= dev
->txreq_free
.next
;
941 spin_unlock(&dev
->pending_lock
);
942 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
943 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
945 if (ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_RECV_OK
&&
946 list_empty(&qp
->iowait
)) {
948 qp
->s_flags
|= QIB_S_WAIT_TX
;
949 list_add_tail(&qp
->iowait
, &dev
->txwait
);
951 qp
->s_flags
&= ~QIB_S_BUSY
;
952 spin_unlock(&dev
->pending_lock
);
953 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
954 tx
= ERR_PTR(-EBUSY
);
959 static inline struct qib_verbs_txreq
*get_txreq(struct qib_ibdev
*dev
,
962 struct qib_verbs_txreq
*tx
;
965 spin_lock_irqsave(&dev
->pending_lock
, flags
);
966 /* assume the list non empty */
967 if (likely(!list_empty(&dev
->txreq_free
))) {
968 struct list_head
*l
= dev
->txreq_free
.next
;
971 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
972 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
974 /* call slow path to get the extra lock */
975 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
976 tx
= __get_txreq(dev
, qp
);
981 void qib_put_txreq(struct qib_verbs_txreq
*tx
)
983 struct qib_ibdev
*dev
;
988 dev
= to_idev(qp
->ibqp
.device
);
990 if (atomic_dec_and_test(&qp
->refcount
))
996 if (tx
->txreq
.flags
& QIB_SDMA_TXREQ_F_FREEBUF
) {
997 tx
->txreq
.flags
&= ~QIB_SDMA_TXREQ_F_FREEBUF
;
998 dma_unmap_single(&dd_from_dev(dev
)->pcidev
->dev
,
999 tx
->txreq
.addr
, tx
->hdr_dwords
<< 2,
1001 kfree(tx
->align_buf
);
1004 spin_lock_irqsave(&dev
->pending_lock
, flags
);
1006 /* Put struct back on free list */
1007 list_add(&tx
->txreq
.list
, &dev
->txreq_free
);
1009 if (!list_empty(&dev
->txwait
)) {
1010 /* Wake up first QP wanting a free struct */
1011 qp
= list_entry(dev
->txwait
.next
, struct qib_qp
, iowait
);
1012 list_del_init(&qp
->iowait
);
1013 atomic_inc(&qp
->refcount
);
1014 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1016 spin_lock_irqsave(&qp
->s_lock
, flags
);
1017 if (qp
->s_flags
& QIB_S_WAIT_TX
) {
1018 qp
->s_flags
&= ~QIB_S_WAIT_TX
;
1019 qib_schedule_send(qp
);
1021 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1023 if (atomic_dec_and_test(&qp
->refcount
))
1026 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1030 * This is called when there are send DMA descriptors that might be
1033 * This is called with ppd->sdma_lock held.
1035 void qib_verbs_sdma_desc_avail(struct qib_pportdata
*ppd
, unsigned avail
)
1037 struct qib_qp
*qp
, *nqp
;
1038 struct qib_qp
*qps
[20];
1039 struct qib_ibdev
*dev
;
1043 dev
= &ppd
->dd
->verbs_dev
;
1044 spin_lock(&dev
->pending_lock
);
1046 /* Search wait list for first QP wanting DMA descriptors. */
1047 list_for_each_entry_safe(qp
, nqp
, &dev
->dmawait
, iowait
) {
1048 if (qp
->port_num
!= ppd
->port
)
1050 if (n
== ARRAY_SIZE(qps
))
1052 if (qp
->s_tx
->txreq
.sg_count
> avail
)
1054 avail
-= qp
->s_tx
->txreq
.sg_count
;
1055 list_del_init(&qp
->iowait
);
1056 atomic_inc(&qp
->refcount
);
1060 spin_unlock(&dev
->pending_lock
);
1062 for (i
= 0; i
< n
; i
++) {
1064 spin_lock(&qp
->s_lock
);
1065 if (qp
->s_flags
& QIB_S_WAIT_DMA_DESC
) {
1066 qp
->s_flags
&= ~QIB_S_WAIT_DMA_DESC
;
1067 qib_schedule_send(qp
);
1069 spin_unlock(&qp
->s_lock
);
1070 if (atomic_dec_and_test(&qp
->refcount
))
1076 * This is called with ppd->sdma_lock held.
1078 static void sdma_complete(struct qib_sdma_txreq
*cookie
, int status
)
1080 struct qib_verbs_txreq
*tx
=
1081 container_of(cookie
, struct qib_verbs_txreq
, txreq
);
1082 struct qib_qp
*qp
= tx
->qp
;
1084 spin_lock(&qp
->s_lock
);
1086 qib_send_complete(qp
, tx
->wqe
, IB_WC_SUCCESS
);
1087 else if (qp
->ibqp
.qp_type
== IB_QPT_RC
) {
1088 struct qib_ib_header
*hdr
;
1090 if (tx
->txreq
.flags
& QIB_SDMA_TXREQ_F_FREEBUF
)
1091 hdr
= &tx
->align_buf
->hdr
;
1093 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
1095 hdr
= &dev
->pio_hdrs
[tx
->hdr_inx
].hdr
;
1097 qib_rc_send_complete(qp
, hdr
);
1099 if (atomic_dec_and_test(&qp
->s_dma_busy
)) {
1100 if (qp
->state
== IB_QPS_RESET
)
1101 wake_up(&qp
->wait_dma
);
1102 else if (qp
->s_flags
& QIB_S_WAIT_DMA
) {
1103 qp
->s_flags
&= ~QIB_S_WAIT_DMA
;
1104 qib_schedule_send(qp
);
1107 spin_unlock(&qp
->s_lock
);
1112 static int wait_kmem(struct qib_ibdev
*dev
, struct qib_qp
*qp
)
1114 unsigned long flags
;
1117 spin_lock_irqsave(&qp
->s_lock
, flags
);
1118 if (ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_RECV_OK
) {
1119 spin_lock(&dev
->pending_lock
);
1120 if (list_empty(&qp
->iowait
)) {
1121 if (list_empty(&dev
->memwait
))
1122 mod_timer(&dev
->mem_timer
, jiffies
+ 1);
1123 qp
->s_flags
|= QIB_S_WAIT_KMEM
;
1124 list_add_tail(&qp
->iowait
, &dev
->memwait
);
1126 spin_unlock(&dev
->pending_lock
);
1127 qp
->s_flags
&= ~QIB_S_BUSY
;
1130 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1135 static int qib_verbs_send_dma(struct qib_qp
*qp
, struct qib_ib_header
*hdr
,
1136 u32 hdrwords
, struct qib_sge_state
*ss
, u32 len
,
1137 u32 plen
, u32 dwords
)
1139 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
1140 struct qib_devdata
*dd
= dd_from_dev(dev
);
1141 struct qib_ibport
*ibp
= to_iport(qp
->ibqp
.device
, qp
->port_num
);
1142 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1143 struct qib_verbs_txreq
*tx
;
1144 struct qib_pio_header
*phdr
;
1152 /* resend previously constructed packet */
1153 ret
= qib_sdma_verbs_send(ppd
, tx
->ss
, tx
->dwords
, tx
);
1157 tx
= get_txreq(dev
, qp
);
1161 control
= dd
->f_setpbc_control(ppd
, plen
, qp
->s_srate
,
1162 be16_to_cpu(hdr
->lrh
[0]) >> 12);
1164 atomic_inc(&qp
->refcount
);
1165 tx
->wqe
= qp
->s_wqe
;
1166 tx
->mr
= qp
->s_rdma_mr
;
1168 qp
->s_rdma_mr
= NULL
;
1169 tx
->txreq
.callback
= sdma_complete
;
1170 if (dd
->flags
& QIB_HAS_SDMA_TIMEOUT
)
1171 tx
->txreq
.flags
= QIB_SDMA_TXREQ_F_HEADTOHOST
;
1173 tx
->txreq
.flags
= QIB_SDMA_TXREQ_F_INTREQ
;
1174 if (plen
+ 1 > dd
->piosize2kmax_dwords
)
1175 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_USELARGEBUF
;
1179 * Don't try to DMA if it takes more descriptors than
1182 ndesc
= qib_count_sge(ss
, len
);
1183 if (ndesc
>= ppd
->sdma_descq_cnt
)
1188 phdr
= &dev
->pio_hdrs
[tx
->hdr_inx
];
1189 phdr
->pbc
[0] = cpu_to_le32(plen
);
1190 phdr
->pbc
[1] = cpu_to_le32(control
);
1191 memcpy(&phdr
->hdr
, hdr
, hdrwords
<< 2);
1192 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_FREEDESC
;
1193 tx
->txreq
.sg_count
= ndesc
;
1194 tx
->txreq
.addr
= dev
->pio_hdrs_phys
+
1195 tx
->hdr_inx
* sizeof(struct qib_pio_header
);
1196 tx
->hdr_dwords
= hdrwords
+ 2; /* add PBC length */
1197 ret
= qib_sdma_verbs_send(ppd
, ss
, dwords
, tx
);
1201 /* Allocate a buffer and copy the header and payload to it. */
1202 tx
->hdr_dwords
= plen
+ 1;
1203 phdr
= kmalloc(tx
->hdr_dwords
<< 2, GFP_ATOMIC
);
1206 phdr
->pbc
[0] = cpu_to_le32(plen
);
1207 phdr
->pbc
[1] = cpu_to_le32(control
);
1208 memcpy(&phdr
->hdr
, hdr
, hdrwords
<< 2);
1209 qib_copy_from_sge((u32
*) &phdr
->hdr
+ hdrwords
, ss
, len
);
1211 tx
->txreq
.addr
= dma_map_single(&dd
->pcidev
->dev
, phdr
,
1212 tx
->hdr_dwords
<< 2, DMA_TO_DEVICE
);
1213 if (dma_mapping_error(&dd
->pcidev
->dev
, tx
->txreq
.addr
))
1215 tx
->align_buf
= phdr
;
1216 tx
->txreq
.flags
|= QIB_SDMA_TXREQ_F_FREEBUF
;
1217 tx
->txreq
.sg_count
= 1;
1218 ret
= qib_sdma_verbs_send(ppd
, NULL
, 0, tx
);
1225 ret
= wait_kmem(dev
, qp
);
1236 * If we are now in the error state, return zero to flush the
1237 * send work request.
1239 static int no_bufs_available(struct qib_qp
*qp
)
1241 struct qib_ibdev
*dev
= to_idev(qp
->ibqp
.device
);
1242 struct qib_devdata
*dd
;
1243 unsigned long flags
;
1247 * Note that as soon as want_buffer() is called and
1248 * possibly before it returns, qib_ib_piobufavail()
1249 * could be called. Therefore, put QP on the I/O wait list before
1250 * enabling the PIO avail interrupt.
1252 spin_lock_irqsave(&qp
->s_lock
, flags
);
1253 if (ib_qib_state_ops
[qp
->state
] & QIB_PROCESS_RECV_OK
) {
1254 spin_lock(&dev
->pending_lock
);
1255 if (list_empty(&qp
->iowait
)) {
1257 qp
->s_flags
|= QIB_S_WAIT_PIO
;
1258 list_add_tail(&qp
->iowait
, &dev
->piowait
);
1259 dd
= dd_from_dev(dev
);
1260 dd
->f_wantpiobuf_intr(dd
, 1);
1262 spin_unlock(&dev
->pending_lock
);
1263 qp
->s_flags
&= ~QIB_S_BUSY
;
1266 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1270 static int qib_verbs_send_pio(struct qib_qp
*qp
, struct qib_ib_header
*ibhdr
,
1271 u32 hdrwords
, struct qib_sge_state
*ss
, u32 len
,
1272 u32 plen
, u32 dwords
)
1274 struct qib_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
1275 struct qib_pportdata
*ppd
= dd
->pport
+ qp
->port_num
- 1;
1276 u32
*hdr
= (u32
*) ibhdr
;
1277 u32 __iomem
*piobuf_orig
;
1278 u32 __iomem
*piobuf
;
1280 unsigned long flags
;
1285 control
= dd
->f_setpbc_control(ppd
, plen
, qp
->s_srate
,
1286 be16_to_cpu(ibhdr
->lrh
[0]) >> 12);
1287 pbc
= ((u64
) control
<< 32) | plen
;
1288 piobuf
= dd
->f_getsendbuf(ppd
, pbc
, &pbufn
);
1289 if (unlikely(piobuf
== NULL
))
1290 return no_bufs_available(qp
);
1294 * We have to flush after the PBC for correctness on some cpus
1295 * or WC buffer can be written out of order.
1297 writeq(pbc
, piobuf
);
1298 piobuf_orig
= piobuf
;
1301 flush_wc
= dd
->flags
& QIB_PIO_FLUSH_WC
;
1304 * If there is just the header portion, must flush before
1305 * writing last word of header for correctness, and after
1306 * the last header word (trigger word).
1310 qib_pio_copy(piobuf
, hdr
, hdrwords
- 1);
1312 __raw_writel(hdr
[hdrwords
- 1], piobuf
+ hdrwords
- 1);
1315 qib_pio_copy(piobuf
, hdr
, hdrwords
);
1321 qib_pio_copy(piobuf
, hdr
, hdrwords
);
1324 /* The common case is aligned and contained in one segment. */
1325 if (likely(ss
->num_sge
== 1 && len
<= ss
->sge
.length
&&
1326 !((unsigned long)ss
->sge
.vaddr
& (sizeof(u32
) - 1)))) {
1327 u32
*addr
= (u32
*) ss
->sge
.vaddr
;
1329 /* Update address before sending packet. */
1330 update_sge(ss
, len
);
1332 qib_pio_copy(piobuf
, addr
, dwords
- 1);
1333 /* must flush early everything before trigger word */
1335 __raw_writel(addr
[dwords
- 1], piobuf
+ dwords
- 1);
1336 /* be sure trigger word is written */
1339 qib_pio_copy(piobuf
, addr
, dwords
);
1342 copy_io(piobuf
, ss
, len
, flush_wc
);
1344 if (dd
->flags
& QIB_USE_SPCL_TRIG
) {
1345 u32 spcl_off
= (pbufn
>= dd
->piobcnt2k
) ? 2047 : 1023;
1348 __raw_writel(0xaebecede, piobuf_orig
+ spcl_off
);
1350 qib_sendbuf_done(dd
, pbufn
);
1351 if (qp
->s_rdma_mr
) {
1352 qib_put_mr(qp
->s_rdma_mr
);
1353 qp
->s_rdma_mr
= NULL
;
1356 spin_lock_irqsave(&qp
->s_lock
, flags
);
1357 qib_send_complete(qp
, qp
->s_wqe
, IB_WC_SUCCESS
);
1358 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1359 } else if (qp
->ibqp
.qp_type
== IB_QPT_RC
) {
1360 spin_lock_irqsave(&qp
->s_lock
, flags
);
1361 qib_rc_send_complete(qp
, ibhdr
);
1362 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1368 * qib_verbs_send - send a packet
1369 * @qp: the QP to send on
1370 * @hdr: the packet header
1371 * @hdrwords: the number of 32-bit words in the header
1372 * @ss: the SGE to send
1373 * @len: the length of the packet in bytes
1375 * Return zero if packet is sent or queued OK.
1376 * Return non-zero and clear qp->s_flags QIB_S_BUSY otherwise.
1378 int qib_verbs_send(struct qib_qp
*qp
, struct qib_ib_header
*hdr
,
1379 u32 hdrwords
, struct qib_sge_state
*ss
, u32 len
)
1381 struct qib_devdata
*dd
= dd_from_ibdev(qp
->ibqp
.device
);
1384 u32 dwords
= (len
+ 3) >> 2;
1387 * Calculate the send buffer trigger address.
1388 * The +1 counts for the pbc control dword following the pbc length.
1390 plen
= hdrwords
+ dwords
+ 1;
1393 * VL15 packets (IB_QPT_SMI) will always use PIO, so we
1394 * can defer SDMA restart until link goes ACTIVE without
1395 * worrying about just how we got there.
1397 if (qp
->ibqp
.qp_type
== IB_QPT_SMI
||
1398 !(dd
->flags
& QIB_HAS_SEND_DMA
))
1399 ret
= qib_verbs_send_pio(qp
, hdr
, hdrwords
, ss
, len
,
1402 ret
= qib_verbs_send_dma(qp
, hdr
, hdrwords
, ss
, len
,
1408 int qib_snapshot_counters(struct qib_pportdata
*ppd
, u64
*swords
,
1409 u64
*rwords
, u64
*spkts
, u64
*rpkts
,
1413 struct qib_devdata
*dd
= ppd
->dd
;
1415 if (!(dd
->flags
& QIB_PRESENT
)) {
1416 /* no hardware, freeze, etc. */
1420 *swords
= dd
->f_portcntr(ppd
, QIBPORTCNTR_WORDSEND
);
1421 *rwords
= dd
->f_portcntr(ppd
, QIBPORTCNTR_WORDRCV
);
1422 *spkts
= dd
->f_portcntr(ppd
, QIBPORTCNTR_PKTSEND
);
1423 *rpkts
= dd
->f_portcntr(ppd
, QIBPORTCNTR_PKTRCV
);
1424 *xmit_wait
= dd
->f_portcntr(ppd
, QIBPORTCNTR_SENDSTALL
);
1433 * qib_get_counters - get various chip counters
1434 * @dd: the qlogic_ib device
1435 * @cntrs: counters are placed here
1437 * Return the counters needed by recv_pma_get_portcounters().
1439 int qib_get_counters(struct qib_pportdata
*ppd
,
1440 struct qib_verbs_counters
*cntrs
)
1444 if (!(ppd
->dd
->flags
& QIB_PRESENT
)) {
1445 /* no hardware, freeze, etc. */
1449 cntrs
->symbol_error_counter
=
1450 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBSYMBOLERR
);
1451 cntrs
->link_error_recovery_counter
=
1452 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBLINKERRRECOV
);
1454 * The link downed counter counts when the other side downs the
1455 * connection. We add in the number of times we downed the link
1456 * due to local link integrity errors to compensate.
1458 cntrs
->link_downed_counter
=
1459 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_IBLINKDOWN
);
1460 cntrs
->port_rcv_errors
=
1461 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXDROPPKT
) +
1462 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RCVOVFL
) +
1463 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERR_RLEN
) +
1464 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_INVALIDRLEN
) +
1465 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRLINK
) +
1466 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRICRC
) +
1467 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRVCRC
) +
1468 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_ERRLPCRC
) +
1469 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_BADFORMAT
);
1470 cntrs
->port_rcv_errors
+=
1471 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXLOCALPHYERR
);
1472 cntrs
->port_rcv_errors
+=
1473 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RXVLERR
);
1474 cntrs
->port_rcv_remphys_errors
=
1475 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_RCVEBP
);
1476 cntrs
->port_xmit_discards
=
1477 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_UNSUPVL
);
1478 cntrs
->port_xmit_data
= ppd
->dd
->f_portcntr(ppd
,
1479 QIBPORTCNTR_WORDSEND
);
1480 cntrs
->port_rcv_data
= ppd
->dd
->f_portcntr(ppd
,
1481 QIBPORTCNTR_WORDRCV
);
1482 cntrs
->port_xmit_packets
= ppd
->dd
->f_portcntr(ppd
,
1483 QIBPORTCNTR_PKTSEND
);
1484 cntrs
->port_rcv_packets
= ppd
->dd
->f_portcntr(ppd
,
1485 QIBPORTCNTR_PKTRCV
);
1486 cntrs
->local_link_integrity_errors
=
1487 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_LLI
);
1488 cntrs
->excessive_buffer_overrun_errors
=
1489 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_EXCESSBUFOVFL
);
1490 cntrs
->vl15_dropped
=
1491 ppd
->dd
->f_portcntr(ppd
, QIBPORTCNTR_VL15PKTDROP
);
1500 * qib_ib_piobufavail - callback when a PIO buffer is available
1501 * @dd: the device pointer
1503 * This is called from qib_intr() at interrupt level when a PIO buffer is
1504 * available after qib_verbs_send() returned an error that no buffers were
1505 * available. Disable the interrupt if there are no more QPs waiting.
1507 void qib_ib_piobufavail(struct qib_devdata
*dd
)
1509 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
1510 struct list_head
*list
;
1511 struct qib_qp
*qps
[5];
1513 unsigned long flags
;
1516 list
= &dev
->piowait
;
1520 * Note: checking that the piowait list is empty and clearing
1521 * the buffer available interrupt needs to be atomic or we
1522 * could end up with QPs on the wait list with the interrupt
1525 spin_lock_irqsave(&dev
->pending_lock
, flags
);
1526 while (!list_empty(list
)) {
1527 if (n
== ARRAY_SIZE(qps
))
1529 qp
= list_entry(list
->next
, struct qib_qp
, iowait
);
1530 list_del_init(&qp
->iowait
);
1531 atomic_inc(&qp
->refcount
);
1534 dd
->f_wantpiobuf_intr(dd
, 0);
1536 spin_unlock_irqrestore(&dev
->pending_lock
, flags
);
1538 for (i
= 0; i
< n
; i
++) {
1541 spin_lock_irqsave(&qp
->s_lock
, flags
);
1542 if (qp
->s_flags
& QIB_S_WAIT_PIO
) {
1543 qp
->s_flags
&= ~QIB_S_WAIT_PIO
;
1544 qib_schedule_send(qp
);
1546 spin_unlock_irqrestore(&qp
->s_lock
, flags
);
1548 /* Notify qib_destroy_qp() if it is waiting. */
1549 if (atomic_dec_and_test(&qp
->refcount
))
1554 static int qib_query_device(struct ib_device
*ibdev
, struct ib_device_attr
*props
,
1555 struct ib_udata
*uhw
)
1557 struct qib_devdata
*dd
= dd_from_ibdev(ibdev
);
1558 struct qib_ibdev
*dev
= to_idev(ibdev
);
1560 if (uhw
->inlen
|| uhw
->outlen
)
1562 memset(props
, 0, sizeof(*props
));
1564 props
->device_cap_flags
= IB_DEVICE_BAD_PKEY_CNTR
|
1565 IB_DEVICE_BAD_QKEY_CNTR
| IB_DEVICE_SHUTDOWN_PORT
|
1566 IB_DEVICE_SYS_IMAGE_GUID
| IB_DEVICE_RC_RNR_NAK_GEN
|
1567 IB_DEVICE_PORT_ACTIVE_EVENT
| IB_DEVICE_SRQ_RESIZE
;
1568 props
->page_size_cap
= PAGE_SIZE
;
1570 QIB_SRC_OUI_1
<< 16 | QIB_SRC_OUI_2
<< 8 | QIB_SRC_OUI_3
;
1571 props
->vendor_part_id
= dd
->deviceid
;
1572 props
->hw_ver
= dd
->minrev
;
1573 props
->sys_image_guid
= ib_qib_sys_image_guid
;
1574 props
->max_mr_size
= ~0ULL;
1575 props
->max_qp
= ib_qib_max_qps
;
1576 props
->max_qp_wr
= ib_qib_max_qp_wrs
;
1577 props
->max_sge
= ib_qib_max_sges
;
1578 props
->max_sge_rd
= ib_qib_max_sges
;
1579 props
->max_cq
= ib_qib_max_cqs
;
1580 props
->max_ah
= ib_qib_max_ahs
;
1581 props
->max_cqe
= ib_qib_max_cqes
;
1582 props
->max_mr
= dev
->lk_table
.max
;
1583 props
->max_fmr
= dev
->lk_table
.max
;
1584 props
->max_map_per_fmr
= 32767;
1585 props
->max_pd
= ib_qib_max_pds
;
1586 props
->max_qp_rd_atom
= QIB_MAX_RDMA_ATOMIC
;
1587 props
->max_qp_init_rd_atom
= 255;
1588 /* props->max_res_rd_atom */
1589 props
->max_srq
= ib_qib_max_srqs
;
1590 props
->max_srq_wr
= ib_qib_max_srq_wrs
;
1591 props
->max_srq_sge
= ib_qib_max_srq_sges
;
1592 /* props->local_ca_ack_delay */
1593 props
->atomic_cap
= IB_ATOMIC_GLOB
;
1594 props
->max_pkeys
= qib_get_npkeys(dd
);
1595 props
->max_mcast_grp
= ib_qib_max_mcast_grps
;
1596 props
->max_mcast_qp_attach
= ib_qib_max_mcast_qp_attached
;
1597 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
1598 props
->max_mcast_grp
;
1603 static int qib_query_port(struct ib_device
*ibdev
, u8 port
,
1604 struct ib_port_attr
*props
)
1606 struct qib_devdata
*dd
= dd_from_ibdev(ibdev
);
1607 struct qib_ibport
*ibp
= to_iport(ibdev
, port
);
1608 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1612 memset(props
, 0, sizeof(*props
));
1613 props
->lid
= lid
? lid
: be16_to_cpu(IB_LID_PERMISSIVE
);
1614 props
->lmc
= ppd
->lmc
;
1615 props
->sm_lid
= ibp
->sm_lid
;
1616 props
->sm_sl
= ibp
->sm_sl
;
1617 props
->state
= dd
->f_iblink_state(ppd
->lastibcstat
);
1618 props
->phys_state
= dd
->f_ibphys_portstate(ppd
->lastibcstat
);
1619 props
->port_cap_flags
= ibp
->port_cap_flags
;
1620 props
->gid_tbl_len
= QIB_GUIDS_PER_PORT
;
1621 props
->max_msg_sz
= 0x80000000;
1622 props
->pkey_tbl_len
= qib_get_npkeys(dd
);
1623 props
->bad_pkey_cntr
= ibp
->pkey_violations
;
1624 props
->qkey_viol_cntr
= ibp
->qkey_violations
;
1625 props
->active_width
= ppd
->link_width_active
;
1626 /* See rate_show() */
1627 props
->active_speed
= ppd
->link_speed_active
;
1628 props
->max_vl_num
= qib_num_vls(ppd
->vls_supported
);
1629 props
->init_type_reply
= 0;
1631 props
->max_mtu
= qib_ibmtu
? qib_ibmtu
: IB_MTU_4096
;
1632 switch (ppd
->ibmtu
) {
1651 props
->active_mtu
= mtu
;
1652 props
->subnet_timeout
= ibp
->subnet_timeout
;
1657 static int qib_modify_device(struct ib_device
*device
,
1658 int device_modify_mask
,
1659 struct ib_device_modify
*device_modify
)
1661 struct qib_devdata
*dd
= dd_from_ibdev(device
);
1665 if (device_modify_mask
& ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID
|
1666 IB_DEVICE_MODIFY_NODE_DESC
)) {
1671 if (device_modify_mask
& IB_DEVICE_MODIFY_NODE_DESC
) {
1672 memcpy(device
->node_desc
, device_modify
->node_desc
, 64);
1673 for (i
= 0; i
< dd
->num_pports
; i
++) {
1674 struct qib_ibport
*ibp
= &dd
->pport
[i
].ibport_data
;
1676 qib_node_desc_chg(ibp
);
1680 if (device_modify_mask
& IB_DEVICE_MODIFY_SYS_IMAGE_GUID
) {
1681 ib_qib_sys_image_guid
=
1682 cpu_to_be64(device_modify
->sys_image_guid
);
1683 for (i
= 0; i
< dd
->num_pports
; i
++) {
1684 struct qib_ibport
*ibp
= &dd
->pport
[i
].ibport_data
;
1686 qib_sys_guid_chg(ibp
);
1696 static int qib_modify_port(struct ib_device
*ibdev
, u8 port
,
1697 int port_modify_mask
, struct ib_port_modify
*props
)
1699 struct qib_ibport
*ibp
= to_iport(ibdev
, port
);
1700 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1702 ibp
->port_cap_flags
|= props
->set_port_cap_mask
;
1703 ibp
->port_cap_flags
&= ~props
->clr_port_cap_mask
;
1704 if (props
->set_port_cap_mask
|| props
->clr_port_cap_mask
)
1705 qib_cap_mask_chg(ibp
);
1706 if (port_modify_mask
& IB_PORT_SHUTDOWN
)
1707 qib_set_linkstate(ppd
, QIB_IB_LINKDOWN
);
1708 if (port_modify_mask
& IB_PORT_RESET_QKEY_CNTR
)
1709 ibp
->qkey_violations
= 0;
1713 static int qib_query_gid(struct ib_device
*ibdev
, u8 port
,
1714 int index
, union ib_gid
*gid
)
1716 struct qib_devdata
*dd
= dd_from_ibdev(ibdev
);
1719 if (!port
|| port
> dd
->num_pports
)
1722 struct qib_ibport
*ibp
= to_iport(ibdev
, port
);
1723 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1725 gid
->global
.subnet_prefix
= ibp
->gid_prefix
;
1727 gid
->global
.interface_id
= ppd
->guid
;
1728 else if (index
< QIB_GUIDS_PER_PORT
)
1729 gid
->global
.interface_id
= ibp
->guids
[index
- 1];
1737 static struct ib_pd
*qib_alloc_pd(struct ib_device
*ibdev
,
1738 struct ib_ucontext
*context
,
1739 struct ib_udata
*udata
)
1741 struct qib_ibdev
*dev
= to_idev(ibdev
);
1746 * This is actually totally arbitrary. Some correctness tests
1747 * assume there's a maximum number of PDs that can be allocated.
1748 * We don't actually have this limit, but we fail the test if
1749 * we allow allocations of more than we report for this value.
1752 pd
= kmalloc(sizeof(*pd
), GFP_KERNEL
);
1754 ret
= ERR_PTR(-ENOMEM
);
1758 spin_lock(&dev
->n_pds_lock
);
1759 if (dev
->n_pds_allocated
== ib_qib_max_pds
) {
1760 spin_unlock(&dev
->n_pds_lock
);
1762 ret
= ERR_PTR(-ENOMEM
);
1766 dev
->n_pds_allocated
++;
1767 spin_unlock(&dev
->n_pds_lock
);
1769 /* ib_alloc_pd() will initialize pd->ibpd. */
1770 pd
->user
= udata
!= NULL
;
1778 static int qib_dealloc_pd(struct ib_pd
*ibpd
)
1780 struct qib_pd
*pd
= to_ipd(ibpd
);
1781 struct qib_ibdev
*dev
= to_idev(ibpd
->device
);
1783 spin_lock(&dev
->n_pds_lock
);
1784 dev
->n_pds_allocated
--;
1785 spin_unlock(&dev
->n_pds_lock
);
1792 int qib_check_ah(struct ib_device
*ibdev
, struct ib_ah_attr
*ah_attr
)
1794 /* A multicast address requires a GRH (see ch. 8.4.1). */
1795 if (ah_attr
->dlid
>= QIB_MULTICAST_LID_BASE
&&
1796 ah_attr
->dlid
!= QIB_PERMISSIVE_LID
&&
1797 !(ah_attr
->ah_flags
& IB_AH_GRH
))
1799 if ((ah_attr
->ah_flags
& IB_AH_GRH
) &&
1800 ah_attr
->grh
.sgid_index
>= QIB_GUIDS_PER_PORT
)
1802 if (ah_attr
->dlid
== 0)
1804 if (ah_attr
->port_num
< 1 ||
1805 ah_attr
->port_num
> ibdev
->phys_port_cnt
)
1807 if (ah_attr
->static_rate
!= IB_RATE_PORT_CURRENT
&&
1808 ib_rate_to_mult(ah_attr
->static_rate
) < 0)
1810 if (ah_attr
->sl
> 15)
1818 * qib_create_ah - create an address handle
1819 * @pd: the protection domain
1820 * @ah_attr: the attributes of the AH
1822 * This may be called from interrupt context.
1824 static struct ib_ah
*qib_create_ah(struct ib_pd
*pd
,
1825 struct ib_ah_attr
*ah_attr
)
1829 struct qib_ibdev
*dev
= to_idev(pd
->device
);
1830 unsigned long flags
;
1832 if (qib_check_ah(pd
->device
, ah_attr
)) {
1833 ret
= ERR_PTR(-EINVAL
);
1837 ah
= kmalloc(sizeof(*ah
), GFP_ATOMIC
);
1839 ret
= ERR_PTR(-ENOMEM
);
1843 spin_lock_irqsave(&dev
->n_ahs_lock
, flags
);
1844 if (dev
->n_ahs_allocated
== ib_qib_max_ahs
) {
1845 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1847 ret
= ERR_PTR(-ENOMEM
);
1851 dev
->n_ahs_allocated
++;
1852 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1854 /* ib_create_ah() will initialize ah->ibah. */
1855 ah
->attr
= *ah_attr
;
1856 atomic_set(&ah
->refcount
, 0);
1864 struct ib_ah
*qib_create_qp0_ah(struct qib_ibport
*ibp
, u16 dlid
)
1866 struct ib_ah_attr attr
;
1867 struct ib_ah
*ah
= ERR_PTR(-EINVAL
);
1870 memset(&attr
, 0, sizeof(attr
));
1872 attr
.port_num
= ppd_from_ibp(ibp
)->port
;
1874 qp0
= rcu_dereference(ibp
->qp0
);
1876 ah
= ib_create_ah(qp0
->ibqp
.pd
, &attr
);
1882 * qib_destroy_ah - destroy an address handle
1883 * @ibah: the AH to destroy
1885 * This may be called from interrupt context.
1887 static int qib_destroy_ah(struct ib_ah
*ibah
)
1889 struct qib_ibdev
*dev
= to_idev(ibah
->device
);
1890 struct qib_ah
*ah
= to_iah(ibah
);
1891 unsigned long flags
;
1893 if (atomic_read(&ah
->refcount
) != 0)
1896 spin_lock_irqsave(&dev
->n_ahs_lock
, flags
);
1897 dev
->n_ahs_allocated
--;
1898 spin_unlock_irqrestore(&dev
->n_ahs_lock
, flags
);
1905 static int qib_modify_ah(struct ib_ah
*ibah
, struct ib_ah_attr
*ah_attr
)
1907 struct qib_ah
*ah
= to_iah(ibah
);
1909 if (qib_check_ah(ibah
->device
, ah_attr
))
1912 ah
->attr
= *ah_attr
;
1917 static int qib_query_ah(struct ib_ah
*ibah
, struct ib_ah_attr
*ah_attr
)
1919 struct qib_ah
*ah
= to_iah(ibah
);
1921 *ah_attr
= ah
->attr
;
1927 * qib_get_npkeys - return the size of the PKEY table for context 0
1928 * @dd: the qlogic_ib device
1930 unsigned qib_get_npkeys(struct qib_devdata
*dd
)
1932 return ARRAY_SIZE(dd
->rcd
[0]->pkeys
);
1936 * Return the indexed PKEY from the port PKEY table.
1937 * No need to validate rcd[ctxt]; the port is setup if we are here.
1939 unsigned qib_get_pkey(struct qib_ibport
*ibp
, unsigned index
)
1941 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
1942 struct qib_devdata
*dd
= ppd
->dd
;
1943 unsigned ctxt
= ppd
->hw_pidx
;
1946 /* dd->rcd null if mini_init or some init failures */
1947 if (!dd
->rcd
|| index
>= ARRAY_SIZE(dd
->rcd
[ctxt
]->pkeys
))
1950 ret
= dd
->rcd
[ctxt
]->pkeys
[index
];
1955 static int qib_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
,
1958 struct qib_devdata
*dd
= dd_from_ibdev(ibdev
);
1961 if (index
>= qib_get_npkeys(dd
)) {
1966 *pkey
= qib_get_pkey(to_iport(ibdev
, port
), index
);
1974 * qib_alloc_ucontext - allocate a ucontest
1975 * @ibdev: the infiniband device
1976 * @udata: not used by the QLogic_IB driver
1979 static struct ib_ucontext
*qib_alloc_ucontext(struct ib_device
*ibdev
,
1980 struct ib_udata
*udata
)
1982 struct qib_ucontext
*context
;
1983 struct ib_ucontext
*ret
;
1985 context
= kmalloc(sizeof(*context
), GFP_KERNEL
);
1987 ret
= ERR_PTR(-ENOMEM
);
1991 ret
= &context
->ibucontext
;
1997 static int qib_dealloc_ucontext(struct ib_ucontext
*context
)
1999 kfree(to_iucontext(context
));
2003 static void init_ibport(struct qib_pportdata
*ppd
)
2005 struct qib_verbs_counters cntrs
;
2006 struct qib_ibport
*ibp
= &ppd
->ibport_data
;
2008 spin_lock_init(&ibp
->lock
);
2009 /* Set the prefix to the default value (see ch. 4.1.1) */
2010 ibp
->gid_prefix
= IB_DEFAULT_GID_PREFIX
;
2011 ibp
->sm_lid
= be16_to_cpu(IB_LID_PERMISSIVE
);
2012 ibp
->port_cap_flags
= IB_PORT_SYS_IMAGE_GUID_SUP
|
2013 IB_PORT_CLIENT_REG_SUP
| IB_PORT_SL_MAP_SUP
|
2014 IB_PORT_TRAP_SUP
| IB_PORT_AUTO_MIGR_SUP
|
2015 IB_PORT_DR_NOTICE_SUP
| IB_PORT_CAP_MASK_NOTICE_SUP
|
2016 IB_PORT_OTHER_LOCAL_CHANGES_SUP
;
2017 if (ppd
->dd
->flags
& QIB_HAS_LINK_LATENCY
)
2018 ibp
->port_cap_flags
|= IB_PORT_LINK_LATENCY_SUP
;
2019 ibp
->pma_counter_select
[0] = IB_PMA_PORT_XMIT_DATA
;
2020 ibp
->pma_counter_select
[1] = IB_PMA_PORT_RCV_DATA
;
2021 ibp
->pma_counter_select
[2] = IB_PMA_PORT_XMIT_PKTS
;
2022 ibp
->pma_counter_select
[3] = IB_PMA_PORT_RCV_PKTS
;
2023 ibp
->pma_counter_select
[4] = IB_PMA_PORT_XMIT_WAIT
;
2025 /* Snapshot current HW counters to "clear" them. */
2026 qib_get_counters(ppd
, &cntrs
);
2027 ibp
->z_symbol_error_counter
= cntrs
.symbol_error_counter
;
2028 ibp
->z_link_error_recovery_counter
=
2029 cntrs
.link_error_recovery_counter
;
2030 ibp
->z_link_downed_counter
= cntrs
.link_downed_counter
;
2031 ibp
->z_port_rcv_errors
= cntrs
.port_rcv_errors
;
2032 ibp
->z_port_rcv_remphys_errors
= cntrs
.port_rcv_remphys_errors
;
2033 ibp
->z_port_xmit_discards
= cntrs
.port_xmit_discards
;
2034 ibp
->z_port_xmit_data
= cntrs
.port_xmit_data
;
2035 ibp
->z_port_rcv_data
= cntrs
.port_rcv_data
;
2036 ibp
->z_port_xmit_packets
= cntrs
.port_xmit_packets
;
2037 ibp
->z_port_rcv_packets
= cntrs
.port_rcv_packets
;
2038 ibp
->z_local_link_integrity_errors
=
2039 cntrs
.local_link_integrity_errors
;
2040 ibp
->z_excessive_buffer_overrun_errors
=
2041 cntrs
.excessive_buffer_overrun_errors
;
2042 ibp
->z_vl15_dropped
= cntrs
.vl15_dropped
;
2043 RCU_INIT_POINTER(ibp
->qp0
, NULL
);
2044 RCU_INIT_POINTER(ibp
->qp1
, NULL
);
2047 static int qib_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
2048 struct ib_port_immutable
*immutable
)
2050 struct ib_port_attr attr
;
2053 err
= qib_query_port(ibdev
, port_num
, &attr
);
2057 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2058 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2059 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_IB
;
2060 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
2066 * qib_register_ib_device - register our device with the infiniband core
2067 * @dd: the device data structure
2068 * Return the allocated qib_ibdev pointer or NULL on error.
2070 int qib_register_ib_device(struct qib_devdata
*dd
)
2072 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
2073 struct ib_device
*ibdev
= &dev
->ibdev
;
2074 struct qib_pportdata
*ppd
= dd
->pport
;
2075 unsigned i
, lk_tab_size
;
2078 dev
->qp_table_size
= ib_qib_qp_table_size
;
2079 get_random_bytes(&dev
->qp_rnd
, sizeof(dev
->qp_rnd
));
2080 dev
->qp_table
= kmalloc_array(
2082 sizeof(*dev
->qp_table
),
2084 if (!dev
->qp_table
) {
2088 for (i
= 0; i
< dev
->qp_table_size
; i
++)
2089 RCU_INIT_POINTER(dev
->qp_table
[i
], NULL
);
2091 for (i
= 0; i
< dd
->num_pports
; i
++)
2092 init_ibport(ppd
+ i
);
2094 /* Only need to initialize non-zero fields. */
2095 spin_lock_init(&dev
->qpt_lock
);
2096 spin_lock_init(&dev
->n_pds_lock
);
2097 spin_lock_init(&dev
->n_ahs_lock
);
2098 spin_lock_init(&dev
->n_cqs_lock
);
2099 spin_lock_init(&dev
->n_qps_lock
);
2100 spin_lock_init(&dev
->n_srqs_lock
);
2101 spin_lock_init(&dev
->n_mcast_grps_lock
);
2102 init_timer(&dev
->mem_timer
);
2103 dev
->mem_timer
.function
= mem_timer
;
2104 dev
->mem_timer
.data
= (unsigned long) dev
;
2106 qib_init_qpn_table(dd
, &dev
->qpn_table
);
2109 * The top ib_qib_lkey_table_size bits are used to index the
2110 * table. The lower 8 bits can be owned by the user (copied from
2111 * the LKEY). The remaining bits act as a generation number or tag.
2113 spin_lock_init(&dev
->lk_table
.lock
);
2114 /* insure generation is at least 4 bits see keys.c */
2115 if (ib_qib_lkey_table_size
> MAX_LKEY_TABLE_BITS
) {
2116 qib_dev_warn(dd
, "lkey bits %u too large, reduced to %u\n",
2117 ib_qib_lkey_table_size
, MAX_LKEY_TABLE_BITS
);
2118 ib_qib_lkey_table_size
= MAX_LKEY_TABLE_BITS
;
2120 dev
->lk_table
.max
= 1 << ib_qib_lkey_table_size
;
2121 lk_tab_size
= dev
->lk_table
.max
* sizeof(*dev
->lk_table
.table
);
2122 dev
->lk_table
.table
= (struct qib_mregion __rcu
**)
2123 vmalloc(lk_tab_size
);
2124 if (dev
->lk_table
.table
== NULL
) {
2128 RCU_INIT_POINTER(dev
->dma_mr
, NULL
);
2129 for (i
= 0; i
< dev
->lk_table
.max
; i
++)
2130 RCU_INIT_POINTER(dev
->lk_table
.table
[i
], NULL
);
2131 INIT_LIST_HEAD(&dev
->pending_mmaps
);
2132 spin_lock_init(&dev
->pending_lock
);
2133 dev
->mmap_offset
= PAGE_SIZE
;
2134 spin_lock_init(&dev
->mmap_offset_lock
);
2135 INIT_LIST_HEAD(&dev
->piowait
);
2136 INIT_LIST_HEAD(&dev
->dmawait
);
2137 INIT_LIST_HEAD(&dev
->txwait
);
2138 INIT_LIST_HEAD(&dev
->memwait
);
2139 INIT_LIST_HEAD(&dev
->txreq_free
);
2141 if (ppd
->sdma_descq_cnt
) {
2142 dev
->pio_hdrs
= dma_alloc_coherent(&dd
->pcidev
->dev
,
2143 ppd
->sdma_descq_cnt
*
2144 sizeof(struct qib_pio_header
),
2145 &dev
->pio_hdrs_phys
,
2147 if (!dev
->pio_hdrs
) {
2153 for (i
= 0; i
< ppd
->sdma_descq_cnt
; i
++) {
2154 struct qib_verbs_txreq
*tx
;
2156 tx
= kzalloc(sizeof(*tx
), GFP_KERNEL
);
2162 list_add(&tx
->txreq
.list
, &dev
->txreq_free
);
2166 * The system image GUID is supposed to be the same for all
2167 * IB HCAs in a single system but since there can be other
2168 * device types in the system, we can't be sure this is unique.
2170 if (!ib_qib_sys_image_guid
)
2171 ib_qib_sys_image_guid
= ppd
->guid
;
2173 strlcpy(ibdev
->name
, "qib%d", IB_DEVICE_NAME_MAX
);
2174 ibdev
->owner
= THIS_MODULE
;
2175 ibdev
->node_guid
= ppd
->guid
;
2176 ibdev
->uverbs_abi_ver
= QIB_UVERBS_ABI_VERSION
;
2177 ibdev
->uverbs_cmd_mask
=
2178 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
2179 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
2180 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
2181 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
2182 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
2183 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
2184 (1ull << IB_USER_VERBS_CMD_MODIFY_AH
) |
2185 (1ull << IB_USER_VERBS_CMD_QUERY_AH
) |
2186 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
2187 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
2188 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
2189 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
2190 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
2191 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
2192 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
2193 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
2194 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
2195 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
2196 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
2197 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
2198 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
2199 (1ull << IB_USER_VERBS_CMD_POST_SEND
) |
2200 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
2201 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST
) |
2202 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST
) |
2203 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
2204 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
2205 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
2206 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
2207 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV
);
2208 ibdev
->node_type
= RDMA_NODE_IB_CA
;
2209 ibdev
->phys_port_cnt
= dd
->num_pports
;
2210 ibdev
->num_comp_vectors
= 1;
2211 ibdev
->dma_device
= &dd
->pcidev
->dev
;
2212 ibdev
->query_device
= qib_query_device
;
2213 ibdev
->modify_device
= qib_modify_device
;
2214 ibdev
->query_port
= qib_query_port
;
2215 ibdev
->modify_port
= qib_modify_port
;
2216 ibdev
->query_pkey
= qib_query_pkey
;
2217 ibdev
->query_gid
= qib_query_gid
;
2218 ibdev
->alloc_ucontext
= qib_alloc_ucontext
;
2219 ibdev
->dealloc_ucontext
= qib_dealloc_ucontext
;
2220 ibdev
->alloc_pd
= qib_alloc_pd
;
2221 ibdev
->dealloc_pd
= qib_dealloc_pd
;
2222 ibdev
->create_ah
= qib_create_ah
;
2223 ibdev
->destroy_ah
= qib_destroy_ah
;
2224 ibdev
->modify_ah
= qib_modify_ah
;
2225 ibdev
->query_ah
= qib_query_ah
;
2226 ibdev
->create_srq
= qib_create_srq
;
2227 ibdev
->modify_srq
= qib_modify_srq
;
2228 ibdev
->query_srq
= qib_query_srq
;
2229 ibdev
->destroy_srq
= qib_destroy_srq
;
2230 ibdev
->create_qp
= qib_create_qp
;
2231 ibdev
->modify_qp
= qib_modify_qp
;
2232 ibdev
->query_qp
= qib_query_qp
;
2233 ibdev
->destroy_qp
= qib_destroy_qp
;
2234 ibdev
->post_send
= qib_post_send
;
2235 ibdev
->post_recv
= qib_post_receive
;
2236 ibdev
->post_srq_recv
= qib_post_srq_receive
;
2237 ibdev
->create_cq
= qib_create_cq
;
2238 ibdev
->destroy_cq
= qib_destroy_cq
;
2239 ibdev
->resize_cq
= qib_resize_cq
;
2240 ibdev
->poll_cq
= qib_poll_cq
;
2241 ibdev
->req_notify_cq
= qib_req_notify_cq
;
2242 ibdev
->get_dma_mr
= qib_get_dma_mr
;
2243 ibdev
->reg_phys_mr
= qib_reg_phys_mr
;
2244 ibdev
->reg_user_mr
= qib_reg_user_mr
;
2245 ibdev
->dereg_mr
= qib_dereg_mr
;
2246 ibdev
->alloc_mr
= qib_alloc_mr
;
2247 ibdev
->alloc_fast_reg_page_list
= qib_alloc_fast_reg_page_list
;
2248 ibdev
->free_fast_reg_page_list
= qib_free_fast_reg_page_list
;
2249 ibdev
->alloc_fmr
= qib_alloc_fmr
;
2250 ibdev
->map_phys_fmr
= qib_map_phys_fmr
;
2251 ibdev
->unmap_fmr
= qib_unmap_fmr
;
2252 ibdev
->dealloc_fmr
= qib_dealloc_fmr
;
2253 ibdev
->attach_mcast
= qib_multicast_attach
;
2254 ibdev
->detach_mcast
= qib_multicast_detach
;
2255 ibdev
->process_mad
= qib_process_mad
;
2256 ibdev
->mmap
= qib_mmap
;
2257 ibdev
->dma_ops
= &qib_dma_mapping_ops
;
2258 ibdev
->get_port_immutable
= qib_port_immutable
;
2260 snprintf(ibdev
->node_desc
, sizeof(ibdev
->node_desc
),
2261 "Intel Infiniband HCA %s", init_utsname()->nodename
);
2263 ret
= ib_register_device(ibdev
, qib_create_port_files
);
2267 ret
= qib_create_agents(dev
);
2271 ret
= qib_verbs_register_sysfs(dd
);
2278 qib_free_agents(dev
);
2280 ib_unregister_device(ibdev
);
2283 while (!list_empty(&dev
->txreq_free
)) {
2284 struct list_head
*l
= dev
->txreq_free
.next
;
2285 struct qib_verbs_txreq
*tx
;
2288 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
2291 if (ppd
->sdma_descq_cnt
)
2292 dma_free_coherent(&dd
->pcidev
->dev
,
2293 ppd
->sdma_descq_cnt
*
2294 sizeof(struct qib_pio_header
),
2295 dev
->pio_hdrs
, dev
->pio_hdrs_phys
);
2297 vfree(dev
->lk_table
.table
);
2299 kfree(dev
->qp_table
);
2301 qib_dev_err(dd
, "cannot register verbs: %d!\n", -ret
);
2306 void qib_unregister_ib_device(struct qib_devdata
*dd
)
2308 struct qib_ibdev
*dev
= &dd
->verbs_dev
;
2309 struct ib_device
*ibdev
= &dev
->ibdev
;
2311 unsigned lk_tab_size
;
2313 qib_verbs_unregister_sysfs(dd
);
2315 qib_free_agents(dev
);
2317 ib_unregister_device(ibdev
);
2319 if (!list_empty(&dev
->piowait
))
2320 qib_dev_err(dd
, "piowait list not empty!\n");
2321 if (!list_empty(&dev
->dmawait
))
2322 qib_dev_err(dd
, "dmawait list not empty!\n");
2323 if (!list_empty(&dev
->txwait
))
2324 qib_dev_err(dd
, "txwait list not empty!\n");
2325 if (!list_empty(&dev
->memwait
))
2326 qib_dev_err(dd
, "memwait list not empty!\n");
2328 qib_dev_err(dd
, "DMA MR not NULL!\n");
2330 qps_inuse
= qib_free_all_qps(dd
);
2332 qib_dev_err(dd
, "QP memory leak! %u still in use\n",
2335 del_timer_sync(&dev
->mem_timer
);
2336 qib_free_qpn_table(&dev
->qpn_table
);
2337 while (!list_empty(&dev
->txreq_free
)) {
2338 struct list_head
*l
= dev
->txreq_free
.next
;
2339 struct qib_verbs_txreq
*tx
;
2342 tx
= list_entry(l
, struct qib_verbs_txreq
, txreq
.list
);
2345 if (dd
->pport
->sdma_descq_cnt
)
2346 dma_free_coherent(&dd
->pcidev
->dev
,
2347 dd
->pport
->sdma_descq_cnt
*
2348 sizeof(struct qib_pio_header
),
2349 dev
->pio_hdrs
, dev
->pio_hdrs_phys
);
2350 lk_tab_size
= dev
->lk_table
.max
* sizeof(*dev
->lk_table
.table
);
2351 vfree(dev
->lk_table
.table
);
2352 kfree(dev
->qp_table
);
2356 * This must be called with s_lock held.
2358 void qib_schedule_send(struct qib_qp
*qp
)
2360 if (qib_send_ok(qp
)) {
2361 struct qib_ibport
*ibp
=
2362 to_iport(qp
->ibqp
.device
, qp
->port_num
);
2363 struct qib_pportdata
*ppd
= ppd_from_ibp(ibp
);
2365 queue_work(ppd
->qib_wq
, &qp
->s_work
);