2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Description: Main component of the bnxt_re driver
39 #include <linux/module.h>
40 #include <linux/netdevice.h>
41 #include <linux/ethtool.h>
42 #include <linux/mutex.h>
43 #include <linux/list.h>
44 #include <linux/rculist.h>
45 #include <linux/spinlock.h>
46 #include <linux/pci.h>
47 #include <net/dcbnl.h>
49 #include <net/addrconf.h>
50 #include <linux/if_ether.h>
52 #include <rdma/ib_verbs.h>
53 #include <rdma/ib_user_verbs.h>
54 #include <rdma/ib_umem.h>
55 #include <rdma/ib_addr.h>
59 #include "qplib_res.h"
62 #include "qplib_rcfw.h"
65 #include <rdma/bnxt_re-abi.h>
67 #include "hw_counters.h"
69 static char version
[] =
72 MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
73 MODULE_DESCRIPTION(BNXT_RE_DESC
" Driver");
74 MODULE_LICENSE("Dual BSD/GPL");
77 static struct list_head bnxt_re_dev_list
= LIST_HEAD_INIT(bnxt_re_dev_list
);
78 /* Mutex to protect the list of bnxt_re devices added */
79 static DEFINE_MUTEX(bnxt_re_dev_lock
);
80 static struct workqueue_struct
*bnxt_re_wq
;
81 static void bnxt_re_remove_device(struct bnxt_re_dev
*rdev
);
82 static void bnxt_re_dealloc_driver(struct ib_device
*ib_dev
);
83 static void bnxt_re_stop_irq(void *handle
);
85 static void bnxt_re_set_drv_mode(struct bnxt_re_dev
*rdev
, u8 mode
)
87 struct bnxt_qplib_chip_ctx
*cctx
;
89 cctx
= rdev
->chip_ctx
;
90 cctx
->modes
.wqe_mode
= bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
) ?
91 mode
: BNXT_QPLIB_WQE_MODE_STATIC
;
94 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev
*rdev
)
96 struct bnxt_qplib_chip_ctx
*chip_ctx
;
100 chip_ctx
= rdev
->chip_ctx
;
101 rdev
->chip_ctx
= NULL
;
102 rdev
->rcfw
.res
= NULL
;
103 rdev
->qplib_res
.cctx
= NULL
;
104 rdev
->qplib_res
.pdev
= NULL
;
105 rdev
->qplib_res
.netdev
= NULL
;
109 static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev
*rdev
, u8 wqe_mode
)
111 struct bnxt_qplib_chip_ctx
*chip_ctx
;
112 struct bnxt_en_dev
*en_dev
;
115 en_dev
= rdev
->en_dev
;
116 bp
= netdev_priv(en_dev
->net
);
118 chip_ctx
= kzalloc(sizeof(*chip_ctx
), GFP_KERNEL
);
121 chip_ctx
->chip_num
= bp
->chip_num
;
123 rdev
->chip_ctx
= chip_ctx
;
124 /* rest members to follow eventually */
126 rdev
->qplib_res
.cctx
= rdev
->chip_ctx
;
127 rdev
->rcfw
.res
= &rdev
->qplib_res
;
129 bnxt_re_set_drv_mode(rdev
, wqe_mode
);
133 /* SR-IOV helper functions */
135 static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev
*rdev
)
139 bp
= netdev_priv(rdev
->en_dev
->net
);
144 /* Set the maximum number of each resource that the driver actually wants
145 * to allocate. This may be up to the maximum number the firmware has
146 * reserved for the function. The driver may choose to allocate fewer
147 * resources than the firmware maximum.
149 static void bnxt_re_limit_pf_res(struct bnxt_re_dev
*rdev
)
151 struct bnxt_qplib_dev_attr
*attr
;
152 struct bnxt_qplib_ctx
*ctx
;
155 attr
= &rdev
->dev_attr
;
156 ctx
= &rdev
->qplib_ctx
;
158 ctx
->qpc_count
= min_t(u32
, BNXT_RE_MAX_QPC_COUNT
,
160 ctx
->mrw_count
= BNXT_RE_MAX_MRW_COUNT_256K
;
161 /* Use max_mr from fw since max_mrw does not get set */
162 ctx
->mrw_count
= min_t(u32
, ctx
->mrw_count
, attr
->max_mr
);
163 ctx
->srqc_count
= min_t(u32
, BNXT_RE_MAX_SRQC_COUNT
,
165 ctx
->cq_count
= min_t(u32
, BNXT_RE_MAX_CQ_COUNT
, attr
->max_cq
);
166 if (!bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
))
167 for (i
= 0; i
< MAX_TQM_ALLOC_REQ
; i
++)
168 rdev
->qplib_ctx
.tqm_ctx
.qcount
[i
] =
169 rdev
->dev_attr
.tqm_alloc_reqs
[i
];
172 static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx
*qplib_ctx
, u32 num_vf
)
174 struct bnxt_qplib_vf_res
*vf_res
;
179 vf_res
= &qplib_ctx
->vf_res
;
181 * Reserve a set of resources for the PF. Divide the remaining
182 * resources among the VFs
184 vf_pct
= 100 - BNXT_RE_PCT_RSVD_FOR_PF
;
186 num_vf
= 100 * num_vf
;
187 vf_res
->max_qp_per_vf
= (qplib_ctx
->qpc_count
* vf_pct
) / num_vf
;
188 vf_res
->max_srq_per_vf
= (qplib_ctx
->srqc_count
* vf_pct
) / num_vf
;
189 vf_res
->max_cq_per_vf
= (qplib_ctx
->cq_count
* vf_pct
) / num_vf
;
191 * The driver allows many more MRs than other resources. If the
192 * firmware does also, then reserve a fixed amount for the PF and
193 * divide the rest among VFs. VFs may use many MRs for NFS
194 * mounts, ISER, NVME applications, etc. If the firmware severely
195 * restricts the number of MRs, then let PF have half and divide
196 * the rest among VFs, as for the other resource types.
198 if (qplib_ctx
->mrw_count
< BNXT_RE_MAX_MRW_COUNT_64K
) {
199 mrws
= qplib_ctx
->mrw_count
* vf_pct
;
202 mrws
= qplib_ctx
->mrw_count
- BNXT_RE_RESVD_MR_FOR_PF
;
204 vf_res
->max_mrw_per_vf
= (mrws
/ nvfs
);
205 vf_res
->max_gid_per_vf
= BNXT_RE_MAX_GID_PER_VF
;
208 static void bnxt_re_set_resource_limits(struct bnxt_re_dev
*rdev
)
212 memset(&rdev
->qplib_ctx
.vf_res
, 0, sizeof(struct bnxt_qplib_vf_res
));
213 bnxt_re_limit_pf_res(rdev
);
215 num_vfs
= bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
) ?
216 BNXT_RE_GEN_P5_MAX_VF
: rdev
->num_vfs
;
218 bnxt_re_limit_vf_res(&rdev
->qplib_ctx
, num_vfs
);
221 /* for handling bnxt_en callbacks later */
222 static void bnxt_re_stop(void *p
)
226 static void bnxt_re_start(void *p
)
230 static void bnxt_re_sriov_config(void *p
, int num_vfs
)
232 struct bnxt_re_dev
*rdev
= p
;
237 rdev
->num_vfs
= num_vfs
;
238 if (!bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
)) {
239 bnxt_re_set_resource_limits(rdev
);
240 bnxt_qplib_set_func_resources(&rdev
->qplib_res
, &rdev
->rcfw
,
245 static void bnxt_re_shutdown(void *p
)
247 struct bnxt_re_dev
*rdev
= p
;
252 /* Release the MSIx vectors before queuing unregister */
253 bnxt_re_stop_irq(rdev
);
254 ib_unregister_device_queued(&rdev
->ibdev
);
257 static void bnxt_re_stop_irq(void *handle
)
259 struct bnxt_re_dev
*rdev
= (struct bnxt_re_dev
*)handle
;
260 struct bnxt_qplib_rcfw
*rcfw
= &rdev
->rcfw
;
261 struct bnxt_qplib_nq
*nq
;
264 for (indx
= BNXT_RE_NQ_IDX
; indx
< rdev
->num_msix
; indx
++) {
265 nq
= &rdev
->nq
[indx
- 1];
266 bnxt_qplib_nq_stop_irq(nq
, false);
269 bnxt_qplib_rcfw_stop_irq(rcfw
, false);
272 static void bnxt_re_start_irq(void *handle
, struct bnxt_msix_entry
*ent
)
274 struct bnxt_re_dev
*rdev
= (struct bnxt_re_dev
*)handle
;
275 struct bnxt_msix_entry
*msix_ent
= rdev
->msix_entries
;
276 struct bnxt_qplib_rcfw
*rcfw
= &rdev
->rcfw
;
277 struct bnxt_qplib_nq
*nq
;
281 /* Not setting the f/w timeout bit in rcfw.
282 * During the driver unload the first command
283 * to f/w will timeout and that will set the
286 ibdev_err(&rdev
->ibdev
, "Failed to re-start IRQs\n");
290 /* Vectors may change after restart, so update with new vectors
291 * in device sctructure.
293 for (indx
= 0; indx
< rdev
->num_msix
; indx
++)
294 rdev
->msix_entries
[indx
].vector
= ent
[indx
].vector
;
296 bnxt_qplib_rcfw_start_irq(rcfw
, msix_ent
[BNXT_RE_AEQ_IDX
].vector
,
298 for (indx
= BNXT_RE_NQ_IDX
; indx
< rdev
->num_msix
; indx
++) {
299 nq
= &rdev
->nq
[indx
- 1];
300 rc
= bnxt_qplib_nq_start_irq(nq
, indx
- 1,
301 msix_ent
[indx
].vector
, false);
303 ibdev_warn(&rdev
->ibdev
, "Failed to reinit NQ index %d\n",
308 static struct bnxt_ulp_ops bnxt_re_ulp_ops
= {
309 .ulp_async_notifier
= NULL
,
310 .ulp_stop
= bnxt_re_stop
,
311 .ulp_start
= bnxt_re_start
,
312 .ulp_sriov_config
= bnxt_re_sriov_config
,
313 .ulp_shutdown
= bnxt_re_shutdown
,
314 .ulp_irq_stop
= bnxt_re_stop_irq
,
315 .ulp_irq_restart
= bnxt_re_start_irq
318 /* RoCE -> Net driver */
320 /* Driver registration routines used to let the networking driver (bnxt_en)
321 * to know that the RoCE driver is now installed
323 static int bnxt_re_unregister_netdev(struct bnxt_re_dev
*rdev
)
325 struct bnxt_en_dev
*en_dev
;
331 en_dev
= rdev
->en_dev
;
333 rc
= en_dev
->en_ops
->bnxt_unregister_device(rdev
->en_dev
,
338 static int bnxt_re_register_netdev(struct bnxt_re_dev
*rdev
)
340 struct bnxt_en_dev
*en_dev
;
346 en_dev
= rdev
->en_dev
;
348 rc
= en_dev
->en_ops
->bnxt_register_device(en_dev
, BNXT_ROCE_ULP
,
349 &bnxt_re_ulp_ops
, rdev
);
350 rdev
->qplib_res
.pdev
= rdev
->en_dev
->pdev
;
354 static int bnxt_re_free_msix(struct bnxt_re_dev
*rdev
)
356 struct bnxt_en_dev
*en_dev
;
362 en_dev
= rdev
->en_dev
;
365 rc
= en_dev
->en_ops
->bnxt_free_msix(rdev
->en_dev
, BNXT_ROCE_ULP
);
370 static int bnxt_re_request_msix(struct bnxt_re_dev
*rdev
)
372 int rc
= 0, num_msix_want
= BNXT_RE_MAX_MSIX
, num_msix_got
;
373 struct bnxt_en_dev
*en_dev
;
378 en_dev
= rdev
->en_dev
;
380 num_msix_want
= min_t(u32
, BNXT_RE_MAX_MSIX
, num_online_cpus());
382 num_msix_got
= en_dev
->en_ops
->bnxt_request_msix(en_dev
, BNXT_ROCE_ULP
,
385 if (num_msix_got
< BNXT_RE_MIN_MSIX
) {
389 if (num_msix_got
!= num_msix_want
) {
390 ibdev_warn(&rdev
->ibdev
,
391 "Requested %d MSI-X vectors, got %d\n",
392 num_msix_want
, num_msix_got
);
394 rdev
->num_msix
= num_msix_got
;
399 static void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev
*rdev
, struct input
*hdr
,
400 u16 opcd
, u16 crid
, u16 trid
)
402 hdr
->req_type
= cpu_to_le16(opcd
);
403 hdr
->cmpl_ring
= cpu_to_le16(crid
);
404 hdr
->target_id
= cpu_to_le16(trid
);
407 static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg
*fw_msg
, void *msg
,
408 int msg_len
, void *resp
, int resp_max_len
,
412 fw_msg
->msg_len
= msg_len
;
414 fw_msg
->resp_max_len
= resp_max_len
;
415 fw_msg
->timeout
= timeout
;
418 static int bnxt_re_net_ring_free(struct bnxt_re_dev
*rdev
,
419 u16 fw_ring_id
, int type
)
421 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
422 struct hwrm_ring_free_input req
= {0};
423 struct hwrm_ring_free_output resp
;
424 struct bnxt_fw_msg fw_msg
;
430 memset(&fw_msg
, 0, sizeof(fw_msg
));
432 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
, HWRM_RING_FREE
, -1, -1);
433 req
.ring_type
= type
;
434 req
.ring_id
= cpu_to_le16(fw_ring_id
);
435 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&resp
,
436 sizeof(resp
), DFLT_HWRM_CMD_TIMEOUT
);
437 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
439 ibdev_err(&rdev
->ibdev
, "Failed to free HW ring:%d :%#x",
444 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev
*rdev
,
445 struct bnxt_re_ring_attr
*ring_attr
,
448 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
449 struct hwrm_ring_alloc_input req
= {0};
450 struct hwrm_ring_alloc_output resp
;
451 struct bnxt_fw_msg fw_msg
;
457 memset(&fw_msg
, 0, sizeof(fw_msg
));
458 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
, HWRM_RING_ALLOC
, -1, -1);
460 req
.page_tbl_addr
= cpu_to_le64(ring_attr
->dma_arr
[0]);
461 if (ring_attr
->pages
> 1) {
462 /* Page size is in log2 units */
463 req
.page_size
= BNXT_PAGE_SHIFT
;
464 req
.page_tbl_depth
= 1;
467 /* Association of ring index with doorbell index and MSIX number */
468 req
.logical_id
= cpu_to_le16(ring_attr
->lrid
);
469 req
.length
= cpu_to_le32(ring_attr
->depth
+ 1);
470 req
.ring_type
= ring_attr
->type
;
471 req
.int_mode
= ring_attr
->mode
;
472 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&resp
,
473 sizeof(resp
), DFLT_HWRM_CMD_TIMEOUT
);
474 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
476 *fw_ring_id
= le16_to_cpu(resp
.ring_id
);
481 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev
*rdev
,
484 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
485 struct hwrm_stat_ctx_free_input req
= {0};
486 struct bnxt_fw_msg fw_msg
;
492 memset(&fw_msg
, 0, sizeof(fw_msg
));
494 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
, HWRM_STAT_CTX_FREE
, -1, -1);
495 req
.stat_ctx_id
= cpu_to_le32(fw_stats_ctx_id
);
496 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&req
,
497 sizeof(req
), DFLT_HWRM_CMD_TIMEOUT
);
498 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
500 ibdev_err(&rdev
->ibdev
, "Failed to free HW stats context %#x",
506 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev
*rdev
,
508 u32
*fw_stats_ctx_id
)
510 struct hwrm_stat_ctx_alloc_output resp
= {0};
511 struct hwrm_stat_ctx_alloc_input req
= {0};
512 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
513 struct bnxt_fw_msg fw_msg
;
516 *fw_stats_ctx_id
= INVALID_STATS_CTX_ID
;
521 memset(&fw_msg
, 0, sizeof(fw_msg
));
523 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
, HWRM_STAT_CTX_ALLOC
, -1, -1);
524 req
.update_period_ms
= cpu_to_le32(1000);
525 req
.stats_dma_addr
= cpu_to_le64(dma_map
);
526 req
.stats_dma_length
= cpu_to_le16(sizeof(struct ctx_hw_stats_ext
));
527 req
.stat_ctx_flags
= STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE
;
528 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&resp
,
529 sizeof(resp
), DFLT_HWRM_CMD_TIMEOUT
);
530 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
532 *fw_stats_ctx_id
= le32_to_cpu(resp
.stat_ctx_id
);
539 static bool is_bnxt_re_dev(struct net_device
*netdev
)
541 struct ethtool_drvinfo drvinfo
;
543 if (netdev
->ethtool_ops
&& netdev
->ethtool_ops
->get_drvinfo
) {
544 memset(&drvinfo
, 0, sizeof(drvinfo
));
545 netdev
->ethtool_ops
->get_drvinfo(netdev
, &drvinfo
);
547 if (strcmp(drvinfo
.driver
, "bnxt_en"))
554 static struct bnxt_re_dev
*bnxt_re_from_netdev(struct net_device
*netdev
)
556 struct ib_device
*ibdev
=
557 ib_device_get_by_netdev(netdev
, RDMA_DRIVER_BNXT_RE
);
561 return container_of(ibdev
, struct bnxt_re_dev
, ibdev
);
564 static void bnxt_re_dev_unprobe(struct net_device
*netdev
,
565 struct bnxt_en_dev
*en_dev
)
568 module_put(en_dev
->pdev
->driver
->driver
.owner
);
571 static struct bnxt_en_dev
*bnxt_re_dev_probe(struct net_device
*netdev
)
573 struct bnxt
*bp
= netdev_priv(netdev
);
574 struct bnxt_en_dev
*en_dev
;
575 struct pci_dev
*pdev
;
577 /* Call bnxt_en's RoCE probe via indirect API */
579 return ERR_PTR(-EINVAL
);
581 en_dev
= bp
->ulp_probe(netdev
);
587 return ERR_PTR(-EINVAL
);
589 if (!(en_dev
->flags
& BNXT_EN_FLAG_ROCE_CAP
)) {
591 "%s: probe error: RoCE is not supported on this device",
592 ROCE_DRV_MODULE_NAME
);
593 return ERR_PTR(-ENODEV
);
596 /* Bump net device reference count */
597 if (!try_module_get(pdev
->driver
->driver
.owner
))
598 return ERR_PTR(-ENODEV
);
605 static ssize_t
hw_rev_show(struct device
*device
, struct device_attribute
*attr
,
608 struct bnxt_re_dev
*rdev
=
609 rdma_device_to_drv_device(device
, struct bnxt_re_dev
, ibdev
);
611 return scnprintf(buf
, PAGE_SIZE
, "0x%x\n", rdev
->en_dev
->pdev
->vendor
);
613 static DEVICE_ATTR_RO(hw_rev
);
615 static ssize_t
hca_type_show(struct device
*device
,
616 struct device_attribute
*attr
, char *buf
)
618 struct bnxt_re_dev
*rdev
=
619 rdma_device_to_drv_device(device
, struct bnxt_re_dev
, ibdev
);
621 return scnprintf(buf
, PAGE_SIZE
, "%s\n", rdev
->ibdev
.node_desc
);
623 static DEVICE_ATTR_RO(hca_type
);
625 static struct attribute
*bnxt_re_attributes
[] = {
626 &dev_attr_hw_rev
.attr
,
627 &dev_attr_hca_type
.attr
,
631 static const struct attribute_group bnxt_re_dev_attr_group
= {
632 .attrs
= bnxt_re_attributes
,
635 static const struct ib_device_ops bnxt_re_dev_ops
= {
636 .owner
= THIS_MODULE
,
637 .driver_id
= RDMA_DRIVER_BNXT_RE
,
638 .uverbs_abi_ver
= BNXT_RE_ABI_VERSION
,
640 .add_gid
= bnxt_re_add_gid
,
641 .alloc_hw_stats
= bnxt_re_ib_alloc_hw_stats
,
642 .alloc_mr
= bnxt_re_alloc_mr
,
643 .alloc_pd
= bnxt_re_alloc_pd
,
644 .alloc_ucontext
= bnxt_re_alloc_ucontext
,
645 .create_ah
= bnxt_re_create_ah
,
646 .create_cq
= bnxt_re_create_cq
,
647 .create_qp
= bnxt_re_create_qp
,
648 .create_srq
= bnxt_re_create_srq
,
649 .dealloc_driver
= bnxt_re_dealloc_driver
,
650 .dealloc_pd
= bnxt_re_dealloc_pd
,
651 .dealloc_ucontext
= bnxt_re_dealloc_ucontext
,
652 .del_gid
= bnxt_re_del_gid
,
653 .dereg_mr
= bnxt_re_dereg_mr
,
654 .destroy_ah
= bnxt_re_destroy_ah
,
655 .destroy_cq
= bnxt_re_destroy_cq
,
656 .destroy_qp
= bnxt_re_destroy_qp
,
657 .destroy_srq
= bnxt_re_destroy_srq
,
658 .get_dev_fw_str
= bnxt_re_query_fw_str
,
659 .get_dma_mr
= bnxt_re_get_dma_mr
,
660 .get_hw_stats
= bnxt_re_ib_get_hw_stats
,
661 .get_link_layer
= bnxt_re_get_link_layer
,
662 .get_port_immutable
= bnxt_re_get_port_immutable
,
663 .map_mr_sg
= bnxt_re_map_mr_sg
,
664 .mmap
= bnxt_re_mmap
,
665 .modify_ah
= bnxt_re_modify_ah
,
666 .modify_qp
= bnxt_re_modify_qp
,
667 .modify_srq
= bnxt_re_modify_srq
,
668 .poll_cq
= bnxt_re_poll_cq
,
669 .post_recv
= bnxt_re_post_recv
,
670 .post_send
= bnxt_re_post_send
,
671 .post_srq_recv
= bnxt_re_post_srq_recv
,
672 .query_ah
= bnxt_re_query_ah
,
673 .query_device
= bnxt_re_query_device
,
674 .query_pkey
= bnxt_re_query_pkey
,
675 .query_port
= bnxt_re_query_port
,
676 .query_qp
= bnxt_re_query_qp
,
677 .query_srq
= bnxt_re_query_srq
,
678 .reg_user_mr
= bnxt_re_reg_user_mr
,
679 .req_notify_cq
= bnxt_re_req_notify_cq
,
680 INIT_RDMA_OBJ_SIZE(ib_ah
, bnxt_re_ah
, ib_ah
),
681 INIT_RDMA_OBJ_SIZE(ib_cq
, bnxt_re_cq
, ib_cq
),
682 INIT_RDMA_OBJ_SIZE(ib_pd
, bnxt_re_pd
, ib_pd
),
683 INIT_RDMA_OBJ_SIZE(ib_srq
, bnxt_re_srq
, ib_srq
),
684 INIT_RDMA_OBJ_SIZE(ib_ucontext
, bnxt_re_ucontext
, ib_uctx
),
687 static int bnxt_re_register_ib(struct bnxt_re_dev
*rdev
)
689 struct ib_device
*ibdev
= &rdev
->ibdev
;
693 ibdev
->node_type
= RDMA_NODE_IB_CA
;
694 strlcpy(ibdev
->node_desc
, BNXT_RE_DESC
" HCA",
695 strlen(BNXT_RE_DESC
) + 5);
696 ibdev
->phys_port_cnt
= 1;
698 bnxt_qplib_get_guid(rdev
->netdev
->dev_addr
, (u8
*)&ibdev
->node_guid
);
700 ibdev
->num_comp_vectors
= rdev
->num_msix
- 1;
701 ibdev
->dev
.parent
= &rdev
->en_dev
->pdev
->dev
;
702 ibdev
->local_dma_lkey
= BNXT_QPLIB_RSVD_LKEY
;
705 ibdev
->uverbs_cmd_mask
=
706 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
707 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
708 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
709 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
710 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
711 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
712 (1ull << IB_USER_VERBS_CMD_REREG_MR
) |
713 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
714 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
715 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
716 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ
) |
717 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
718 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
719 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
720 (1ull << IB_USER_VERBS_CMD_QUERY_QP
) |
721 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
722 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ
) |
723 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ
) |
724 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ
) |
725 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ
) |
726 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
727 (1ull << IB_USER_VERBS_CMD_MODIFY_AH
) |
728 (1ull << IB_USER_VERBS_CMD_QUERY_AH
) |
729 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
);
730 /* POLL_CQ and REQ_NOTIFY_CQ is directly handled in libbnxt_re */
733 rdma_set_device_sysfs_group(ibdev
, &bnxt_re_dev_attr_group
);
734 ib_set_device_ops(ibdev
, &bnxt_re_dev_ops
);
735 ret
= ib_device_set_netdev(&rdev
->ibdev
, rdev
->netdev
, 1);
739 return ib_register_device(ibdev
, "bnxt_re%d");
742 static void bnxt_re_dev_remove(struct bnxt_re_dev
*rdev
)
744 dev_put(rdev
->netdev
);
746 mutex_lock(&bnxt_re_dev_lock
);
747 list_del_rcu(&rdev
->list
);
748 mutex_unlock(&bnxt_re_dev_lock
);
753 static struct bnxt_re_dev
*bnxt_re_dev_add(struct net_device
*netdev
,
754 struct bnxt_en_dev
*en_dev
)
756 struct bnxt_re_dev
*rdev
;
758 /* Allocate bnxt_re_dev instance here */
759 rdev
= ib_alloc_device(bnxt_re_dev
, ibdev
);
761 ibdev_err(NULL
, "%s: bnxt_re_dev allocation failure!",
762 ROCE_DRV_MODULE_NAME
);
766 rdev
->netdev
= netdev
;
767 dev_hold(rdev
->netdev
);
768 rdev
->en_dev
= en_dev
;
769 rdev
->id
= rdev
->en_dev
->pdev
->devfn
;
770 INIT_LIST_HEAD(&rdev
->qp_list
);
771 mutex_init(&rdev
->qp_lock
);
772 atomic_set(&rdev
->qp_count
, 0);
773 atomic_set(&rdev
->cq_count
, 0);
774 atomic_set(&rdev
->srq_count
, 0);
775 atomic_set(&rdev
->mr_count
, 0);
776 atomic_set(&rdev
->mw_count
, 0);
777 rdev
->cosq
[0] = 0xFFFF;
778 rdev
->cosq
[1] = 0xFFFF;
780 mutex_lock(&bnxt_re_dev_lock
);
781 list_add_tail_rcu(&rdev
->list
, &bnxt_re_dev_list
);
782 mutex_unlock(&bnxt_re_dev_lock
);
786 static int bnxt_re_handle_unaffi_async_event(struct creq_func_event
789 switch (unaffi_async
->event
) {
790 case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR
:
792 case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR
:
794 case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR
:
796 case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR
:
798 case CREQ_FUNC_EVENT_EVENT_CQ_ERROR
:
800 case CREQ_FUNC_EVENT_EVENT_TQM_ERROR
:
802 case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR
:
804 case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR
:
806 case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR
:
808 case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR
:
810 case CREQ_FUNC_EVENT_EVENT_TIM_ERROR
:
818 static int bnxt_re_handle_qp_async_event(struct creq_qp_event
*qp_event
,
819 struct bnxt_re_qp
*qp
)
821 struct ib_event event
;
824 if (qp
->qplib_qp
.state
== CMDQ_MODIFY_QP_NEW_STATE_ERR
&&
825 rdma_is_kernel_res(&qp
->ib_qp
.res
)) {
826 flags
= bnxt_re_lock_cqs(qp
);
827 bnxt_qplib_add_flush_qp(&qp
->qplib_qp
);
828 bnxt_re_unlock_cqs(qp
, flags
);
831 memset(&event
, 0, sizeof(event
));
832 if (qp
->qplib_qp
.srq
) {
833 event
.device
= &qp
->rdev
->ibdev
;
834 event
.element
.qp
= &qp
->ib_qp
;
835 event
.event
= IB_EVENT_QP_LAST_WQE_REACHED
;
838 if (event
.device
&& qp
->ib_qp
.event_handler
)
839 qp
->ib_qp
.event_handler(&event
, qp
->ib_qp
.qp_context
);
844 static int bnxt_re_handle_affi_async_event(struct creq_qp_event
*affi_async
,
851 return rc
; /* QP was already dead, still return success */
853 event
= affi_async
->event
;
854 if (event
== CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION
) {
855 struct bnxt_qplib_qp
*lib_qp
= obj
;
856 struct bnxt_re_qp
*qp
= container_of(lib_qp
, struct bnxt_re_qp
,
858 rc
= bnxt_re_handle_qp_async_event(affi_async
, qp
);
863 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw
*rcfw
,
864 void *aeqe
, void *obj
)
866 struct creq_qp_event
*affi_async
;
867 struct creq_func_event
*unaffi_async
;
871 type
= ((struct creq_base
*)aeqe
)->type
;
872 if (type
== CREQ_BASE_TYPE_FUNC_EVENT
) {
874 rc
= bnxt_re_handle_unaffi_async_event(unaffi_async
);
877 rc
= bnxt_re_handle_affi_async_event(affi_async
, obj
);
883 static int bnxt_re_srqn_handler(struct bnxt_qplib_nq
*nq
,
884 struct bnxt_qplib_srq
*handle
, u8 event
)
886 struct bnxt_re_srq
*srq
= container_of(handle
, struct bnxt_re_srq
,
888 struct ib_event ib_event
;
892 ibdev_err(NULL
, "%s: SRQ is NULL, SRQN not handled",
893 ROCE_DRV_MODULE_NAME
);
897 ib_event
.device
= &srq
->rdev
->ibdev
;
898 ib_event
.element
.srq
= &srq
->ib_srq
;
899 if (event
== NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT
)
900 ib_event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
902 ib_event
.event
= IB_EVENT_SRQ_ERR
;
904 if (srq
->ib_srq
.event_handler
) {
905 /* Lock event_handler? */
906 (*srq
->ib_srq
.event_handler
)(&ib_event
,
907 srq
->ib_srq
.srq_context
);
913 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq
*nq
,
914 struct bnxt_qplib_cq
*handle
)
916 struct bnxt_re_cq
*cq
= container_of(handle
, struct bnxt_re_cq
,
920 ibdev_err(NULL
, "%s: CQ is NULL, CQN not handled",
921 ROCE_DRV_MODULE_NAME
);
924 if (cq
->ib_cq
.comp_handler
) {
925 /* Lock comp_handler? */
926 (*cq
->ib_cq
.comp_handler
)(&cq
->ib_cq
, cq
->ib_cq
.cq_context
);
932 #define BNXT_RE_GEN_P5_PF_NQ_DB 0x10000
933 #define BNXT_RE_GEN_P5_VF_NQ_DB 0x4000
934 static u32
bnxt_re_get_nqdb_offset(struct bnxt_re_dev
*rdev
, u16 indx
)
936 return bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
) ?
937 (rdev
->is_virtfn
? BNXT_RE_GEN_P5_VF_NQ_DB
:
938 BNXT_RE_GEN_P5_PF_NQ_DB
) :
939 rdev
->msix_entries
[indx
].db_offset
;
942 static void bnxt_re_cleanup_res(struct bnxt_re_dev
*rdev
)
946 for (i
= 1; i
< rdev
->num_msix
; i
++)
947 bnxt_qplib_disable_nq(&rdev
->nq
[i
- 1]);
949 if (rdev
->qplib_res
.rcfw
)
950 bnxt_qplib_cleanup_res(&rdev
->qplib_res
);
953 static int bnxt_re_init_res(struct bnxt_re_dev
*rdev
)
955 int num_vec_enabled
= 0;
959 bnxt_qplib_init_res(&rdev
->qplib_res
);
961 for (i
= 1; i
< rdev
->num_msix
; i
++) {
962 db_offt
= bnxt_re_get_nqdb_offset(rdev
, i
);
963 rc
= bnxt_qplib_enable_nq(rdev
->en_dev
->pdev
, &rdev
->nq
[i
- 1],
964 i
- 1, rdev
->msix_entries
[i
].vector
,
965 db_offt
, &bnxt_re_cqn_handler
,
966 &bnxt_re_srqn_handler
);
968 ibdev_err(&rdev
->ibdev
,
969 "Failed to enable NQ with rc = 0x%x", rc
);
976 for (i
= num_vec_enabled
; i
>= 0; i
--)
977 bnxt_qplib_disable_nq(&rdev
->nq
[i
]);
981 static void bnxt_re_free_nq_res(struct bnxt_re_dev
*rdev
)
986 for (i
= 0; i
< rdev
->num_msix
- 1; i
++) {
987 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
988 bnxt_re_net_ring_free(rdev
, rdev
->nq
[i
].ring_id
, type
);
989 bnxt_qplib_free_nq(&rdev
->nq
[i
]);
990 rdev
->nq
[i
].res
= NULL
;
994 static void bnxt_re_free_res(struct bnxt_re_dev
*rdev
)
996 bnxt_re_free_nq_res(rdev
);
998 if (rdev
->qplib_res
.dpi_tbl
.max
) {
999 bnxt_qplib_dealloc_dpi(&rdev
->qplib_res
,
1000 &rdev
->qplib_res
.dpi_tbl
,
1001 &rdev
->dpi_privileged
);
1003 if (rdev
->qplib_res
.rcfw
) {
1004 bnxt_qplib_free_res(&rdev
->qplib_res
);
1005 rdev
->qplib_res
.rcfw
= NULL
;
1009 static int bnxt_re_alloc_res(struct bnxt_re_dev
*rdev
)
1011 struct bnxt_re_ring_attr rattr
= {};
1012 struct bnxt_qplib_ctx
*qplib_ctx
;
1013 int num_vec_created
= 0;
1017 /* Configure and allocate resources for qplib */
1018 rdev
->qplib_res
.rcfw
= &rdev
->rcfw
;
1019 rc
= bnxt_qplib_get_dev_attr(&rdev
->rcfw
, &rdev
->dev_attr
,
1024 rc
= bnxt_qplib_alloc_res(&rdev
->qplib_res
, rdev
->en_dev
->pdev
,
1025 rdev
->netdev
, &rdev
->dev_attr
);
1029 rc
= bnxt_qplib_alloc_dpi(&rdev
->qplib_res
.dpi_tbl
,
1030 &rdev
->dpi_privileged
,
1035 qplib_ctx
= &rdev
->qplib_ctx
;
1036 for (i
= 0; i
< rdev
->num_msix
- 1; i
++) {
1037 struct bnxt_qplib_nq
*nq
;
1040 nq
->hwq
.max_elements
= (qplib_ctx
->cq_count
+
1041 qplib_ctx
->srqc_count
+ 2);
1042 rc
= bnxt_qplib_alloc_nq(&rdev
->qplib_res
, &rdev
->nq
[i
]);
1044 ibdev_err(&rdev
->ibdev
, "Alloc Failed NQ%d rc:%#x",
1048 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
1049 rattr
.dma_arr
= nq
->hwq
.pbl
[PBL_LVL_0
].pg_map_arr
;
1050 rattr
.pages
= nq
->hwq
.pbl
[rdev
->nq
[i
].hwq
.level
].pg_count
;
1052 rattr
.mode
= RING_ALLOC_REQ_INT_MODE_MSIX
;
1053 rattr
.depth
= BNXT_QPLIB_NQE_MAX_CNT
- 1;
1054 rattr
.lrid
= rdev
->msix_entries
[i
+ 1].ring_idx
;
1055 rc
= bnxt_re_net_ring_alloc(rdev
, &rattr
, &nq
->ring_id
);
1057 ibdev_err(&rdev
->ibdev
,
1058 "Failed to allocate NQ fw id with rc = 0x%x",
1060 bnxt_qplib_free_nq(&rdev
->nq
[i
]);
1067 for (i
= num_vec_created
- 1; i
>= 0; i
--) {
1068 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
1069 bnxt_re_net_ring_free(rdev
, rdev
->nq
[i
].ring_id
, type
);
1070 bnxt_qplib_free_nq(&rdev
->nq
[i
]);
1072 bnxt_qplib_dealloc_dpi(&rdev
->qplib_res
,
1073 &rdev
->qplib_res
.dpi_tbl
,
1074 &rdev
->dpi_privileged
);
1076 bnxt_qplib_free_res(&rdev
->qplib_res
);
1079 rdev
->qplib_res
.rcfw
= NULL
;
1083 static void bnxt_re_dispatch_event(struct ib_device
*ibdev
, struct ib_qp
*qp
,
1084 u8 port_num
, enum ib_event_type event
)
1086 struct ib_event ib_event
;
1088 ib_event
.device
= ibdev
;
1090 ib_event
.element
.qp
= qp
;
1091 ib_event
.event
= event
;
1092 if (qp
->event_handler
)
1093 qp
->event_handler(&ib_event
, qp
->qp_context
);
1096 ib_event
.element
.port_num
= port_num
;
1097 ib_event
.event
= event
;
1098 ib_dispatch_event(&ib_event
);
1102 #define HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN 0x02
1103 static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev
*rdev
, u8 dir
,
1106 struct hwrm_queue_pri2cos_qcfg_input req
= {0};
1107 struct bnxt
*bp
= netdev_priv(rdev
->netdev
);
1108 struct hwrm_queue_pri2cos_qcfg_output resp
;
1109 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
1110 struct bnxt_fw_msg fw_msg
;
1112 u8
*qcfgmap
, *tmp_map
;
1118 memset(&fw_msg
, 0, sizeof(fw_msg
));
1119 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
,
1120 HWRM_QUEUE_PRI2COS_QCFG
, -1, -1);
1121 flags
|= (dir
& 0x01);
1122 flags
|= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN
;
1123 req
.flags
= cpu_to_le32(flags
);
1124 req
.port_id
= bp
->pf
.port_id
;
1126 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&resp
,
1127 sizeof(resp
), DFLT_HWRM_CMD_TIMEOUT
);
1128 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
1132 if (resp
.queue_cfg_info
) {
1133 ibdev_warn(&rdev
->ibdev
,
1134 "Asymmetric cos queue configuration detected");
1135 ibdev_warn(&rdev
->ibdev
,
1136 " on device, QoS may not be fully functional\n");
1138 qcfgmap
= &resp
.pri0_cos_queue_id
;
1139 tmp_map
= (u8
*)cid_map
;
1140 for (i
= 0; i
< IEEE_8021QAZ_MAX_TCS
; i
++)
1141 tmp_map
[i
] = qcfgmap
[i
];
1146 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev
*rdev
,
1147 struct bnxt_re_qp
*qp
)
1149 return (qp
->ib_qp
.qp_type
== IB_QPT_GSI
) ||
1150 (qp
== rdev
->gsi_ctx
.gsi_sqp
);
1153 static void bnxt_re_dev_stop(struct bnxt_re_dev
*rdev
)
1155 int mask
= IB_QP_STATE
;
1156 struct ib_qp_attr qp_attr
;
1157 struct bnxt_re_qp
*qp
;
1159 qp_attr
.qp_state
= IB_QPS_ERR
;
1160 mutex_lock(&rdev
->qp_lock
);
1161 list_for_each_entry(qp
, &rdev
->qp_list
, list
) {
1162 /* Modify the state of all QPs except QP1/Shadow QP */
1163 if (!bnxt_re_is_qp1_or_shadow_qp(rdev
, qp
)) {
1164 if (qp
->qplib_qp
.state
!=
1165 CMDQ_MODIFY_QP_NEW_STATE_RESET
&&
1166 qp
->qplib_qp
.state
!=
1167 CMDQ_MODIFY_QP_NEW_STATE_ERR
) {
1168 bnxt_re_dispatch_event(&rdev
->ibdev
, &qp
->ib_qp
,
1169 1, IB_EVENT_QP_FATAL
);
1170 bnxt_re_modify_qp(&qp
->ib_qp
, &qp_attr
, mask
,
1175 mutex_unlock(&rdev
->qp_lock
);
1178 static int bnxt_re_update_gid(struct bnxt_re_dev
*rdev
)
1180 struct bnxt_qplib_sgid_tbl
*sgid_tbl
= &rdev
->qplib_res
.sgid_tbl
;
1181 struct bnxt_qplib_gid gid
;
1185 if (!ib_device_try_get(&rdev
->ibdev
))
1189 ibdev_err(&rdev
->ibdev
, "QPLIB: SGID table not allocated");
1194 for (index
= 0; index
< sgid_tbl
->active
; index
++) {
1195 gid_idx
= sgid_tbl
->hw_id
[index
];
1197 if (!memcmp(&sgid_tbl
->tbl
[index
], &bnxt_qplib_gid_zero
,
1198 sizeof(bnxt_qplib_gid_zero
)))
1200 /* need to modify the VLAN enable setting of non VLAN GID only
1201 * as setting is done for VLAN GID while adding GID
1203 if (sgid_tbl
->vlan
[index
])
1206 memcpy(&gid
, &sgid_tbl
->tbl
[index
], sizeof(gid
));
1208 rc
= bnxt_qplib_update_sgid(sgid_tbl
, &gid
, gid_idx
,
1209 rdev
->qplib_res
.netdev
->dev_addr
);
1212 ib_device_put(&rdev
->ibdev
);
1216 static u32
bnxt_re_get_priority_mask(struct bnxt_re_dev
*rdev
)
1218 u32 prio_map
= 0, tmp_map
= 0;
1219 struct net_device
*netdev
;
1222 netdev
= rdev
->netdev
;
1224 memset(&app
, 0, sizeof(app
));
1225 app
.selector
= IEEE_8021QAZ_APP_SEL_ETHERTYPE
;
1226 app
.protocol
= ETH_P_IBOE
;
1227 tmp_map
= dcb_ieee_getapp_mask(netdev
, &app
);
1230 app
.selector
= IEEE_8021QAZ_APP_SEL_DGRAM
;
1231 app
.protocol
= ROCE_V2_UDP_DPORT
;
1232 tmp_map
= dcb_ieee_getapp_mask(netdev
, &app
);
1233 prio_map
|= tmp_map
;
1238 static void bnxt_re_parse_cid_map(u8 prio_map
, u8
*cid_map
, u16
*cosq
)
1243 for (prio
= 0, id
= 0; prio
< 8; prio
++) {
1244 if (prio_map
& (1 << prio
)) {
1245 cosq
[id
] = cid_map
[prio
];
1247 if (id
== 2) /* Max 2 tcs supported */
1253 static int bnxt_re_setup_qos(struct bnxt_re_dev
*rdev
)
1259 /* Get priority for roce */
1260 prio_map
= bnxt_re_get_priority_mask(rdev
);
1262 if (prio_map
== rdev
->cur_prio_map
)
1264 rdev
->cur_prio_map
= prio_map
;
1265 /* Get cosq id for this priority */
1266 rc
= bnxt_re_query_hwrm_pri2cos(rdev
, 0, &cid_map
);
1268 ibdev_warn(&rdev
->ibdev
, "no cos for p_mask %x\n", prio_map
);
1271 /* Parse CoS IDs for app priority */
1272 bnxt_re_parse_cid_map(prio_map
, (u8
*)&cid_map
, rdev
->cosq
);
1275 rc
= bnxt_qplib_map_tc2cos(&rdev
->qplib_res
, rdev
->cosq
);
1277 ibdev_warn(&rdev
->ibdev
, "no tc for cos{%x, %x}\n",
1278 rdev
->cosq
[0], rdev
->cosq
[1]);
1282 /* Actual priorities are not programmed as they are already
1283 * done by L2 driver; just enable or disable priority vlan tagging
1285 if ((prio_map
== 0 && rdev
->qplib_res
.prio
) ||
1286 (prio_map
!= 0 && !rdev
->qplib_res
.prio
)) {
1287 rdev
->qplib_res
.prio
= prio_map
? true : false;
1289 bnxt_re_update_gid(rdev
);
1295 static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev
*rdev
)
1297 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
1298 struct hwrm_ver_get_output resp
= {0};
1299 struct hwrm_ver_get_input req
= {0};
1300 struct bnxt_fw_msg fw_msg
;
1303 memset(&fw_msg
, 0, sizeof(fw_msg
));
1304 bnxt_re_init_hwrm_hdr(rdev
, (void *)&req
,
1305 HWRM_VER_GET
, -1, -1);
1306 req
.hwrm_intf_maj
= HWRM_VERSION_MAJOR
;
1307 req
.hwrm_intf_min
= HWRM_VERSION_MINOR
;
1308 req
.hwrm_intf_upd
= HWRM_VERSION_UPDATE
;
1309 bnxt_re_fill_fw_msg(&fw_msg
, (void *)&req
, sizeof(req
), (void *)&resp
,
1310 sizeof(resp
), DFLT_HWRM_CMD_TIMEOUT
);
1311 rc
= en_dev
->en_ops
->bnxt_send_fw_msg(en_dev
, BNXT_ROCE_ULP
, &fw_msg
);
1313 ibdev_err(&rdev
->ibdev
, "Failed to query HW version, rc = 0x%x",
1317 rdev
->qplib_ctx
.hwrm_intf_ver
=
1318 (u64
)le16_to_cpu(resp
.hwrm_intf_major
) << 48 |
1319 (u64
)le16_to_cpu(resp
.hwrm_intf_minor
) << 32 |
1320 (u64
)le16_to_cpu(resp
.hwrm_intf_build
) << 16 |
1321 le16_to_cpu(resp
.hwrm_intf_patch
);
1324 static int bnxt_re_ib_init(struct bnxt_re_dev
*rdev
)
1329 /* Register ib dev */
1330 rc
= bnxt_re_register_ib(rdev
);
1332 pr_err("Failed to register with IB: %#x\n", rc
);
1335 dev_info(rdev_to_dev(rdev
), "Device registered successfully");
1336 ib_get_eth_speed(&rdev
->ibdev
, 1, &rdev
->active_speed
,
1337 &rdev
->active_width
);
1338 set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS
, &rdev
->flags
);
1340 event
= netif_running(rdev
->netdev
) && netif_carrier_ok(rdev
->netdev
) ?
1341 IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
1343 bnxt_re_dispatch_event(&rdev
->ibdev
, NULL
, 1, event
);
1348 static void bnxt_re_dev_uninit(struct bnxt_re_dev
*rdev
)
1353 if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG
, &rdev
->flags
))
1354 cancel_delayed_work_sync(&rdev
->worker
);
1356 if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED
,
1358 bnxt_re_cleanup_res(rdev
);
1359 if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED
, &rdev
->flags
))
1360 bnxt_re_free_res(rdev
);
1362 if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN
, &rdev
->flags
)) {
1363 rc
= bnxt_qplib_deinit_rcfw(&rdev
->rcfw
);
1365 ibdev_warn(&rdev
->ibdev
,
1366 "Failed to deinitialize RCFW: %#x", rc
);
1367 bnxt_re_net_stats_ctx_free(rdev
, rdev
->qplib_ctx
.stats
.fw_id
);
1368 bnxt_qplib_free_ctx(&rdev
->qplib_res
, &rdev
->qplib_ctx
);
1369 bnxt_qplib_disable_rcfw_channel(&rdev
->rcfw
);
1370 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
1371 bnxt_re_net_ring_free(rdev
, rdev
->rcfw
.creq
.ring_id
, type
);
1372 bnxt_qplib_free_rcfw_channel(&rdev
->rcfw
);
1374 if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX
, &rdev
->flags
)) {
1375 rc
= bnxt_re_free_msix(rdev
);
1377 ibdev_warn(&rdev
->ibdev
,
1378 "Failed to free MSI-X vectors: %#x", rc
);
1381 bnxt_re_destroy_chip_ctx(rdev
);
1382 if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED
, &rdev
->flags
)) {
1383 rc
= bnxt_re_unregister_netdev(rdev
);
1385 ibdev_warn(&rdev
->ibdev
,
1386 "Failed to unregister with netdev: %#x", rc
);
1390 /* worker thread for polling periodic events. Now used for QoS programming*/
1391 static void bnxt_re_worker(struct work_struct
*work
)
1393 struct bnxt_re_dev
*rdev
= container_of(work
, struct bnxt_re_dev
,
1396 bnxt_re_setup_qos(rdev
);
1397 schedule_delayed_work(&rdev
->worker
, msecs_to_jiffies(30000));
1400 static int bnxt_re_dev_init(struct bnxt_re_dev
*rdev
, u8 wqe_mode
)
1402 struct bnxt_qplib_creq_ctx
*creq
;
1403 struct bnxt_re_ring_attr rattr
;
1409 /* Registered a new RoCE device instance to netdev */
1410 memset(&rattr
, 0, sizeof(rattr
));
1411 rc
= bnxt_re_register_netdev(rdev
);
1414 ibdev_err(&rdev
->ibdev
,
1415 "Failed to register with netedev: %#x\n", rc
);
1418 set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED
, &rdev
->flags
);
1420 rc
= bnxt_re_setup_chip_ctx(rdev
, wqe_mode
);
1422 ibdev_err(&rdev
->ibdev
, "Failed to get chip context\n");
1426 /* Check whether VF or PF */
1427 bnxt_re_get_sriov_func_type(rdev
);
1429 rc
= bnxt_re_request_msix(rdev
);
1431 ibdev_err(&rdev
->ibdev
,
1432 "Failed to get MSI-X vectors: %#x\n", rc
);
1436 set_bit(BNXT_RE_FLAG_GOT_MSIX
, &rdev
->flags
);
1438 bnxt_re_query_hwrm_intf_version(rdev
);
1440 /* Establish RCFW Communication Channel to initialize the context
1441 * memory for the function and all child VFs
1443 rc
= bnxt_qplib_alloc_rcfw_channel(&rdev
->qplib_res
, &rdev
->rcfw
,
1445 BNXT_RE_MAX_QPC_COUNT
);
1447 ibdev_err(&rdev
->ibdev
,
1448 "Failed to allocate RCFW Channel: %#x\n", rc
);
1452 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
1453 creq
= &rdev
->rcfw
.creq
;
1454 rattr
.dma_arr
= creq
->hwq
.pbl
[PBL_LVL_0
].pg_map_arr
;
1455 rattr
.pages
= creq
->hwq
.pbl
[creq
->hwq
.level
].pg_count
;
1457 rattr
.mode
= RING_ALLOC_REQ_INT_MODE_MSIX
;
1458 rattr
.depth
= BNXT_QPLIB_CREQE_MAX_CNT
- 1;
1459 rattr
.lrid
= rdev
->msix_entries
[BNXT_RE_AEQ_IDX
].ring_idx
;
1460 rc
= bnxt_re_net_ring_alloc(rdev
, &rattr
, &creq
->ring_id
);
1462 ibdev_err(&rdev
->ibdev
, "Failed to allocate CREQ: %#x\n", rc
);
1465 db_offt
= bnxt_re_get_nqdb_offset(rdev
, BNXT_RE_AEQ_IDX
);
1466 vid
= rdev
->msix_entries
[BNXT_RE_AEQ_IDX
].vector
;
1467 rc
= bnxt_qplib_enable_rcfw_channel(&rdev
->rcfw
,
1468 vid
, db_offt
, rdev
->is_virtfn
,
1469 &bnxt_re_aeq_handler
);
1471 ibdev_err(&rdev
->ibdev
, "Failed to enable RCFW channel: %#x\n",
1476 rc
= bnxt_qplib_get_dev_attr(&rdev
->rcfw
, &rdev
->dev_attr
,
1481 bnxt_re_set_resource_limits(rdev
);
1483 rc
= bnxt_qplib_alloc_ctx(&rdev
->qplib_res
, &rdev
->qplib_ctx
, 0,
1484 bnxt_qplib_is_chip_gen_p5(rdev
->chip_ctx
));
1486 ibdev_err(&rdev
->ibdev
,
1487 "Failed to allocate QPLIB context: %#x\n", rc
);
1490 rc
= bnxt_re_net_stats_ctx_alloc(rdev
,
1491 rdev
->qplib_ctx
.stats
.dma_map
,
1492 &rdev
->qplib_ctx
.stats
.fw_id
);
1494 ibdev_err(&rdev
->ibdev
,
1495 "Failed to allocate stats context: %#x\n", rc
);
1499 rc
= bnxt_qplib_init_rcfw(&rdev
->rcfw
, &rdev
->qplib_ctx
,
1502 ibdev_err(&rdev
->ibdev
,
1503 "Failed to initialize RCFW: %#x\n", rc
);
1506 set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN
, &rdev
->flags
);
1508 /* Resources based on the 'new' device caps */
1509 rc
= bnxt_re_alloc_res(rdev
);
1511 ibdev_err(&rdev
->ibdev
,
1512 "Failed to allocate resources: %#x\n", rc
);
1515 set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED
, &rdev
->flags
);
1516 rc
= bnxt_re_init_res(rdev
);
1518 ibdev_err(&rdev
->ibdev
,
1519 "Failed to initialize resources: %#x\n", rc
);
1523 set_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED
, &rdev
->flags
);
1525 if (!rdev
->is_virtfn
) {
1526 rc
= bnxt_re_setup_qos(rdev
);
1528 ibdev_info(&rdev
->ibdev
,
1529 "RoCE priority not yet configured\n");
1531 INIT_DELAYED_WORK(&rdev
->worker
, bnxt_re_worker
);
1532 set_bit(BNXT_RE_FLAG_QOS_WORK_REG
, &rdev
->flags
);
1533 schedule_delayed_work(&rdev
->worker
, msecs_to_jiffies(30000));
1538 bnxt_re_net_stats_ctx_free(rdev
, rdev
->qplib_ctx
.stats
.fw_id
);
1540 bnxt_qplib_free_ctx(&rdev
->qplib_res
, &rdev
->qplib_ctx
);
1542 bnxt_qplib_disable_rcfw_channel(&rdev
->rcfw
);
1544 type
= bnxt_qplib_get_ring_type(rdev
->chip_ctx
);
1545 bnxt_re_net_ring_free(rdev
, rdev
->rcfw
.creq
.ring_id
, type
);
1547 bnxt_qplib_free_rcfw_channel(&rdev
->rcfw
);
1549 bnxt_re_dev_uninit(rdev
);
1554 static void bnxt_re_dev_unreg(struct bnxt_re_dev
*rdev
)
1556 struct bnxt_en_dev
*en_dev
= rdev
->en_dev
;
1557 struct net_device
*netdev
= rdev
->netdev
;
1559 bnxt_re_dev_remove(rdev
);
1562 bnxt_re_dev_unprobe(netdev
, en_dev
);
1565 static int bnxt_re_dev_reg(struct bnxt_re_dev
**rdev
, struct net_device
*netdev
)
1567 struct bnxt_en_dev
*en_dev
;
1570 if (!is_bnxt_re_dev(netdev
))
1573 en_dev
= bnxt_re_dev_probe(netdev
);
1574 if (IS_ERR(en_dev
)) {
1575 if (en_dev
!= ERR_PTR(-ENODEV
))
1576 ibdev_err(&(*rdev
)->ibdev
, "%s: Failed to probe\n",
1577 ROCE_DRV_MODULE_NAME
);
1578 rc
= PTR_ERR(en_dev
);
1581 *rdev
= bnxt_re_dev_add(netdev
, en_dev
);
1584 bnxt_re_dev_unprobe(netdev
, en_dev
);
1591 static void bnxt_re_remove_device(struct bnxt_re_dev
*rdev
)
1593 bnxt_re_dev_uninit(rdev
);
1594 pci_dev_put(rdev
->en_dev
->pdev
);
1595 bnxt_re_dev_unreg(rdev
);
1598 static int bnxt_re_add_device(struct bnxt_re_dev
**rdev
,
1599 struct net_device
*netdev
, u8 wqe_mode
)
1603 rc
= bnxt_re_dev_reg(rdev
, netdev
);
1607 pr_err("Failed to register with the device %s: %#x\n",
1612 pci_dev_get((*rdev
)->en_dev
->pdev
);
1613 rc
= bnxt_re_dev_init(*rdev
, wqe_mode
);
1615 pci_dev_put((*rdev
)->en_dev
->pdev
);
1616 bnxt_re_dev_unreg(*rdev
);
1622 static void bnxt_re_dealloc_driver(struct ib_device
*ib_dev
)
1624 struct bnxt_re_dev
*rdev
=
1625 container_of(ib_dev
, struct bnxt_re_dev
, ibdev
);
1627 dev_info(rdev_to_dev(rdev
), "Unregistering Device");
1630 bnxt_re_remove_device(rdev
);
1634 /* Handle all deferred netevents tasks */
1635 static void bnxt_re_task(struct work_struct
*work
)
1637 struct bnxt_re_work
*re_work
;
1638 struct bnxt_re_dev
*rdev
;
1641 re_work
= container_of(work
, struct bnxt_re_work
, work
);
1642 rdev
= re_work
->rdev
;
1644 if (re_work
->event
== NETDEV_REGISTER
) {
1645 rc
= bnxt_re_ib_init(rdev
);
1647 ibdev_err(&rdev
->ibdev
,
1648 "Failed to register with IB: %#x", rc
);
1650 bnxt_re_remove_device(rdev
);
1657 if (!ib_device_try_get(&rdev
->ibdev
))
1660 switch (re_work
->event
) {
1662 bnxt_re_dispatch_event(&rdev
->ibdev
, NULL
, 1,
1663 IB_EVENT_PORT_ACTIVE
);
1666 bnxt_re_dev_stop(rdev
);
1669 if (!netif_carrier_ok(rdev
->netdev
))
1670 bnxt_re_dev_stop(rdev
);
1671 else if (netif_carrier_ok(rdev
->netdev
))
1672 bnxt_re_dispatch_event(&rdev
->ibdev
, NULL
, 1,
1673 IB_EVENT_PORT_ACTIVE
);
1674 ib_get_eth_speed(&rdev
->ibdev
, 1, &rdev
->active_speed
,
1675 &rdev
->active_width
);
1680 ib_device_put(&rdev
->ibdev
);
1682 put_device(&rdev
->ibdev
.dev
);
1687 * "Notifier chain callback can be invoked for the same chain from
1688 * different CPUs at the same time".
1690 * For cases when the netdev is already present, our call to the
1691 * register_netdevice_notifier() will actually get the rtnl_lock()
1692 * before sending NETDEV_REGISTER and (if up) NETDEV_UP
1695 * But for cases when the netdev is not already present, the notifier
1696 * chain is subjected to be invoked from different CPUs simultaneously.
1698 * This is protected by the netdev_mutex.
1700 static int bnxt_re_netdev_event(struct notifier_block
*notifier
,
1701 unsigned long event
, void *ptr
)
1703 struct net_device
*real_dev
, *netdev
= netdev_notifier_info_to_dev(ptr
);
1704 struct bnxt_re_work
*re_work
;
1705 struct bnxt_re_dev
*rdev
;
1707 bool sch_work
= false;
1708 bool release
= true;
1710 real_dev
= rdma_vlan_dev_real_dev(netdev
);
1714 rdev
= bnxt_re_from_netdev(real_dev
);
1715 if (!rdev
&& event
!= NETDEV_REGISTER
)
1718 if (real_dev
!= netdev
)
1722 case NETDEV_REGISTER
:
1725 rc
= bnxt_re_add_device(&rdev
, real_dev
,
1726 BNXT_QPLIB_WQE_MODE_STATIC
);
1732 case NETDEV_UNREGISTER
:
1733 ib_unregister_device_queued(&rdev
->ibdev
);
1741 /* Allocate for the deferred task */
1742 re_work
= kzalloc(sizeof(*re_work
), GFP_ATOMIC
);
1744 get_device(&rdev
->ibdev
.dev
);
1745 re_work
->rdev
= rdev
;
1746 re_work
->event
= event
;
1747 re_work
->vlan_dev
= (real_dev
== netdev
?
1749 INIT_WORK(&re_work
->work
, bnxt_re_task
);
1750 queue_work(bnxt_re_wq
, &re_work
->work
);
1755 if (rdev
&& release
)
1756 ib_device_put(&rdev
->ibdev
);
1760 static struct notifier_block bnxt_re_netdev_notifier
= {
1761 .notifier_call
= bnxt_re_netdev_event
1764 static int __init
bnxt_re_mod_init(void)
1768 pr_info("%s: %s", ROCE_DRV_MODULE_NAME
, version
);
1770 bnxt_re_wq
= create_singlethread_workqueue("bnxt_re");
1774 INIT_LIST_HEAD(&bnxt_re_dev_list
);
1776 rc
= register_netdevice_notifier(&bnxt_re_netdev_notifier
);
1778 pr_err("%s: Cannot register to netdevice_notifier",
1779 ROCE_DRV_MODULE_NAME
);
1785 destroy_workqueue(bnxt_re_wq
);
1790 static void __exit
bnxt_re_mod_exit(void)
1792 struct bnxt_re_dev
*rdev
;
1794 unregister_netdevice_notifier(&bnxt_re_netdev_notifier
);
1796 destroy_workqueue(bnxt_re_wq
);
1797 list_for_each_entry(rdev
, &bnxt_re_dev_list
, list
) {
1798 /* VF device removal should be called before the removal
1799 * of PF device. Queue VFs unregister first, so that VFs
1800 * shall be removed before the PF during the call of
1801 * ib_unregister_driver.
1803 if (rdev
->is_virtfn
)
1804 ib_unregister_device(&rdev
->ibdev
);
1806 ib_unregister_driver(RDMA_DRIVER_BNXT_RE
);
1809 module_init(bnxt_re_mod_init
);
1810 module_exit(bnxt_re_mod_exit
);