1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Admin Function driver
4 * Copyright (C) 2018 Marvell.
8 #include <linux/module.h>
9 #include <linux/interrupt.h>
10 #include <linux/delay.h>
11 #include <linux/irq.h>
12 #include <linux/pci.h>
13 #include <linux/sysfs.h>
21 #include "rvu_trace.h"
22 #include "rvu_npc_hash.h"
24 #define DRV_NAME "rvu_af"
25 #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver"
27 static void rvu_set_msix_offset(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
28 struct rvu_block
*block
, int lf
);
29 static void rvu_clear_msix_offset(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
30 struct rvu_block
*block
, int lf
);
31 static void __rvu_flr_handler(struct rvu
*rvu
, u16 pcifunc
);
33 static int rvu_mbox_init(struct rvu
*rvu
, struct mbox_wq_info
*mw
,
35 void (mbox_handler
)(struct work_struct
*),
36 void (mbox_up_handler
)(struct work_struct
*));
42 /* Supported devices */
43 static const struct pci_device_id rvu_id_table
[] = {
44 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM
, PCI_DEVID_OCTEONTX2_RVU_AF
) },
45 { 0, } /* end of table */
48 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
49 MODULE_DESCRIPTION(DRV_STRING
);
50 MODULE_LICENSE("GPL v2");
51 MODULE_DEVICE_TABLE(pci
, rvu_id_table
);
53 static char *mkex_profile
; /* MKEX profile name */
54 module_param(mkex_profile
, charp
, 0000);
55 MODULE_PARM_DESC(mkex_profile
, "MKEX profile name string");
57 static char *kpu_profile
; /* KPU profile name */
58 module_param(kpu_profile
, charp
, 0000);
59 MODULE_PARM_DESC(kpu_profile
, "KPU profile name string");
61 static void rvu_setup_hw_capabilities(struct rvu
*rvu
)
63 struct rvu_hwinfo
*hw
= rvu
->hw
;
65 hw
->cap
.nix_tx_aggr_lvl
= NIX_TXSCH_LVL_TL1
;
66 hw
->cap
.nix_fixed_txschq_mapping
= false;
67 hw
->cap
.nix_shaping
= true;
68 hw
->cap
.nix_tx_link_bp
= true;
69 hw
->cap
.nix_rx_multicast
= true;
70 hw
->cap
.nix_shaper_toggle_wait
= false;
71 hw
->cap
.npc_hash_extract
= false;
72 hw
->cap
.npc_exact_match_enabled
= false;
75 if (is_rvu_pre_96xx_C0(rvu
)) {
76 hw
->cap
.nix_fixed_txschq_mapping
= true;
77 hw
->cap
.nix_txsch_per_cgx_lmac
= 4;
78 hw
->cap
.nix_txsch_per_lbk_lmac
= 132;
79 hw
->cap
.nix_txsch_per_sdp_lmac
= 76;
80 hw
->cap
.nix_shaping
= false;
81 hw
->cap
.nix_tx_link_bp
= false;
82 if (is_rvu_96xx_A0(rvu
) || is_rvu_95xx_A0(rvu
))
83 hw
->cap
.nix_rx_multicast
= false;
85 if (!is_rvu_pre_96xx_C0(rvu
))
86 hw
->cap
.nix_shaper_toggle_wait
= true;
88 if (!is_rvu_otx2(rvu
))
89 hw
->cap
.per_pf_mbox_regs
= true;
91 if (is_rvu_npc_hash_extract_en(rvu
))
92 hw
->cap
.npc_hash_extract
= true;
95 /* Poll a RVU block's register 'offset', for a 'zero'
96 * or 'nonzero' at bits specified by 'mask'
98 int rvu_poll_reg(struct rvu
*rvu
, u64 block
, u64 offset
, u64 mask
, bool zero
)
100 unsigned long timeout
= jiffies
+ usecs_to_jiffies(20000);
105 reg
= rvu
->afreg_base
+ ((block
<< 28) | offset
);
107 reg_val
= readq(reg
);
108 if (zero
&& !(reg_val
& mask
))
110 if (!zero
&& (reg_val
& mask
))
112 if (time_before(jiffies
, timeout
)) {
116 /* In scenarios where CPU is scheduled out before checking
117 * 'time_before' (above) and gets scheduled in such that
118 * jiffies are beyond timeout value, then check again if HW is
119 * done with the operation in the meantime.
128 int rvu_alloc_rsrc(struct rsrc_bmap
*rsrc
)
135 id
= find_first_zero_bit(rsrc
->bmap
, rsrc
->max
);
139 __set_bit(id
, rsrc
->bmap
);
144 int rvu_alloc_rsrc_contig(struct rsrc_bmap
*rsrc
, int nrsrc
)
151 start
= bitmap_find_next_zero_area(rsrc
->bmap
, rsrc
->max
, 0, nrsrc
, 0);
152 if (start
>= rsrc
->max
)
155 bitmap_set(rsrc
->bmap
, start
, nrsrc
);
159 void rvu_free_rsrc_contig(struct rsrc_bmap
*rsrc
, int nrsrc
, int start
)
163 if (start
>= rsrc
->max
)
166 bitmap_clear(rsrc
->bmap
, start
, nrsrc
);
169 bool rvu_rsrc_check_contig(struct rsrc_bmap
*rsrc
, int nrsrc
)
176 start
= bitmap_find_next_zero_area(rsrc
->bmap
, rsrc
->max
, 0, nrsrc
, 0);
177 if (start
>= rsrc
->max
)
183 void rvu_free_rsrc(struct rsrc_bmap
*rsrc
, int id
)
188 __clear_bit(id
, rsrc
->bmap
);
191 int rvu_rsrc_free_count(struct rsrc_bmap
*rsrc
)
198 used
= bitmap_weight(rsrc
->bmap
, rsrc
->max
);
199 return (rsrc
->max
- used
);
202 bool is_rsrc_free(struct rsrc_bmap
*rsrc
, int id
)
207 return !test_bit(id
, rsrc
->bmap
);
210 int rvu_alloc_bitmap(struct rsrc_bmap
*rsrc
)
212 rsrc
->bmap
= kcalloc(BITS_TO_LONGS(rsrc
->max
),
213 sizeof(long), GFP_KERNEL
);
219 void rvu_free_bitmap(struct rsrc_bmap
*rsrc
)
224 /* Get block LF's HW index from a PF_FUNC's block slot number */
225 int rvu_get_lf(struct rvu
*rvu
, struct rvu_block
*block
, u16 pcifunc
, u16 slot
)
230 mutex_lock(&rvu
->rsrc_lock
);
231 for (lf
= 0; lf
< block
->lf
.max
; lf
++) {
232 if (block
->fn_map
[lf
] == pcifunc
) {
234 mutex_unlock(&rvu
->rsrc_lock
);
240 mutex_unlock(&rvu
->rsrc_lock
);
244 /* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.
245 * Some silicon variants of OcteonTX2 supports
246 * multiple blocks of same type.
248 * @pcifunc has to be zero when no LF is yet attached.
250 * For a pcifunc if LFs are attached from multiple blocks of same type, then
251 * return blkaddr of first encountered block.
253 int rvu_get_blkaddr(struct rvu
*rvu
, int blktype
, u16 pcifunc
)
255 int devnum
, blkaddr
= -ENODEV
;
261 blkaddr
= BLKADDR_NPC
;
264 blkaddr
= BLKADDR_NPA
;
267 /* For now assume NIX0 */
269 blkaddr
= BLKADDR_NIX0
;
274 blkaddr
= BLKADDR_SSO
;
277 blkaddr
= BLKADDR_SSOW
;
280 blkaddr
= BLKADDR_TIM
;
283 /* For now assume CPT0 */
285 blkaddr
= BLKADDR_CPT0
;
291 /* Check if this is a RVU PF or VF */
292 if (pcifunc
& RVU_PFVF_FUNC_MASK
) {
294 devnum
= rvu_get_hwvf(rvu
, pcifunc
);
297 devnum
= rvu_get_pf(pcifunc
);
300 /* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' or
303 if (blktype
== BLKTYPE_NIX
) {
304 reg
= is_pf
? RVU_PRIV_PFX_NIXX_CFG(0) :
305 RVU_PRIV_HWVFX_NIXX_CFG(0);
306 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, reg
| (devnum
<< 16));
308 blkaddr
= BLKADDR_NIX0
;
312 reg
= is_pf
? RVU_PRIV_PFX_NIXX_CFG(1) :
313 RVU_PRIV_HWVFX_NIXX_CFG(1);
314 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, reg
| (devnum
<< 16));
316 blkaddr
= BLKADDR_NIX1
;
319 if (blktype
== BLKTYPE_CPT
) {
320 reg
= is_pf
? RVU_PRIV_PFX_CPTX_CFG(0) :
321 RVU_PRIV_HWVFX_CPTX_CFG(0);
322 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, reg
| (devnum
<< 16));
324 blkaddr
= BLKADDR_CPT0
;
328 reg
= is_pf
? RVU_PRIV_PFX_CPTX_CFG(1) :
329 RVU_PRIV_HWVFX_CPTX_CFG(1);
330 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, reg
| (devnum
<< 16));
332 blkaddr
= BLKADDR_CPT1
;
336 if (is_block_implemented(rvu
->hw
, blkaddr
))
341 static void rvu_update_rsrc_map(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
342 struct rvu_block
*block
, u16 pcifunc
,
345 int devnum
, num_lfs
= 0;
349 if (lf
>= block
->lf
.max
) {
350 dev_err(&rvu
->pdev
->dev
,
351 "%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
352 __func__
, lf
, block
->name
, block
->lf
.max
);
356 /* Check if this is for a RVU PF or VF */
357 if (pcifunc
& RVU_PFVF_FUNC_MASK
) {
359 devnum
= rvu_get_hwvf(rvu
, pcifunc
);
362 devnum
= rvu_get_pf(pcifunc
);
365 block
->fn_map
[lf
] = attach
? pcifunc
: 0;
367 switch (block
->addr
) {
369 pfvf
->npalf
= attach
? true : false;
370 num_lfs
= pfvf
->npalf
;
374 pfvf
->nixlf
= attach
? true : false;
375 num_lfs
= pfvf
->nixlf
;
378 attach
? pfvf
->sso
++ : pfvf
->sso
--;
382 attach
? pfvf
->ssow
++ : pfvf
->ssow
--;
383 num_lfs
= pfvf
->ssow
;
386 attach
? pfvf
->timlfs
++ : pfvf
->timlfs
--;
387 num_lfs
= pfvf
->timlfs
;
390 attach
? pfvf
->cptlfs
++ : pfvf
->cptlfs
--;
391 num_lfs
= pfvf
->cptlfs
;
394 attach
? pfvf
->cpt1_lfs
++ : pfvf
->cpt1_lfs
--;
395 num_lfs
= pfvf
->cpt1_lfs
;
399 reg
= is_pf
? block
->pf_lfcnt_reg
: block
->vf_lfcnt_reg
;
400 rvu_write64(rvu
, BLKADDR_RVUM
, reg
| (devnum
<< 16), num_lfs
);
403 inline int rvu_get_pf(u16 pcifunc
)
405 return (pcifunc
>> RVU_PFVF_PF_SHIFT
) & RVU_PFVF_PF_MASK
;
408 void rvu_get_pf_numvfs(struct rvu
*rvu
, int pf
, int *numvfs
, int *hwvf
)
412 /* Get numVFs attached to this PF and first HWVF */
413 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
415 *numvfs
= (cfg
>> 12) & 0xFF;
420 int rvu_get_hwvf(struct rvu
*rvu
, int pcifunc
)
425 pf
= rvu_get_pf(pcifunc
);
426 func
= pcifunc
& RVU_PFVF_FUNC_MASK
;
428 /* Get first HWVF attached to this PF */
429 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
431 return ((cfg
& 0xFFF) + func
- 1);
434 struct rvu_pfvf
*rvu_get_pfvf(struct rvu
*rvu
, int pcifunc
)
436 /* Check if it is a PF or VF */
437 if (pcifunc
& RVU_PFVF_FUNC_MASK
)
438 return &rvu
->hwvf
[rvu_get_hwvf(rvu
, pcifunc
)];
440 return &rvu
->pf
[rvu_get_pf(pcifunc
)];
443 static bool is_pf_func_valid(struct rvu
*rvu
, u16 pcifunc
)
448 pf
= rvu_get_pf(pcifunc
);
449 if (pf
>= rvu
->hw
->total_pfs
)
452 if (!(pcifunc
& RVU_PFVF_FUNC_MASK
))
455 /* Check if VF is within number of VFs attached to this PF */
456 vf
= (pcifunc
& RVU_PFVF_FUNC_MASK
) - 1;
457 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
458 nvfs
= (cfg
>> 12) & 0xFF;
465 bool is_block_implemented(struct rvu_hwinfo
*hw
, int blkaddr
)
467 struct rvu_block
*block
;
469 if (blkaddr
< BLKADDR_RVUM
|| blkaddr
>= BLK_COUNT
)
472 block
= &hw
->block
[blkaddr
];
473 return block
->implemented
;
476 static void rvu_check_block_implemented(struct rvu
*rvu
)
478 struct rvu_hwinfo
*hw
= rvu
->hw
;
479 struct rvu_block
*block
;
483 /* For each block check if 'implemented' bit is set */
484 for (blkid
= 0; blkid
< BLK_COUNT
; blkid
++) {
485 block
= &hw
->block
[blkid
];
486 cfg
= rvupf_read64(rvu
, RVU_PF_BLOCK_ADDRX_DISC(blkid
));
487 if (cfg
& BIT_ULL(11))
488 block
->implemented
= true;
492 static void rvu_setup_rvum_blk_revid(struct rvu
*rvu
)
494 rvu_write64(rvu
, BLKADDR_RVUM
,
495 RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM
),
499 static void rvu_clear_rvum_blk_revid(struct rvu
*rvu
)
501 rvu_write64(rvu
, BLKADDR_RVUM
,
502 RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM
), 0x00);
505 int rvu_lf_reset(struct rvu
*rvu
, struct rvu_block
*block
, int lf
)
509 if (!block
->implemented
)
512 rvu_write64(rvu
, block
->addr
, block
->lfreset_reg
, lf
| BIT_ULL(12));
513 err
= rvu_poll_reg(rvu
, block
->addr
, block
->lfreset_reg
, BIT_ULL(12),
518 static void rvu_block_reset(struct rvu
*rvu
, int blkaddr
, u64 rst_reg
)
520 struct rvu_block
*block
= &rvu
->hw
->block
[blkaddr
];
523 if (!block
->implemented
)
526 rvu_write64(rvu
, blkaddr
, rst_reg
, BIT_ULL(0));
527 err
= rvu_poll_reg(rvu
, blkaddr
, rst_reg
, BIT_ULL(63), true);
529 dev_err(rvu
->dev
, "HW block:%d reset timeout retrying again\n", blkaddr
);
530 while (rvu_poll_reg(rvu
, blkaddr
, rst_reg
, BIT_ULL(63), true) == -EBUSY
)
535 static void rvu_reset_all_blocks(struct rvu
*rvu
)
537 /* Do a HW reset of all RVU blocks */
538 rvu_block_reset(rvu
, BLKADDR_NPA
, NPA_AF_BLK_RST
);
539 rvu_block_reset(rvu
, BLKADDR_NIX0
, NIX_AF_BLK_RST
);
540 rvu_block_reset(rvu
, BLKADDR_NIX1
, NIX_AF_BLK_RST
);
541 rvu_block_reset(rvu
, BLKADDR_NPC
, NPC_AF_BLK_RST
);
542 rvu_block_reset(rvu
, BLKADDR_SSO
, SSO_AF_BLK_RST
);
543 rvu_block_reset(rvu
, BLKADDR_TIM
, TIM_AF_BLK_RST
);
544 rvu_block_reset(rvu
, BLKADDR_CPT0
, CPT_AF_BLK_RST
);
545 rvu_block_reset(rvu
, BLKADDR_CPT1
, CPT_AF_BLK_RST
);
546 rvu_block_reset(rvu
, BLKADDR_NDC_NIX0_RX
, NDC_AF_BLK_RST
);
547 rvu_block_reset(rvu
, BLKADDR_NDC_NIX0_TX
, NDC_AF_BLK_RST
);
548 rvu_block_reset(rvu
, BLKADDR_NDC_NIX1_RX
, NDC_AF_BLK_RST
);
549 rvu_block_reset(rvu
, BLKADDR_NDC_NIX1_TX
, NDC_AF_BLK_RST
);
550 rvu_block_reset(rvu
, BLKADDR_NDC_NPA0
, NDC_AF_BLK_RST
);
553 static void rvu_scan_block(struct rvu
*rvu
, struct rvu_block
*block
)
555 struct rvu_pfvf
*pfvf
;
559 for (lf
= 0; lf
< block
->lf
.max
; lf
++) {
560 cfg
= rvu_read64(rvu
, block
->addr
,
561 block
->lfcfg_reg
| (lf
<< block
->lfshift
));
562 if (!(cfg
& BIT_ULL(63)))
565 /* Set this resource as being used */
566 __set_bit(lf
, block
->lf
.bmap
);
568 /* Get, to whom this LF is attached */
569 pfvf
= rvu_get_pfvf(rvu
, (cfg
>> 8) & 0xFFFF);
570 rvu_update_rsrc_map(rvu
, pfvf
, block
,
571 (cfg
>> 8) & 0xFFFF, lf
, true);
573 /* Set start MSIX vector for this LF within this PF/VF */
574 rvu_set_msix_offset(rvu
, pfvf
, block
, lf
);
578 static void rvu_check_min_msix_vec(struct rvu
*rvu
, int nvecs
, int pf
, int vf
)
587 "PF%d:VF%d is configured with zero msix vectors, %d\n",
594 min_vecs
= RVU_AF_INT_VEC_CNT
+ RVU_PF_INT_VEC_CNT
;
596 min_vecs
= RVU_PF_INT_VEC_CNT
;
598 if (!(nvecs
< min_vecs
))
601 "PF%d is configured with too few vectors, %d, min is %d\n",
602 pf
, nvecs
, min_vecs
);
605 static int rvu_setup_msix_resources(struct rvu
*rvu
)
607 struct rvu_hwinfo
*hw
= rvu
->hw
;
608 int pf
, vf
, numvfs
, hwvf
, err
;
609 int nvecs
, offset
, max_msix
;
610 struct rvu_pfvf
*pfvf
;
614 for (pf
= 0; pf
< hw
->total_pfs
; pf
++) {
615 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
616 /* If PF is not enabled, nothing to do */
617 if (!((cfg
>> 20) & 0x01))
620 rvu_get_pf_numvfs(rvu
, pf
, &numvfs
, &hwvf
);
623 /* Get num of MSIX vectors attached to this PF */
624 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_MSIX_CFG(pf
));
625 pfvf
->msix
.max
= ((cfg
>> 32) & 0xFFF) + 1;
626 rvu_check_min_msix_vec(rvu
, pfvf
->msix
.max
, pf
, 0);
628 /* Alloc msix bitmap for this PF */
629 err
= rvu_alloc_bitmap(&pfvf
->msix
);
633 /* Allocate memory for MSIX vector to RVU block LF mapping */
634 pfvf
->msix_lfmap
= devm_kcalloc(rvu
->dev
, pfvf
->msix
.max
,
635 sizeof(u16
), GFP_KERNEL
);
636 if (!pfvf
->msix_lfmap
)
639 /* For PF0 (AF) firmware will set msix vector offsets for
640 * AF, block AF and PF0_INT vectors, so jump to VFs.
645 /* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.
646 * These are allocated on driver init and never freed,
647 * so no need to set 'msix_lfmap' for these.
649 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_INT_CFG(pf
));
650 nvecs
= (cfg
>> 12) & 0xFF;
652 offset
= rvu_alloc_rsrc_contig(&pfvf
->msix
, nvecs
);
653 rvu_write64(rvu
, BLKADDR_RVUM
,
654 RVU_PRIV_PFX_INT_CFG(pf
), cfg
| offset
);
656 /* Alloc msix bitmap for VFs */
657 for (vf
= 0; vf
< numvfs
; vf
++) {
658 pfvf
= &rvu
->hwvf
[hwvf
+ vf
];
659 /* Get num of MSIX vectors attached to this VF */
660 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
,
661 RVU_PRIV_PFX_MSIX_CFG(pf
));
662 pfvf
->msix
.max
= (cfg
& 0xFFF) + 1;
663 rvu_check_min_msix_vec(rvu
, pfvf
->msix
.max
, pf
, vf
+ 1);
665 /* Alloc msix bitmap for this VF */
666 err
= rvu_alloc_bitmap(&pfvf
->msix
);
671 devm_kcalloc(rvu
->dev
, pfvf
->msix
.max
,
672 sizeof(u16
), GFP_KERNEL
);
673 if (!pfvf
->msix_lfmap
)
676 /* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.
677 * These are allocated on driver init and never freed,
678 * so no need to set 'msix_lfmap' for these.
680 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
,
681 RVU_PRIV_HWVFX_INT_CFG(hwvf
+ vf
));
682 nvecs
= (cfg
>> 12) & 0xFF;
684 offset
= rvu_alloc_rsrc_contig(&pfvf
->msix
, nvecs
);
685 rvu_write64(rvu
, BLKADDR_RVUM
,
686 RVU_PRIV_HWVFX_INT_CFG(hwvf
+ vf
),
691 /* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence
692 * create an IOMMU mapping for the physical address configured by
693 * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.
695 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_CONST
);
696 max_msix
= cfg
& 0xFFFFF;
697 if (rvu
->fwdata
&& rvu
->fwdata
->msixtr_base
)
698 phy_addr
= rvu
->fwdata
->msixtr_base
;
700 phy_addr
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_AF_MSIXTR_BASE
);
702 iova
= dma_map_resource(rvu
->dev
, phy_addr
,
703 max_msix
* PCI_MSIX_ENTRY_SIZE
,
704 DMA_BIDIRECTIONAL
, 0);
706 if (dma_mapping_error(rvu
->dev
, iova
))
709 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_MSIXTR_BASE
, (u64
)iova
);
710 rvu
->msix_base_iova
= iova
;
711 rvu
->msixtr_base_phy
= phy_addr
;
716 static void rvu_reset_msix(struct rvu
*rvu
)
718 /* Restore msixtr base register */
719 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_MSIXTR_BASE
,
720 rvu
->msixtr_base_phy
);
723 static void rvu_free_hw_resources(struct rvu
*rvu
)
725 struct rvu_hwinfo
*hw
= rvu
->hw
;
726 struct rvu_block
*block
;
727 struct rvu_pfvf
*pfvf
;
731 rvu_npa_freemem(rvu
);
732 rvu_npc_freemem(rvu
);
733 rvu_nix_freemem(rvu
);
735 /* Free block LF bitmaps */
736 for (id
= 0; id
< BLK_COUNT
; id
++) {
737 block
= &hw
->block
[id
];
738 kfree(block
->lf
.bmap
);
741 /* Free MSIX bitmaps */
742 for (id
= 0; id
< hw
->total_pfs
; id
++) {
744 kfree(pfvf
->msix
.bmap
);
747 for (id
= 0; id
< hw
->total_vfs
; id
++) {
748 pfvf
= &rvu
->hwvf
[id
];
749 kfree(pfvf
->msix
.bmap
);
752 /* Unmap MSIX vector base IOVA mapping */
753 if (!rvu
->msix_base_iova
)
755 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_CONST
);
756 max_msix
= cfg
& 0xFFFFF;
757 dma_unmap_resource(rvu
->dev
, rvu
->msix_base_iova
,
758 max_msix
* PCI_MSIX_ENTRY_SIZE
,
759 DMA_BIDIRECTIONAL
, 0);
762 mutex_destroy(&rvu
->rsrc_lock
);
765 static void rvu_setup_pfvf_macaddress(struct rvu
*rvu
)
767 struct rvu_hwinfo
*hw
= rvu
->hw
;
768 int pf
, vf
, numvfs
, hwvf
;
769 struct rvu_pfvf
*pfvf
;
772 for (pf
= 0; pf
< hw
->total_pfs
; pf
++) {
773 /* For PF0(AF), Assign MAC address to only VFs (LBKVFs) */
777 if (!is_pf_cgxmapped(rvu
, pf
))
779 /* Assign MAC address to PF */
781 if (rvu
->fwdata
&& pf
< PF_MACNUM_MAX
) {
782 mac
= &rvu
->fwdata
->pf_macs
[pf
];
784 u64_to_ether_addr(*mac
, pfvf
->mac_addr
);
786 eth_random_addr(pfvf
->mac_addr
);
788 eth_random_addr(pfvf
->mac_addr
);
790 ether_addr_copy(pfvf
->default_mac
, pfvf
->mac_addr
);
793 /* Assign MAC address to VFs*/
794 rvu_get_pf_numvfs(rvu
, pf
, &numvfs
, &hwvf
);
795 for (vf
= 0; vf
< numvfs
; vf
++, hwvf
++) {
796 pfvf
= &rvu
->hwvf
[hwvf
];
797 if (rvu
->fwdata
&& hwvf
< VF_MACNUM_MAX
) {
798 mac
= &rvu
->fwdata
->vf_macs
[hwvf
];
800 u64_to_ether_addr(*mac
, pfvf
->mac_addr
);
802 eth_random_addr(pfvf
->mac_addr
);
804 eth_random_addr(pfvf
->mac_addr
);
806 ether_addr_copy(pfvf
->default_mac
, pfvf
->mac_addr
);
811 static int rvu_fwdata_init(struct rvu
*rvu
)
816 /* Get firmware data base address */
817 err
= cgx_get_fwdata_base(&fwdbase
);
821 BUILD_BUG_ON(offsetof(struct rvu_fwdata
, cgx_fw_data
) > FWDATA_CGX_LMAC_OFFSET
);
822 rvu
->fwdata
= ioremap_wc(fwdbase
, sizeof(struct rvu_fwdata
));
825 if (!is_rvu_fwdata_valid(rvu
)) {
827 "Mismatch in 'fwdata' struct btw kernel and firmware\n");
828 iounmap(rvu
->fwdata
);
834 dev_info(rvu
->dev
, "Unable to fetch 'fwdata' from firmware\n");
838 static void rvu_fwdata_exit(struct rvu
*rvu
)
841 iounmap(rvu
->fwdata
);
844 static int rvu_setup_nix_hw_resource(struct rvu
*rvu
, int blkaddr
)
846 struct rvu_hwinfo
*hw
= rvu
->hw
;
847 struct rvu_block
*block
;
851 /* Init NIX LF's bitmap */
852 block
= &hw
->block
[blkaddr
];
853 if (!block
->implemented
)
855 blkid
= (blkaddr
== BLKADDR_NIX0
) ? 0 : 1;
856 cfg
= rvu_read64(rvu
, blkaddr
, NIX_AF_CONST2
);
857 block
->lf
.max
= cfg
& 0xFFF;
858 block
->addr
= blkaddr
;
859 block
->type
= BLKTYPE_NIX
;
861 block
->lookup_reg
= NIX_AF_RVU_LF_CFG_DEBUG
;
862 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_NIXX_CFG(blkid
);
863 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_NIXX_CFG(blkid
);
864 block
->lfcfg_reg
= NIX_PRIV_LFX_CFG
;
865 block
->msixcfg_reg
= NIX_PRIV_LFX_INT_CFG
;
866 block
->lfreset_reg
= NIX_AF_LF_RST
;
868 sprintf(block
->name
, "NIX%d", blkid
);
869 rvu
->nix_blkaddr
[blkid
] = blkaddr
;
870 return rvu_alloc_bitmap(&block
->lf
);
873 static int rvu_setup_cpt_hw_resource(struct rvu
*rvu
, int blkaddr
)
875 struct rvu_hwinfo
*hw
= rvu
->hw
;
876 struct rvu_block
*block
;
880 /* Init CPT LF's bitmap */
881 block
= &hw
->block
[blkaddr
];
882 if (!block
->implemented
)
884 blkid
= (blkaddr
== BLKADDR_CPT0
) ? 0 : 1;
885 cfg
= rvu_read64(rvu
, blkaddr
, CPT_AF_CONSTANTS0
);
886 block
->lf
.max
= cfg
& 0xFF;
887 block
->addr
= blkaddr
;
888 block
->type
= BLKTYPE_CPT
;
889 block
->multislot
= true;
891 block
->lookup_reg
= CPT_AF_RVU_LF_CFG_DEBUG
;
892 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_CPTX_CFG(blkid
);
893 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_CPTX_CFG(blkid
);
894 block
->lfcfg_reg
= CPT_PRIV_LFX_CFG
;
895 block
->msixcfg_reg
= CPT_PRIV_LFX_INT_CFG
;
896 block
->lfreset_reg
= CPT_AF_LF_RST
;
898 sprintf(block
->name
, "CPT%d", blkid
);
899 return rvu_alloc_bitmap(&block
->lf
);
902 static void rvu_get_lbk_bufsize(struct rvu
*rvu
)
904 struct pci_dev
*pdev
= NULL
;
908 pdev
= pci_get_device(PCI_VENDOR_ID_CAVIUM
,
909 PCI_DEVID_OCTEONTX2_LBK
, pdev
);
913 base
= pci_ioremap_bar(pdev
, 0);
917 lbk_const
= readq(base
+ LBK_CONST
);
919 /* cache fifo size */
920 rvu
->hw
->lbk_bufsize
= FIELD_GET(LBK_CONST_BUF_SIZE
, lbk_const
);
927 static int rvu_setup_hw_resources(struct rvu
*rvu
)
929 struct rvu_hwinfo
*hw
= rvu
->hw
;
930 struct rvu_block
*block
;
934 /* Get HW supported max RVU PF & VF count */
935 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_CONST
);
936 hw
->total_pfs
= (cfg
>> 32) & 0xFF;
937 hw
->total_vfs
= (cfg
>> 20) & 0xFFF;
938 hw
->max_vfs_per_pf
= (cfg
>> 40) & 0xFF;
940 if (!is_rvu_otx2(rvu
))
941 rvu_apr_block_cn10k_init(rvu
);
943 /* Init NPA LF's bitmap */
944 block
= &hw
->block
[BLKADDR_NPA
];
945 if (!block
->implemented
)
947 cfg
= rvu_read64(rvu
, BLKADDR_NPA
, NPA_AF_CONST
);
948 block
->lf
.max
= (cfg
>> 16) & 0xFFF;
949 block
->addr
= BLKADDR_NPA
;
950 block
->type
= BLKTYPE_NPA
;
952 block
->lookup_reg
= NPA_AF_RVU_LF_CFG_DEBUG
;
953 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_NPA_CFG
;
954 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_NPA_CFG
;
955 block
->lfcfg_reg
= NPA_PRIV_LFX_CFG
;
956 block
->msixcfg_reg
= NPA_PRIV_LFX_INT_CFG
;
957 block
->lfreset_reg
= NPA_AF_LF_RST
;
959 sprintf(block
->name
, "NPA");
960 err
= rvu_alloc_bitmap(&block
->lf
);
963 "%s: Failed to allocate NPA LF bitmap\n", __func__
);
968 err
= rvu_setup_nix_hw_resource(rvu
, BLKADDR_NIX0
);
971 "%s: Failed to allocate NIX0 LFs bitmap\n", __func__
);
975 err
= rvu_setup_nix_hw_resource(rvu
, BLKADDR_NIX1
);
978 "%s: Failed to allocate NIX1 LFs bitmap\n", __func__
);
982 /* Init SSO group's bitmap */
983 block
= &hw
->block
[BLKADDR_SSO
];
984 if (!block
->implemented
)
986 cfg
= rvu_read64(rvu
, BLKADDR_SSO
, SSO_AF_CONST
);
987 block
->lf
.max
= cfg
& 0xFFFF;
988 block
->addr
= BLKADDR_SSO
;
989 block
->type
= BLKTYPE_SSO
;
990 block
->multislot
= true;
992 block
->lookup_reg
= SSO_AF_RVU_LF_CFG_DEBUG
;
993 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_SSO_CFG
;
994 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_SSO_CFG
;
995 block
->lfcfg_reg
= SSO_PRIV_LFX_HWGRP_CFG
;
996 block
->msixcfg_reg
= SSO_PRIV_LFX_HWGRP_INT_CFG
;
997 block
->lfreset_reg
= SSO_AF_LF_HWGRP_RST
;
999 sprintf(block
->name
, "SSO GROUP");
1000 err
= rvu_alloc_bitmap(&block
->lf
);
1003 "%s: Failed to allocate SSO LF bitmap\n", __func__
);
1008 /* Init SSO workslot's bitmap */
1009 block
= &hw
->block
[BLKADDR_SSOW
];
1010 if (!block
->implemented
)
1012 block
->lf
.max
= (cfg
>> 56) & 0xFF;
1013 block
->addr
= BLKADDR_SSOW
;
1014 block
->type
= BLKTYPE_SSOW
;
1015 block
->multislot
= true;
1017 block
->lookup_reg
= SSOW_AF_RVU_LF_HWS_CFG_DEBUG
;
1018 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_SSOW_CFG
;
1019 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_SSOW_CFG
;
1020 block
->lfcfg_reg
= SSOW_PRIV_LFX_HWS_CFG
;
1021 block
->msixcfg_reg
= SSOW_PRIV_LFX_HWS_INT_CFG
;
1022 block
->lfreset_reg
= SSOW_AF_LF_HWS_RST
;
1024 sprintf(block
->name
, "SSOWS");
1025 err
= rvu_alloc_bitmap(&block
->lf
);
1028 "%s: Failed to allocate SSOW LF bitmap\n", __func__
);
1033 /* Init TIM LF's bitmap */
1034 block
= &hw
->block
[BLKADDR_TIM
];
1035 if (!block
->implemented
)
1037 cfg
= rvu_read64(rvu
, BLKADDR_TIM
, TIM_AF_CONST
);
1038 block
->lf
.max
= cfg
& 0xFFFF;
1039 block
->addr
= BLKADDR_TIM
;
1040 block
->type
= BLKTYPE_TIM
;
1041 block
->multislot
= true;
1043 block
->lookup_reg
= TIM_AF_RVU_LF_CFG_DEBUG
;
1044 block
->pf_lfcnt_reg
= RVU_PRIV_PFX_TIM_CFG
;
1045 block
->vf_lfcnt_reg
= RVU_PRIV_HWVFX_TIM_CFG
;
1046 block
->lfcfg_reg
= TIM_PRIV_LFX_CFG
;
1047 block
->msixcfg_reg
= TIM_PRIV_LFX_INT_CFG
;
1048 block
->lfreset_reg
= TIM_AF_LF_RST
;
1050 sprintf(block
->name
, "TIM");
1051 err
= rvu_alloc_bitmap(&block
->lf
);
1054 "%s: Failed to allocate TIM LF bitmap\n", __func__
);
1059 err
= rvu_setup_cpt_hw_resource(rvu
, BLKADDR_CPT0
);
1062 "%s: Failed to allocate CPT0 LF bitmap\n", __func__
);
1065 err
= rvu_setup_cpt_hw_resource(rvu
, BLKADDR_CPT1
);
1068 "%s: Failed to allocate CPT1 LF bitmap\n", __func__
);
1072 /* Allocate memory for PFVF data */
1073 rvu
->pf
= devm_kcalloc(rvu
->dev
, hw
->total_pfs
,
1074 sizeof(struct rvu_pfvf
), GFP_KERNEL
);
1077 "%s: Failed to allocate memory for PF's rvu_pfvf struct\n", __func__
);
1081 rvu
->hwvf
= devm_kcalloc(rvu
->dev
, hw
->total_vfs
,
1082 sizeof(struct rvu_pfvf
), GFP_KERNEL
);
1085 "%s: Failed to allocate memory for VF's rvu_pfvf struct\n", __func__
);
1089 mutex_init(&rvu
->rsrc_lock
);
1091 rvu_fwdata_init(rvu
);
1093 err
= rvu_setup_msix_resources(rvu
);
1096 "%s: Failed to setup MSIX resources\n", __func__
);
1100 for (blkid
= 0; blkid
< BLK_COUNT
; blkid
++) {
1101 block
= &hw
->block
[blkid
];
1102 if (!block
->lf
.bmap
)
1105 /* Allocate memory for block LF/slot to pcifunc mapping info */
1106 block
->fn_map
= devm_kcalloc(rvu
->dev
, block
->lf
.max
,
1107 sizeof(u16
), GFP_KERNEL
);
1108 if (!block
->fn_map
) {
1113 /* Scan all blocks to check if low level firmware has
1114 * already provisioned any of the resources to a PF/VF.
1116 rvu_scan_block(rvu
, block
);
1119 err
= rvu_set_channels_base(rvu
);
1123 err
= rvu_npc_init(rvu
);
1125 dev_err(rvu
->dev
, "%s: Failed to initialize npc\n", __func__
);
1129 err
= rvu_cgx_init(rvu
);
1131 dev_err(rvu
->dev
, "%s: Failed to initialize cgx\n", __func__
);
1135 err
= rvu_npc_exact_init(rvu
);
1137 dev_err(rvu
->dev
, "failed to initialize exact match table\n");
1141 /* Assign MACs for CGX mapped functions */
1142 rvu_setup_pfvf_macaddress(rvu
);
1144 err
= rvu_npa_init(rvu
);
1146 dev_err(rvu
->dev
, "%s: Failed to initialize npa\n", __func__
);
1150 rvu_get_lbk_bufsize(rvu
);
1152 err
= rvu_nix_init(rvu
);
1154 dev_err(rvu
->dev
, "%s: Failed to initialize nix\n", __func__
);
1158 err
= rvu_sdp_init(rvu
);
1160 dev_err(rvu
->dev
, "%s: Failed to initialize sdp\n", __func__
);
1164 rvu_program_channels(rvu
);
1166 err
= rvu_mcs_init(rvu
);
1168 dev_err(rvu
->dev
, "%s: Failed to initialize mcs\n", __func__
);
1172 err
= rvu_cpt_init(rvu
);
1174 dev_err(rvu
->dev
, "%s: Failed to initialize cpt\n", __func__
);
1183 rvu_nix_freemem(rvu
);
1185 rvu_npa_freemem(rvu
);
1189 rvu_npc_freemem(rvu
);
1190 rvu_fwdata_exit(rvu
);
1192 rvu_reset_msix(rvu
);
1196 /* NPA and NIX admin queue APIs */
1197 void rvu_aq_free(struct rvu
*rvu
, struct admin_queue
*aq
)
1202 qmem_free(rvu
->dev
, aq
->inst
);
1203 qmem_free(rvu
->dev
, aq
->res
);
1204 devm_kfree(rvu
->dev
, aq
);
1207 int rvu_aq_alloc(struct rvu
*rvu
, struct admin_queue
**ad_queue
,
1208 int qsize
, int inst_size
, int res_size
)
1210 struct admin_queue
*aq
;
1213 *ad_queue
= devm_kzalloc(rvu
->dev
, sizeof(*aq
), GFP_KERNEL
);
1218 /* Alloc memory for instructions i.e AQ */
1219 err
= qmem_alloc(rvu
->dev
, &aq
->inst
, qsize
, inst_size
);
1221 devm_kfree(rvu
->dev
, aq
);
1225 /* Alloc memory for results */
1226 err
= qmem_alloc(rvu
->dev
, &aq
->res
, qsize
, res_size
);
1228 rvu_aq_free(rvu
, aq
);
1232 spin_lock_init(&aq
->lock
);
1236 int rvu_mbox_handler_ready(struct rvu
*rvu
, struct msg_req
*req
,
1237 struct ready_msg_rsp
*rsp
)
1240 rsp
->rclk_freq
= rvu
->fwdata
->rclk
;
1241 rsp
->sclk_freq
= rvu
->fwdata
->sclk
;
1246 /* Get current count of a RVU block's LF/slots
1247 * provisioned to a given RVU func.
1249 u16
rvu_get_rsrc_mapcount(struct rvu_pfvf
*pfvf
, int blkaddr
)
1253 return pfvf
->npalf
? 1 : 0;
1256 return pfvf
->nixlf
? 1 : 0;
1262 return pfvf
->timlfs
;
1264 return pfvf
->cptlfs
;
1266 return pfvf
->cpt1_lfs
;
1271 /* Return true if LFs of block type are attached to pcifunc */
1272 static bool is_blktype_attached(struct rvu_pfvf
*pfvf
, int blktype
)
1276 return pfvf
->npalf
? 1 : 0;
1278 return pfvf
->nixlf
? 1 : 0;
1282 return !!pfvf
->ssow
;
1284 return !!pfvf
->timlfs
;
1286 return pfvf
->cptlfs
|| pfvf
->cpt1_lfs
;
1292 bool is_pffunc_map_valid(struct rvu
*rvu
, u16 pcifunc
, int blktype
)
1294 struct rvu_pfvf
*pfvf
;
1296 if (!is_pf_func_valid(rvu
, pcifunc
))
1299 pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1301 /* Check if this PFFUNC has a LF of type blktype attached */
1302 if (!is_blktype_attached(pfvf
, blktype
))
1308 static int rvu_lookup_rsrc(struct rvu
*rvu
, struct rvu_block
*block
,
1309 int pcifunc
, int slot
)
1313 val
= ((u64
)pcifunc
<< 24) | (slot
<< 16) | (1ULL << 13);
1314 rvu_write64(rvu
, block
->addr
, block
->lookup_reg
, val
);
1315 /* Wait for the lookup to finish */
1316 /* TODO: put some timeout here */
1317 while (rvu_read64(rvu
, block
->addr
, block
->lookup_reg
) & (1ULL << 13))
1320 val
= rvu_read64(rvu
, block
->addr
, block
->lookup_reg
);
1322 /* Check LF valid bit */
1323 if (!(val
& (1ULL << 12)))
1326 return (val
& 0xFFF);
1329 int rvu_get_blkaddr_from_slot(struct rvu
*rvu
, int blktype
, u16 pcifunc
,
1330 u16 global_slot
, u16
*slot_in_block
)
1332 struct rvu_pfvf
*pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1333 int numlfs
, total_lfs
= 0, nr_blocks
= 0;
1334 int i
, num_blkaddr
[BLK_COUNT
] = { 0 };
1335 struct rvu_block
*block
;
1339 if (!is_blktype_attached(pfvf
, blktype
))
1342 /* Get all the block addresses from which LFs are attached to
1343 * the given pcifunc in num_blkaddr[].
1345 for (blkaddr
= BLKADDR_RVUM
; blkaddr
< BLK_COUNT
; blkaddr
++) {
1346 block
= &rvu
->hw
->block
[blkaddr
];
1347 if (block
->type
!= blktype
)
1349 if (!is_block_implemented(rvu
->hw
, blkaddr
))
1352 numlfs
= rvu_get_rsrc_mapcount(pfvf
, blkaddr
);
1354 total_lfs
+= numlfs
;
1355 num_blkaddr
[nr_blocks
] = blkaddr
;
1360 if (global_slot
>= total_lfs
)
1363 /* Based on the given global slot number retrieve the
1364 * correct block address out of all attached block
1365 * addresses and slot number in that block.
1369 for (i
= 0; i
< nr_blocks
; i
++) {
1370 numlfs
= rvu_get_rsrc_mapcount(pfvf
, num_blkaddr
[i
]);
1371 total_lfs
+= numlfs
;
1372 if (global_slot
< total_lfs
) {
1373 blkaddr
= num_blkaddr
[i
];
1374 start_slot
= total_lfs
- numlfs
;
1375 *slot_in_block
= global_slot
- start_slot
;
1383 static void rvu_detach_block(struct rvu
*rvu
, int pcifunc
, int blktype
)
1385 struct rvu_pfvf
*pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1386 struct rvu_hwinfo
*hw
= rvu
->hw
;
1387 struct rvu_block
*block
;
1388 int slot
, lf
, num_lfs
;
1391 blkaddr
= rvu_get_blkaddr(rvu
, blktype
, pcifunc
);
1395 if (blktype
== BLKTYPE_NIX
)
1396 rvu_nix_reset_mac(pfvf
, pcifunc
);
1398 block
= &hw
->block
[blkaddr
];
1400 num_lfs
= rvu_get_rsrc_mapcount(pfvf
, block
->addr
);
1404 for (slot
= 0; slot
< num_lfs
; slot
++) {
1405 lf
= rvu_lookup_rsrc(rvu
, block
, pcifunc
, slot
);
1406 if (lf
< 0) /* This should never happen */
1409 /* Disable the LF */
1410 rvu_write64(rvu
, blkaddr
, block
->lfcfg_reg
|
1411 (lf
<< block
->lfshift
), 0x00ULL
);
1413 /* Update SW maintained mapping info as well */
1414 rvu_update_rsrc_map(rvu
, pfvf
, block
,
1415 pcifunc
, lf
, false);
1417 /* Free the resource */
1418 rvu_free_rsrc(&block
->lf
, lf
);
1420 /* Clear MSIX vector offset for this LF */
1421 rvu_clear_msix_offset(rvu
, pfvf
, block
, lf
);
1425 static int rvu_detach_rsrcs(struct rvu
*rvu
, struct rsrc_detach
*detach
,
1428 struct rvu_hwinfo
*hw
= rvu
->hw
;
1429 bool detach_all
= true;
1430 struct rvu_block
*block
;
1433 mutex_lock(&rvu
->rsrc_lock
);
1435 /* Check for partial resource detach */
1436 if (detach
&& detach
->partial
)
1439 /* Check for RVU block's LFs attached to this func,
1440 * if so, detach them.
1442 for (blkid
= 0; blkid
< BLK_COUNT
; blkid
++) {
1443 block
= &hw
->block
[blkid
];
1444 if (!block
->lf
.bmap
)
1446 if (!detach_all
&& detach
) {
1447 if (blkid
== BLKADDR_NPA
&& !detach
->npalf
)
1449 else if ((blkid
== BLKADDR_NIX0
) && !detach
->nixlf
)
1451 else if ((blkid
== BLKADDR_NIX1
) && !detach
->nixlf
)
1453 else if ((blkid
== BLKADDR_SSO
) && !detach
->sso
)
1455 else if ((blkid
== BLKADDR_SSOW
) && !detach
->ssow
)
1457 else if ((blkid
== BLKADDR_TIM
) && !detach
->timlfs
)
1459 else if ((blkid
== BLKADDR_CPT0
) && !detach
->cptlfs
)
1461 else if ((blkid
== BLKADDR_CPT1
) && !detach
->cptlfs
)
1464 rvu_detach_block(rvu
, pcifunc
, block
->type
);
1467 mutex_unlock(&rvu
->rsrc_lock
);
1471 int rvu_mbox_handler_detach_resources(struct rvu
*rvu
,
1472 struct rsrc_detach
*detach
,
1473 struct msg_rsp
*rsp
)
1475 return rvu_detach_rsrcs(rvu
, detach
, detach
->hdr
.pcifunc
);
1478 int rvu_get_nix_blkaddr(struct rvu
*rvu
, u16 pcifunc
)
1480 struct rvu_pfvf
*pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1481 int blkaddr
= BLKADDR_NIX0
, vf
;
1482 struct rvu_pfvf
*pf
;
1484 pf
= rvu_get_pfvf(rvu
, pcifunc
& ~RVU_PFVF_FUNC_MASK
);
1486 /* All CGX mapped PFs are set with assigned NIX block during init */
1487 if (is_pf_cgxmapped(rvu
, rvu_get_pf(pcifunc
))) {
1488 blkaddr
= pf
->nix_blkaddr
;
1489 } else if (is_lbk_vf(rvu
, pcifunc
)) {
1491 /* Assign NIX based on VF number. All even numbered VFs get
1492 * NIX0 and odd numbered gets NIX1
1494 blkaddr
= (vf
& 1) ? BLKADDR_NIX1
: BLKADDR_NIX0
;
1495 /* NIX1 is not present on all silicons */
1496 if (!is_block_implemented(rvu
->hw
, BLKADDR_NIX1
))
1497 blkaddr
= BLKADDR_NIX0
;
1500 /* if SDP1 then the blkaddr is NIX1 */
1501 if (is_sdp_pfvf(pcifunc
) && pf
->sdp_info
->node_id
== 1)
1502 blkaddr
= BLKADDR_NIX1
;
1506 pfvf
->nix_blkaddr
= BLKADDR_NIX1
;
1507 pfvf
->nix_rx_intf
= NIX_INTFX_RX(1);
1508 pfvf
->nix_tx_intf
= NIX_INTFX_TX(1);
1512 pfvf
->nix_blkaddr
= BLKADDR_NIX0
;
1513 pfvf
->nix_rx_intf
= NIX_INTFX_RX(0);
1514 pfvf
->nix_tx_intf
= NIX_INTFX_TX(0);
1518 return pfvf
->nix_blkaddr
;
1521 static int rvu_get_attach_blkaddr(struct rvu
*rvu
, int blktype
,
1522 u16 pcifunc
, struct rsrc_attach
*attach
)
1528 blkaddr
= rvu_get_nix_blkaddr(rvu
, pcifunc
);
1531 if (attach
->hdr
.ver
< RVU_MULTI_BLK_VER
)
1532 return rvu_get_blkaddr(rvu
, blktype
, 0);
1533 blkaddr
= attach
->cpt_blkaddr
? attach
->cpt_blkaddr
:
1535 if (blkaddr
!= BLKADDR_CPT0
&& blkaddr
!= BLKADDR_CPT1
)
1539 return rvu_get_blkaddr(rvu
, blktype
, 0);
1542 if (is_block_implemented(rvu
->hw
, blkaddr
))
1548 static void rvu_attach_block(struct rvu
*rvu
, int pcifunc
, int blktype
,
1549 int num_lfs
, struct rsrc_attach
*attach
)
1551 struct rvu_pfvf
*pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1552 struct rvu_hwinfo
*hw
= rvu
->hw
;
1553 struct rvu_block
*block
;
1561 blkaddr
= rvu_get_attach_blkaddr(rvu
, blktype
, pcifunc
, attach
);
1565 block
= &hw
->block
[blkaddr
];
1566 if (!block
->lf
.bmap
)
1569 for (slot
= 0; slot
< num_lfs
; slot
++) {
1570 /* Allocate the resource */
1571 lf
= rvu_alloc_rsrc(&block
->lf
);
1575 cfg
= (1ULL << 63) | (pcifunc
<< 8) | slot
;
1576 rvu_write64(rvu
, blkaddr
, block
->lfcfg_reg
|
1577 (lf
<< block
->lfshift
), cfg
);
1578 rvu_update_rsrc_map(rvu
, pfvf
, block
,
1581 /* Set start MSIX vector for this LF within this PF/VF */
1582 rvu_set_msix_offset(rvu
, pfvf
, block
, lf
);
1586 static int rvu_check_rsrc_availability(struct rvu
*rvu
,
1587 struct rsrc_attach
*req
, u16 pcifunc
)
1589 struct rvu_pfvf
*pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1590 int free_lfs
, mappedlfs
, blkaddr
;
1591 struct rvu_hwinfo
*hw
= rvu
->hw
;
1592 struct rvu_block
*block
;
1594 /* Only one NPA LF can be attached */
1595 if (req
->npalf
&& !is_blktype_attached(pfvf
, BLKTYPE_NPA
)) {
1596 block
= &hw
->block
[BLKADDR_NPA
];
1597 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1600 } else if (req
->npalf
) {
1601 dev_err(&rvu
->pdev
->dev
,
1602 "Func 0x%x: Invalid req, already has NPA\n",
1607 /* Only one NIX LF can be attached */
1608 if (req
->nixlf
&& !is_blktype_attached(pfvf
, BLKTYPE_NIX
)) {
1609 blkaddr
= rvu_get_attach_blkaddr(rvu
, BLKTYPE_NIX
,
1613 block
= &hw
->block
[blkaddr
];
1614 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1617 } else if (req
->nixlf
) {
1618 dev_err(&rvu
->pdev
->dev
,
1619 "Func 0x%x: Invalid req, already has NIX\n",
1625 block
= &hw
->block
[BLKADDR_SSO
];
1626 /* Is request within limits ? */
1627 if (req
->sso
> block
->lf
.max
) {
1628 dev_err(&rvu
->pdev
->dev
,
1629 "Func 0x%x: Invalid SSO req, %d > max %d\n",
1630 pcifunc
, req
->sso
, block
->lf
.max
);
1633 mappedlfs
= rvu_get_rsrc_mapcount(pfvf
, block
->addr
);
1634 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1635 /* Check if additional resources are available */
1636 if (req
->sso
> mappedlfs
&&
1637 ((req
->sso
- mappedlfs
) > free_lfs
))
1642 block
= &hw
->block
[BLKADDR_SSOW
];
1643 if (req
->ssow
> block
->lf
.max
) {
1644 dev_err(&rvu
->pdev
->dev
,
1645 "Func 0x%x: Invalid SSOW req, %d > max %d\n",
1646 pcifunc
, req
->sso
, block
->lf
.max
);
1649 mappedlfs
= rvu_get_rsrc_mapcount(pfvf
, block
->addr
);
1650 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1651 if (req
->ssow
> mappedlfs
&&
1652 ((req
->ssow
- mappedlfs
) > free_lfs
))
1657 block
= &hw
->block
[BLKADDR_TIM
];
1658 if (req
->timlfs
> block
->lf
.max
) {
1659 dev_err(&rvu
->pdev
->dev
,
1660 "Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1661 pcifunc
, req
->timlfs
, block
->lf
.max
);
1664 mappedlfs
= rvu_get_rsrc_mapcount(pfvf
, block
->addr
);
1665 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1666 if (req
->timlfs
> mappedlfs
&&
1667 ((req
->timlfs
- mappedlfs
) > free_lfs
))
1672 blkaddr
= rvu_get_attach_blkaddr(rvu
, BLKTYPE_CPT
,
1676 block
= &hw
->block
[blkaddr
];
1677 if (req
->cptlfs
> block
->lf
.max
) {
1678 dev_err(&rvu
->pdev
->dev
,
1679 "Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1680 pcifunc
, req
->cptlfs
, block
->lf
.max
);
1683 mappedlfs
= rvu_get_rsrc_mapcount(pfvf
, block
->addr
);
1684 free_lfs
= rvu_rsrc_free_count(&block
->lf
);
1685 if (req
->cptlfs
> mappedlfs
&&
1686 ((req
->cptlfs
- mappedlfs
) > free_lfs
))
1693 dev_info(rvu
->dev
, "Request for %s failed\n", block
->name
);
1697 static bool rvu_attach_from_same_block(struct rvu
*rvu
, int blktype
,
1698 struct rsrc_attach
*attach
)
1700 int blkaddr
, num_lfs
;
1702 blkaddr
= rvu_get_attach_blkaddr(rvu
, blktype
,
1703 attach
->hdr
.pcifunc
, attach
);
1707 num_lfs
= rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu
, attach
->hdr
.pcifunc
),
1709 /* Requester already has LFs from given block ? */
1713 int rvu_mbox_handler_attach_resources(struct rvu
*rvu
,
1714 struct rsrc_attach
*attach
,
1715 struct msg_rsp
*rsp
)
1717 u16 pcifunc
= attach
->hdr
.pcifunc
;
1720 /* If first request, detach all existing attached resources */
1721 if (!attach
->modify
)
1722 rvu_detach_rsrcs(rvu
, NULL
, pcifunc
);
1724 mutex_lock(&rvu
->rsrc_lock
);
1726 /* Check if the request can be accommodated */
1727 err
= rvu_check_rsrc_availability(rvu
, attach
, pcifunc
);
1731 /* Now attach the requested resources */
1733 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_NPA
, 1, attach
);
1736 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_NIX
, 1, attach
);
1739 /* RVU func doesn't know which exact LF or slot is attached
1740 * to it, it always sees as slot 0,1,2. So for a 'modify'
1741 * request, simply detach all existing attached LFs/slots
1742 * and attach a fresh.
1745 rvu_detach_block(rvu
, pcifunc
, BLKTYPE_SSO
);
1746 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_SSO
,
1747 attach
->sso
, attach
);
1752 rvu_detach_block(rvu
, pcifunc
, BLKTYPE_SSOW
);
1753 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_SSOW
,
1754 attach
->ssow
, attach
);
1757 if (attach
->timlfs
) {
1759 rvu_detach_block(rvu
, pcifunc
, BLKTYPE_TIM
);
1760 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_TIM
,
1761 attach
->timlfs
, attach
);
1764 if (attach
->cptlfs
) {
1765 if (attach
->modify
&&
1766 rvu_attach_from_same_block(rvu
, BLKTYPE_CPT
, attach
))
1767 rvu_detach_block(rvu
, pcifunc
, BLKTYPE_CPT
);
1768 rvu_attach_block(rvu
, pcifunc
, BLKTYPE_CPT
,
1769 attach
->cptlfs
, attach
);
1773 mutex_unlock(&rvu
->rsrc_lock
);
1777 static u16
rvu_get_msix_offset(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
1778 int blkaddr
, int lf
)
1783 return MSIX_VECTOR_INVALID
;
1785 for (vec
= 0; vec
< pfvf
->msix
.max
; vec
++) {
1786 if (pfvf
->msix_lfmap
[vec
] == MSIX_BLKLF(blkaddr
, lf
))
1789 return MSIX_VECTOR_INVALID
;
1792 static void rvu_set_msix_offset(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
1793 struct rvu_block
*block
, int lf
)
1795 u16 nvecs
, vec
, offset
;
1798 cfg
= rvu_read64(rvu
, block
->addr
, block
->msixcfg_reg
|
1799 (lf
<< block
->lfshift
));
1800 nvecs
= (cfg
>> 12) & 0xFF;
1802 /* Check and alloc MSIX vectors, must be contiguous */
1803 if (!rvu_rsrc_check_contig(&pfvf
->msix
, nvecs
))
1806 offset
= rvu_alloc_rsrc_contig(&pfvf
->msix
, nvecs
);
1808 /* Config MSIX offset in LF */
1809 rvu_write64(rvu
, block
->addr
, block
->msixcfg_reg
|
1810 (lf
<< block
->lfshift
), (cfg
& ~0x7FFULL
) | offset
);
1812 /* Update the bitmap as well */
1813 for (vec
= 0; vec
< nvecs
; vec
++)
1814 pfvf
->msix_lfmap
[offset
+ vec
] = MSIX_BLKLF(block
->addr
, lf
);
1817 static void rvu_clear_msix_offset(struct rvu
*rvu
, struct rvu_pfvf
*pfvf
,
1818 struct rvu_block
*block
, int lf
)
1820 u16 nvecs
, vec
, offset
;
1823 cfg
= rvu_read64(rvu
, block
->addr
, block
->msixcfg_reg
|
1824 (lf
<< block
->lfshift
));
1825 nvecs
= (cfg
>> 12) & 0xFF;
1827 /* Clear MSIX offset in LF */
1828 rvu_write64(rvu
, block
->addr
, block
->msixcfg_reg
|
1829 (lf
<< block
->lfshift
), cfg
& ~0x7FFULL
);
1831 offset
= rvu_get_msix_offset(rvu
, pfvf
, block
->addr
, lf
);
1833 /* Update the mapping */
1834 for (vec
= 0; vec
< nvecs
; vec
++)
1835 pfvf
->msix_lfmap
[offset
+ vec
] = 0;
1837 /* Free the same in MSIX bitmap */
1838 rvu_free_rsrc_contig(&pfvf
->msix
, nvecs
, offset
);
1841 int rvu_mbox_handler_msix_offset(struct rvu
*rvu
, struct msg_req
*req
,
1842 struct msix_offset_rsp
*rsp
)
1844 struct rvu_hwinfo
*hw
= rvu
->hw
;
1845 u16 pcifunc
= req
->hdr
.pcifunc
;
1846 struct rvu_pfvf
*pfvf
;
1847 int lf
, slot
, blkaddr
;
1849 pfvf
= rvu_get_pfvf(rvu
, pcifunc
);
1850 if (!pfvf
->msix
.bmap
)
1853 /* Set MSIX offsets for each block's LFs attached to this PF/VF */
1854 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_NPA
], pcifunc
, 0);
1855 rsp
->npa_msixoff
= rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_NPA
, lf
);
1857 /* Get BLKADDR from which LFs are attached to pcifunc */
1858 blkaddr
= rvu_get_blkaddr(rvu
, BLKTYPE_NIX
, pcifunc
);
1860 rsp
->nix_msixoff
= MSIX_VECTOR_INVALID
;
1862 lf
= rvu_get_lf(rvu
, &hw
->block
[blkaddr
], pcifunc
, 0);
1863 rsp
->nix_msixoff
= rvu_get_msix_offset(rvu
, pfvf
, blkaddr
, lf
);
1866 rsp
->sso
= pfvf
->sso
;
1867 for (slot
= 0; slot
< rsp
->sso
; slot
++) {
1868 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_SSO
], pcifunc
, slot
);
1869 rsp
->sso_msixoff
[slot
] =
1870 rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_SSO
, lf
);
1873 rsp
->ssow
= pfvf
->ssow
;
1874 for (slot
= 0; slot
< rsp
->ssow
; slot
++) {
1875 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_SSOW
], pcifunc
, slot
);
1876 rsp
->ssow_msixoff
[slot
] =
1877 rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_SSOW
, lf
);
1880 rsp
->timlfs
= pfvf
->timlfs
;
1881 for (slot
= 0; slot
< rsp
->timlfs
; slot
++) {
1882 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_TIM
], pcifunc
, slot
);
1883 rsp
->timlf_msixoff
[slot
] =
1884 rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_TIM
, lf
);
1887 rsp
->cptlfs
= pfvf
->cptlfs
;
1888 for (slot
= 0; slot
< rsp
->cptlfs
; slot
++) {
1889 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_CPT0
], pcifunc
, slot
);
1890 rsp
->cptlf_msixoff
[slot
] =
1891 rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_CPT0
, lf
);
1894 rsp
->cpt1_lfs
= pfvf
->cpt1_lfs
;
1895 for (slot
= 0; slot
< rsp
->cpt1_lfs
; slot
++) {
1896 lf
= rvu_get_lf(rvu
, &hw
->block
[BLKADDR_CPT1
], pcifunc
, slot
);
1897 rsp
->cpt1_lf_msixoff
[slot
] =
1898 rvu_get_msix_offset(rvu
, pfvf
, BLKADDR_CPT1
, lf
);
1904 int rvu_mbox_handler_free_rsrc_cnt(struct rvu
*rvu
, struct msg_req
*req
,
1905 struct free_rsrcs_rsp
*rsp
)
1907 struct rvu_hwinfo
*hw
= rvu
->hw
;
1908 struct rvu_block
*block
;
1909 struct nix_txsch
*txsch
;
1910 struct nix_hw
*nix_hw
;
1912 mutex_lock(&rvu
->rsrc_lock
);
1914 block
= &hw
->block
[BLKADDR_NPA
];
1915 rsp
->npa
= rvu_rsrc_free_count(&block
->lf
);
1917 block
= &hw
->block
[BLKADDR_NIX0
];
1918 rsp
->nix
= rvu_rsrc_free_count(&block
->lf
);
1920 block
= &hw
->block
[BLKADDR_NIX1
];
1921 rsp
->nix1
= rvu_rsrc_free_count(&block
->lf
);
1923 block
= &hw
->block
[BLKADDR_SSO
];
1924 rsp
->sso
= rvu_rsrc_free_count(&block
->lf
);
1926 block
= &hw
->block
[BLKADDR_SSOW
];
1927 rsp
->ssow
= rvu_rsrc_free_count(&block
->lf
);
1929 block
= &hw
->block
[BLKADDR_TIM
];
1930 rsp
->tim
= rvu_rsrc_free_count(&block
->lf
);
1932 block
= &hw
->block
[BLKADDR_CPT0
];
1933 rsp
->cpt
= rvu_rsrc_free_count(&block
->lf
);
1935 block
= &hw
->block
[BLKADDR_CPT1
];
1936 rsp
->cpt1
= rvu_rsrc_free_count(&block
->lf
);
1938 if (rvu
->hw
->cap
.nix_fixed_txschq_mapping
) {
1939 rsp
->schq
[NIX_TXSCH_LVL_SMQ
] = 1;
1940 rsp
->schq
[NIX_TXSCH_LVL_TL4
] = 1;
1941 rsp
->schq
[NIX_TXSCH_LVL_TL3
] = 1;
1942 rsp
->schq
[NIX_TXSCH_LVL_TL2
] = 1;
1944 if (!is_block_implemented(rvu
->hw
, BLKADDR_NIX1
))
1946 rsp
->schq_nix1
[NIX_TXSCH_LVL_SMQ
] = 1;
1947 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL4
] = 1;
1948 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL3
] = 1;
1949 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL2
] = 1;
1951 nix_hw
= get_nix_hw(hw
, BLKADDR_NIX0
);
1952 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_SMQ
];
1953 rsp
->schq
[NIX_TXSCH_LVL_SMQ
] =
1954 rvu_rsrc_free_count(&txsch
->schq
);
1956 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL4
];
1957 rsp
->schq
[NIX_TXSCH_LVL_TL4
] =
1958 rvu_rsrc_free_count(&txsch
->schq
);
1960 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL3
];
1961 rsp
->schq
[NIX_TXSCH_LVL_TL3
] =
1962 rvu_rsrc_free_count(&txsch
->schq
);
1964 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL2
];
1965 rsp
->schq
[NIX_TXSCH_LVL_TL2
] =
1966 rvu_rsrc_free_count(&txsch
->schq
);
1968 if (!is_block_implemented(rvu
->hw
, BLKADDR_NIX1
))
1971 nix_hw
= get_nix_hw(hw
, BLKADDR_NIX1
);
1972 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_SMQ
];
1973 rsp
->schq_nix1
[NIX_TXSCH_LVL_SMQ
] =
1974 rvu_rsrc_free_count(&txsch
->schq
);
1976 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL4
];
1977 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL4
] =
1978 rvu_rsrc_free_count(&txsch
->schq
);
1980 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL3
];
1981 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL3
] =
1982 rvu_rsrc_free_count(&txsch
->schq
);
1984 txsch
= &nix_hw
->txsch
[NIX_TXSCH_LVL_TL2
];
1985 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL2
] =
1986 rvu_rsrc_free_count(&txsch
->schq
);
1989 rsp
->schq_nix1
[NIX_TXSCH_LVL_TL1
] = 1;
1991 rsp
->schq
[NIX_TXSCH_LVL_TL1
] = 1;
1992 mutex_unlock(&rvu
->rsrc_lock
);
1997 int rvu_mbox_handler_vf_flr(struct rvu
*rvu
, struct msg_req
*req
,
1998 struct msg_rsp
*rsp
)
2000 u16 pcifunc
= req
->hdr
.pcifunc
;
2004 vf
= pcifunc
& RVU_PFVF_FUNC_MASK
;
2005 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
,
2006 RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc
)));
2007 numvfs
= (cfg
>> 12) & 0xFF;
2009 if (vf
&& vf
<= numvfs
)
2010 __rvu_flr_handler(rvu
, pcifunc
);
2012 return RVU_INVALID_VF_ID
;
2017 int rvu_mbox_handler_get_hw_cap(struct rvu
*rvu
, struct msg_req
*req
,
2018 struct get_hw_cap_rsp
*rsp
)
2020 struct rvu_hwinfo
*hw
= rvu
->hw
;
2022 rsp
->nix_fixed_txschq_mapping
= hw
->cap
.nix_fixed_txschq_mapping
;
2023 rsp
->nix_shaping
= hw
->cap
.nix_shaping
;
2024 rsp
->npc_hash_extract
= hw
->cap
.npc_hash_extract
;
2029 int rvu_mbox_handler_set_vf_perm(struct rvu
*rvu
, struct set_vf_perm
*req
,
2030 struct msg_rsp
*rsp
)
2032 struct rvu_hwinfo
*hw
= rvu
->hw
;
2033 u16 pcifunc
= req
->hdr
.pcifunc
;
2034 struct rvu_pfvf
*pfvf
;
2038 /* Only PF can add VF permissions */
2039 if ((pcifunc
& RVU_PFVF_FUNC_MASK
) || is_lbk_vf(rvu
, pcifunc
))
2042 target
= (pcifunc
& ~RVU_PFVF_FUNC_MASK
) | (req
->vf
+ 1);
2043 pfvf
= rvu_get_pfvf(rvu
, target
);
2045 if (req
->flags
& RESET_VF_PERM
) {
2046 pfvf
->flags
&= RVU_CLEAR_VF_PERM
;
2047 } else if (test_bit(PF_SET_VF_TRUSTED
, &pfvf
->flags
) ^
2048 (req
->flags
& VF_TRUSTED
)) {
2049 change_bit(PF_SET_VF_TRUSTED
, &pfvf
->flags
);
2050 /* disable multicast and promisc entries */
2051 if (!test_bit(PF_SET_VF_TRUSTED
, &pfvf
->flags
)) {
2052 blkaddr
= rvu_get_blkaddr(rvu
, BLKTYPE_NIX
, target
);
2055 nixlf
= rvu_get_lf(rvu
, &hw
->block
[blkaddr
],
2059 npc_enadis_default_mce_entry(rvu
, target
, nixlf
,
2060 NIXLF_ALLMULTI_ENTRY
,
2062 npc_enadis_default_mce_entry(rvu
, target
, nixlf
,
2063 NIXLF_PROMISC_ENTRY
,
2071 static int rvu_process_mbox_msg(struct otx2_mbox
*mbox
, int devid
,
2072 struct mbox_msghdr
*req
)
2074 struct rvu
*rvu
= pci_get_drvdata(mbox
->pdev
);
2076 /* Check if valid, if not reply with a invalid msg */
2077 if (req
->sig
!= OTX2_MBOX_REQ_SIG
)
2081 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
2083 struct _rsp_type *rsp; \
2086 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
2088 sizeof(struct _rsp_type)); \
2089 /* some handlers should complete even if reply */ \
2090 /* could not be allocated */ \
2092 _id != MBOX_MSG_DETACH_RESOURCES && \
2093 _id != MBOX_MSG_NIX_TXSCH_FREE && \
2094 _id != MBOX_MSG_VF_FLR) \
2097 rsp->hdr.id = _id; \
2098 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
2099 rsp->hdr.pcifunc = req->pcifunc; \
2103 err = rvu_mbox_handler_ ## _fn_name(rvu, \
2104 (struct _req_type *)req, \
2107 rsp->hdr.rc = err; \
2109 trace_otx2_msg_process(mbox->pdev, _id, err); \
2110 return rsp ? err : -ENOMEM; \
2117 otx2_reply_invalid_msg(mbox
, devid
, req
->pcifunc
, req
->id
);
2122 static void __rvu_mbox_handler(struct rvu_work
*mwork
, int type
, bool poll
)
2124 struct rvu
*rvu
= mwork
->rvu
;
2125 int offset
, err
, id
, devid
;
2126 struct otx2_mbox_dev
*mdev
;
2127 struct mbox_hdr
*req_hdr
;
2128 struct mbox_msghdr
*msg
;
2129 struct mbox_wq_info
*mw
;
2130 struct otx2_mbox
*mbox
;
2134 mw
= &rvu
->afpf_wq_info
;
2137 mw
= &rvu
->afvf_wq_info
;
2143 devid
= mwork
- mw
->mbox_wrk
;
2145 mdev
= &mbox
->dev
[devid
];
2147 /* Process received mbox messages */
2148 req_hdr
= mdev
->mbase
+ mbox
->rx_start
;
2149 if (mw
->mbox_wrk
[devid
].num_msgs
== 0)
2152 offset
= mbox
->rx_start
+ ALIGN(sizeof(*req_hdr
), MBOX_MSG_ALIGN
);
2154 for (id
= 0; id
< mw
->mbox_wrk
[devid
].num_msgs
; id
++) {
2155 msg
= mdev
->mbase
+ offset
;
2157 /* Set which PF/VF sent this message based on mbox IRQ */
2161 ~(RVU_PFVF_PF_MASK
<< RVU_PFVF_PF_SHIFT
);
2162 msg
->pcifunc
|= (devid
<< RVU_PFVF_PF_SHIFT
);
2166 ~(RVU_PFVF_FUNC_MASK
<< RVU_PFVF_FUNC_SHIFT
);
2167 msg
->pcifunc
|= (devid
<< RVU_PFVF_FUNC_SHIFT
) + 1;
2171 err
= rvu_process_mbox_msg(mbox
, devid
, msg
);
2173 offset
= mbox
->rx_start
+ msg
->next_msgoff
;
2177 if (msg
->pcifunc
& RVU_PFVF_FUNC_MASK
)
2178 dev_warn(rvu
->dev
, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
2179 err
, otx2_mbox_id2name(msg
->id
),
2180 msg
->id
, rvu_get_pf(msg
->pcifunc
),
2181 (msg
->pcifunc
& RVU_PFVF_FUNC_MASK
) - 1);
2183 dev_warn(rvu
->dev
, "Error %d when processing message %s (0x%x) from PF%d\n",
2184 err
, otx2_mbox_id2name(msg
->id
),
2187 mw
->mbox_wrk
[devid
].num_msgs
= 0;
2190 otx2_mbox_wait_for_zero(mbox
, devid
);
2192 /* Send mbox responses to VF/PF */
2193 otx2_mbox_msg_send(mbox
, devid
);
2196 static inline void rvu_afpf_mbox_handler(struct work_struct
*work
)
2198 struct rvu_work
*mwork
= container_of(work
, struct rvu_work
, work
);
2199 struct rvu
*rvu
= mwork
->rvu
;
2201 mutex_lock(&rvu
->mbox_lock
);
2202 __rvu_mbox_handler(mwork
, TYPE_AFPF
, true);
2203 mutex_unlock(&rvu
->mbox_lock
);
2206 static inline void rvu_afvf_mbox_handler(struct work_struct
*work
)
2208 struct rvu_work
*mwork
= container_of(work
, struct rvu_work
, work
);
2210 __rvu_mbox_handler(mwork
, TYPE_AFVF
, false);
2213 static void __rvu_mbox_up_handler(struct rvu_work
*mwork
, int type
)
2215 struct rvu
*rvu
= mwork
->rvu
;
2216 struct otx2_mbox_dev
*mdev
;
2217 struct mbox_hdr
*rsp_hdr
;
2218 struct mbox_msghdr
*msg
;
2219 struct mbox_wq_info
*mw
;
2220 struct otx2_mbox
*mbox
;
2221 int offset
, id
, devid
;
2225 mw
= &rvu
->afpf_wq_info
;
2228 mw
= &rvu
->afvf_wq_info
;
2234 devid
= mwork
- mw
->mbox_wrk_up
;
2235 mbox
= &mw
->mbox_up
;
2236 mdev
= &mbox
->dev
[devid
];
2238 rsp_hdr
= mdev
->mbase
+ mbox
->rx_start
;
2239 if (mw
->mbox_wrk_up
[devid
].up_num_msgs
== 0) {
2240 dev_warn(rvu
->dev
, "mbox up handler: num_msgs = 0\n");
2244 offset
= mbox
->rx_start
+ ALIGN(sizeof(*rsp_hdr
), MBOX_MSG_ALIGN
);
2246 for (id
= 0; id
< mw
->mbox_wrk_up
[devid
].up_num_msgs
; id
++) {
2247 msg
= mdev
->mbase
+ offset
;
2249 if (msg
->id
>= MBOX_MSG_MAX
) {
2251 "Mbox msg with unknown ID 0x%x\n", msg
->id
);
2255 if (msg
->sig
!= OTX2_MBOX_RSP_SIG
) {
2257 "Mbox msg with wrong signature %x, ID 0x%x\n",
2263 case MBOX_MSG_CGX_LINK_EVENT
:
2268 "Mbox msg response has err %d, ID 0x%x\n",
2273 offset
= mbox
->rx_start
+ msg
->next_msgoff
;
2276 mw
->mbox_wrk_up
[devid
].up_num_msgs
= 0;
2278 otx2_mbox_reset(mbox
, devid
);
2281 static inline void rvu_afpf_mbox_up_handler(struct work_struct
*work
)
2283 struct rvu_work
*mwork
= container_of(work
, struct rvu_work
, work
);
2285 __rvu_mbox_up_handler(mwork
, TYPE_AFPF
);
2288 static inline void rvu_afvf_mbox_up_handler(struct work_struct
*work
)
2290 struct rvu_work
*mwork
= container_of(work
, struct rvu_work
, work
);
2292 __rvu_mbox_up_handler(mwork
, TYPE_AFVF
);
2295 static int rvu_get_mbox_regions(struct rvu
*rvu
, void **mbox_addr
,
2296 int num
, int type
, unsigned long *pf_bmap
)
2298 struct rvu_hwinfo
*hw
= rvu
->hw
;
2302 /* For cn10k platform VF mailbox regions of a PF follows after the
2303 * PF <-> AF mailbox region. Whereas for Octeontx2 it is read from
2304 * RVU_PF_VF_BAR4_ADDR register.
2306 if (type
== TYPE_AFVF
) {
2307 for (region
= 0; region
< num
; region
++) {
2308 if (!test_bit(region
, pf_bmap
))
2311 if (hw
->cap
.per_pf_mbox_regs
) {
2312 bar4
= rvu_read64(rvu
, BLKADDR_RVUM
,
2313 RVU_AF_PFX_BAR4_ADDR(0)) +
2315 bar4
+= region
* MBOX_SIZE
;
2317 bar4
= rvupf_read64(rvu
, RVU_PF_VF_BAR4_ADDR
);
2318 bar4
+= region
* MBOX_SIZE
;
2320 mbox_addr
[region
] = (void *)ioremap_wc(bar4
, MBOX_SIZE
);
2321 if (!mbox_addr
[region
])
2327 /* For cn10k platform AF <-> PF mailbox region of a PF is read from per
2328 * PF registers. Whereas for Octeontx2 it is read from
2329 * RVU_AF_PF_BAR4_ADDR register.
2331 for (region
= 0; region
< num
; region
++) {
2332 if (!test_bit(region
, pf_bmap
))
2335 if (hw
->cap
.per_pf_mbox_regs
) {
2336 bar4
= rvu_read64(rvu
, BLKADDR_RVUM
,
2337 RVU_AF_PFX_BAR4_ADDR(region
));
2339 bar4
= rvu_read64(rvu
, BLKADDR_RVUM
,
2340 RVU_AF_PF_BAR4_ADDR
);
2341 bar4
+= region
* MBOX_SIZE
;
2343 mbox_addr
[region
] = (void *)ioremap_wc(bar4
, MBOX_SIZE
);
2344 if (!mbox_addr
[region
])
2351 iounmap((void __iomem
*)mbox_addr
[region
]);
2355 static int rvu_mbox_init(struct rvu
*rvu
, struct mbox_wq_info
*mw
,
2357 void (mbox_handler
)(struct work_struct
*),
2358 void (mbox_up_handler
)(struct work_struct
*))
2360 int err
= -EINVAL
, i
, dir
, dir_up
;
2361 void __iomem
*reg_base
;
2362 struct rvu_work
*mwork
;
2363 unsigned long *pf_bmap
;
2364 void **mbox_regions
;
2368 pf_bmap
= bitmap_zalloc(num
, GFP_KERNEL
);
2373 if (type
== TYPE_AFVF
)
2374 bitmap_set(pf_bmap
, 0, num
);
2376 if (type
== TYPE_AFPF
) {
2377 /* Mark enabled PFs in bitmap */
2378 for (i
= 0; i
< num
; i
++) {
2379 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(i
));
2380 if (cfg
& BIT_ULL(20))
2381 set_bit(i
, pf_bmap
);
2385 mutex_init(&rvu
->mbox_lock
);
2387 mbox_regions
= kcalloc(num
, sizeof(void *), GFP_KERNEL
);
2388 if (!mbox_regions
) {
2395 name
= "rvu_afpf_mailbox";
2396 dir
= MBOX_DIR_AFPF
;
2397 dir_up
= MBOX_DIR_AFPF_UP
;
2398 reg_base
= rvu
->afreg_base
;
2399 err
= rvu_get_mbox_regions(rvu
, mbox_regions
, num
, TYPE_AFPF
, pf_bmap
);
2404 name
= "rvu_afvf_mailbox";
2405 dir
= MBOX_DIR_PFVF
;
2406 dir_up
= MBOX_DIR_PFVF_UP
;
2407 reg_base
= rvu
->pfreg_base
;
2408 err
= rvu_get_mbox_regions(rvu
, mbox_regions
, num
, TYPE_AFVF
, pf_bmap
);
2416 mw
->mbox_wq
= alloc_workqueue(name
,
2417 WQ_UNBOUND
| WQ_HIGHPRI
| WQ_MEM_RECLAIM
,
2424 mw
->mbox_wrk
= devm_kcalloc(rvu
->dev
, num
,
2425 sizeof(struct rvu_work
), GFP_KERNEL
);
2426 if (!mw
->mbox_wrk
) {
2431 mw
->mbox_wrk_up
= devm_kcalloc(rvu
->dev
, num
,
2432 sizeof(struct rvu_work
), GFP_KERNEL
);
2433 if (!mw
->mbox_wrk_up
) {
2438 err
= otx2_mbox_regions_init(&mw
->mbox
, mbox_regions
, rvu
->pdev
,
2439 reg_base
, dir
, num
, pf_bmap
);
2443 err
= otx2_mbox_regions_init(&mw
->mbox_up
, mbox_regions
, rvu
->pdev
,
2444 reg_base
, dir_up
, num
, pf_bmap
);
2448 for (i
= 0; i
< num
; i
++) {
2449 if (!test_bit(i
, pf_bmap
))
2452 mwork
= &mw
->mbox_wrk
[i
];
2454 INIT_WORK(&mwork
->work
, mbox_handler
);
2456 mwork
= &mw
->mbox_wrk_up
[i
];
2458 INIT_WORK(&mwork
->work
, mbox_up_handler
);
2463 destroy_workqueue(mw
->mbox_wq
);
2466 iounmap((void __iomem
*)mbox_regions
[num
]);
2468 kfree(mbox_regions
);
2470 bitmap_free(pf_bmap
);
2474 static void rvu_mbox_destroy(struct mbox_wq_info
*mw
)
2476 struct otx2_mbox
*mbox
= &mw
->mbox
;
2477 struct otx2_mbox_dev
*mdev
;
2481 destroy_workqueue(mw
->mbox_wq
);
2485 for (devid
= 0; devid
< mbox
->ndevs
; devid
++) {
2486 mdev
= &mbox
->dev
[devid
];
2488 iounmap((void __iomem
*)mdev
->hwbase
);
2491 otx2_mbox_destroy(&mw
->mbox
);
2492 otx2_mbox_destroy(&mw
->mbox_up
);
2495 static void rvu_queue_work(struct mbox_wq_info
*mw
, int first
,
2496 int mdevs
, u64 intr
)
2498 struct otx2_mbox_dev
*mdev
;
2499 struct otx2_mbox
*mbox
;
2500 struct mbox_hdr
*hdr
;
2503 for (i
= first
; i
< mdevs
; i
++) {
2505 if (!(intr
& BIT_ULL(i
- first
)))
2509 mdev
= &mbox
->dev
[i
];
2510 hdr
= mdev
->mbase
+ mbox
->rx_start
;
2512 /*The hdr->num_msgs is set to zero immediately in the interrupt
2513 * handler to ensure that it holds a correct value next time
2514 * when the interrupt handler is called.
2515 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
2516 * pf>mbox.up_num_msgs holds the data for use in
2517 * pfaf_mbox_up_handler.
2520 if (hdr
->num_msgs
) {
2521 mw
->mbox_wrk
[i
].num_msgs
= hdr
->num_msgs
;
2523 queue_work(mw
->mbox_wq
, &mw
->mbox_wrk
[i
].work
);
2525 mbox
= &mw
->mbox_up
;
2526 mdev
= &mbox
->dev
[i
];
2527 hdr
= mdev
->mbase
+ mbox
->rx_start
;
2528 if (hdr
->num_msgs
) {
2529 mw
->mbox_wrk_up
[i
].up_num_msgs
= hdr
->num_msgs
;
2531 queue_work(mw
->mbox_wq
, &mw
->mbox_wrk_up
[i
].work
);
2536 static irqreturn_t
rvu_mbox_intr_handler(int irq
, void *rvu_irq
)
2538 struct rvu
*rvu
= (struct rvu
*)rvu_irq
;
2542 intr
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_AF_PFAF_MBOX_INT
);
2543 /* Clear interrupts */
2544 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFAF_MBOX_INT
, intr
);
2546 trace_otx2_msg_interrupt(rvu
->pdev
, "PF(s) to AF", intr
);
2548 /* Sync with mbox memory region */
2551 rvu_queue_work(&rvu
->afpf_wq_info
, 0, rvu
->hw
->total_pfs
, intr
);
2553 /* Handle VF interrupts */
2555 intr
= rvupf_read64(rvu
, RVU_PF_VFPF_MBOX_INTX(1));
2556 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INTX(1), intr
);
2558 rvu_queue_work(&rvu
->afvf_wq_info
, 64, vfs
, intr
);
2562 intr
= rvupf_read64(rvu
, RVU_PF_VFPF_MBOX_INTX(0));
2563 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INTX(0), intr
);
2565 trace_otx2_msg_interrupt(rvu
->pdev
, "VF(s) to AF", intr
);
2567 rvu_queue_work(&rvu
->afvf_wq_info
, 0, vfs
, intr
);
2572 static void rvu_enable_mbox_intr(struct rvu
*rvu
)
2574 struct rvu_hwinfo
*hw
= rvu
->hw
;
2576 /* Clear spurious irqs, if any */
2577 rvu_write64(rvu
, BLKADDR_RVUM
,
2578 RVU_AF_PFAF_MBOX_INT
, INTR_MASK(hw
->total_pfs
));
2580 /* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */
2581 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFAF_MBOX_INT_ENA_W1S
,
2582 INTR_MASK(hw
->total_pfs
) & ~1ULL);
2585 static void rvu_blklf_teardown(struct rvu
*rvu
, u16 pcifunc
, u8 blkaddr
)
2587 struct rvu_block
*block
;
2588 int slot
, lf
, num_lfs
;
2591 block
= &rvu
->hw
->block
[blkaddr
];
2592 num_lfs
= rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu
, pcifunc
),
2596 for (slot
= 0; slot
< num_lfs
; slot
++) {
2597 lf
= rvu_get_lf(rvu
, block
, pcifunc
, slot
);
2601 /* Cleanup LF and reset it */
2602 if (block
->addr
== BLKADDR_NIX0
|| block
->addr
== BLKADDR_NIX1
)
2603 rvu_nix_lf_teardown(rvu
, pcifunc
, block
->addr
, lf
);
2604 else if (block
->addr
== BLKADDR_NPA
)
2605 rvu_npa_lf_teardown(rvu
, pcifunc
, lf
);
2606 else if ((block
->addr
== BLKADDR_CPT0
) ||
2607 (block
->addr
== BLKADDR_CPT1
))
2608 rvu_cpt_lf_teardown(rvu
, pcifunc
, block
->addr
, lf
,
2611 err
= rvu_lf_reset(rvu
, block
, lf
);
2613 dev_err(rvu
->dev
, "Failed to reset blkaddr %d LF%d\n",
2619 static void __rvu_flr_handler(struct rvu
*rvu
, u16 pcifunc
)
2621 if (rvu_npc_exact_has_match_table(rvu
))
2622 rvu_npc_exact_reset(rvu
, pcifunc
);
2624 mutex_lock(&rvu
->flr_lock
);
2625 /* Reset order should reflect inter-block dependencies:
2626 * 1. Reset any packet/work sources (NIX, CPT, TIM)
2627 * 2. Flush and reset SSO/SSOW
2628 * 3. Cleanup pools (NPA)
2631 /* Free allocated BPIDs */
2632 rvu_nix_flr_free_bpids(rvu
, pcifunc
);
2634 /* Free multicast/mirror node associated with the 'pcifunc' */
2635 rvu_nix_mcast_flr_free_entries(rvu
, pcifunc
);
2637 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_NIX0
);
2638 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_NIX1
);
2639 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_CPT0
);
2640 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_CPT1
);
2641 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_TIM
);
2642 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_SSOW
);
2643 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_SSO
);
2644 rvu_blklf_teardown(rvu
, pcifunc
, BLKADDR_NPA
);
2645 rvu_reset_lmt_map_tbl(rvu
, pcifunc
);
2646 rvu_detach_rsrcs(rvu
, NULL
, pcifunc
);
2647 /* In scenarios where PF/VF drivers detach NIXLF without freeing MCAM
2648 * entries, check and free the MCAM entries explicitly to avoid leak.
2649 * Since LF is detached use LF number as -1.
2651 rvu_npc_free_mcam_entries(rvu
, pcifunc
, -1);
2652 rvu_mac_reset(rvu
, pcifunc
);
2654 if (rvu
->mcs_blk_cnt
)
2655 rvu_mcs_flr_handler(rvu
, pcifunc
);
2657 mutex_unlock(&rvu
->flr_lock
);
2660 static void rvu_afvf_flr_handler(struct rvu
*rvu
, int vf
)
2664 /* pcifunc = 0(PF0) | (vf + 1) */
2665 __rvu_flr_handler(rvu
, vf
+ 1);
2672 /* Signal FLR finish and enable IRQ */
2673 rvupf_write64(rvu
, RVU_PF_VFTRPENDX(reg
), BIT_ULL(vf
));
2674 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1SX(reg
), BIT_ULL(vf
));
2677 static void rvu_flr_handler(struct work_struct
*work
)
2679 struct rvu_work
*flrwork
= container_of(work
, struct rvu_work
, work
);
2680 struct rvu
*rvu
= flrwork
->rvu
;
2681 u16 pcifunc
, numvfs
, vf
;
2685 pf
= flrwork
- rvu
->flr_wrk
;
2686 if (pf
>= rvu
->hw
->total_pfs
) {
2687 rvu_afvf_flr_handler(rvu
, pf
- rvu
->hw
->total_pfs
);
2691 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
2692 numvfs
= (cfg
>> 12) & 0xFF;
2693 pcifunc
= pf
<< RVU_PFVF_PF_SHIFT
;
2695 for (vf
= 0; vf
< numvfs
; vf
++)
2696 __rvu_flr_handler(rvu
, (pcifunc
| (vf
+ 1)));
2698 __rvu_flr_handler(rvu
, pcifunc
);
2700 /* Signal FLR finish */
2701 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFTRPEND
, BIT_ULL(pf
));
2703 /* Enable interrupt */
2704 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT_ENA_W1S
, BIT_ULL(pf
));
2707 static void rvu_afvf_queue_flr_work(struct rvu
*rvu
, int start_vf
, int numvfs
)
2709 int dev
, vf
, reg
= 0;
2715 intr
= rvupf_read64(rvu
, RVU_PF_VFFLR_INTX(reg
));
2719 for (vf
= 0; vf
< numvfs
; vf
++) {
2720 if (!(intr
& BIT_ULL(vf
)))
2722 /* Clear and disable the interrupt */
2723 rvupf_write64(rvu
, RVU_PF_VFFLR_INTX(reg
), BIT_ULL(vf
));
2724 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1CX(reg
), BIT_ULL(vf
));
2726 dev
= vf
+ start_vf
+ rvu
->hw
->total_pfs
;
2727 queue_work(rvu
->flr_wq
, &rvu
->flr_wrk
[dev
].work
);
2731 static irqreturn_t
rvu_flr_intr_handler(int irq
, void *rvu_irq
)
2733 struct rvu
*rvu
= (struct rvu
*)rvu_irq
;
2737 intr
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT
);
2741 for (pf
= 0; pf
< rvu
->hw
->total_pfs
; pf
++) {
2742 if (intr
& (1ULL << pf
)) {
2743 /* clear interrupt */
2744 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT
,
2746 /* Disable the interrupt */
2747 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT_ENA_W1C
,
2749 /* PF is already dead do only AF related operations */
2750 queue_work(rvu
->flr_wq
, &rvu
->flr_wrk
[pf
].work
);
2755 rvu_afvf_queue_flr_work(rvu
, 0, 64);
2757 rvu_afvf_queue_flr_work(rvu
, 64, rvu
->vfs
- 64);
2762 static void rvu_me_handle_vfset(struct rvu
*rvu
, int idx
, u64 intr
)
2766 /* Nothing to be done here other than clearing the
2769 for (vf
= 0; vf
< 64; vf
++) {
2770 if (intr
& (1ULL << vf
)) {
2771 /* clear the trpend due to ME(master enable) */
2772 rvupf_write64(rvu
, RVU_PF_VFTRPENDX(idx
), BIT_ULL(vf
));
2773 /* clear interrupt */
2774 rvupf_write64(rvu
, RVU_PF_VFME_INTX(idx
), BIT_ULL(vf
));
2779 /* Handles ME interrupts from VFs of AF */
2780 static irqreturn_t
rvu_me_vf_intr_handler(int irq
, void *rvu_irq
)
2782 struct rvu
*rvu
= (struct rvu
*)rvu_irq
;
2786 intr
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_AF_PFME_INT
);
2788 for (vfset
= 0; vfset
<= 1; vfset
++) {
2789 intr
= rvupf_read64(rvu
, RVU_PF_VFME_INTX(vfset
));
2791 rvu_me_handle_vfset(rvu
, vfset
, intr
);
2797 /* Handles ME interrupts from PFs */
2798 static irqreturn_t
rvu_me_pf_intr_handler(int irq
, void *rvu_irq
)
2800 struct rvu
*rvu
= (struct rvu
*)rvu_irq
;
2804 intr
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_AF_PFME_INT
);
2806 /* Nothing to be done here other than clearing the
2809 for (pf
= 0; pf
< rvu
->hw
->total_pfs
; pf
++) {
2810 if (intr
& (1ULL << pf
)) {
2811 /* clear the trpend due to ME(master enable) */
2812 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFTRPEND
,
2814 /* clear interrupt */
2815 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFME_INT
,
2823 static void rvu_unregister_interrupts(struct rvu
*rvu
)
2827 rvu_cpt_unregister_interrupts(rvu
);
2829 /* Disable the Mbox interrupt */
2830 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFAF_MBOX_INT_ENA_W1C
,
2831 INTR_MASK(rvu
->hw
->total_pfs
) & ~1ULL);
2833 /* Disable the PF FLR interrupt */
2834 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT_ENA_W1C
,
2835 INTR_MASK(rvu
->hw
->total_pfs
) & ~1ULL);
2837 /* Disable the PF ME interrupt */
2838 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFME_INT_ENA_W1C
,
2839 INTR_MASK(rvu
->hw
->total_pfs
) & ~1ULL);
2841 for (irq
= 0; irq
< rvu
->num_vec
; irq
++) {
2842 if (rvu
->irq_allocated
[irq
]) {
2843 free_irq(pci_irq_vector(rvu
->pdev
, irq
), rvu
);
2844 rvu
->irq_allocated
[irq
] = false;
2848 pci_free_irq_vectors(rvu
->pdev
);
2852 static int rvu_afvf_msix_vectors_num_ok(struct rvu
*rvu
)
2854 struct rvu_pfvf
*pfvf
= &rvu
->pf
[0];
2858 offset
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2860 /* Make sure there are enough MSIX vectors configured so that
2861 * VF interrupts can be handled. Offset equal to zero means
2862 * that PF vectors are not configured and overlapping AF vectors.
2864 return (pfvf
->msix
.max
>= RVU_AF_INT_VEC_CNT
+ RVU_PF_INT_VEC_CNT
) &&
2868 static int rvu_register_interrupts(struct rvu
*rvu
)
2870 int ret
, offset
, pf_vec_start
;
2872 rvu
->num_vec
= pci_msix_vec_count(rvu
->pdev
);
2874 rvu
->irq_name
= devm_kmalloc_array(rvu
->dev
, rvu
->num_vec
,
2875 NAME_SIZE
, GFP_KERNEL
);
2879 rvu
->irq_allocated
= devm_kcalloc(rvu
->dev
, rvu
->num_vec
,
2880 sizeof(bool), GFP_KERNEL
);
2881 if (!rvu
->irq_allocated
)
2885 ret
= pci_alloc_irq_vectors(rvu
->pdev
, rvu
->num_vec
,
2886 rvu
->num_vec
, PCI_IRQ_MSIX
);
2889 "RVUAF: Request for %d msix vectors failed, ret %d\n",
2894 /* Register mailbox interrupt handler */
2895 sprintf(&rvu
->irq_name
[RVU_AF_INT_VEC_MBOX
* NAME_SIZE
], "RVUAF Mbox");
2896 ret
= request_irq(pci_irq_vector(rvu
->pdev
, RVU_AF_INT_VEC_MBOX
),
2897 rvu_mbox_intr_handler
, 0,
2898 &rvu
->irq_name
[RVU_AF_INT_VEC_MBOX
* NAME_SIZE
], rvu
);
2901 "RVUAF: IRQ registration failed for mbox irq\n");
2905 rvu
->irq_allocated
[RVU_AF_INT_VEC_MBOX
] = true;
2907 /* Enable mailbox interrupts from all PFs */
2908 rvu_enable_mbox_intr(rvu
);
2910 /* Register FLR interrupt handler */
2911 sprintf(&rvu
->irq_name
[RVU_AF_INT_VEC_PFFLR
* NAME_SIZE
],
2913 ret
= request_irq(pci_irq_vector(rvu
->pdev
, RVU_AF_INT_VEC_PFFLR
),
2914 rvu_flr_intr_handler
, 0,
2915 &rvu
->irq_name
[RVU_AF_INT_VEC_PFFLR
* NAME_SIZE
],
2919 "RVUAF: IRQ registration failed for FLR\n");
2922 rvu
->irq_allocated
[RVU_AF_INT_VEC_PFFLR
] = true;
2924 /* Enable FLR interrupt for all PFs*/
2925 rvu_write64(rvu
, BLKADDR_RVUM
,
2926 RVU_AF_PFFLR_INT
, INTR_MASK(rvu
->hw
->total_pfs
));
2928 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFFLR_INT_ENA_W1S
,
2929 INTR_MASK(rvu
->hw
->total_pfs
) & ~1ULL);
2931 /* Register ME interrupt handler */
2932 sprintf(&rvu
->irq_name
[RVU_AF_INT_VEC_PFME
* NAME_SIZE
],
2934 ret
= request_irq(pci_irq_vector(rvu
->pdev
, RVU_AF_INT_VEC_PFME
),
2935 rvu_me_pf_intr_handler
, 0,
2936 &rvu
->irq_name
[RVU_AF_INT_VEC_PFME
* NAME_SIZE
],
2940 "RVUAF: IRQ registration failed for ME\n");
2942 rvu
->irq_allocated
[RVU_AF_INT_VEC_PFME
] = true;
2944 /* Clear TRPEND bit for all PF */
2945 rvu_write64(rvu
, BLKADDR_RVUM
,
2946 RVU_AF_PFTRPEND
, INTR_MASK(rvu
->hw
->total_pfs
));
2947 /* Enable ME interrupt for all PFs*/
2948 rvu_write64(rvu
, BLKADDR_RVUM
,
2949 RVU_AF_PFME_INT
, INTR_MASK(rvu
->hw
->total_pfs
));
2951 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_AF_PFME_INT_ENA_W1S
,
2952 INTR_MASK(rvu
->hw
->total_pfs
) & ~1ULL);
2954 if (!rvu_afvf_msix_vectors_num_ok(rvu
))
2957 /* Get PF MSIX vectors offset. */
2958 pf_vec_start
= rvu_read64(rvu
, BLKADDR_RVUM
,
2959 RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2961 /* Register MBOX0 interrupt. */
2962 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFPF_MBOX0
;
2963 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF Mbox0");
2964 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
2965 rvu_mbox_intr_handler
, 0,
2966 &rvu
->irq_name
[offset
* NAME_SIZE
],
2970 "RVUAF: IRQ registration failed for Mbox0\n");
2972 rvu
->irq_allocated
[offset
] = true;
2974 /* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so
2975 * simply increment current offset by 1.
2977 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFPF_MBOX1
;
2978 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF Mbox1");
2979 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
2980 rvu_mbox_intr_handler
, 0,
2981 &rvu
->irq_name
[offset
* NAME_SIZE
],
2985 "RVUAF: IRQ registration failed for Mbox1\n");
2987 rvu
->irq_allocated
[offset
] = true;
2989 /* Register FLR interrupt handler for AF's VFs */
2990 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFFLR0
;
2991 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF FLR0");
2992 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
2993 rvu_flr_intr_handler
, 0,
2994 &rvu
->irq_name
[offset
* NAME_SIZE
], rvu
);
2997 "RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
3000 rvu
->irq_allocated
[offset
] = true;
3002 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFFLR1
;
3003 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF FLR1");
3004 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
3005 rvu_flr_intr_handler
, 0,
3006 &rvu
->irq_name
[offset
* NAME_SIZE
], rvu
);
3009 "RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
3012 rvu
->irq_allocated
[offset
] = true;
3014 /* Register ME interrupt handler for AF's VFs */
3015 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFME0
;
3016 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF ME0");
3017 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
3018 rvu_me_vf_intr_handler
, 0,
3019 &rvu
->irq_name
[offset
* NAME_SIZE
], rvu
);
3022 "RVUAF: IRQ registration failed for RVUAFVF ME0\n");
3025 rvu
->irq_allocated
[offset
] = true;
3027 offset
= pf_vec_start
+ RVU_PF_INT_VEC_VFME1
;
3028 sprintf(&rvu
->irq_name
[offset
* NAME_SIZE
], "RVUAFVF ME1");
3029 ret
= request_irq(pci_irq_vector(rvu
->pdev
, offset
),
3030 rvu_me_vf_intr_handler
, 0,
3031 &rvu
->irq_name
[offset
* NAME_SIZE
], rvu
);
3034 "RVUAF: IRQ registration failed for RVUAFVF ME1\n");
3037 rvu
->irq_allocated
[offset
] = true;
3039 ret
= rvu_cpt_register_interrupts(rvu
);
3046 rvu_unregister_interrupts(rvu
);
3050 static void rvu_flr_wq_destroy(struct rvu
*rvu
)
3053 destroy_workqueue(rvu
->flr_wq
);
3058 static int rvu_flr_init(struct rvu
*rvu
)
3064 /* Enable FLR for all PFs*/
3065 for (pf
= 0; pf
< rvu
->hw
->total_pfs
; pf
++) {
3066 cfg
= rvu_read64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
));
3067 rvu_write64(rvu
, BLKADDR_RVUM
, RVU_PRIV_PFX_CFG(pf
),
3071 rvu
->flr_wq
= alloc_ordered_workqueue("rvu_afpf_flr",
3072 WQ_HIGHPRI
| WQ_MEM_RECLAIM
);
3076 num_devs
= rvu
->hw
->total_pfs
+ pci_sriov_get_totalvfs(rvu
->pdev
);
3077 rvu
->flr_wrk
= devm_kcalloc(rvu
->dev
, num_devs
,
3078 sizeof(struct rvu_work
), GFP_KERNEL
);
3079 if (!rvu
->flr_wrk
) {
3080 destroy_workqueue(rvu
->flr_wq
);
3084 for (dev
= 0; dev
< num_devs
; dev
++) {
3085 rvu
->flr_wrk
[dev
].rvu
= rvu
;
3086 INIT_WORK(&rvu
->flr_wrk
[dev
].work
, rvu_flr_handler
);
3089 mutex_init(&rvu
->flr_lock
);
3094 static void rvu_disable_afvf_intr(struct rvu
*rvu
)
3098 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs
));
3099 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs
));
3100 rvupf_write64(rvu
, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs
));
3104 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
3105 INTR_MASK(vfs
- 64));
3106 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs
- 64));
3107 rvupf_write64(rvu
, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs
- 64));
3110 static void rvu_enable_afvf_intr(struct rvu
*rvu
)
3114 /* Clear any pending interrupts and enable AF VF interrupts for
3118 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs
));
3119 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs
));
3122 rvupf_write64(rvu
, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs
));
3123 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs
));
3124 rvupf_write64(rvu
, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs
));
3126 /* Same for remaining VFs, if any. */
3130 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs
- 64));
3131 rvupf_write64(rvu
, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
3132 INTR_MASK(vfs
- 64));
3134 rvupf_write64(rvu
, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs
- 64));
3135 rvupf_write64(rvu
, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs
- 64));
3136 rvupf_write64(rvu
, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs
- 64));
3139 int rvu_get_num_lbk_chans(void)
3141 struct pci_dev
*pdev
;
3145 pdev
= pci_get_device(PCI_VENDOR_ID_CAVIUM
, PCI_DEVID_OCTEONTX2_LBK
,
3150 base
= pci_ioremap_bar(pdev
, 0);
3154 /* Read number of available LBK channels from LBK(0)_CONST register. */
3155 ret
= (readq(base
+ 0x10) >> 32) & 0xffff;
3163 static int rvu_enable_sriov(struct rvu
*rvu
)
3165 struct pci_dev
*pdev
= rvu
->pdev
;
3166 int err
, chans
, vfs
;
3169 if (!rvu_afvf_msix_vectors_num_ok(rvu
)) {
3170 dev_warn(&pdev
->dev
,
3171 "Skipping SRIOV enablement since not enough IRQs are available\n");
3175 /* Get RVU VFs device id */
3176 pos
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_SRIOV
);
3179 pci_read_config_word(pdev
, pos
+ PCI_SRIOV_VF_DID
, &rvu
->vf_devid
);
3181 chans
= rvu_get_num_lbk_chans();
3185 vfs
= pci_sriov_get_totalvfs(pdev
);
3187 /* Limit VFs in case we have more VFs than LBK channels available. */
3194 /* LBK channel number 63 is used for switching packets between
3195 * CGX mapped VFs. Hence limit LBK pairs till 62 only.
3200 /* Save VFs number for reference in VF interrupts handlers.
3201 * Since interrupts might start arriving during SRIOV enablement
3202 * ordinary API cannot be used to get number of enabled VFs.
3206 err
= rvu_mbox_init(rvu
, &rvu
->afvf_wq_info
, TYPE_AFVF
, vfs
,
3207 rvu_afvf_mbox_handler
, rvu_afvf_mbox_up_handler
);
3211 rvu_enable_afvf_intr(rvu
);
3212 /* Make sure IRQs are enabled before SRIOV. */
3215 err
= pci_enable_sriov(pdev
, vfs
);
3217 rvu_disable_afvf_intr(rvu
);
3218 rvu_mbox_destroy(&rvu
->afvf_wq_info
);
3225 static void rvu_disable_sriov(struct rvu
*rvu
)
3227 rvu_disable_afvf_intr(rvu
);
3228 rvu_mbox_destroy(&rvu
->afvf_wq_info
);
3229 pci_disable_sriov(rvu
->pdev
);
3232 static void rvu_update_module_params(struct rvu
*rvu
)
3234 const char *default_pfl_name
= "default";
3236 strscpy(rvu
->mkex_pfl_name
,
3237 mkex_profile
? mkex_profile
: default_pfl_name
, MKEX_NAME_LEN
);
3238 strscpy(rvu
->kpu_pfl_name
,
3239 kpu_profile
? kpu_profile
: default_pfl_name
, KPU_NAME_LEN
);
3242 static int rvu_probe(struct pci_dev
*pdev
, const struct pci_device_id
*id
)
3244 struct device
*dev
= &pdev
->dev
;
3248 rvu
= devm_kzalloc(dev
, sizeof(*rvu
), GFP_KERNEL
);
3252 rvu
->hw
= devm_kzalloc(dev
, sizeof(struct rvu_hwinfo
), GFP_KERNEL
);
3254 devm_kfree(dev
, rvu
);
3258 pci_set_drvdata(pdev
, rvu
);
3260 rvu
->dev
= &pdev
->dev
;
3262 err
= pci_enable_device(pdev
);
3264 dev_err(dev
, "Failed to enable PCI device\n");
3268 err
= pci_request_regions(pdev
, DRV_NAME
);
3270 dev_err(dev
, "PCI request regions failed 0x%x\n", err
);
3271 goto err_disable_device
;
3274 err
= dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(48));
3276 dev_err(dev
, "DMA mask config failed, abort\n");
3277 goto err_release_regions
;
3280 pci_set_master(pdev
);
3282 rvu
->ptp
= ptp_get();
3283 if (IS_ERR(rvu
->ptp
)) {
3284 err
= PTR_ERR(rvu
->ptp
);
3286 goto err_release_regions
;
3290 /* Map Admin function CSRs */
3291 rvu
->afreg_base
= pcim_iomap(pdev
, PCI_AF_REG_BAR_NUM
, 0);
3292 rvu
->pfreg_base
= pcim_iomap(pdev
, PCI_PF_REG_BAR_NUM
, 0);
3293 if (!rvu
->afreg_base
|| !rvu
->pfreg_base
) {
3294 dev_err(dev
, "Unable to map admin function CSRs, aborting\n");
3299 /* Store module params in rvu structure */
3300 rvu_update_module_params(rvu
);
3302 /* Check which blocks the HW supports */
3303 rvu_check_block_implemented(rvu
);
3305 rvu_reset_all_blocks(rvu
);
3307 rvu_setup_hw_capabilities(rvu
);
3309 err
= rvu_setup_hw_resources(rvu
);
3313 /* Init mailbox btw AF and PFs */
3314 err
= rvu_mbox_init(rvu
, &rvu
->afpf_wq_info
, TYPE_AFPF
,
3315 rvu
->hw
->total_pfs
, rvu_afpf_mbox_handler
,
3316 rvu_afpf_mbox_up_handler
);
3318 dev_err(dev
, "%s: Failed to initialize mbox\n", __func__
);
3322 err
= rvu_flr_init(rvu
);
3324 dev_err(dev
, "%s: Failed to initialize flr\n", __func__
);
3328 err
= rvu_register_interrupts(rvu
);
3330 dev_err(dev
, "%s: Failed to register interrupts\n", __func__
);
3334 err
= rvu_register_dl(rvu
);
3336 dev_err(dev
, "%s: Failed to register devlink\n", __func__
);
3340 rvu_setup_rvum_blk_revid(rvu
);
3342 /* Enable AF's VFs (if any) */
3343 err
= rvu_enable_sriov(rvu
);
3345 dev_err(dev
, "%s: Failed to enable sriov\n", __func__
);
3349 /* Initialize debugfs */
3352 mutex_init(&rvu
->rswitch
.switch_lock
);
3355 ptp_start(rvu
, rvu
->fwdata
->sclk
, rvu
->fwdata
->ptp_ext_clk_rate
,
3356 rvu
->fwdata
->ptp_ext_tstamp
);
3360 rvu_unregister_dl(rvu
);
3362 rvu_unregister_interrupts(rvu
);
3364 rvu_flr_wq_destroy(rvu
);
3366 rvu_mbox_destroy(&rvu
->afpf_wq_info
);
3369 rvu_fwdata_exit(rvu
);
3371 rvu_reset_all_blocks(rvu
);
3372 rvu_free_hw_resources(rvu
);
3373 rvu_clear_rvum_blk_revid(rvu
);
3376 err_release_regions
:
3377 pci_release_regions(pdev
);
3379 pci_disable_device(pdev
);
3381 pci_set_drvdata(pdev
, NULL
);
3382 devm_kfree(&pdev
->dev
, rvu
->hw
);
3383 devm_kfree(dev
, rvu
);
3387 static void rvu_remove(struct pci_dev
*pdev
)
3389 struct rvu
*rvu
= pci_get_drvdata(pdev
);
3392 rvu_unregister_dl(rvu
);
3393 rvu_unregister_interrupts(rvu
);
3394 rvu_flr_wq_destroy(rvu
);
3396 rvu_fwdata_exit(rvu
);
3398 rvu_mbox_destroy(&rvu
->afpf_wq_info
);
3399 rvu_disable_sriov(rvu
);
3400 rvu_reset_all_blocks(rvu
);
3401 rvu_free_hw_resources(rvu
);
3402 rvu_clear_rvum_blk_revid(rvu
);
3404 pci_release_regions(pdev
);
3405 pci_disable_device(pdev
);
3406 pci_set_drvdata(pdev
, NULL
);
3408 devm_kfree(&pdev
->dev
, rvu
->hw
);
3409 devm_kfree(&pdev
->dev
, rvu
);
3412 static struct pci_driver rvu_driver
= {
3414 .id_table
= rvu_id_table
,
3416 .remove
= rvu_remove
,
3419 static int __init
rvu_init_module(void)
3423 pr_info("%s: %s\n", DRV_NAME
, DRV_STRING
);
3425 err
= pci_register_driver(&cgx_driver
);
3429 err
= pci_register_driver(&ptp_driver
);
3433 err
= pci_register_driver(&mcs_driver
);
3437 err
= pci_register_driver(&rvu_driver
);
3443 pci_unregister_driver(&mcs_driver
);
3445 pci_unregister_driver(&ptp_driver
);
3447 pci_unregister_driver(&cgx_driver
);
3452 static void __exit
rvu_cleanup_module(void)
3454 pci_unregister_driver(&rvu_driver
);
3455 pci_unregister_driver(&mcs_driver
);
3456 pci_unregister_driver(&ptp_driver
);
3457 pci_unregister_driver(&cgx_driver
);
3460 module_init(rvu_init_module
);
3461 module_exit(rvu_cleanup_module
);