1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018 Intel Corporation */
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
10 #include <linux/pm_runtime.h>
11 #include <net/pkt_sched.h>
12 #include <linux/bpf_trace.h>
13 #include <net/xdp_sock_drv.h>
14 #include <linux/pci.h>
23 #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver"
25 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
27 #define IGC_XDP_PASS 0
28 #define IGC_XDP_CONSUMED BIT(0)
29 #define IGC_XDP_TX BIT(1)
30 #define IGC_XDP_REDIRECT BIT(2)
32 static int debug
= -1;
34 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
35 MODULE_DESCRIPTION(DRV_SUMMARY
);
36 MODULE_LICENSE("GPL v2");
37 module_param(debug
, int, 0);
38 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
40 char igc_driver_name
[] = "igc";
41 static const char igc_driver_string
[] = DRV_SUMMARY
;
42 static const char igc_copyright
[] =
43 "Copyright(c) 2018 Intel Corporation.";
45 static const struct igc_info
*igc_info_tbl
[] = {
46 [board_base
] = &igc_base_info
,
49 static const struct pci_device_id igc_pci_tbl
[] = {
50 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_LM
), board_base
},
51 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_V
), board_base
},
52 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_I
), board_base
},
53 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I220_V
), board_base
},
54 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_K
), board_base
},
55 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_K2
), board_base
},
56 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_K
), board_base
},
57 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_LMVP
), board_base
},
58 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_LMVP
), board_base
},
59 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_IT
), board_base
},
60 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_LM
), board_base
},
61 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_V
), board_base
},
62 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_IT
), board_base
},
63 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I221_V
), board_base
},
64 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I226_BLANK_NVM
), board_base
},
65 { PCI_VDEVICE(INTEL
, IGC_DEV_ID_I225_BLANK_NVM
), board_base
},
66 /* required last entry */
70 MODULE_DEVICE_TABLE(pci
, igc_pci_tbl
);
79 void igc_reset(struct igc_adapter
*adapter
)
81 struct net_device
*dev
= adapter
->netdev
;
82 struct igc_hw
*hw
= &adapter
->hw
;
83 struct igc_fc_info
*fc
= &hw
->fc
;
86 /* Repartition PBA for greater than 9k MTU if required */
89 /* flow control settings
90 * The high water mark must be low enough to fit one full frame
91 * after transmitting the pause frame. As such we must have enough
92 * space to allow for us to complete our current transmit and then
93 * receive the frame that is in progress from the link partner.
95 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
97 hwm
= (pba
<< 10) - (adapter
->max_frame_size
+ MAX_JUMBO_FRAME_SIZE
);
99 fc
->high_water
= hwm
& 0xFFFFFFF0; /* 16-byte granularity */
100 fc
->low_water
= fc
->high_water
- 16;
101 fc
->pause_time
= 0xFFFF;
103 fc
->current_mode
= fc
->requested_mode
;
105 hw
->mac
.ops
.reset_hw(hw
);
107 if (hw
->mac
.ops
.init_hw(hw
))
108 netdev_err(dev
, "Error on hardware initialization\n");
110 /* Re-establish EEE setting */
111 igc_set_eee_i225(hw
, true, true, true);
113 if (!netif_running(adapter
->netdev
))
114 igc_power_down_phy_copper_base(&adapter
->hw
);
116 /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
117 wr32(IGC_VET
, ETH_P_8021Q
);
119 /* Re-enable PTP, where applicable. */
120 igc_ptp_reset(adapter
);
122 /* Re-enable TSN offloading, where applicable. */
123 igc_tsn_reset(adapter
);
125 igc_get_phy_info(hw
);
129 * igc_power_up_link - Power up the phy link
130 * @adapter: address of board private structure
132 static void igc_power_up_link(struct igc_adapter
*adapter
)
134 igc_reset_phy(&adapter
->hw
);
136 igc_power_up_phy_copper(&adapter
->hw
);
138 igc_setup_link(&adapter
->hw
);
142 * igc_release_hw_control - release control of the h/w to f/w
143 * @adapter: address of board private structure
145 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
146 * For ASF and Pass Through versions of f/w this means that the
147 * driver is no longer loaded.
149 static void igc_release_hw_control(struct igc_adapter
*adapter
)
151 struct igc_hw
*hw
= &adapter
->hw
;
154 if (!pci_device_is_present(adapter
->pdev
))
157 /* Let firmware take over control of h/w */
158 ctrl_ext
= rd32(IGC_CTRL_EXT
);
160 ctrl_ext
& ~IGC_CTRL_EXT_DRV_LOAD
);
164 * igc_get_hw_control - get control of the h/w from f/w
165 * @adapter: address of board private structure
167 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
168 * For ASF and Pass Through versions of f/w this means that
169 * the driver is loaded.
171 static void igc_get_hw_control(struct igc_adapter
*adapter
)
173 struct igc_hw
*hw
= &adapter
->hw
;
176 /* Let firmware know the driver has taken over */
177 ctrl_ext
= rd32(IGC_CTRL_EXT
);
179 ctrl_ext
| IGC_CTRL_EXT_DRV_LOAD
);
182 static void igc_unmap_tx_buffer(struct device
*dev
, struct igc_tx_buffer
*buf
)
184 dma_unmap_single(dev
, dma_unmap_addr(buf
, dma
),
185 dma_unmap_len(buf
, len
), DMA_TO_DEVICE
);
187 dma_unmap_len_set(buf
, len
, 0);
191 * igc_clean_tx_ring - Free Tx Buffers
192 * @tx_ring: ring to be cleaned
194 static void igc_clean_tx_ring(struct igc_ring
*tx_ring
)
196 u16 i
= tx_ring
->next_to_clean
;
197 struct igc_tx_buffer
*tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
200 while (i
!= tx_ring
->next_to_use
) {
201 union igc_adv_tx_desc
*eop_desc
, *tx_desc
;
203 switch (tx_buffer
->type
) {
204 case IGC_TX_BUFFER_TYPE_XSK
:
207 case IGC_TX_BUFFER_TYPE_XDP
:
208 xdp_return_frame(tx_buffer
->xdpf
);
209 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
211 case IGC_TX_BUFFER_TYPE_SKB
:
212 dev_kfree_skb_any(tx_buffer
->skb
);
213 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
216 netdev_warn_once(tx_ring
->netdev
, "Unknown Tx buffer type\n");
220 /* check for eop_desc to determine the end of the packet */
221 eop_desc
= tx_buffer
->next_to_watch
;
222 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
224 /* unmap remaining buffers */
225 while (tx_desc
!= eop_desc
) {
229 if (unlikely(i
== tx_ring
->count
)) {
231 tx_buffer
= tx_ring
->tx_buffer_info
;
232 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
235 /* unmap any remaining paged data */
236 if (dma_unmap_len(tx_buffer
, len
))
237 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
240 tx_buffer
->next_to_watch
= NULL
;
242 /* move us one more past the eop_desc for start of next pkt */
245 if (unlikely(i
== tx_ring
->count
)) {
247 tx_buffer
= tx_ring
->tx_buffer_info
;
251 if (tx_ring
->xsk_pool
&& xsk_frames
)
252 xsk_tx_completed(tx_ring
->xsk_pool
, xsk_frames
);
254 /* reset BQL for queue */
255 netdev_tx_reset_queue(txring_txq(tx_ring
));
257 /* Zero out the buffer ring */
258 memset(tx_ring
->tx_buffer_info
, 0,
259 sizeof(*tx_ring
->tx_buffer_info
) * tx_ring
->count
);
261 /* Zero out the descriptor ring */
262 memset(tx_ring
->desc
, 0, tx_ring
->size
);
264 /* reset next_to_use and next_to_clean */
265 tx_ring
->next_to_use
= 0;
266 tx_ring
->next_to_clean
= 0;
270 * igc_free_tx_resources - Free Tx Resources per Queue
271 * @tx_ring: Tx descriptor ring for a specific queue
273 * Free all transmit software resources
275 void igc_free_tx_resources(struct igc_ring
*tx_ring
)
277 igc_disable_tx_ring(tx_ring
);
279 vfree(tx_ring
->tx_buffer_info
);
280 tx_ring
->tx_buffer_info
= NULL
;
282 /* if not set, then don't free */
286 dma_free_coherent(tx_ring
->dev
, tx_ring
->size
,
287 tx_ring
->desc
, tx_ring
->dma
);
289 tx_ring
->desc
= NULL
;
293 * igc_free_all_tx_resources - Free Tx Resources for All Queues
294 * @adapter: board private structure
296 * Free all transmit software resources
298 static void igc_free_all_tx_resources(struct igc_adapter
*adapter
)
302 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
303 igc_free_tx_resources(adapter
->tx_ring
[i
]);
307 * igc_clean_all_tx_rings - Free Tx Buffers for all queues
308 * @adapter: board private structure
310 static void igc_clean_all_tx_rings(struct igc_adapter
*adapter
)
314 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
315 if (adapter
->tx_ring
[i
])
316 igc_clean_tx_ring(adapter
->tx_ring
[i
]);
319 static void igc_disable_tx_ring_hw(struct igc_ring
*ring
)
321 struct igc_hw
*hw
= &ring
->q_vector
->adapter
->hw
;
322 u8 idx
= ring
->reg_idx
;
325 txdctl
= rd32(IGC_TXDCTL(idx
));
326 txdctl
&= ~IGC_TXDCTL_QUEUE_ENABLE
;
327 txdctl
|= IGC_TXDCTL_SWFLUSH
;
328 wr32(IGC_TXDCTL(idx
), txdctl
);
332 * igc_disable_all_tx_rings_hw - Disable all transmit queue operation
333 * @adapter: board private structure
335 static void igc_disable_all_tx_rings_hw(struct igc_adapter
*adapter
)
339 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
340 struct igc_ring
*tx_ring
= adapter
->tx_ring
[i
];
342 igc_disable_tx_ring_hw(tx_ring
);
347 * igc_setup_tx_resources - allocate Tx resources (Descriptors)
348 * @tx_ring: tx descriptor ring (for a specific queue) to setup
350 * Return 0 on success, negative on failure
352 int igc_setup_tx_resources(struct igc_ring
*tx_ring
)
354 struct net_device
*ndev
= tx_ring
->netdev
;
355 struct device
*dev
= tx_ring
->dev
;
358 size
= sizeof(struct igc_tx_buffer
) * tx_ring
->count
;
359 tx_ring
->tx_buffer_info
= vzalloc(size
);
360 if (!tx_ring
->tx_buffer_info
)
363 /* round up to nearest 4K */
364 tx_ring
->size
= tx_ring
->count
* sizeof(union igc_adv_tx_desc
);
365 tx_ring
->size
= ALIGN(tx_ring
->size
, 4096);
367 tx_ring
->desc
= dma_alloc_coherent(dev
, tx_ring
->size
,
368 &tx_ring
->dma
, GFP_KERNEL
);
373 tx_ring
->next_to_use
= 0;
374 tx_ring
->next_to_clean
= 0;
379 vfree(tx_ring
->tx_buffer_info
);
380 netdev_err(ndev
, "Unable to allocate memory for Tx descriptor ring\n");
385 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
386 * @adapter: board private structure
388 * Return 0 on success, negative on failure
390 static int igc_setup_all_tx_resources(struct igc_adapter
*adapter
)
392 struct net_device
*dev
= adapter
->netdev
;
395 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
396 err
= igc_setup_tx_resources(adapter
->tx_ring
[i
]);
398 netdev_err(dev
, "Error on Tx queue %u setup\n", i
);
399 for (i
--; i
>= 0; i
--)
400 igc_free_tx_resources(adapter
->tx_ring
[i
]);
408 static void igc_clean_rx_ring_page_shared(struct igc_ring
*rx_ring
)
410 u16 i
= rx_ring
->next_to_clean
;
412 dev_kfree_skb(rx_ring
->skb
);
415 /* Free all the Rx ring sk_buffs */
416 while (i
!= rx_ring
->next_to_alloc
) {
417 struct igc_rx_buffer
*buffer_info
= &rx_ring
->rx_buffer_info
[i
];
419 /* Invalidate cache lines that may have been written to by
420 * device so that we avoid corrupting memory.
422 dma_sync_single_range_for_cpu(rx_ring
->dev
,
424 buffer_info
->page_offset
,
425 igc_rx_bufsz(rx_ring
),
428 /* free resources associated with mapping */
429 dma_unmap_page_attrs(rx_ring
->dev
,
431 igc_rx_pg_size(rx_ring
),
434 __page_frag_cache_drain(buffer_info
->page
,
435 buffer_info
->pagecnt_bias
);
438 if (i
== rx_ring
->count
)
443 static void igc_clean_rx_ring_xsk_pool(struct igc_ring
*ring
)
445 struct igc_rx_buffer
*bi
;
448 for (i
= 0; i
< ring
->count
; i
++) {
449 bi
= &ring
->rx_buffer_info
[i
];
453 xsk_buff_free(bi
->xdp
);
459 * igc_clean_rx_ring - Free Rx Buffers per Queue
460 * @ring: ring to free buffers from
462 static void igc_clean_rx_ring(struct igc_ring
*ring
)
465 igc_clean_rx_ring_xsk_pool(ring
);
467 igc_clean_rx_ring_page_shared(ring
);
469 clear_ring_uses_large_buffer(ring
);
471 ring
->next_to_alloc
= 0;
472 ring
->next_to_clean
= 0;
473 ring
->next_to_use
= 0;
477 * igc_clean_all_rx_rings - Free Rx Buffers for all queues
478 * @adapter: board private structure
480 static void igc_clean_all_rx_rings(struct igc_adapter
*adapter
)
484 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
485 if (adapter
->rx_ring
[i
])
486 igc_clean_rx_ring(adapter
->rx_ring
[i
]);
490 * igc_free_rx_resources - Free Rx Resources
491 * @rx_ring: ring to clean the resources from
493 * Free all receive software resources
495 void igc_free_rx_resources(struct igc_ring
*rx_ring
)
497 igc_clean_rx_ring(rx_ring
);
499 xdp_rxq_info_unreg(&rx_ring
->xdp_rxq
);
501 vfree(rx_ring
->rx_buffer_info
);
502 rx_ring
->rx_buffer_info
= NULL
;
504 /* if not set, then don't free */
508 dma_free_coherent(rx_ring
->dev
, rx_ring
->size
,
509 rx_ring
->desc
, rx_ring
->dma
);
511 rx_ring
->desc
= NULL
;
515 * igc_free_all_rx_resources - Free Rx Resources for All Queues
516 * @adapter: board private structure
518 * Free all receive software resources
520 static void igc_free_all_rx_resources(struct igc_adapter
*adapter
)
524 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
525 igc_free_rx_resources(adapter
->rx_ring
[i
]);
529 * igc_setup_rx_resources - allocate Rx resources (Descriptors)
530 * @rx_ring: rx descriptor ring (for a specific queue) to setup
532 * Returns 0 on success, negative on failure
534 int igc_setup_rx_resources(struct igc_ring
*rx_ring
)
536 struct net_device
*ndev
= rx_ring
->netdev
;
537 struct device
*dev
= rx_ring
->dev
;
538 u8 index
= rx_ring
->queue_index
;
539 int size
, desc_len
, res
;
541 /* XDP RX-queue info */
542 if (xdp_rxq_info_is_reg(&rx_ring
->xdp_rxq
))
543 xdp_rxq_info_unreg(&rx_ring
->xdp_rxq
);
544 res
= xdp_rxq_info_reg(&rx_ring
->xdp_rxq
, ndev
, index
,
545 rx_ring
->q_vector
->napi
.napi_id
);
547 netdev_err(ndev
, "Failed to register xdp_rxq index %u\n",
552 size
= sizeof(struct igc_rx_buffer
) * rx_ring
->count
;
553 rx_ring
->rx_buffer_info
= vzalloc(size
);
554 if (!rx_ring
->rx_buffer_info
)
557 desc_len
= sizeof(union igc_adv_rx_desc
);
559 /* Round up to nearest 4K */
560 rx_ring
->size
= rx_ring
->count
* desc_len
;
561 rx_ring
->size
= ALIGN(rx_ring
->size
, 4096);
563 rx_ring
->desc
= dma_alloc_coherent(dev
, rx_ring
->size
,
564 &rx_ring
->dma
, GFP_KERNEL
);
569 rx_ring
->next_to_alloc
= 0;
570 rx_ring
->next_to_clean
= 0;
571 rx_ring
->next_to_use
= 0;
576 xdp_rxq_info_unreg(&rx_ring
->xdp_rxq
);
577 vfree(rx_ring
->rx_buffer_info
);
578 rx_ring
->rx_buffer_info
= NULL
;
579 netdev_err(ndev
, "Unable to allocate memory for Rx descriptor ring\n");
584 * igc_setup_all_rx_resources - wrapper to allocate Rx resources
585 * (Descriptors) for all queues
586 * @adapter: board private structure
588 * Return 0 on success, negative on failure
590 static int igc_setup_all_rx_resources(struct igc_adapter
*adapter
)
592 struct net_device
*dev
= adapter
->netdev
;
595 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
596 err
= igc_setup_rx_resources(adapter
->rx_ring
[i
]);
598 netdev_err(dev
, "Error on Rx queue %u setup\n", i
);
599 for (i
--; i
>= 0; i
--)
600 igc_free_rx_resources(adapter
->rx_ring
[i
]);
608 static struct xsk_buff_pool
*igc_get_xsk_pool(struct igc_adapter
*adapter
,
609 struct igc_ring
*ring
)
611 if (!igc_xdp_is_enabled(adapter
) ||
612 !test_bit(IGC_RING_FLAG_AF_XDP_ZC
, &ring
->flags
))
615 return xsk_get_pool_from_qid(ring
->netdev
, ring
->queue_index
);
619 * igc_configure_rx_ring - Configure a receive ring after Reset
620 * @adapter: board private structure
621 * @ring: receive ring to be configured
623 * Configure the Rx unit of the MAC after a reset.
625 static void igc_configure_rx_ring(struct igc_adapter
*adapter
,
626 struct igc_ring
*ring
)
628 struct igc_hw
*hw
= &adapter
->hw
;
629 union igc_adv_rx_desc
*rx_desc
;
630 int reg_idx
= ring
->reg_idx
;
631 u32 srrctl
= 0, rxdctl
= 0;
632 u64 rdba
= ring
->dma
;
635 xdp_rxq_info_unreg_mem_model(&ring
->xdp_rxq
);
636 ring
->xsk_pool
= igc_get_xsk_pool(adapter
, ring
);
637 if (ring
->xsk_pool
) {
638 WARN_ON(xdp_rxq_info_reg_mem_model(&ring
->xdp_rxq
,
639 MEM_TYPE_XSK_BUFF_POOL
,
641 xsk_pool_set_rxq_info(ring
->xsk_pool
, &ring
->xdp_rxq
);
643 WARN_ON(xdp_rxq_info_reg_mem_model(&ring
->xdp_rxq
,
644 MEM_TYPE_PAGE_SHARED
,
648 if (igc_xdp_is_enabled(adapter
))
649 set_ring_uses_large_buffer(ring
);
651 /* disable the queue */
652 wr32(IGC_RXDCTL(reg_idx
), 0);
654 /* Set DMA base address registers */
655 wr32(IGC_RDBAL(reg_idx
),
656 rdba
& 0x00000000ffffffffULL
);
657 wr32(IGC_RDBAH(reg_idx
), rdba
>> 32);
658 wr32(IGC_RDLEN(reg_idx
),
659 ring
->count
* sizeof(union igc_adv_rx_desc
));
661 /* initialize head and tail */
662 ring
->tail
= adapter
->io_addr
+ IGC_RDT(reg_idx
);
663 wr32(IGC_RDH(reg_idx
), 0);
664 writel(0, ring
->tail
);
666 /* reset next-to- use/clean to place SW in sync with hardware */
667 ring
->next_to_clean
= 0;
668 ring
->next_to_use
= 0;
671 buf_size
= xsk_pool_get_rx_frame_size(ring
->xsk_pool
);
672 else if (ring_uses_large_buffer(ring
))
673 buf_size
= IGC_RXBUFFER_3072
;
675 buf_size
= IGC_RXBUFFER_2048
;
677 srrctl
= rd32(IGC_SRRCTL(reg_idx
));
678 srrctl
&= ~(IGC_SRRCTL_BSIZEPKT_MASK
| IGC_SRRCTL_BSIZEHDR_MASK
|
679 IGC_SRRCTL_DESCTYPE_MASK
);
680 srrctl
|= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN
);
681 srrctl
|= IGC_SRRCTL_BSIZEPKT(buf_size
);
682 srrctl
|= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF
;
684 wr32(IGC_SRRCTL(reg_idx
), srrctl
);
686 rxdctl
|= IGC_RX_PTHRESH
;
687 rxdctl
|= IGC_RX_HTHRESH
<< 8;
688 rxdctl
|= IGC_RX_WTHRESH
<< 16;
690 /* initialize rx_buffer_info */
691 memset(ring
->rx_buffer_info
, 0,
692 sizeof(struct igc_rx_buffer
) * ring
->count
);
694 /* initialize Rx descriptor 0 */
695 rx_desc
= IGC_RX_DESC(ring
, 0);
696 rx_desc
->wb
.upper
.length
= 0;
698 /* enable receive descriptor fetching */
699 rxdctl
|= IGC_RXDCTL_QUEUE_ENABLE
;
701 wr32(IGC_RXDCTL(reg_idx
), rxdctl
);
705 * igc_configure_rx - Configure receive Unit after Reset
706 * @adapter: board private structure
708 * Configure the Rx unit of the MAC after a reset.
710 static void igc_configure_rx(struct igc_adapter
*adapter
)
714 /* Setup the HW Rx Head and Tail Descriptor Pointers and
715 * the Base and Length of the Rx Descriptor Ring
717 for (i
= 0; i
< adapter
->num_rx_queues
; i
++)
718 igc_configure_rx_ring(adapter
, adapter
->rx_ring
[i
]);
722 * igc_configure_tx_ring - Configure transmit ring after Reset
723 * @adapter: board private structure
724 * @ring: tx ring to configure
726 * Configure a transmit ring after a reset.
728 static void igc_configure_tx_ring(struct igc_adapter
*adapter
,
729 struct igc_ring
*ring
)
731 struct igc_hw
*hw
= &adapter
->hw
;
732 int reg_idx
= ring
->reg_idx
;
733 u64 tdba
= ring
->dma
;
736 ring
->xsk_pool
= igc_get_xsk_pool(adapter
, ring
);
738 /* disable the queue */
739 wr32(IGC_TXDCTL(reg_idx
), 0);
742 wr32(IGC_TDLEN(reg_idx
),
743 ring
->count
* sizeof(union igc_adv_tx_desc
));
744 wr32(IGC_TDBAL(reg_idx
),
745 tdba
& 0x00000000ffffffffULL
);
746 wr32(IGC_TDBAH(reg_idx
), tdba
>> 32);
748 ring
->tail
= adapter
->io_addr
+ IGC_TDT(reg_idx
);
749 wr32(IGC_TDH(reg_idx
), 0);
750 writel(0, ring
->tail
);
752 txdctl
|= IGC_TX_PTHRESH
;
753 txdctl
|= IGC_TX_HTHRESH
<< 8;
754 txdctl
|= IGC_TX_WTHRESH
<< 16;
756 txdctl
|= IGC_TXDCTL_QUEUE_ENABLE
;
757 wr32(IGC_TXDCTL(reg_idx
), txdctl
);
761 * igc_configure_tx - Configure transmit Unit after Reset
762 * @adapter: board private structure
764 * Configure the Tx unit of the MAC after a reset.
766 static void igc_configure_tx(struct igc_adapter
*adapter
)
770 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
771 igc_configure_tx_ring(adapter
, adapter
->tx_ring
[i
]);
775 * igc_setup_mrqc - configure the multiple receive queue control registers
776 * @adapter: Board private structure
778 static void igc_setup_mrqc(struct igc_adapter
*adapter
)
780 struct igc_hw
*hw
= &adapter
->hw
;
781 u32 j
, num_rx_queues
;
785 netdev_rss_key_fill(rss_key
, sizeof(rss_key
));
786 for (j
= 0; j
< 10; j
++)
787 wr32(IGC_RSSRK(j
), rss_key
[j
]);
789 num_rx_queues
= adapter
->rss_queues
;
791 if (adapter
->rss_indir_tbl_init
!= num_rx_queues
) {
792 for (j
= 0; j
< IGC_RETA_SIZE
; j
++)
793 adapter
->rss_indir_tbl
[j
] =
794 (j
* num_rx_queues
) / IGC_RETA_SIZE
;
795 adapter
->rss_indir_tbl_init
= num_rx_queues
;
797 igc_write_rss_indir_tbl(adapter
);
799 /* Disable raw packet checksumming so that RSS hash is placed in
800 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
801 * offloads as they are enabled by default
803 rxcsum
= rd32(IGC_RXCSUM
);
804 rxcsum
|= IGC_RXCSUM_PCSD
;
806 /* Enable Receive Checksum Offload for SCTP */
807 rxcsum
|= IGC_RXCSUM_CRCOFL
;
809 /* Don't need to set TUOFL or IPOFL, they default to 1 */
810 wr32(IGC_RXCSUM
, rxcsum
);
812 /* Generate RSS hash based on packet types, TCP/UDP
813 * port numbers and/or IPv4/v6 src and dst addresses
815 mrqc
= IGC_MRQC_RSS_FIELD_IPV4
|
816 IGC_MRQC_RSS_FIELD_IPV4_TCP
|
817 IGC_MRQC_RSS_FIELD_IPV6
|
818 IGC_MRQC_RSS_FIELD_IPV6_TCP
|
819 IGC_MRQC_RSS_FIELD_IPV6_TCP_EX
;
821 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV4_UDP
)
822 mrqc
|= IGC_MRQC_RSS_FIELD_IPV4_UDP
;
823 if (adapter
->flags
& IGC_FLAG_RSS_FIELD_IPV6_UDP
)
824 mrqc
|= IGC_MRQC_RSS_FIELD_IPV6_UDP
;
826 mrqc
|= IGC_MRQC_ENABLE_RSS_MQ
;
828 wr32(IGC_MRQC
, mrqc
);
832 * igc_setup_rctl - configure the receive control registers
833 * @adapter: Board private structure
835 static void igc_setup_rctl(struct igc_adapter
*adapter
)
837 struct igc_hw
*hw
= &adapter
->hw
;
840 rctl
= rd32(IGC_RCTL
);
842 rctl
&= ~(3 << IGC_RCTL_MO_SHIFT
);
843 rctl
&= ~(IGC_RCTL_LBM_TCVR
| IGC_RCTL_LBM_MAC
);
845 rctl
|= IGC_RCTL_EN
| IGC_RCTL_BAM
| IGC_RCTL_RDMTS_HALF
|
846 (hw
->mac
.mc_filter_type
<< IGC_RCTL_MO_SHIFT
);
848 /* enable stripping of CRC. Newer features require
849 * that the HW strips the CRC.
851 rctl
|= IGC_RCTL_SECRC
;
853 /* disable store bad packets and clear size bits. */
854 rctl
&= ~(IGC_RCTL_SBP
| IGC_RCTL_SZ_256
);
856 /* enable LPE to allow for reception of jumbo frames */
857 rctl
|= IGC_RCTL_LPE
;
859 /* disable queue 0 to prevent tail write w/o re-config */
860 wr32(IGC_RXDCTL(0), 0);
862 /* This is useful for sniffing bad packets. */
863 if (adapter
->netdev
->features
& NETIF_F_RXALL
) {
864 /* UPE and MPE will be handled by normal PROMISC logic
867 rctl
|= (IGC_RCTL_SBP
| /* Receive bad packets */
868 IGC_RCTL_BAM
| /* RX All Bcast Pkts */
869 IGC_RCTL_PMCF
); /* RX All MAC Ctrl Pkts */
871 rctl
&= ~(IGC_RCTL_DPF
| /* Allow filtered pause */
872 IGC_RCTL_CFIEN
); /* Disable VLAN CFIEN Filter */
875 wr32(IGC_RCTL
, rctl
);
879 * igc_setup_tctl - configure the transmit control registers
880 * @adapter: Board private structure
882 static void igc_setup_tctl(struct igc_adapter
*adapter
)
884 struct igc_hw
*hw
= &adapter
->hw
;
887 /* disable queue 0 which icould be enabled by default */
888 wr32(IGC_TXDCTL(0), 0);
890 /* Program the Transmit Control Register */
891 tctl
= rd32(IGC_TCTL
);
892 tctl
&= ~IGC_TCTL_CT
;
893 tctl
|= IGC_TCTL_PSP
| IGC_TCTL_RTLC
|
894 (IGC_COLLISION_THRESHOLD
<< IGC_CT_SHIFT
);
896 /* Enable transmits */
899 wr32(IGC_TCTL
, tctl
);
903 * igc_set_mac_filter_hw() - Set MAC address filter in hardware
904 * @adapter: Pointer to adapter where the filter should be set
905 * @index: Filter index
906 * @type: MAC address filter type (source or destination)
908 * @queue: If non-negative, queue assignment feature is enabled and frames
909 * matching the filter are enqueued onto 'queue'. Otherwise, queue
910 * assignment is disabled.
912 static void igc_set_mac_filter_hw(struct igc_adapter
*adapter
, int index
,
913 enum igc_mac_filter_type type
,
914 const u8
*addr
, int queue
)
916 struct net_device
*dev
= adapter
->netdev
;
917 struct igc_hw
*hw
= &adapter
->hw
;
920 if (WARN_ON(index
>= hw
->mac
.rar_entry_count
))
923 ral
= le32_to_cpup((__le32
*)(addr
));
924 rah
= le16_to_cpup((__le16
*)(addr
+ 4));
926 if (type
== IGC_MAC_FILTER_TYPE_SRC
) {
927 rah
&= ~IGC_RAH_ASEL_MASK
;
928 rah
|= IGC_RAH_ASEL_SRC_ADDR
;
932 rah
&= ~IGC_RAH_QSEL_MASK
;
933 rah
|= (queue
<< IGC_RAH_QSEL_SHIFT
);
934 rah
|= IGC_RAH_QSEL_ENABLE
;
939 wr32(IGC_RAL(index
), ral
);
940 wr32(IGC_RAH(index
), rah
);
942 netdev_dbg(dev
, "MAC address filter set in HW: index %d", index
);
946 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
947 * @adapter: Pointer to adapter where the filter should be cleared
948 * @index: Filter index
950 static void igc_clear_mac_filter_hw(struct igc_adapter
*adapter
, int index
)
952 struct net_device
*dev
= adapter
->netdev
;
953 struct igc_hw
*hw
= &adapter
->hw
;
955 if (WARN_ON(index
>= hw
->mac
.rar_entry_count
))
958 wr32(IGC_RAL(index
), 0);
959 wr32(IGC_RAH(index
), 0);
961 netdev_dbg(dev
, "MAC address filter cleared in HW: index %d", index
);
964 /* Set default MAC address for the PF in the first RAR entry */
965 static void igc_set_default_mac_filter(struct igc_adapter
*adapter
)
967 struct net_device
*dev
= adapter
->netdev
;
968 u8
*addr
= adapter
->hw
.mac
.addr
;
970 netdev_dbg(dev
, "Set default MAC address filter: address %pM", addr
);
972 igc_set_mac_filter_hw(adapter
, 0, IGC_MAC_FILTER_TYPE_DST
, addr
, -1);
976 * igc_set_mac - Change the Ethernet Address of the NIC
977 * @netdev: network interface device structure
978 * @p: pointer to an address structure
980 * Returns 0 on success, negative on failure
982 static int igc_set_mac(struct net_device
*netdev
, void *p
)
984 struct igc_adapter
*adapter
= netdev_priv(netdev
);
985 struct igc_hw
*hw
= &adapter
->hw
;
986 struct sockaddr
*addr
= p
;
988 if (!is_valid_ether_addr(addr
->sa_data
))
989 return -EADDRNOTAVAIL
;
991 eth_hw_addr_set(netdev
, addr
->sa_data
);
992 memcpy(hw
->mac
.addr
, addr
->sa_data
, netdev
->addr_len
);
994 /* set the correct pool for the new PF MAC address in entry 0 */
995 igc_set_default_mac_filter(adapter
);
1001 * igc_write_mc_addr_list - write multicast addresses to MTA
1002 * @netdev: network interface device structure
1004 * Writes multicast address list to the MTA hash table.
1005 * Returns: -ENOMEM on failure
1006 * 0 on no addresses written
1007 * X on writing X addresses to MTA
1009 static int igc_write_mc_addr_list(struct net_device
*netdev
)
1011 struct igc_adapter
*adapter
= netdev_priv(netdev
);
1012 struct igc_hw
*hw
= &adapter
->hw
;
1013 struct netdev_hw_addr
*ha
;
1017 if (netdev_mc_empty(netdev
)) {
1018 /* nothing to program, so clear mc list */
1019 igc_update_mc_addr_list(hw
, NULL
, 0);
1023 mta_list
= kcalloc(netdev_mc_count(netdev
), 6, GFP_ATOMIC
);
1027 /* The shared function expects a packed array of only addresses. */
1029 netdev_for_each_mc_addr(ha
, netdev
)
1030 memcpy(mta_list
+ (i
++ * ETH_ALEN
), ha
->addr
, ETH_ALEN
);
1032 igc_update_mc_addr_list(hw
, mta_list
, i
);
1035 return netdev_mc_count(netdev
);
1038 static __le32
igc_tx_launchtime(struct igc_ring
*ring
, ktime_t txtime
,
1039 bool *first_flag
, bool *insert_empty
)
1041 struct igc_adapter
*adapter
= netdev_priv(ring
->netdev
);
1042 ktime_t cycle_time
= adapter
->cycle_time
;
1043 ktime_t base_time
= adapter
->base_time
;
1044 ktime_t now
= ktime_get_clocktai();
1045 ktime_t baset_est
, end_of_cycle
;
1049 n
= div64_s64(ktime_sub_ns(now
, base_time
), cycle_time
);
1051 baset_est
= ktime_add_ns(base_time
, cycle_time
* (n
));
1052 end_of_cycle
= ktime_add_ns(baset_est
, cycle_time
);
1054 if (ktime_compare(txtime
, end_of_cycle
) >= 0) {
1055 if (baset_est
!= ring
->last_ff_cycle
) {
1057 ring
->last_ff_cycle
= baset_est
;
1059 if (ktime_compare(end_of_cycle
, ring
->last_tx_cycle
) > 0)
1060 *insert_empty
= true;
1064 /* Introducing a window at end of cycle on which packets
1065 * potentially not honor launchtime. Window of 5us chosen
1066 * considering software update the tail pointer and packets
1067 * are dma'ed to packet buffer.
1069 if ((ktime_sub_ns(end_of_cycle
, now
) < 5 * NSEC_PER_USEC
))
1070 netdev_warn(ring
->netdev
, "Packet with txtime=%llu may not be honoured\n",
1073 ring
->last_tx_cycle
= end_of_cycle
;
1075 launchtime
= ktime_sub_ns(txtime
, baset_est
);
1077 div_s64_rem(launchtime
, cycle_time
, &launchtime
);
1081 return cpu_to_le32(launchtime
);
1084 static int igc_init_empty_frame(struct igc_ring
*ring
,
1085 struct igc_tx_buffer
*buffer
,
1086 struct sk_buff
*skb
)
1091 size
= skb_headlen(skb
);
1093 dma
= dma_map_single(ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
1094 if (dma_mapping_error(ring
->dev
, dma
)) {
1095 netdev_err_once(ring
->netdev
, "Failed to map DMA for TX\n");
1100 buffer
->protocol
= 0;
1101 buffer
->bytecount
= skb
->len
;
1102 buffer
->gso_segs
= 1;
1103 buffer
->time_stamp
= jiffies
;
1104 dma_unmap_len_set(buffer
, len
, skb
->len
);
1105 dma_unmap_addr_set(buffer
, dma
, dma
);
1110 static int igc_init_tx_empty_descriptor(struct igc_ring
*ring
,
1111 struct sk_buff
*skb
,
1112 struct igc_tx_buffer
*first
)
1114 union igc_adv_tx_desc
*desc
;
1115 u32 cmd_type
, olinfo_status
;
1118 if (!igc_desc_unused(ring
))
1121 err
= igc_init_empty_frame(ring
, first
, skb
);
1125 cmd_type
= IGC_ADVTXD_DTYP_DATA
| IGC_ADVTXD_DCMD_DEXT
|
1126 IGC_ADVTXD_DCMD_IFCS
| IGC_TXD_DCMD
|
1128 olinfo_status
= first
->bytecount
<< IGC_ADVTXD_PAYLEN_SHIFT
;
1130 desc
= IGC_TX_DESC(ring
, ring
->next_to_use
);
1131 desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
1132 desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
1133 desc
->read
.buffer_addr
= cpu_to_le64(dma_unmap_addr(first
, dma
));
1135 netdev_tx_sent_queue(txring_txq(ring
), skb
->len
);
1137 first
->next_to_watch
= desc
;
1139 ring
->next_to_use
++;
1140 if (ring
->next_to_use
== ring
->count
)
1141 ring
->next_to_use
= 0;
1146 #define IGC_EMPTY_FRAME_SIZE 60
1148 static void igc_tx_ctxtdesc(struct igc_ring
*tx_ring
,
1149 __le32 launch_time
, bool first_flag
,
1150 u32 vlan_macip_lens
, u32 type_tucmd
,
1153 struct igc_adv_tx_context_desc
*context_desc
;
1154 u16 i
= tx_ring
->next_to_use
;
1156 context_desc
= IGC_TX_CTXTDESC(tx_ring
, i
);
1159 tx_ring
->next_to_use
= (i
< tx_ring
->count
) ? i
: 0;
1161 /* set bits to identify this as an advanced context descriptor */
1162 type_tucmd
|= IGC_TXD_CMD_DEXT
| IGC_ADVTXD_DTYP_CTXT
;
1164 /* For i225, context index must be unique per ring. */
1165 if (test_bit(IGC_RING_FLAG_TX_CTX_IDX
, &tx_ring
->flags
))
1166 mss_l4len_idx
|= tx_ring
->reg_idx
<< 4;
1169 mss_l4len_idx
|= IGC_ADVTXD_TSN_CNTX_FIRST
;
1171 context_desc
->vlan_macip_lens
= cpu_to_le32(vlan_macip_lens
);
1172 context_desc
->type_tucmd_mlhl
= cpu_to_le32(type_tucmd
);
1173 context_desc
->mss_l4len_idx
= cpu_to_le32(mss_l4len_idx
);
1174 context_desc
->launch_time
= launch_time
;
1177 static void igc_tx_csum(struct igc_ring
*tx_ring
, struct igc_tx_buffer
*first
,
1178 __le32 launch_time
, bool first_flag
)
1180 struct sk_buff
*skb
= first
->skb
;
1181 u32 vlan_macip_lens
= 0;
1184 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
) {
1186 if (!(first
->tx_flags
& IGC_TX_FLAGS_VLAN
) &&
1187 !tx_ring
->launchtime_enable
)
1192 switch (skb
->csum_offset
) {
1193 case offsetof(struct tcphdr
, check
):
1194 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_TCP
;
1196 case offsetof(struct udphdr
, check
):
1198 case offsetof(struct sctphdr
, checksum
):
1199 /* validate that this is actually an SCTP request */
1200 if (skb_csum_is_sctp(skb
)) {
1201 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_SCTP
;
1206 skb_checksum_help(skb
);
1210 /* update TX checksum flag */
1211 first
->tx_flags
|= IGC_TX_FLAGS_CSUM
;
1212 vlan_macip_lens
= skb_checksum_start_offset(skb
) -
1213 skb_network_offset(skb
);
1215 vlan_macip_lens
|= skb_network_offset(skb
) << IGC_ADVTXD_MACLEN_SHIFT
;
1216 vlan_macip_lens
|= first
->tx_flags
& IGC_TX_FLAGS_VLAN_MASK
;
1218 igc_tx_ctxtdesc(tx_ring
, launch_time
, first_flag
,
1219 vlan_macip_lens
, type_tucmd
, 0);
1222 static int __igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
1224 struct net_device
*netdev
= tx_ring
->netdev
;
1226 netif_stop_subqueue(netdev
, tx_ring
->queue_index
);
1228 /* memory barriier comment */
1231 /* We need to check again in a case another CPU has just
1232 * made room available.
1234 if (igc_desc_unused(tx_ring
) < size
)
1238 netif_wake_subqueue(netdev
, tx_ring
->queue_index
);
1240 u64_stats_update_begin(&tx_ring
->tx_syncp2
);
1241 tx_ring
->tx_stats
.restart_queue2
++;
1242 u64_stats_update_end(&tx_ring
->tx_syncp2
);
1247 static inline int igc_maybe_stop_tx(struct igc_ring
*tx_ring
, const u16 size
)
1249 if (igc_desc_unused(tx_ring
) >= size
)
1251 return __igc_maybe_stop_tx(tx_ring
, size
);
1254 #define IGC_SET_FLAG(_input, _flag, _result) \
1255 (((_flag) <= (_result)) ? \
1256 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \
1257 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1259 static u32
igc_tx_cmd_type(struct sk_buff
*skb
, u32 tx_flags
)
1261 /* set type for advanced descriptor with frame checksum insertion */
1262 u32 cmd_type
= IGC_ADVTXD_DTYP_DATA
|
1263 IGC_ADVTXD_DCMD_DEXT
|
1264 IGC_ADVTXD_DCMD_IFCS
;
1266 /* set HW vlan bit if vlan is present */
1267 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_VLAN
,
1268 IGC_ADVTXD_DCMD_VLE
);
1270 /* set segmentation bits for TSO */
1271 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSO
,
1272 (IGC_ADVTXD_DCMD_TSE
));
1274 /* set timestamp bit if present, will select the register set
1275 * based on the _TSTAMP(_X) bit.
1277 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSTAMP
,
1278 (IGC_ADVTXD_MAC_TSTAMP
));
1280 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSTAMP_1
,
1281 (IGC_ADVTXD_TSTAMP_REG_1
));
1283 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSTAMP_2
,
1284 (IGC_ADVTXD_TSTAMP_REG_2
));
1286 cmd_type
|= IGC_SET_FLAG(tx_flags
, IGC_TX_FLAGS_TSTAMP_3
,
1287 (IGC_ADVTXD_TSTAMP_REG_3
));
1289 /* insert frame checksum */
1290 cmd_type
^= IGC_SET_FLAG(skb
->no_fcs
, 1, IGC_ADVTXD_DCMD_IFCS
);
1295 static void igc_tx_olinfo_status(struct igc_ring
*tx_ring
,
1296 union igc_adv_tx_desc
*tx_desc
,
1297 u32 tx_flags
, unsigned int paylen
)
1299 u32 olinfo_status
= paylen
<< IGC_ADVTXD_PAYLEN_SHIFT
;
1301 /* insert L4 checksum */
1302 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_CSUM
) *
1303 ((IGC_TXD_POPTS_TXSM
<< 8) /
1306 /* insert IPv4 checksum */
1307 olinfo_status
|= (tx_flags
& IGC_TX_FLAGS_IPV4
) *
1308 (((IGC_TXD_POPTS_IXSM
<< 8)) /
1311 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
1314 static int igc_tx_map(struct igc_ring
*tx_ring
,
1315 struct igc_tx_buffer
*first
,
1318 struct sk_buff
*skb
= first
->skb
;
1319 struct igc_tx_buffer
*tx_buffer
;
1320 union igc_adv_tx_desc
*tx_desc
;
1321 u32 tx_flags
= first
->tx_flags
;
1323 u16 i
= tx_ring
->next_to_use
;
1324 unsigned int data_len
, size
;
1328 cmd_type
= igc_tx_cmd_type(skb
, tx_flags
);
1329 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
1331 igc_tx_olinfo_status(tx_ring
, tx_desc
, tx_flags
, skb
->len
- hdr_len
);
1333 size
= skb_headlen(skb
);
1334 data_len
= skb
->data_len
;
1336 dma
= dma_map_single(tx_ring
->dev
, skb
->data
, size
, DMA_TO_DEVICE
);
1340 for (frag
= &skb_shinfo(skb
)->frags
[0];; frag
++) {
1341 if (dma_mapping_error(tx_ring
->dev
, dma
))
1344 /* record length, and DMA address */
1345 dma_unmap_len_set(tx_buffer
, len
, size
);
1346 dma_unmap_addr_set(tx_buffer
, dma
, dma
);
1348 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1350 while (unlikely(size
> IGC_MAX_DATA_PER_TXD
)) {
1351 tx_desc
->read
.cmd_type_len
=
1352 cpu_to_le32(cmd_type
^ IGC_MAX_DATA_PER_TXD
);
1356 if (i
== tx_ring
->count
) {
1357 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1360 tx_desc
->read
.olinfo_status
= 0;
1362 dma
+= IGC_MAX_DATA_PER_TXD
;
1363 size
-= IGC_MAX_DATA_PER_TXD
;
1365 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
1368 if (likely(!data_len
))
1371 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
^ size
);
1375 if (i
== tx_ring
->count
) {
1376 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
1379 tx_desc
->read
.olinfo_status
= 0;
1381 size
= skb_frag_size(frag
);
1384 dma
= skb_frag_dma_map(tx_ring
->dev
, frag
, 0,
1385 size
, DMA_TO_DEVICE
);
1387 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1390 /* write last descriptor with RS and EOP bits */
1391 cmd_type
|= size
| IGC_TXD_DCMD
;
1392 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
1394 netdev_tx_sent_queue(txring_txq(tx_ring
), first
->bytecount
);
1396 /* set the timestamp */
1397 first
->time_stamp
= jiffies
;
1399 skb_tx_timestamp(skb
);
1401 /* Force memory writes to complete before letting h/w know there
1402 * are new descriptors to fetch. (Only applicable for weak-ordered
1403 * memory model archs, such as IA-64).
1405 * We also need this memory barrier to make certain all of the
1406 * status bits have been updated before next_to_watch is written.
1410 /* set next_to_watch value indicating a packet is present */
1411 first
->next_to_watch
= tx_desc
;
1414 if (i
== tx_ring
->count
)
1417 tx_ring
->next_to_use
= i
;
1419 /* Make sure there is space in the ring for the next send. */
1420 igc_maybe_stop_tx(tx_ring
, DESC_NEEDED
);
1422 if (netif_xmit_stopped(txring_txq(tx_ring
)) || !netdev_xmit_more()) {
1423 writel(i
, tx_ring
->tail
);
1428 netdev_err(tx_ring
->netdev
, "TX DMA map failed\n");
1429 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1431 /* clear dma mappings for failed tx_buffer_info map */
1432 while (tx_buffer
!= first
) {
1433 if (dma_unmap_len(tx_buffer
, len
))
1434 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
1437 i
+= tx_ring
->count
;
1438 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
1441 if (dma_unmap_len(tx_buffer
, len
))
1442 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
1444 dev_kfree_skb_any(tx_buffer
->skb
);
1445 tx_buffer
->skb
= NULL
;
1447 tx_ring
->next_to_use
= i
;
1452 static int igc_tso(struct igc_ring
*tx_ring
,
1453 struct igc_tx_buffer
*first
,
1454 __le32 launch_time
, bool first_flag
,
1457 u32 vlan_macip_lens
, type_tucmd
, mss_l4len_idx
;
1458 struct sk_buff
*skb
= first
->skb
;
1469 u32 paylen
, l4_offset
;
1472 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1475 if (!skb_is_gso(skb
))
1478 err
= skb_cow_head(skb
, 0);
1482 ip
.hdr
= skb_network_header(skb
);
1483 l4
.hdr
= skb_checksum_start(skb
);
1485 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1486 type_tucmd
= IGC_ADVTXD_TUCMD_L4T_TCP
;
1488 /* initialize outer IP header fields */
1489 if (ip
.v4
->version
== 4) {
1490 unsigned char *csum_start
= skb_checksum_start(skb
);
1491 unsigned char *trans_start
= ip
.hdr
+ (ip
.v4
->ihl
* 4);
1493 /* IP header will have to cancel out any data that
1494 * is not a part of the outer IP header
1496 ip
.v4
->check
= csum_fold(csum_partial(trans_start
,
1497 csum_start
- trans_start
,
1499 type_tucmd
|= IGC_ADVTXD_TUCMD_IPV4
;
1502 first
->tx_flags
|= IGC_TX_FLAGS_TSO
|
1506 ip
.v6
->payload_len
= 0;
1507 first
->tx_flags
|= IGC_TX_FLAGS_TSO
|
1511 /* determine offset of inner transport header */
1512 l4_offset
= l4
.hdr
- skb
->data
;
1514 /* remove payload length from inner checksum */
1515 paylen
= skb
->len
- l4_offset
;
1516 if (type_tucmd
& IGC_ADVTXD_TUCMD_L4T_TCP
) {
1517 /* compute length of segmentation header */
1518 *hdr_len
= (l4
.tcp
->doff
* 4) + l4_offset
;
1519 csum_replace_by_diff(&l4
.tcp
->check
,
1520 (__force __wsum
)htonl(paylen
));
1522 /* compute length of segmentation header */
1523 *hdr_len
= sizeof(*l4
.udp
) + l4_offset
;
1524 csum_replace_by_diff(&l4
.udp
->check
,
1525 (__force __wsum
)htonl(paylen
));
1528 /* update gso size and bytecount with header size */
1529 first
->gso_segs
= skb_shinfo(skb
)->gso_segs
;
1530 first
->bytecount
+= (first
->gso_segs
- 1) * *hdr_len
;
1533 mss_l4len_idx
= (*hdr_len
- l4_offset
) << IGC_ADVTXD_L4LEN_SHIFT
;
1534 mss_l4len_idx
|= skb_shinfo(skb
)->gso_size
<< IGC_ADVTXD_MSS_SHIFT
;
1536 /* VLAN MACLEN IPLEN */
1537 vlan_macip_lens
= l4
.hdr
- ip
.hdr
;
1538 vlan_macip_lens
|= (ip
.hdr
- skb
->data
) << IGC_ADVTXD_MACLEN_SHIFT
;
1539 vlan_macip_lens
|= first
->tx_flags
& IGC_TX_FLAGS_VLAN_MASK
;
1541 igc_tx_ctxtdesc(tx_ring
, launch_time
, first_flag
,
1542 vlan_macip_lens
, type_tucmd
, mss_l4len_idx
);
1547 static bool igc_request_tx_tstamp(struct igc_adapter
*adapter
, struct sk_buff
*skb
, u32
*flags
)
1551 for (i
= 0; i
< IGC_MAX_TX_TSTAMP_REGS
; i
++) {
1552 struct igc_tx_timestamp_request
*tstamp
= &adapter
->tx_tstamp
[i
];
1557 tstamp
->skb
= skb_get(skb
);
1558 tstamp
->start
= jiffies
;
1559 *flags
= tstamp
->flags
;
1567 static netdev_tx_t
igc_xmit_frame_ring(struct sk_buff
*skb
,
1568 struct igc_ring
*tx_ring
)
1570 struct igc_adapter
*adapter
= netdev_priv(tx_ring
->netdev
);
1571 bool first_flag
= false, insert_empty
= false;
1572 u16 count
= TXD_USE_COUNT(skb_headlen(skb
));
1573 __be16 protocol
= vlan_get_protocol(skb
);
1574 struct igc_tx_buffer
*first
;
1575 __le32 launch_time
= 0;
1582 /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1583 * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1584 * + 2 desc gap to keep tail from touching head,
1585 * + 1 desc for context descriptor,
1586 * otherwise try next time
1588 for (f
= 0; f
< skb_shinfo(skb
)->nr_frags
; f
++)
1589 count
+= TXD_USE_COUNT(skb_frag_size(
1590 &skb_shinfo(skb
)->frags
[f
]));
1592 if (igc_maybe_stop_tx(tx_ring
, count
+ 5)) {
1593 /* this is a hard error */
1594 return NETDEV_TX_BUSY
;
1597 if (!tx_ring
->launchtime_enable
)
1600 txtime
= skb
->tstamp
;
1601 skb
->tstamp
= ktime_set(0, 0);
1602 launch_time
= igc_tx_launchtime(tx_ring
, txtime
, &first_flag
, &insert_empty
);
1605 struct igc_tx_buffer
*empty_info
;
1606 struct sk_buff
*empty
;
1609 empty_info
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
1610 empty
= alloc_skb(IGC_EMPTY_FRAME_SIZE
, GFP_ATOMIC
);
1614 data
= skb_put(empty
, IGC_EMPTY_FRAME_SIZE
);
1615 memset(data
, 0, IGC_EMPTY_FRAME_SIZE
);
1617 igc_tx_ctxtdesc(tx_ring
, 0, false, 0, 0, 0);
1619 if (igc_init_tx_empty_descriptor(tx_ring
,
1622 dev_kfree_skb_any(empty
);
1626 /* record the location of the first descriptor for this packet */
1627 first
= &tx_ring
->tx_buffer_info
[tx_ring
->next_to_use
];
1628 first
->type
= IGC_TX_BUFFER_TYPE_SKB
;
1630 first
->bytecount
= skb
->len
;
1631 first
->gso_segs
= 1;
1633 if (adapter
->qbv_transition
|| tx_ring
->oper_gate_closed
)
1636 if (tx_ring
->max_sdu
> 0 && first
->bytecount
> tx_ring
->max_sdu
) {
1637 adapter
->stats
.txdrop
++;
1641 if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP
, &tx_ring
->flags
) &&
1642 skb_shinfo(skb
)->tx_flags
& SKBTX_HW_TSTAMP
)) {
1643 /* FIXME: add support for retrieving timestamps from
1644 * the other timer registers before skipping the
1645 * timestamping request.
1647 unsigned long flags
;
1650 spin_lock_irqsave(&adapter
->ptp_tx_lock
, flags
);
1651 if (igc_request_tx_tstamp(adapter
, skb
, &tstamp_flags
)) {
1652 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
1653 tx_flags
|= IGC_TX_FLAGS_TSTAMP
| tstamp_flags
;
1655 adapter
->tx_hwtstamp_skipped
++;
1658 spin_unlock_irqrestore(&adapter
->ptp_tx_lock
, flags
);
1661 if (skb_vlan_tag_present(skb
)) {
1662 tx_flags
|= IGC_TX_FLAGS_VLAN
;
1663 tx_flags
|= (skb_vlan_tag_get(skb
) << IGC_TX_FLAGS_VLAN_SHIFT
);
1666 /* record initial flags and protocol */
1667 first
->tx_flags
= tx_flags
;
1668 first
->protocol
= protocol
;
1670 tso
= igc_tso(tx_ring
, first
, launch_time
, first_flag
, &hdr_len
);
1674 igc_tx_csum(tx_ring
, first
, launch_time
, first_flag
);
1676 igc_tx_map(tx_ring
, first
, hdr_len
);
1678 return NETDEV_TX_OK
;
1681 dev_kfree_skb_any(first
->skb
);
1684 return NETDEV_TX_OK
;
1687 static inline struct igc_ring
*igc_tx_queue_mapping(struct igc_adapter
*adapter
,
1688 struct sk_buff
*skb
)
1690 unsigned int r_idx
= skb
->queue_mapping
;
1692 if (r_idx
>= adapter
->num_tx_queues
)
1693 r_idx
= r_idx
% adapter
->num_tx_queues
;
1695 return adapter
->tx_ring
[r_idx
];
1698 static netdev_tx_t
igc_xmit_frame(struct sk_buff
*skb
,
1699 struct net_device
*netdev
)
1701 struct igc_adapter
*adapter
= netdev_priv(netdev
);
1703 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1704 * in order to meet this minimum size requirement.
1706 if (skb
->len
< 17) {
1707 if (skb_padto(skb
, 17))
1708 return NETDEV_TX_OK
;
1712 return igc_xmit_frame_ring(skb
, igc_tx_queue_mapping(adapter
, skb
));
1715 static void igc_rx_checksum(struct igc_ring
*ring
,
1716 union igc_adv_rx_desc
*rx_desc
,
1717 struct sk_buff
*skb
)
1719 skb_checksum_none_assert(skb
);
1721 /* Ignore Checksum bit is set */
1722 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_IXSM
))
1725 /* Rx checksum disabled via ethtool */
1726 if (!(ring
->netdev
->features
& NETIF_F_RXCSUM
))
1729 /* TCP/UDP checksum error bit is set */
1730 if (igc_test_staterr(rx_desc
,
1731 IGC_RXDEXT_STATERR_L4E
|
1732 IGC_RXDEXT_STATERR_IPE
)) {
1733 /* work around errata with sctp packets where the TCPE aka
1734 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1735 * packets (aka let the stack check the crc32c)
1737 if (!(skb
->len
== 60 &&
1738 test_bit(IGC_RING_FLAG_RX_SCTP_CSUM
, &ring
->flags
))) {
1739 u64_stats_update_begin(&ring
->rx_syncp
);
1740 ring
->rx_stats
.csum_err
++;
1741 u64_stats_update_end(&ring
->rx_syncp
);
1743 /* let the stack verify checksum errors */
1746 /* It must be a TCP or UDP packet with a valid checksum */
1747 if (igc_test_staterr(rx_desc
, IGC_RXD_STAT_TCPCS
|
1748 IGC_RXD_STAT_UDPCS
))
1749 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1751 netdev_dbg(ring
->netdev
, "cksum success: bits %08X\n",
1752 le32_to_cpu(rx_desc
->wb
.upper
.status_error
));
1755 /* Mapping HW RSS Type to enum pkt_hash_types */
1756 static const enum pkt_hash_types igc_rss_type_table
[IGC_RSS_TYPE_MAX_TABLE
] = {
1757 [IGC_RSS_TYPE_NO_HASH
] = PKT_HASH_TYPE_L2
,
1758 [IGC_RSS_TYPE_HASH_TCP_IPV4
] = PKT_HASH_TYPE_L4
,
1759 [IGC_RSS_TYPE_HASH_IPV4
] = PKT_HASH_TYPE_L3
,
1760 [IGC_RSS_TYPE_HASH_TCP_IPV6
] = PKT_HASH_TYPE_L4
,
1761 [IGC_RSS_TYPE_HASH_IPV6_EX
] = PKT_HASH_TYPE_L3
,
1762 [IGC_RSS_TYPE_HASH_IPV6
] = PKT_HASH_TYPE_L3
,
1763 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX
] = PKT_HASH_TYPE_L4
,
1764 [IGC_RSS_TYPE_HASH_UDP_IPV4
] = PKT_HASH_TYPE_L4
,
1765 [IGC_RSS_TYPE_HASH_UDP_IPV6
] = PKT_HASH_TYPE_L4
,
1766 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX
] = PKT_HASH_TYPE_L4
,
1767 [10] = PKT_HASH_TYPE_NONE
, /* RSS Type above 9 "Reserved" by HW */
1768 [11] = PKT_HASH_TYPE_NONE
, /* keep array sized for SW bit-mask */
1769 [12] = PKT_HASH_TYPE_NONE
, /* to handle future HW revisons */
1770 [13] = PKT_HASH_TYPE_NONE
,
1771 [14] = PKT_HASH_TYPE_NONE
,
1772 [15] = PKT_HASH_TYPE_NONE
,
1775 static inline void igc_rx_hash(struct igc_ring
*ring
,
1776 union igc_adv_rx_desc
*rx_desc
,
1777 struct sk_buff
*skb
)
1779 if (ring
->netdev
->features
& NETIF_F_RXHASH
) {
1780 u32 rss_hash
= le32_to_cpu(rx_desc
->wb
.lower
.hi_dword
.rss
);
1781 u32 rss_type
= igc_rss_type(rx_desc
);
1783 skb_set_hash(skb
, rss_hash
, igc_rss_type_table
[rss_type
]);
1787 static void igc_rx_vlan(struct igc_ring
*rx_ring
,
1788 union igc_adv_rx_desc
*rx_desc
,
1789 struct sk_buff
*skb
)
1791 struct net_device
*dev
= rx_ring
->netdev
;
1794 if ((dev
->features
& NETIF_F_HW_VLAN_CTAG_RX
) &&
1795 igc_test_staterr(rx_desc
, IGC_RXD_STAT_VP
)) {
1796 if (igc_test_staterr(rx_desc
, IGC_RXDEXT_STATERR_LB
) &&
1797 test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP
, &rx_ring
->flags
))
1798 vid
= be16_to_cpu((__force __be16
)rx_desc
->wb
.upper
.vlan
);
1800 vid
= le16_to_cpu(rx_desc
->wb
.upper
.vlan
);
1802 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), vid
);
1807 * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1808 * @rx_ring: rx descriptor ring packet is being transacted on
1809 * @rx_desc: pointer to the EOP Rx descriptor
1810 * @skb: pointer to current skb being populated
1812 * This function checks the ring, descriptor, and packet information in order
1813 * to populate the hash, checksum, VLAN, protocol, and other fields within the
1816 static void igc_process_skb_fields(struct igc_ring
*rx_ring
,
1817 union igc_adv_rx_desc
*rx_desc
,
1818 struct sk_buff
*skb
)
1820 igc_rx_hash(rx_ring
, rx_desc
, skb
);
1822 igc_rx_checksum(rx_ring
, rx_desc
, skb
);
1824 igc_rx_vlan(rx_ring
, rx_desc
, skb
);
1826 skb_record_rx_queue(skb
, rx_ring
->queue_index
);
1828 skb
->protocol
= eth_type_trans(skb
, rx_ring
->netdev
);
1831 static void igc_vlan_mode(struct net_device
*netdev
, netdev_features_t features
)
1833 bool enable
= !!(features
& NETIF_F_HW_VLAN_CTAG_RX
);
1834 struct igc_adapter
*adapter
= netdev_priv(netdev
);
1835 struct igc_hw
*hw
= &adapter
->hw
;
1838 ctrl
= rd32(IGC_CTRL
);
1841 /* enable VLAN tag insert/strip */
1842 ctrl
|= IGC_CTRL_VME
;
1844 /* disable VLAN tag insert/strip */
1845 ctrl
&= ~IGC_CTRL_VME
;
1847 wr32(IGC_CTRL
, ctrl
);
1850 static void igc_restore_vlan(struct igc_adapter
*adapter
)
1852 igc_vlan_mode(adapter
->netdev
, adapter
->netdev
->features
);
1855 static struct igc_rx_buffer
*igc_get_rx_buffer(struct igc_ring
*rx_ring
,
1856 const unsigned int size
,
1857 int *rx_buffer_pgcnt
)
1859 struct igc_rx_buffer
*rx_buffer
;
1861 rx_buffer
= &rx_ring
->rx_buffer_info
[rx_ring
->next_to_clean
];
1863 #if (PAGE_SIZE < 8192)
1864 page_count(rx_buffer
->page
);
1868 prefetchw(rx_buffer
->page
);
1870 /* we are reusing so sync this buffer for CPU use */
1871 dma_sync_single_range_for_cpu(rx_ring
->dev
,
1873 rx_buffer
->page_offset
,
1877 rx_buffer
->pagecnt_bias
--;
1882 static void igc_rx_buffer_flip(struct igc_rx_buffer
*buffer
,
1883 unsigned int truesize
)
1885 #if (PAGE_SIZE < 8192)
1886 buffer
->page_offset
^= truesize
;
1888 buffer
->page_offset
+= truesize
;
1892 static unsigned int igc_get_rx_frame_truesize(struct igc_ring
*ring
,
1895 unsigned int truesize
;
1897 #if (PAGE_SIZE < 8192)
1898 truesize
= igc_rx_pg_size(ring
) / 2;
1900 truesize
= ring_uses_build_skb(ring
) ?
1901 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)) +
1902 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
) :
1903 SKB_DATA_ALIGN(size
);
1909 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1910 * @rx_ring: rx descriptor ring to transact packets on
1911 * @rx_buffer: buffer containing page to add
1912 * @skb: sk_buff to place the data into
1913 * @size: size of buffer to be added
1915 * This function will add the data contained in rx_buffer->page to the skb.
1917 static void igc_add_rx_frag(struct igc_ring
*rx_ring
,
1918 struct igc_rx_buffer
*rx_buffer
,
1919 struct sk_buff
*skb
,
1922 unsigned int truesize
;
1924 #if (PAGE_SIZE < 8192)
1925 truesize
= igc_rx_pg_size(rx_ring
) / 2;
1927 truesize
= ring_uses_build_skb(rx_ring
) ?
1928 SKB_DATA_ALIGN(IGC_SKB_PAD
+ size
) :
1929 SKB_DATA_ALIGN(size
);
1931 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, rx_buffer
->page
,
1932 rx_buffer
->page_offset
, size
, truesize
);
1934 igc_rx_buffer_flip(rx_buffer
, truesize
);
1937 static struct sk_buff
*igc_build_skb(struct igc_ring
*rx_ring
,
1938 struct igc_rx_buffer
*rx_buffer
,
1939 struct xdp_buff
*xdp
)
1941 unsigned int size
= xdp
->data_end
- xdp
->data
;
1942 unsigned int truesize
= igc_get_rx_frame_truesize(rx_ring
, size
);
1943 unsigned int metasize
= xdp
->data
- xdp
->data_meta
;
1944 struct sk_buff
*skb
;
1946 /* prefetch first cache line of first page */
1947 net_prefetch(xdp
->data_meta
);
1949 /* build an skb around the page buffer */
1950 skb
= napi_build_skb(xdp
->data_hard_start
, truesize
);
1954 /* update pointers within the skb to store the data */
1955 skb_reserve(skb
, xdp
->data
- xdp
->data_hard_start
);
1956 __skb_put(skb
, size
);
1958 skb_metadata_set(skb
, metasize
);
1960 igc_rx_buffer_flip(rx_buffer
, truesize
);
1964 static struct sk_buff
*igc_construct_skb(struct igc_ring
*rx_ring
,
1965 struct igc_rx_buffer
*rx_buffer
,
1966 struct xdp_buff
*xdp
,
1969 unsigned int metasize
= xdp
->data
- xdp
->data_meta
;
1970 unsigned int size
= xdp
->data_end
- xdp
->data
;
1971 unsigned int truesize
= igc_get_rx_frame_truesize(rx_ring
, size
);
1972 void *va
= xdp
->data
;
1973 unsigned int headlen
;
1974 struct sk_buff
*skb
;
1976 /* prefetch first cache line of first page */
1977 net_prefetch(xdp
->data_meta
);
1979 /* allocate a skb to store the frags */
1980 skb
= napi_alloc_skb(&rx_ring
->q_vector
->napi
,
1981 IGC_RX_HDR_LEN
+ metasize
);
1986 skb_hwtstamps(skb
)->hwtstamp
= timestamp
;
1988 /* Determine available headroom for copy */
1990 if (headlen
> IGC_RX_HDR_LEN
)
1991 headlen
= eth_get_headlen(skb
->dev
, va
, IGC_RX_HDR_LEN
);
1993 /* align pull length to size of long to optimize memcpy performance */
1994 memcpy(__skb_put(skb
, headlen
+ metasize
), xdp
->data_meta
,
1995 ALIGN(headlen
+ metasize
, sizeof(long)));
1998 skb_metadata_set(skb
, metasize
);
1999 __skb_pull(skb
, metasize
);
2002 /* update all of the pointers */
2005 skb_add_rx_frag(skb
, 0, rx_buffer
->page
,
2006 (va
+ headlen
) - page_address(rx_buffer
->page
),
2008 igc_rx_buffer_flip(rx_buffer
, truesize
);
2010 rx_buffer
->pagecnt_bias
++;
2017 * igc_reuse_rx_page - page flip buffer and store it back on the ring
2018 * @rx_ring: rx descriptor ring to store buffers on
2019 * @old_buff: donor buffer to have page reused
2021 * Synchronizes page for reuse by the adapter
2023 static void igc_reuse_rx_page(struct igc_ring
*rx_ring
,
2024 struct igc_rx_buffer
*old_buff
)
2026 u16 nta
= rx_ring
->next_to_alloc
;
2027 struct igc_rx_buffer
*new_buff
;
2029 new_buff
= &rx_ring
->rx_buffer_info
[nta
];
2031 /* update, and store next to alloc */
2033 rx_ring
->next_to_alloc
= (nta
< rx_ring
->count
) ? nta
: 0;
2035 /* Transfer page from old buffer to new buffer.
2036 * Move each member individually to avoid possible store
2037 * forwarding stalls.
2039 new_buff
->dma
= old_buff
->dma
;
2040 new_buff
->page
= old_buff
->page
;
2041 new_buff
->page_offset
= old_buff
->page_offset
;
2042 new_buff
->pagecnt_bias
= old_buff
->pagecnt_bias
;
2045 static bool igc_can_reuse_rx_page(struct igc_rx_buffer
*rx_buffer
,
2046 int rx_buffer_pgcnt
)
2048 unsigned int pagecnt_bias
= rx_buffer
->pagecnt_bias
;
2049 struct page
*page
= rx_buffer
->page
;
2051 /* avoid re-using remote and pfmemalloc pages */
2052 if (!dev_page_is_reusable(page
))
2055 #if (PAGE_SIZE < 8192)
2056 /* if we are only owner of page we can reuse it */
2057 if (unlikely((rx_buffer_pgcnt
- pagecnt_bias
) > 1))
2060 #define IGC_LAST_OFFSET \
2061 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
2063 if (rx_buffer
->page_offset
> IGC_LAST_OFFSET
)
2067 /* If we have drained the page fragment pool we need to update
2068 * the pagecnt_bias and page count so that we fully restock the
2069 * number of references the driver holds.
2071 if (unlikely(pagecnt_bias
== 1)) {
2072 page_ref_add(page
, USHRT_MAX
- 1);
2073 rx_buffer
->pagecnt_bias
= USHRT_MAX
;
2080 * igc_is_non_eop - process handling of non-EOP buffers
2081 * @rx_ring: Rx ring being processed
2082 * @rx_desc: Rx descriptor for current buffer
2084 * This function updates next to clean. If the buffer is an EOP buffer
2085 * this function exits returning false, otherwise it will place the
2086 * sk_buff in the next buffer to be chained and return true indicating
2087 * that this is in fact a non-EOP buffer.
2089 static bool igc_is_non_eop(struct igc_ring
*rx_ring
,
2090 union igc_adv_rx_desc
*rx_desc
)
2092 u32 ntc
= rx_ring
->next_to_clean
+ 1;
2094 /* fetch, update, and store next to clean */
2095 ntc
= (ntc
< rx_ring
->count
) ? ntc
: 0;
2096 rx_ring
->next_to_clean
= ntc
;
2098 prefetch(IGC_RX_DESC(rx_ring
, ntc
));
2100 if (likely(igc_test_staterr(rx_desc
, IGC_RXD_STAT_EOP
)))
2107 * igc_cleanup_headers - Correct corrupted or empty headers
2108 * @rx_ring: rx descriptor ring packet is being transacted on
2109 * @rx_desc: pointer to the EOP Rx descriptor
2110 * @skb: pointer to current skb being fixed
2112 * Address the case where we are pulling data in on pages only
2113 * and as such no data is present in the skb header.
2115 * In addition if skb is not at least 60 bytes we need to pad it so that
2116 * it is large enough to qualify as a valid Ethernet frame.
2118 * Returns true if an error was encountered and skb was freed.
2120 static bool igc_cleanup_headers(struct igc_ring
*rx_ring
,
2121 union igc_adv_rx_desc
*rx_desc
,
2122 struct sk_buff
*skb
)
2124 /* XDP packets use error pointer so abort at this point */
2128 if (unlikely(igc_test_staterr(rx_desc
, IGC_RXDEXT_STATERR_RXE
))) {
2129 struct net_device
*netdev
= rx_ring
->netdev
;
2131 if (!(netdev
->features
& NETIF_F_RXALL
)) {
2132 dev_kfree_skb_any(skb
);
2137 /* if eth_skb_pad returns an error the skb was freed */
2138 if (eth_skb_pad(skb
))
2144 static void igc_put_rx_buffer(struct igc_ring
*rx_ring
,
2145 struct igc_rx_buffer
*rx_buffer
,
2146 int rx_buffer_pgcnt
)
2148 if (igc_can_reuse_rx_page(rx_buffer
, rx_buffer_pgcnt
)) {
2149 /* hand second half of page back to the ring */
2150 igc_reuse_rx_page(rx_ring
, rx_buffer
);
2152 /* We are not reusing the buffer so unmap it and free
2153 * any references we are holding to it
2155 dma_unmap_page_attrs(rx_ring
->dev
, rx_buffer
->dma
,
2156 igc_rx_pg_size(rx_ring
), DMA_FROM_DEVICE
,
2158 __page_frag_cache_drain(rx_buffer
->page
,
2159 rx_buffer
->pagecnt_bias
);
2162 /* clear contents of rx_buffer */
2163 rx_buffer
->page
= NULL
;
2166 static inline unsigned int igc_rx_offset(struct igc_ring
*rx_ring
)
2168 struct igc_adapter
*adapter
= rx_ring
->q_vector
->adapter
;
2170 if (ring_uses_build_skb(rx_ring
))
2172 if (igc_xdp_is_enabled(adapter
))
2173 return XDP_PACKET_HEADROOM
;
2178 static bool igc_alloc_mapped_page(struct igc_ring
*rx_ring
,
2179 struct igc_rx_buffer
*bi
)
2181 struct page
*page
= bi
->page
;
2184 /* since we are recycling buffers we should seldom need to alloc */
2188 /* alloc new page for storage */
2189 page
= dev_alloc_pages(igc_rx_pg_order(rx_ring
));
2190 if (unlikely(!page
)) {
2191 rx_ring
->rx_stats
.alloc_failed
++;
2195 /* map page for use */
2196 dma
= dma_map_page_attrs(rx_ring
->dev
, page
, 0,
2197 igc_rx_pg_size(rx_ring
),
2201 /* if mapping failed free memory back to system since
2202 * there isn't much point in holding memory we can't use
2204 if (dma_mapping_error(rx_ring
->dev
, dma
)) {
2207 rx_ring
->rx_stats
.alloc_failed
++;
2213 bi
->page_offset
= igc_rx_offset(rx_ring
);
2214 page_ref_add(page
, USHRT_MAX
- 1);
2215 bi
->pagecnt_bias
= USHRT_MAX
;
2221 * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2222 * @rx_ring: rx descriptor ring
2223 * @cleaned_count: number of buffers to clean
2225 static void igc_alloc_rx_buffers(struct igc_ring
*rx_ring
, u16 cleaned_count
)
2227 union igc_adv_rx_desc
*rx_desc
;
2228 u16 i
= rx_ring
->next_to_use
;
2229 struct igc_rx_buffer
*bi
;
2236 rx_desc
= IGC_RX_DESC(rx_ring
, i
);
2237 bi
= &rx_ring
->rx_buffer_info
[i
];
2238 i
-= rx_ring
->count
;
2240 bufsz
= igc_rx_bufsz(rx_ring
);
2243 if (!igc_alloc_mapped_page(rx_ring
, bi
))
2246 /* sync the buffer for use by the device */
2247 dma_sync_single_range_for_device(rx_ring
->dev
, bi
->dma
,
2248 bi
->page_offset
, bufsz
,
2251 /* Refresh the desc even if buffer_addrs didn't change
2252 * because each write-back erases this info.
2254 rx_desc
->read
.pkt_addr
= cpu_to_le64(bi
->dma
+ bi
->page_offset
);
2260 rx_desc
= IGC_RX_DESC(rx_ring
, 0);
2261 bi
= rx_ring
->rx_buffer_info
;
2262 i
-= rx_ring
->count
;
2265 /* clear the length for the next_to_use descriptor */
2266 rx_desc
->wb
.upper
.length
= 0;
2269 } while (cleaned_count
);
2271 i
+= rx_ring
->count
;
2273 if (rx_ring
->next_to_use
!= i
) {
2274 /* record the next descriptor to use */
2275 rx_ring
->next_to_use
= i
;
2277 /* update next to alloc since we have filled the ring */
2278 rx_ring
->next_to_alloc
= i
;
2280 /* Force memory writes to complete before letting h/w
2281 * know there are new descriptors to fetch. (Only
2282 * applicable for weak-ordered memory model archs,
2286 writel(i
, rx_ring
->tail
);
2290 static bool igc_alloc_rx_buffers_zc(struct igc_ring
*ring
, u16 count
)
2292 union igc_adv_rx_desc
*desc
;
2293 u16 i
= ring
->next_to_use
;
2294 struct igc_rx_buffer
*bi
;
2301 XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff
);
2303 desc
= IGC_RX_DESC(ring
, i
);
2304 bi
= &ring
->rx_buffer_info
[i
];
2308 bi
->xdp
= xsk_buff_alloc(ring
->xsk_pool
);
2314 dma
= xsk_buff_xdp_get_dma(bi
->xdp
);
2315 desc
->read
.pkt_addr
= cpu_to_le64(dma
);
2321 desc
= IGC_RX_DESC(ring
, 0);
2322 bi
= ring
->rx_buffer_info
;
2326 /* Clear the length for the next_to_use descriptor. */
2327 desc
->wb
.upper
.length
= 0;
2334 if (ring
->next_to_use
!= i
) {
2335 ring
->next_to_use
= i
;
2337 /* Force memory writes to complete before letting h/w
2338 * know there are new descriptors to fetch. (Only
2339 * applicable for weak-ordered memory model archs,
2343 writel(i
, ring
->tail
);
2349 /* This function requires __netif_tx_lock is held by the caller. */
2350 static int igc_xdp_init_tx_descriptor(struct igc_ring
*ring
,
2351 struct xdp_frame
*xdpf
)
2353 struct skb_shared_info
*sinfo
= xdp_get_shared_info_from_frame(xdpf
);
2354 u8 nr_frags
= unlikely(xdp_frame_has_frags(xdpf
)) ? sinfo
->nr_frags
: 0;
2355 u16 count
, index
= ring
->next_to_use
;
2356 struct igc_tx_buffer
*head
= &ring
->tx_buffer_info
[index
];
2357 struct igc_tx_buffer
*buffer
= head
;
2358 union igc_adv_tx_desc
*desc
= IGC_TX_DESC(ring
, index
);
2359 u32 olinfo_status
, len
= xdpf
->len
, cmd_type
;
2360 void *data
= xdpf
->data
;
2363 count
= TXD_USE_COUNT(len
);
2364 for (i
= 0; i
< nr_frags
; i
++)
2365 count
+= TXD_USE_COUNT(skb_frag_size(&sinfo
->frags
[i
]));
2367 if (igc_maybe_stop_tx(ring
, count
+ 3)) {
2368 /* this is a hard error */
2373 head
->bytecount
= xdp_get_frame_len(xdpf
);
2374 head
->type
= IGC_TX_BUFFER_TYPE_XDP
;
2378 olinfo_status
= head
->bytecount
<< IGC_ADVTXD_PAYLEN_SHIFT
;
2379 desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
2384 dma
= dma_map_single(ring
->dev
, data
, len
, DMA_TO_DEVICE
);
2385 if (dma_mapping_error(ring
->dev
, dma
)) {
2386 netdev_err_once(ring
->netdev
,
2387 "Failed to map DMA for TX\n");
2391 dma_unmap_len_set(buffer
, len
, len
);
2392 dma_unmap_addr_set(buffer
, dma
, dma
);
2394 cmd_type
= IGC_ADVTXD_DTYP_DATA
| IGC_ADVTXD_DCMD_DEXT
|
2395 IGC_ADVTXD_DCMD_IFCS
| len
;
2397 desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
2398 desc
->read
.buffer_addr
= cpu_to_le64(dma
);
2400 buffer
->protocol
= 0;
2402 if (++index
== ring
->count
)
2408 buffer
= &ring
->tx_buffer_info
[index
];
2409 desc
= IGC_TX_DESC(ring
, index
);
2410 desc
->read
.olinfo_status
= 0;
2412 data
= skb_frag_address(&sinfo
->frags
[i
]);
2413 len
= skb_frag_size(&sinfo
->frags
[i
]);
2416 desc
->read
.cmd_type_len
|= cpu_to_le32(IGC_TXD_DCMD
);
2418 netdev_tx_sent_queue(txring_txq(ring
), head
->bytecount
);
2419 /* set the timestamp */
2420 head
->time_stamp
= jiffies
;
2421 /* set next_to_watch value indicating a packet is present */
2422 head
->next_to_watch
= desc
;
2423 ring
->next_to_use
= index
;
2429 buffer
= &ring
->tx_buffer_info
[index
];
2430 if (dma_unmap_len(buffer
, len
))
2431 dma_unmap_page(ring
->dev
,
2432 dma_unmap_addr(buffer
, dma
),
2433 dma_unmap_len(buffer
, len
),
2435 dma_unmap_len_set(buffer
, len
, 0);
2440 index
+= ring
->count
;
2447 static struct igc_ring
*igc_xdp_get_tx_ring(struct igc_adapter
*adapter
,
2452 if (unlikely(index
< 0))
2455 while (index
>= adapter
->num_tx_queues
)
2456 index
-= adapter
->num_tx_queues
;
2458 return adapter
->tx_ring
[index
];
2461 static int igc_xdp_xmit_back(struct igc_adapter
*adapter
, struct xdp_buff
*xdp
)
2463 struct xdp_frame
*xdpf
= xdp_convert_buff_to_frame(xdp
);
2464 int cpu
= smp_processor_id();
2465 struct netdev_queue
*nq
;
2466 struct igc_ring
*ring
;
2469 if (unlikely(!xdpf
))
2472 ring
= igc_xdp_get_tx_ring(adapter
, cpu
);
2473 nq
= txring_txq(ring
);
2475 __netif_tx_lock(nq
, cpu
);
2476 /* Avoid transmit queue timeout since we share it with the slow path */
2477 txq_trans_cond_update(nq
);
2478 res
= igc_xdp_init_tx_descriptor(ring
, xdpf
);
2479 __netif_tx_unlock(nq
);
2483 /* This function assumes rcu_read_lock() is held by the caller. */
2484 static int __igc_xdp_run_prog(struct igc_adapter
*adapter
,
2485 struct bpf_prog
*prog
,
2486 struct xdp_buff
*xdp
)
2488 u32 act
= bpf_prog_run_xdp(prog
, xdp
);
2492 return IGC_XDP_PASS
;
2494 if (igc_xdp_xmit_back(adapter
, xdp
) < 0)
2498 if (xdp_do_redirect(adapter
->netdev
, xdp
, prog
) < 0)
2500 return IGC_XDP_REDIRECT
;
2503 bpf_warn_invalid_xdp_action(adapter
->netdev
, prog
, act
);
2507 trace_xdp_exception(adapter
->netdev
, prog
, act
);
2510 return IGC_XDP_CONSUMED
;
2514 static struct sk_buff
*igc_xdp_run_prog(struct igc_adapter
*adapter
,
2515 struct xdp_buff
*xdp
)
2517 struct bpf_prog
*prog
;
2520 prog
= READ_ONCE(adapter
->xdp_prog
);
2526 res
= __igc_xdp_run_prog(adapter
, prog
, xdp
);
2529 return ERR_PTR(-res
);
2532 /* This function assumes __netif_tx_lock is held by the caller. */
2533 static void igc_flush_tx_descriptors(struct igc_ring
*ring
)
2535 /* Once tail pointer is updated, hardware can fetch the descriptors
2536 * any time so we issue a write membar here to ensure all memory
2537 * writes are complete before the tail pointer is updated.
2540 writel(ring
->next_to_use
, ring
->tail
);
2543 static void igc_finalize_xdp(struct igc_adapter
*adapter
, int status
)
2545 int cpu
= smp_processor_id();
2546 struct netdev_queue
*nq
;
2547 struct igc_ring
*ring
;
2549 if (status
& IGC_XDP_TX
) {
2550 ring
= igc_xdp_get_tx_ring(adapter
, cpu
);
2551 nq
= txring_txq(ring
);
2553 __netif_tx_lock(nq
, cpu
);
2554 igc_flush_tx_descriptors(ring
);
2555 __netif_tx_unlock(nq
);
2558 if (status
& IGC_XDP_REDIRECT
)
2562 static void igc_update_rx_stats(struct igc_q_vector
*q_vector
,
2563 unsigned int packets
, unsigned int bytes
)
2565 struct igc_ring
*ring
= q_vector
->rx
.ring
;
2567 u64_stats_update_begin(&ring
->rx_syncp
);
2568 ring
->rx_stats
.packets
+= packets
;
2569 ring
->rx_stats
.bytes
+= bytes
;
2570 u64_stats_update_end(&ring
->rx_syncp
);
2572 q_vector
->rx
.total_packets
+= packets
;
2573 q_vector
->rx
.total_bytes
+= bytes
;
2576 static int igc_clean_rx_irq(struct igc_q_vector
*q_vector
, const int budget
)
2578 unsigned int total_bytes
= 0, total_packets
= 0;
2579 struct igc_adapter
*adapter
= q_vector
->adapter
;
2580 struct igc_ring
*rx_ring
= q_vector
->rx
.ring
;
2581 struct sk_buff
*skb
= rx_ring
->skb
;
2582 u16 cleaned_count
= igc_desc_unused(rx_ring
);
2583 int xdp_status
= 0, rx_buffer_pgcnt
;
2585 while (likely(total_packets
< budget
)) {
2586 union igc_adv_rx_desc
*rx_desc
;
2587 struct igc_rx_buffer
*rx_buffer
;
2588 unsigned int size
, truesize
;
2589 struct igc_xdp_buff ctx
;
2590 ktime_t timestamp
= 0;
2594 /* return some buffers to hardware, one at a time is too slow */
2595 if (cleaned_count
>= IGC_RX_BUFFER_WRITE
) {
2596 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
2600 rx_desc
= IGC_RX_DESC(rx_ring
, rx_ring
->next_to_clean
);
2601 size
= le16_to_cpu(rx_desc
->wb
.upper
.length
);
2605 /* This memory barrier is needed to keep us from reading
2606 * any other fields out of the rx_desc until we know the
2607 * descriptor has been written back
2611 rx_buffer
= igc_get_rx_buffer(rx_ring
, size
, &rx_buffer_pgcnt
);
2612 truesize
= igc_get_rx_frame_truesize(rx_ring
, size
);
2614 pktbuf
= page_address(rx_buffer
->page
) + rx_buffer
->page_offset
;
2616 if (igc_test_staterr(rx_desc
, IGC_RXDADV_STAT_TSIP
)) {
2617 timestamp
= igc_ptp_rx_pktstamp(q_vector
->adapter
,
2619 ctx
.rx_ts
= timestamp
;
2620 pkt_offset
= IGC_TS_HDR_LEN
;
2621 size
-= IGC_TS_HDR_LEN
;
2625 xdp_init_buff(&ctx
.xdp
, truesize
, &rx_ring
->xdp_rxq
);
2626 xdp_prepare_buff(&ctx
.xdp
, pktbuf
- igc_rx_offset(rx_ring
),
2627 igc_rx_offset(rx_ring
) + pkt_offset
,
2629 xdp_buff_clear_frags_flag(&ctx
.xdp
);
2630 ctx
.rx_desc
= rx_desc
;
2632 skb
= igc_xdp_run_prog(adapter
, &ctx
.xdp
);
2636 unsigned int xdp_res
= -PTR_ERR(skb
);
2639 case IGC_XDP_CONSUMED
:
2640 rx_buffer
->pagecnt_bias
++;
2643 case IGC_XDP_REDIRECT
:
2644 igc_rx_buffer_flip(rx_buffer
, truesize
);
2645 xdp_status
|= xdp_res
;
2650 total_bytes
+= size
;
2652 igc_add_rx_frag(rx_ring
, rx_buffer
, skb
, size
);
2653 else if (ring_uses_build_skb(rx_ring
))
2654 skb
= igc_build_skb(rx_ring
, rx_buffer
, &ctx
.xdp
);
2656 skb
= igc_construct_skb(rx_ring
, rx_buffer
, &ctx
.xdp
,
2659 /* exit if we failed to retrieve a buffer */
2661 rx_ring
->rx_stats
.alloc_failed
++;
2662 rx_buffer
->pagecnt_bias
++;
2666 igc_put_rx_buffer(rx_ring
, rx_buffer
, rx_buffer_pgcnt
);
2669 /* fetch next buffer in frame if non-eop */
2670 if (igc_is_non_eop(rx_ring
, rx_desc
))
2673 /* verify the packet layout is correct */
2674 if (igc_cleanup_headers(rx_ring
, rx_desc
, skb
)) {
2679 /* probably a little skewed due to removing CRC */
2680 total_bytes
+= skb
->len
;
2682 /* populate checksum, VLAN, and protocol */
2683 igc_process_skb_fields(rx_ring
, rx_desc
, skb
);
2685 napi_gro_receive(&q_vector
->napi
, skb
);
2687 /* reset skb pointer */
2690 /* update budget accounting */
2695 igc_finalize_xdp(adapter
, xdp_status
);
2697 /* place incomplete frames back on ring for completion */
2700 igc_update_rx_stats(q_vector
, total_packets
, total_bytes
);
2703 igc_alloc_rx_buffers(rx_ring
, cleaned_count
);
2705 return total_packets
;
2708 static struct sk_buff
*igc_construct_skb_zc(struct igc_ring
*ring
,
2709 struct xdp_buff
*xdp
)
2711 unsigned int totalsize
= xdp
->data_end
- xdp
->data_meta
;
2712 unsigned int metasize
= xdp
->data
- xdp
->data_meta
;
2713 struct sk_buff
*skb
;
2715 net_prefetch(xdp
->data_meta
);
2717 skb
= __napi_alloc_skb(&ring
->q_vector
->napi
, totalsize
,
2718 GFP_ATOMIC
| __GFP_NOWARN
);
2722 memcpy(__skb_put(skb
, totalsize
), xdp
->data_meta
,
2723 ALIGN(totalsize
, sizeof(long)));
2726 skb_metadata_set(skb
, metasize
);
2727 __skb_pull(skb
, metasize
);
2733 static void igc_dispatch_skb_zc(struct igc_q_vector
*q_vector
,
2734 union igc_adv_rx_desc
*desc
,
2735 struct xdp_buff
*xdp
,
2738 struct igc_ring
*ring
= q_vector
->rx
.ring
;
2739 struct sk_buff
*skb
;
2741 skb
= igc_construct_skb_zc(ring
, xdp
);
2743 ring
->rx_stats
.alloc_failed
++;
2748 skb_hwtstamps(skb
)->hwtstamp
= timestamp
;
2750 if (igc_cleanup_headers(ring
, desc
, skb
))
2753 igc_process_skb_fields(ring
, desc
, skb
);
2754 napi_gro_receive(&q_vector
->napi
, skb
);
2757 static struct igc_xdp_buff
*xsk_buff_to_igc_ctx(struct xdp_buff
*xdp
)
2759 /* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The
2760 * igc_xdp_buff shares its layout with xdp_buff_xsk and private
2761 * igc_xdp_buff fields fall into xdp_buff_xsk->cb
2763 return (struct igc_xdp_buff
*)xdp
;
2766 static int igc_clean_rx_irq_zc(struct igc_q_vector
*q_vector
, const int budget
)
2768 struct igc_adapter
*adapter
= q_vector
->adapter
;
2769 struct igc_ring
*ring
= q_vector
->rx
.ring
;
2770 u16 cleaned_count
= igc_desc_unused(ring
);
2771 int total_bytes
= 0, total_packets
= 0;
2772 u16 ntc
= ring
->next_to_clean
;
2773 struct bpf_prog
*prog
;
2774 bool failure
= false;
2779 prog
= READ_ONCE(adapter
->xdp_prog
);
2781 while (likely(total_packets
< budget
)) {
2782 union igc_adv_rx_desc
*desc
;
2783 struct igc_rx_buffer
*bi
;
2784 struct igc_xdp_buff
*ctx
;
2785 ktime_t timestamp
= 0;
2789 desc
= IGC_RX_DESC(ring
, ntc
);
2790 size
= le16_to_cpu(desc
->wb
.upper
.length
);
2794 /* This memory barrier is needed to keep us from reading
2795 * any other fields out of the rx_desc until we know the
2796 * descriptor has been written back
2800 bi
= &ring
->rx_buffer_info
[ntc
];
2802 ctx
= xsk_buff_to_igc_ctx(bi
->xdp
);
2803 ctx
->rx_desc
= desc
;
2805 if (igc_test_staterr(desc
, IGC_RXDADV_STAT_TSIP
)) {
2806 timestamp
= igc_ptp_rx_pktstamp(q_vector
->adapter
,
2808 ctx
->rx_ts
= timestamp
;
2810 bi
->xdp
->data
+= IGC_TS_HDR_LEN
;
2812 /* HW timestamp has been copied into local variable. Metadata
2813 * length when XDP program is called should be 0.
2815 bi
->xdp
->data_meta
+= IGC_TS_HDR_LEN
;
2816 size
-= IGC_TS_HDR_LEN
;
2819 bi
->xdp
->data_end
= bi
->xdp
->data
+ size
;
2820 xsk_buff_dma_sync_for_cpu(bi
->xdp
, ring
->xsk_pool
);
2822 res
= __igc_xdp_run_prog(adapter
, prog
, bi
->xdp
);
2825 igc_dispatch_skb_zc(q_vector
, desc
, bi
->xdp
, timestamp
);
2827 case IGC_XDP_CONSUMED
:
2828 xsk_buff_free(bi
->xdp
);
2831 case IGC_XDP_REDIRECT
:
2837 total_bytes
+= size
;
2841 if (ntc
== ring
->count
)
2845 ring
->next_to_clean
= ntc
;
2848 if (cleaned_count
>= IGC_RX_BUFFER_WRITE
)
2849 failure
= !igc_alloc_rx_buffers_zc(ring
, cleaned_count
);
2852 igc_finalize_xdp(adapter
, xdp_status
);
2854 igc_update_rx_stats(q_vector
, total_packets
, total_bytes
);
2856 if (xsk_uses_need_wakeup(ring
->xsk_pool
)) {
2857 if (failure
|| ring
->next_to_clean
== ring
->next_to_use
)
2858 xsk_set_rx_need_wakeup(ring
->xsk_pool
);
2860 xsk_clear_rx_need_wakeup(ring
->xsk_pool
);
2861 return total_packets
;
2864 return failure
? budget
: total_packets
;
2867 static void igc_update_tx_stats(struct igc_q_vector
*q_vector
,
2868 unsigned int packets
, unsigned int bytes
)
2870 struct igc_ring
*ring
= q_vector
->tx
.ring
;
2872 u64_stats_update_begin(&ring
->tx_syncp
);
2873 ring
->tx_stats
.bytes
+= bytes
;
2874 ring
->tx_stats
.packets
+= packets
;
2875 u64_stats_update_end(&ring
->tx_syncp
);
2877 q_vector
->tx
.total_bytes
+= bytes
;
2878 q_vector
->tx
.total_packets
+= packets
;
2881 static void igc_xdp_xmit_zc(struct igc_ring
*ring
)
2883 struct xsk_buff_pool
*pool
= ring
->xsk_pool
;
2884 struct netdev_queue
*nq
= txring_txq(ring
);
2885 union igc_adv_tx_desc
*tx_desc
= NULL
;
2886 int cpu
= smp_processor_id();
2887 struct xdp_desc xdp_desc
;
2890 if (!netif_carrier_ok(ring
->netdev
))
2893 __netif_tx_lock(nq
, cpu
);
2895 /* Avoid transmit queue timeout since we share it with the slow path */
2896 txq_trans_cond_update(nq
);
2898 ntu
= ring
->next_to_use
;
2899 budget
= igc_desc_unused(ring
);
2901 while (xsk_tx_peek_desc(pool
, &xdp_desc
) && budget
--) {
2902 u32 cmd_type
, olinfo_status
;
2903 struct igc_tx_buffer
*bi
;
2906 cmd_type
= IGC_ADVTXD_DTYP_DATA
| IGC_ADVTXD_DCMD_DEXT
|
2907 IGC_ADVTXD_DCMD_IFCS
| IGC_TXD_DCMD
|
2909 olinfo_status
= xdp_desc
.len
<< IGC_ADVTXD_PAYLEN_SHIFT
;
2911 dma
= xsk_buff_raw_get_dma(pool
, xdp_desc
.addr
);
2912 xsk_buff_raw_dma_sync_for_device(pool
, dma
, xdp_desc
.len
);
2914 tx_desc
= IGC_TX_DESC(ring
, ntu
);
2915 tx_desc
->read
.cmd_type_len
= cpu_to_le32(cmd_type
);
2916 tx_desc
->read
.olinfo_status
= cpu_to_le32(olinfo_status
);
2917 tx_desc
->read
.buffer_addr
= cpu_to_le64(dma
);
2919 bi
= &ring
->tx_buffer_info
[ntu
];
2920 bi
->type
= IGC_TX_BUFFER_TYPE_XSK
;
2922 bi
->bytecount
= xdp_desc
.len
;
2924 bi
->time_stamp
= jiffies
;
2925 bi
->next_to_watch
= tx_desc
;
2927 netdev_tx_sent_queue(txring_txq(ring
), xdp_desc
.len
);
2930 if (ntu
== ring
->count
)
2934 ring
->next_to_use
= ntu
;
2936 igc_flush_tx_descriptors(ring
);
2937 xsk_tx_release(pool
);
2940 __netif_tx_unlock(nq
);
2944 * igc_clean_tx_irq - Reclaim resources after transmit completes
2945 * @q_vector: pointer to q_vector containing needed info
2946 * @napi_budget: Used to determine if we are in netpoll
2948 * returns true if ring is completely cleaned
2950 static bool igc_clean_tx_irq(struct igc_q_vector
*q_vector
, int napi_budget
)
2952 struct igc_adapter
*adapter
= q_vector
->adapter
;
2953 unsigned int total_bytes
= 0, total_packets
= 0;
2954 unsigned int budget
= q_vector
->tx
.work_limit
;
2955 struct igc_ring
*tx_ring
= q_vector
->tx
.ring
;
2956 unsigned int i
= tx_ring
->next_to_clean
;
2957 struct igc_tx_buffer
*tx_buffer
;
2958 union igc_adv_tx_desc
*tx_desc
;
2961 if (test_bit(__IGC_DOWN
, &adapter
->state
))
2964 tx_buffer
= &tx_ring
->tx_buffer_info
[i
];
2965 tx_desc
= IGC_TX_DESC(tx_ring
, i
);
2966 i
-= tx_ring
->count
;
2969 union igc_adv_tx_desc
*eop_desc
= tx_buffer
->next_to_watch
;
2971 /* if next_to_watch is not set then there is no work pending */
2975 /* prevent any other reads prior to eop_desc */
2978 /* if DD is not set pending work has not been completed */
2979 if (!(eop_desc
->wb
.status
& cpu_to_le32(IGC_TXD_STAT_DD
)))
2982 /* clear next_to_watch to prevent false hangs */
2983 tx_buffer
->next_to_watch
= NULL
;
2985 /* update the statistics for this packet */
2986 total_bytes
+= tx_buffer
->bytecount
;
2987 total_packets
+= tx_buffer
->gso_segs
;
2989 switch (tx_buffer
->type
) {
2990 case IGC_TX_BUFFER_TYPE_XSK
:
2993 case IGC_TX_BUFFER_TYPE_XDP
:
2994 xdp_return_frame(tx_buffer
->xdpf
);
2995 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
2997 case IGC_TX_BUFFER_TYPE_SKB
:
2998 napi_consume_skb(tx_buffer
->skb
, napi_budget
);
2999 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
3002 netdev_warn_once(tx_ring
->netdev
, "Unknown Tx buffer type\n");
3006 /* clear last DMA location and unmap remaining buffers */
3007 while (tx_desc
!= eop_desc
) {
3012 i
-= tx_ring
->count
;
3013 tx_buffer
= tx_ring
->tx_buffer_info
;
3014 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
3017 /* unmap any remaining paged data */
3018 if (dma_unmap_len(tx_buffer
, len
))
3019 igc_unmap_tx_buffer(tx_ring
->dev
, tx_buffer
);
3022 /* move us one more past the eop_desc for start of next pkt */
3027 i
-= tx_ring
->count
;
3028 tx_buffer
= tx_ring
->tx_buffer_info
;
3029 tx_desc
= IGC_TX_DESC(tx_ring
, 0);
3032 /* issue prefetch for next Tx descriptor */
3035 /* update budget accounting */
3037 } while (likely(budget
));
3039 netdev_tx_completed_queue(txring_txq(tx_ring
),
3040 total_packets
, total_bytes
);
3042 i
+= tx_ring
->count
;
3043 tx_ring
->next_to_clean
= i
;
3045 igc_update_tx_stats(q_vector
, total_packets
, total_bytes
);
3047 if (tx_ring
->xsk_pool
) {
3049 xsk_tx_completed(tx_ring
->xsk_pool
, xsk_frames
);
3050 if (xsk_uses_need_wakeup(tx_ring
->xsk_pool
))
3051 xsk_set_tx_need_wakeup(tx_ring
->xsk_pool
);
3052 igc_xdp_xmit_zc(tx_ring
);
3055 if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
)) {
3056 struct igc_hw
*hw
= &adapter
->hw
;
3058 /* Detect a transmit hang in hardware, this serializes the
3059 * check with the clearing of time_stamp and movement of i
3061 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
3062 if (tx_buffer
->next_to_watch
&&
3063 time_after(jiffies
, tx_buffer
->time_stamp
+
3064 (adapter
->tx_timeout_factor
* HZ
)) &&
3065 !(rd32(IGC_STATUS
) & IGC_STATUS_TXOFF
) &&
3066 (rd32(IGC_TDH(tx_ring
->reg_idx
)) != readl(tx_ring
->tail
)) &&
3067 !tx_ring
->oper_gate_closed
) {
3068 /* detected Tx unit hang */
3069 netdev_err(tx_ring
->netdev
,
3070 "Detected Tx Unit Hang\n"
3074 " next_to_use <%x>\n"
3075 " next_to_clean <%x>\n"
3076 "buffer_info[next_to_clean]\n"
3077 " time_stamp <%lx>\n"
3078 " next_to_watch <%p>\n"
3080 " desc.status <%x>\n",
3081 tx_ring
->queue_index
,
3082 rd32(IGC_TDH(tx_ring
->reg_idx
)),
3083 readl(tx_ring
->tail
),
3084 tx_ring
->next_to_use
,
3085 tx_ring
->next_to_clean
,
3086 tx_buffer
->time_stamp
,
3087 tx_buffer
->next_to_watch
,
3089 tx_buffer
->next_to_watch
->wb
.status
);
3090 netif_stop_subqueue(tx_ring
->netdev
,
3091 tx_ring
->queue_index
);
3093 /* we are about to reset, no point in enabling stuff */
3098 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
3099 if (unlikely(total_packets
&&
3100 netif_carrier_ok(tx_ring
->netdev
) &&
3101 igc_desc_unused(tx_ring
) >= TX_WAKE_THRESHOLD
)) {
3102 /* Make sure that anybody stopping the queue after this
3103 * sees the new next_to_clean.
3106 if (__netif_subqueue_stopped(tx_ring
->netdev
,
3107 tx_ring
->queue_index
) &&
3108 !(test_bit(__IGC_DOWN
, &adapter
->state
))) {
3109 netif_wake_subqueue(tx_ring
->netdev
,
3110 tx_ring
->queue_index
);
3112 u64_stats_update_begin(&tx_ring
->tx_syncp
);
3113 tx_ring
->tx_stats
.restart_queue
++;
3114 u64_stats_update_end(&tx_ring
->tx_syncp
);
3121 static int igc_find_mac_filter(struct igc_adapter
*adapter
,
3122 enum igc_mac_filter_type type
, const u8
*addr
)
3124 struct igc_hw
*hw
= &adapter
->hw
;
3125 int max_entries
= hw
->mac
.rar_entry_count
;
3129 for (i
= 0; i
< max_entries
; i
++) {
3130 ral
= rd32(IGC_RAL(i
));
3131 rah
= rd32(IGC_RAH(i
));
3133 if (!(rah
& IGC_RAH_AV
))
3135 if (!!(rah
& IGC_RAH_ASEL_SRC_ADDR
) != type
)
3137 if ((rah
& IGC_RAH_RAH_MASK
) !=
3138 le16_to_cpup((__le16
*)(addr
+ 4)))
3140 if (ral
!= le32_to_cpup((__le32
*)(addr
)))
3149 static int igc_get_avail_mac_filter_slot(struct igc_adapter
*adapter
)
3151 struct igc_hw
*hw
= &adapter
->hw
;
3152 int max_entries
= hw
->mac
.rar_entry_count
;
3156 for (i
= 0; i
< max_entries
; i
++) {
3157 rah
= rd32(IGC_RAH(i
));
3159 if (!(rah
& IGC_RAH_AV
))
3167 * igc_add_mac_filter() - Add MAC address filter
3168 * @adapter: Pointer to adapter where the filter should be added
3169 * @type: MAC address filter type (source or destination)
3170 * @addr: MAC address
3171 * @queue: If non-negative, queue assignment feature is enabled and frames
3172 * matching the filter are enqueued onto 'queue'. Otherwise, queue
3173 * assignment is disabled.
3175 * Return: 0 in case of success, negative errno code otherwise.
3177 static int igc_add_mac_filter(struct igc_adapter
*adapter
,
3178 enum igc_mac_filter_type type
, const u8
*addr
,
3181 struct net_device
*dev
= adapter
->netdev
;
3184 index
= igc_find_mac_filter(adapter
, type
, addr
);
3188 index
= igc_get_avail_mac_filter_slot(adapter
);
3192 netdev_dbg(dev
, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3193 index
, type
== IGC_MAC_FILTER_TYPE_DST
? "dst" : "src",
3197 igc_set_mac_filter_hw(adapter
, index
, type
, addr
, queue
);
3202 * igc_del_mac_filter() - Delete MAC address filter
3203 * @adapter: Pointer to adapter where the filter should be deleted from
3204 * @type: MAC address filter type (source or destination)
3205 * @addr: MAC address
3207 static void igc_del_mac_filter(struct igc_adapter
*adapter
,
3208 enum igc_mac_filter_type type
, const u8
*addr
)
3210 struct net_device
*dev
= adapter
->netdev
;
3213 index
= igc_find_mac_filter(adapter
, type
, addr
);
3218 /* If this is the default filter, we don't actually delete it.
3219 * We just reset to its default value i.e. disable queue
3222 netdev_dbg(dev
, "Disable default MAC filter queue assignment");
3224 igc_set_mac_filter_hw(adapter
, 0, type
, addr
, -1);
3226 netdev_dbg(dev
, "Delete MAC address filter: index %d type %s address %pM\n",
3228 type
== IGC_MAC_FILTER_TYPE_DST
? "dst" : "src",
3231 igc_clear_mac_filter_hw(adapter
, index
);
3236 * igc_add_vlan_prio_filter() - Add VLAN priority filter
3237 * @adapter: Pointer to adapter where the filter should be added
3238 * @prio: VLAN priority value
3239 * @queue: Queue number which matching frames are assigned to
3241 * Return: 0 in case of success, negative errno code otherwise.
3243 static int igc_add_vlan_prio_filter(struct igc_adapter
*adapter
, int prio
,
3246 struct net_device
*dev
= adapter
->netdev
;
3247 struct igc_hw
*hw
= &adapter
->hw
;
3250 vlanpqf
= rd32(IGC_VLANPQF
);
3252 if (vlanpqf
& IGC_VLANPQF_VALID(prio
)) {
3253 netdev_dbg(dev
, "VLAN priority filter already in use\n");
3257 vlanpqf
|= IGC_VLANPQF_QSEL(prio
, queue
);
3258 vlanpqf
|= IGC_VLANPQF_VALID(prio
);
3260 wr32(IGC_VLANPQF
, vlanpqf
);
3262 netdev_dbg(dev
, "Add VLAN priority filter: prio %d queue %d\n",
3268 * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3269 * @adapter: Pointer to adapter where the filter should be deleted from
3270 * @prio: VLAN priority value
3272 static void igc_del_vlan_prio_filter(struct igc_adapter
*adapter
, int prio
)
3274 struct igc_hw
*hw
= &adapter
->hw
;
3277 vlanpqf
= rd32(IGC_VLANPQF
);
3279 vlanpqf
&= ~IGC_VLANPQF_VALID(prio
);
3280 vlanpqf
&= ~IGC_VLANPQF_QSEL(prio
, IGC_VLANPQF_QUEUE_MASK
);
3282 wr32(IGC_VLANPQF
, vlanpqf
);
3284 netdev_dbg(adapter
->netdev
, "Delete VLAN priority filter: prio %d\n",
3288 static int igc_get_avail_etype_filter_slot(struct igc_adapter
*adapter
)
3290 struct igc_hw
*hw
= &adapter
->hw
;
3293 for (i
= 0; i
< MAX_ETYPE_FILTER
; i
++) {
3294 u32 etqf
= rd32(IGC_ETQF(i
));
3296 if (!(etqf
& IGC_ETQF_FILTER_ENABLE
))
3304 * igc_add_etype_filter() - Add ethertype filter
3305 * @adapter: Pointer to adapter where the filter should be added
3306 * @etype: Ethertype value
3307 * @queue: If non-negative, queue assignment feature is enabled and frames
3308 * matching the filter are enqueued onto 'queue'. Otherwise, queue
3309 * assignment is disabled.
3311 * Return: 0 in case of success, negative errno code otherwise.
3313 static int igc_add_etype_filter(struct igc_adapter
*adapter
, u16 etype
,
3316 struct igc_hw
*hw
= &adapter
->hw
;
3320 index
= igc_get_avail_etype_filter_slot(adapter
);
3324 etqf
= rd32(IGC_ETQF(index
));
3326 etqf
&= ~IGC_ETQF_ETYPE_MASK
;
3330 etqf
&= ~IGC_ETQF_QUEUE_MASK
;
3331 etqf
|= (queue
<< IGC_ETQF_QUEUE_SHIFT
);
3332 etqf
|= IGC_ETQF_QUEUE_ENABLE
;
3335 etqf
|= IGC_ETQF_FILTER_ENABLE
;
3337 wr32(IGC_ETQF(index
), etqf
);
3339 netdev_dbg(adapter
->netdev
, "Add ethertype filter: etype %04x queue %d\n",
3344 static int igc_find_etype_filter(struct igc_adapter
*adapter
, u16 etype
)
3346 struct igc_hw
*hw
= &adapter
->hw
;
3349 for (i
= 0; i
< MAX_ETYPE_FILTER
; i
++) {
3350 u32 etqf
= rd32(IGC_ETQF(i
));
3352 if ((etqf
& IGC_ETQF_ETYPE_MASK
) == etype
)
3360 * igc_del_etype_filter() - Delete ethertype filter
3361 * @adapter: Pointer to adapter where the filter should be deleted from
3362 * @etype: Ethertype value
3364 static void igc_del_etype_filter(struct igc_adapter
*adapter
, u16 etype
)
3366 struct igc_hw
*hw
= &adapter
->hw
;
3369 index
= igc_find_etype_filter(adapter
, etype
);
3373 wr32(IGC_ETQF(index
), 0);
3375 netdev_dbg(adapter
->netdev
, "Delete ethertype filter: etype %04x\n",
3379 static int igc_flex_filter_select(struct igc_adapter
*adapter
,
3380 struct igc_flex_filter
*input
,
3383 struct igc_hw
*hw
= &adapter
->hw
;
3387 if (input
->index
>= MAX_FLEX_FILTER
) {
3388 dev_err(&adapter
->pdev
->dev
, "Wrong Flex Filter index selected!\n");
3392 /* Indirect table select register */
3393 fhftsl
= rd32(IGC_FHFTSL
);
3394 fhftsl
&= ~IGC_FHFTSL_FTSL_MASK
;
3395 switch (input
->index
) {
3409 wr32(IGC_FHFTSL
, fhftsl
);
3411 /* Normalize index down to host table register */
3412 fhft_index
= input
->index
% 8;
3414 *fhft
= (fhft_index
< 4) ? IGC_FHFT(fhft_index
) :
3415 IGC_FHFT_EXT(fhft_index
- 4);
3420 static int igc_write_flex_filter_ll(struct igc_adapter
*adapter
,
3421 struct igc_flex_filter
*input
)
3423 struct device
*dev
= &adapter
->pdev
->dev
;
3424 struct igc_hw
*hw
= &adapter
->hw
;
3425 u8
*data
= input
->data
;
3426 u8
*mask
= input
->mask
;
3433 /* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3434 * out early to avoid surprises later.
3436 if (input
->length
% 8 != 0) {
3437 dev_err(dev
, "The length of a flex filter has to be 8 byte aligned!\n");
3441 /* Select corresponding flex filter register and get base for host table. */
3442 ret
= igc_flex_filter_select(adapter
, input
, &fhft
);
3446 /* When adding a filter globally disable flex filter feature. That is
3447 * recommended within the datasheet.
3449 wufc
= rd32(IGC_WUFC
);
3450 wufc
&= ~IGC_WUFC_FLEX_HQ
;
3451 wr32(IGC_WUFC
, wufc
);
3453 /* Configure filter */
3454 queuing
= input
->length
& IGC_FHFT_LENGTH_MASK
;
3455 queuing
|= (input
->rx_queue
<< IGC_FHFT_QUEUE_SHIFT
) & IGC_FHFT_QUEUE_MASK
;
3456 queuing
|= (input
->prio
<< IGC_FHFT_PRIO_SHIFT
) & IGC_FHFT_PRIO_MASK
;
3458 if (input
->immediate_irq
)
3459 queuing
|= IGC_FHFT_IMM_INT
;
3462 queuing
|= IGC_FHFT_DROP
;
3464 wr32(fhft
+ 0xFC, queuing
);
3466 /* Write data (128 byte) and mask (128 bit) */
3467 for (i
= 0; i
< 16; ++i
) {
3468 const size_t data_idx
= i
* 8;
3469 const size_t row_idx
= i
* 16;
3471 (data
[data_idx
+ 0] << 0) |
3472 (data
[data_idx
+ 1] << 8) |
3473 (data
[data_idx
+ 2] << 16) |
3474 (data
[data_idx
+ 3] << 24);
3476 (data
[data_idx
+ 4] << 0) |
3477 (data
[data_idx
+ 5] << 8) |
3478 (data
[data_idx
+ 6] << 16) |
3479 (data
[data_idx
+ 7] << 24);
3482 /* Write row: dw0, dw1 and mask */
3483 wr32(fhft
+ row_idx
, dw0
);
3484 wr32(fhft
+ row_idx
+ 4, dw1
);
3486 /* mask is only valid for MASK(7, 0) */
3487 tmp
= rd32(fhft
+ row_idx
+ 8);
3488 tmp
&= ~GENMASK(7, 0);
3490 wr32(fhft
+ row_idx
+ 8, tmp
);
3493 /* Enable filter. */
3494 wufc
|= IGC_WUFC_FLEX_HQ
;
3495 if (input
->index
> 8) {
3496 /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3497 u32 wufc_ext
= rd32(IGC_WUFC_EXT
);
3499 wufc_ext
|= (IGC_WUFC_EXT_FLX8
<< (input
->index
- 8));
3501 wr32(IGC_WUFC_EXT
, wufc_ext
);
3503 wufc
|= (IGC_WUFC_FLX0
<< input
->index
);
3505 wr32(IGC_WUFC
, wufc
);
3507 dev_dbg(&adapter
->pdev
->dev
, "Added flex filter %u to HW.\n",
3513 static void igc_flex_filter_add_field(struct igc_flex_filter
*flex
,
3514 const void *src
, unsigned int offset
,
3515 size_t len
, const void *mask
)
3520 memcpy(&flex
->data
[offset
], src
, len
);
3523 for (i
= 0; i
< len
; ++i
) {
3524 const unsigned int idx
= i
+ offset
;
3525 const u8
*ptr
= mask
;
3529 flex
->mask
[idx
/ 8] |= BIT(idx
% 8);
3534 flex
->mask
[idx
/ 8] |= BIT(idx
% 8);
3538 static int igc_find_avail_flex_filter_slot(struct igc_adapter
*adapter
)
3540 struct igc_hw
*hw
= &adapter
->hw
;
3544 wufc
= rd32(IGC_WUFC
);
3545 wufc_ext
= rd32(IGC_WUFC_EXT
);
3547 for (i
= 0; i
< MAX_FLEX_FILTER
; i
++) {
3549 if (!(wufc
& (IGC_WUFC_FLX0
<< i
)))
3552 if (!(wufc_ext
& (IGC_WUFC_EXT_FLX8
<< (i
- 8))))
3560 static bool igc_flex_filter_in_use(struct igc_adapter
*adapter
)
3562 struct igc_hw
*hw
= &adapter
->hw
;
3565 wufc
= rd32(IGC_WUFC
);
3566 wufc_ext
= rd32(IGC_WUFC_EXT
);
3568 if (wufc
& IGC_WUFC_FILTER_MASK
)
3571 if (wufc_ext
& IGC_WUFC_EXT_FILTER_MASK
)
3577 static int igc_add_flex_filter(struct igc_adapter
*adapter
,
3578 struct igc_nfc_rule
*rule
)
3580 struct igc_flex_filter flex
= { };
3581 struct igc_nfc_filter
*filter
= &rule
->filter
;
3582 unsigned int eth_offset
, user_offset
;
3586 index
= igc_find_avail_flex_filter_slot(adapter
);
3590 /* Construct the flex filter:
3597 * -> = 26 bytes => 32 length
3601 flex
.rx_queue
= rule
->action
;
3603 vlan
= rule
->filter
.vlan_tci
|| rule
->filter
.vlan_etype
;
3604 eth_offset
= vlan
? 16 : 12;
3605 user_offset
= vlan
? 18 : 14;
3607 /* Add destination MAC */
3608 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_DST_MAC_ADDR
)
3609 igc_flex_filter_add_field(&flex
, &filter
->dst_addr
, 0,
3612 /* Add source MAC */
3613 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_SRC_MAC_ADDR
)
3614 igc_flex_filter_add_field(&flex
, &filter
->src_addr
, 6,
3617 /* Add VLAN etype */
3618 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_ETYPE
)
3619 igc_flex_filter_add_field(&flex
, &filter
->vlan_etype
, 12,
3620 sizeof(filter
->vlan_etype
),
3624 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_TCI
)
3625 igc_flex_filter_add_field(&flex
, &filter
->vlan_tci
, 14,
3626 sizeof(filter
->vlan_tci
), NULL
);
3628 /* Add Ether type */
3629 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_ETHER_TYPE
) {
3630 __be16 etype
= cpu_to_be16(filter
->etype
);
3632 igc_flex_filter_add_field(&flex
, &etype
, eth_offset
,
3633 sizeof(etype
), NULL
);
3637 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_USER_DATA
)
3638 igc_flex_filter_add_field(&flex
, &filter
->user_data
,
3640 sizeof(filter
->user_data
),
3643 /* Add it down to the hardware and enable it. */
3644 ret
= igc_write_flex_filter_ll(adapter
, &flex
);
3648 filter
->flex_index
= index
;
3653 static void igc_del_flex_filter(struct igc_adapter
*adapter
,
3656 struct igc_hw
*hw
= &adapter
->hw
;
3659 /* Just disable the filter. The filter table itself is kept
3660 * intact. Another flex_filter_add() should override the "old" data
3663 if (reg_index
> 8) {
3664 u32 wufc_ext
= rd32(IGC_WUFC_EXT
);
3666 wufc_ext
&= ~(IGC_WUFC_EXT_FLX8
<< (reg_index
- 8));
3667 wr32(IGC_WUFC_EXT
, wufc_ext
);
3669 wufc
= rd32(IGC_WUFC
);
3671 wufc
&= ~(IGC_WUFC_FLX0
<< reg_index
);
3672 wr32(IGC_WUFC
, wufc
);
3675 if (igc_flex_filter_in_use(adapter
))
3678 /* No filters are in use, we may disable flex filters */
3679 wufc
= rd32(IGC_WUFC
);
3680 wufc
&= ~IGC_WUFC_FLEX_HQ
;
3681 wr32(IGC_WUFC
, wufc
);
3684 static int igc_enable_nfc_rule(struct igc_adapter
*adapter
,
3685 struct igc_nfc_rule
*rule
)
3690 return igc_add_flex_filter(adapter
, rule
);
3693 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_ETHER_TYPE
) {
3694 err
= igc_add_etype_filter(adapter
, rule
->filter
.etype
,
3700 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_SRC_MAC_ADDR
) {
3701 err
= igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_SRC
,
3702 rule
->filter
.src_addr
, rule
->action
);
3707 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_DST_MAC_ADDR
) {
3708 err
= igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
,
3709 rule
->filter
.dst_addr
, rule
->action
);
3714 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_TCI
) {
3715 int prio
= (rule
->filter
.vlan_tci
& VLAN_PRIO_MASK
) >>
3718 err
= igc_add_vlan_prio_filter(adapter
, prio
, rule
->action
);
3726 static void igc_disable_nfc_rule(struct igc_adapter
*adapter
,
3727 const struct igc_nfc_rule
*rule
)
3730 igc_del_flex_filter(adapter
, rule
->filter
.flex_index
);
3734 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_ETHER_TYPE
)
3735 igc_del_etype_filter(adapter
, rule
->filter
.etype
);
3737 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_VLAN_TCI
) {
3738 int prio
= (rule
->filter
.vlan_tci
& VLAN_PRIO_MASK
) >>
3741 igc_del_vlan_prio_filter(adapter
, prio
);
3744 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_SRC_MAC_ADDR
)
3745 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_SRC
,
3746 rule
->filter
.src_addr
);
3748 if (rule
->filter
.match_flags
& IGC_FILTER_FLAG_DST_MAC_ADDR
)
3749 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
,
3750 rule
->filter
.dst_addr
);
3754 * igc_get_nfc_rule() - Get NFC rule
3755 * @adapter: Pointer to adapter
3756 * @location: Rule location
3758 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3760 * Return: Pointer to NFC rule at @location. If not found, NULL.
3762 struct igc_nfc_rule
*igc_get_nfc_rule(struct igc_adapter
*adapter
,
3765 struct igc_nfc_rule
*rule
;
3767 list_for_each_entry(rule
, &adapter
->nfc_rule_list
, list
) {
3768 if (rule
->location
== location
)
3770 if (rule
->location
> location
)
3778 * igc_del_nfc_rule() - Delete NFC rule
3779 * @adapter: Pointer to adapter
3780 * @rule: Pointer to rule to be deleted
3782 * Disable NFC rule in hardware and delete it from adapter.
3784 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3786 void igc_del_nfc_rule(struct igc_adapter
*adapter
, struct igc_nfc_rule
*rule
)
3788 igc_disable_nfc_rule(adapter
, rule
);
3790 list_del(&rule
->list
);
3791 adapter
->nfc_rule_count
--;
3796 static void igc_flush_nfc_rules(struct igc_adapter
*adapter
)
3798 struct igc_nfc_rule
*rule
, *tmp
;
3800 mutex_lock(&adapter
->nfc_rule_lock
);
3802 list_for_each_entry_safe(rule
, tmp
, &adapter
->nfc_rule_list
, list
)
3803 igc_del_nfc_rule(adapter
, rule
);
3805 mutex_unlock(&adapter
->nfc_rule_lock
);
3809 * igc_add_nfc_rule() - Add NFC rule
3810 * @adapter: Pointer to adapter
3811 * @rule: Pointer to rule to be added
3813 * Enable NFC rule in hardware and add it to adapter.
3815 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3817 * Return: 0 on success, negative errno on failure.
3819 int igc_add_nfc_rule(struct igc_adapter
*adapter
, struct igc_nfc_rule
*rule
)
3821 struct igc_nfc_rule
*pred
, *cur
;
3824 err
= igc_enable_nfc_rule(adapter
, rule
);
3829 list_for_each_entry(cur
, &adapter
->nfc_rule_list
, list
) {
3830 if (cur
->location
>= rule
->location
)
3835 list_add(&rule
->list
, pred
? &pred
->list
: &adapter
->nfc_rule_list
);
3836 adapter
->nfc_rule_count
++;
3840 static void igc_restore_nfc_rules(struct igc_adapter
*adapter
)
3842 struct igc_nfc_rule
*rule
;
3844 mutex_lock(&adapter
->nfc_rule_lock
);
3846 list_for_each_entry_reverse(rule
, &adapter
->nfc_rule_list
, list
)
3847 igc_enable_nfc_rule(adapter
, rule
);
3849 mutex_unlock(&adapter
->nfc_rule_lock
);
3852 static int igc_uc_sync(struct net_device
*netdev
, const unsigned char *addr
)
3854 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3856 return igc_add_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
, addr
, -1);
3859 static int igc_uc_unsync(struct net_device
*netdev
, const unsigned char *addr
)
3861 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3863 igc_del_mac_filter(adapter
, IGC_MAC_FILTER_TYPE_DST
, addr
);
3868 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3869 * @netdev: network interface device structure
3871 * The set_rx_mode entry point is called whenever the unicast or multicast
3872 * address lists or the network interface flags are updated. This routine is
3873 * responsible for configuring the hardware for proper unicast, multicast,
3874 * promiscuous mode, and all-multi behavior.
3876 static void igc_set_rx_mode(struct net_device
*netdev
)
3878 struct igc_adapter
*adapter
= netdev_priv(netdev
);
3879 struct igc_hw
*hw
= &adapter
->hw
;
3880 u32 rctl
= 0, rlpml
= MAX_JUMBO_FRAME_SIZE
;
3883 /* Check for Promiscuous and All Multicast modes */
3884 if (netdev
->flags
& IFF_PROMISC
) {
3885 rctl
|= IGC_RCTL_UPE
| IGC_RCTL_MPE
;
3887 if (netdev
->flags
& IFF_ALLMULTI
) {
3888 rctl
|= IGC_RCTL_MPE
;
3890 /* Write addresses to the MTA, if the attempt fails
3891 * then we should just turn on promiscuous mode so
3892 * that we can at least receive multicast traffic
3894 count
= igc_write_mc_addr_list(netdev
);
3896 rctl
|= IGC_RCTL_MPE
;
3900 /* Write addresses to available RAR registers, if there is not
3901 * sufficient space to store all the addresses then enable
3902 * unicast promiscuous mode
3904 if (__dev_uc_sync(netdev
, igc_uc_sync
, igc_uc_unsync
))
3905 rctl
|= IGC_RCTL_UPE
;
3907 /* update state of unicast and multicast */
3908 rctl
|= rd32(IGC_RCTL
) & ~(IGC_RCTL_UPE
| IGC_RCTL_MPE
);
3909 wr32(IGC_RCTL
, rctl
);
3911 #if (PAGE_SIZE < 8192)
3912 if (adapter
->max_frame_size
<= IGC_MAX_FRAME_BUILD_SKB
)
3913 rlpml
= IGC_MAX_FRAME_BUILD_SKB
;
3915 wr32(IGC_RLPML
, rlpml
);
3919 * igc_configure - configure the hardware for RX and TX
3920 * @adapter: private board structure
3922 static void igc_configure(struct igc_adapter
*adapter
)
3924 struct net_device
*netdev
= adapter
->netdev
;
3927 igc_get_hw_control(adapter
);
3928 igc_set_rx_mode(netdev
);
3930 igc_restore_vlan(adapter
);
3932 igc_setup_tctl(adapter
);
3933 igc_setup_mrqc(adapter
);
3934 igc_setup_rctl(adapter
);
3936 igc_set_default_mac_filter(adapter
);
3937 igc_restore_nfc_rules(adapter
);
3939 igc_configure_tx(adapter
);
3940 igc_configure_rx(adapter
);
3942 igc_rx_fifo_flush_base(&adapter
->hw
);
3944 /* call igc_desc_unused which always leaves
3945 * at least 1 descriptor unused to make sure
3946 * next_to_use != next_to_clean
3948 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
3949 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
3952 igc_alloc_rx_buffers_zc(ring
, igc_desc_unused(ring
));
3954 igc_alloc_rx_buffers(ring
, igc_desc_unused(ring
));
3959 * igc_write_ivar - configure ivar for given MSI-X vector
3960 * @hw: pointer to the HW structure
3961 * @msix_vector: vector number we are allocating to a given ring
3962 * @index: row index of IVAR register to write within IVAR table
3963 * @offset: column offset of in IVAR, should be multiple of 8
3965 * The IVAR table consists of 2 columns,
3966 * each containing an cause allocation for an Rx and Tx ring, and a
3967 * variable number of rows depending on the number of queues supported.
3969 static void igc_write_ivar(struct igc_hw
*hw
, int msix_vector
,
3970 int index
, int offset
)
3972 u32 ivar
= array_rd32(IGC_IVAR0
, index
);
3974 /* clear any bits that are currently set */
3975 ivar
&= ~((u32
)0xFF << offset
);
3977 /* write vector and valid bit */
3978 ivar
|= (msix_vector
| IGC_IVAR_VALID
) << offset
;
3980 array_wr32(IGC_IVAR0
, index
, ivar
);
3983 static void igc_assign_vector(struct igc_q_vector
*q_vector
, int msix_vector
)
3985 struct igc_adapter
*adapter
= q_vector
->adapter
;
3986 struct igc_hw
*hw
= &adapter
->hw
;
3987 int rx_queue
= IGC_N0_QUEUE
;
3988 int tx_queue
= IGC_N0_QUEUE
;
3990 if (q_vector
->rx
.ring
)
3991 rx_queue
= q_vector
->rx
.ring
->reg_idx
;
3992 if (q_vector
->tx
.ring
)
3993 tx_queue
= q_vector
->tx
.ring
->reg_idx
;
3995 switch (hw
->mac
.type
) {
3997 if (rx_queue
> IGC_N0_QUEUE
)
3998 igc_write_ivar(hw
, msix_vector
,
4000 (rx_queue
& 0x1) << 4);
4001 if (tx_queue
> IGC_N0_QUEUE
)
4002 igc_write_ivar(hw
, msix_vector
,
4004 ((tx_queue
& 0x1) << 4) + 8);
4005 q_vector
->eims_value
= BIT(msix_vector
);
4008 WARN_ONCE(hw
->mac
.type
!= igc_i225
, "Wrong MAC type\n");
4012 /* add q_vector eims value to global eims_enable_mask */
4013 adapter
->eims_enable_mask
|= q_vector
->eims_value
;
4015 /* configure q_vector to set itr on first interrupt */
4016 q_vector
->set_itr
= 1;
4020 * igc_configure_msix - Configure MSI-X hardware
4021 * @adapter: Pointer to adapter structure
4023 * igc_configure_msix sets up the hardware to properly
4024 * generate MSI-X interrupts.
4026 static void igc_configure_msix(struct igc_adapter
*adapter
)
4028 struct igc_hw
*hw
= &adapter
->hw
;
4032 adapter
->eims_enable_mask
= 0;
4034 /* set vector for other causes, i.e. link changes */
4035 switch (hw
->mac
.type
) {
4037 /* Turn on MSI-X capability first, or our settings
4038 * won't stick. And it will take days to debug.
4040 wr32(IGC_GPIE
, IGC_GPIE_MSIX_MODE
|
4041 IGC_GPIE_PBA
| IGC_GPIE_EIAME
|
4044 /* enable msix_other interrupt */
4045 adapter
->eims_other
= BIT(vector
);
4046 tmp
= (vector
++ | IGC_IVAR_VALID
) << 8;
4048 wr32(IGC_IVAR_MISC
, tmp
);
4051 /* do nothing, since nothing else supports MSI-X */
4053 } /* switch (hw->mac.type) */
4055 adapter
->eims_enable_mask
|= adapter
->eims_other
;
4057 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4058 igc_assign_vector(adapter
->q_vector
[i
], vector
++);
4064 * igc_irq_enable - Enable default interrupt generation settings
4065 * @adapter: board private structure
4067 static void igc_irq_enable(struct igc_adapter
*adapter
)
4069 struct igc_hw
*hw
= &adapter
->hw
;
4071 if (adapter
->msix_entries
) {
4072 u32 ims
= IGC_IMS_LSC
| IGC_IMS_DOUTSYNC
| IGC_IMS_DRSTA
;
4073 u32 regval
= rd32(IGC_EIAC
);
4075 wr32(IGC_EIAC
, regval
| adapter
->eims_enable_mask
);
4076 regval
= rd32(IGC_EIAM
);
4077 wr32(IGC_EIAM
, regval
| adapter
->eims_enable_mask
);
4078 wr32(IGC_EIMS
, adapter
->eims_enable_mask
);
4081 wr32(IGC_IMS
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
4082 wr32(IGC_IAM
, IMS_ENABLE_MASK
| IGC_IMS_DRSTA
);
4087 * igc_irq_disable - Mask off interrupt generation on the NIC
4088 * @adapter: board private structure
4090 static void igc_irq_disable(struct igc_adapter
*adapter
)
4092 struct igc_hw
*hw
= &adapter
->hw
;
4094 if (adapter
->msix_entries
) {
4095 u32 regval
= rd32(IGC_EIAM
);
4097 wr32(IGC_EIAM
, regval
& ~adapter
->eims_enable_mask
);
4098 wr32(IGC_EIMC
, adapter
->eims_enable_mask
);
4099 regval
= rd32(IGC_EIAC
);
4100 wr32(IGC_EIAC
, regval
& ~adapter
->eims_enable_mask
);
4107 if (adapter
->msix_entries
) {
4110 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
4112 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4113 synchronize_irq(adapter
->msix_entries
[vector
++].vector
);
4115 synchronize_irq(adapter
->pdev
->irq
);
4119 void igc_set_flag_queue_pairs(struct igc_adapter
*adapter
,
4120 const u32 max_rss_queues
)
4122 /* Determine if we need to pair queues. */
4123 /* If rss_queues > half of max_rss_queues, pair the queues in
4124 * order to conserve interrupts due to limited supply.
4126 if (adapter
->rss_queues
> (max_rss_queues
/ 2))
4127 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
4129 adapter
->flags
&= ~IGC_FLAG_QUEUE_PAIRS
;
4132 unsigned int igc_get_max_rss_queues(struct igc_adapter
*adapter
)
4134 return IGC_MAX_RX_QUEUES
;
4137 static void igc_init_queue_configuration(struct igc_adapter
*adapter
)
4141 max_rss_queues
= igc_get_max_rss_queues(adapter
);
4142 adapter
->rss_queues
= min_t(u32
, max_rss_queues
, num_online_cpus());
4144 igc_set_flag_queue_pairs(adapter
, max_rss_queues
);
4148 * igc_reset_q_vector - Reset config for interrupt vector
4149 * @adapter: board private structure to initialize
4150 * @v_idx: Index of vector to be reset
4152 * If NAPI is enabled it will delete any references to the
4153 * NAPI struct. This is preparation for igc_free_q_vector.
4155 static void igc_reset_q_vector(struct igc_adapter
*adapter
, int v_idx
)
4157 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
4159 /* if we're coming from igc_set_interrupt_capability, the vectors are
4165 if (q_vector
->tx
.ring
)
4166 adapter
->tx_ring
[q_vector
->tx
.ring
->queue_index
] = NULL
;
4168 if (q_vector
->rx
.ring
)
4169 adapter
->rx_ring
[q_vector
->rx
.ring
->queue_index
] = NULL
;
4171 netif_napi_del(&q_vector
->napi
);
4175 * igc_free_q_vector - Free memory allocated for specific interrupt vector
4176 * @adapter: board private structure to initialize
4177 * @v_idx: Index of vector to be freed
4179 * This function frees the memory allocated to the q_vector.
4181 static void igc_free_q_vector(struct igc_adapter
*adapter
, int v_idx
)
4183 struct igc_q_vector
*q_vector
= adapter
->q_vector
[v_idx
];
4185 adapter
->q_vector
[v_idx
] = NULL
;
4187 /* igc_get_stats64() might access the rings on this vector,
4188 * we must wait a grace period before freeing it.
4191 kfree_rcu(q_vector
, rcu
);
4195 * igc_free_q_vectors - Free memory allocated for interrupt vectors
4196 * @adapter: board private structure to initialize
4198 * This function frees the memory allocated to the q_vectors. In addition if
4199 * NAPI is enabled it will delete any references to the NAPI struct prior
4200 * to freeing the q_vector.
4202 static void igc_free_q_vectors(struct igc_adapter
*adapter
)
4204 int v_idx
= adapter
->num_q_vectors
;
4206 adapter
->num_tx_queues
= 0;
4207 adapter
->num_rx_queues
= 0;
4208 adapter
->num_q_vectors
= 0;
4211 igc_reset_q_vector(adapter
, v_idx
);
4212 igc_free_q_vector(adapter
, v_idx
);
4217 * igc_update_itr - update the dynamic ITR value based on statistics
4218 * @q_vector: pointer to q_vector
4219 * @ring_container: ring info to update the itr for
4221 * Stores a new ITR value based on packets and byte
4222 * counts during the last interrupt. The advantage of per interrupt
4223 * computation is faster updates and more accurate ITR for the current
4224 * traffic pattern. Constants in this function were computed
4225 * based on theoretical maximum wire speed and thresholds were set based
4226 * on testing data as well as attempting to minimize response time
4227 * while increasing bulk throughput.
4228 * NOTE: These calculations are only valid when operating in a single-
4229 * queue environment.
4231 static void igc_update_itr(struct igc_q_vector
*q_vector
,
4232 struct igc_ring_container
*ring_container
)
4234 unsigned int packets
= ring_container
->total_packets
;
4235 unsigned int bytes
= ring_container
->total_bytes
;
4236 u8 itrval
= ring_container
->itr
;
4238 /* no packets, exit with status unchanged */
4243 case lowest_latency
:
4244 /* handle TSO and jumbo frames */
4245 if (bytes
/ packets
> 8000)
4246 itrval
= bulk_latency
;
4247 else if ((packets
< 5) && (bytes
> 512))
4248 itrval
= low_latency
;
4250 case low_latency
: /* 50 usec aka 20000 ints/s */
4251 if (bytes
> 10000) {
4252 /* this if handles the TSO accounting */
4253 if (bytes
/ packets
> 8000)
4254 itrval
= bulk_latency
;
4255 else if ((packets
< 10) || ((bytes
/ packets
) > 1200))
4256 itrval
= bulk_latency
;
4257 else if ((packets
> 35))
4258 itrval
= lowest_latency
;
4259 } else if (bytes
/ packets
> 2000) {
4260 itrval
= bulk_latency
;
4261 } else if (packets
<= 2 && bytes
< 512) {
4262 itrval
= lowest_latency
;
4265 case bulk_latency
: /* 250 usec aka 4000 ints/s */
4266 if (bytes
> 25000) {
4268 itrval
= low_latency
;
4269 } else if (bytes
< 1500) {
4270 itrval
= low_latency
;
4275 /* clear work counters since we have the values we need */
4276 ring_container
->total_bytes
= 0;
4277 ring_container
->total_packets
= 0;
4279 /* write updated itr to ring container */
4280 ring_container
->itr
= itrval
;
4283 static void igc_set_itr(struct igc_q_vector
*q_vector
)
4285 struct igc_adapter
*adapter
= q_vector
->adapter
;
4286 u32 new_itr
= q_vector
->itr_val
;
4289 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4290 switch (adapter
->link_speed
) {
4294 new_itr
= IGC_4K_ITR
;
4300 igc_update_itr(q_vector
, &q_vector
->tx
);
4301 igc_update_itr(q_vector
, &q_vector
->rx
);
4303 current_itr
= max(q_vector
->rx
.itr
, q_vector
->tx
.itr
);
4305 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4306 if (current_itr
== lowest_latency
&&
4307 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
4308 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
4309 current_itr
= low_latency
;
4311 switch (current_itr
) {
4312 /* counts and packets in update_itr are dependent on these numbers */
4313 case lowest_latency
:
4314 new_itr
= IGC_70K_ITR
; /* 70,000 ints/sec */
4317 new_itr
= IGC_20K_ITR
; /* 20,000 ints/sec */
4320 new_itr
= IGC_4K_ITR
; /* 4,000 ints/sec */
4327 if (new_itr
!= q_vector
->itr_val
) {
4328 /* this attempts to bias the interrupt rate towards Bulk
4329 * by adding intermediate steps when interrupt rate is
4332 new_itr
= new_itr
> q_vector
->itr_val
?
4333 max((new_itr
* q_vector
->itr_val
) /
4334 (new_itr
+ (q_vector
->itr_val
>> 2)),
4336 /* Don't write the value here; it resets the adapter's
4337 * internal timer, and causes us to delay far longer than
4338 * we should between interrupts. Instead, we write the ITR
4339 * value at the beginning of the next interrupt so the timing
4340 * ends up being correct.
4342 q_vector
->itr_val
= new_itr
;
4343 q_vector
->set_itr
= 1;
4347 static void igc_reset_interrupt_capability(struct igc_adapter
*adapter
)
4349 int v_idx
= adapter
->num_q_vectors
;
4351 if (adapter
->msix_entries
) {
4352 pci_disable_msix(adapter
->pdev
);
4353 kfree(adapter
->msix_entries
);
4354 adapter
->msix_entries
= NULL
;
4355 } else if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
4356 pci_disable_msi(adapter
->pdev
);
4360 igc_reset_q_vector(adapter
, v_idx
);
4364 * igc_set_interrupt_capability - set MSI or MSI-X if supported
4365 * @adapter: Pointer to adapter structure
4366 * @msix: boolean value for MSI-X capability
4368 * Attempt to configure interrupts using the best available
4369 * capabilities of the hardware and kernel.
4371 static void igc_set_interrupt_capability(struct igc_adapter
*adapter
,
4379 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
4381 /* Number of supported queues. */
4382 adapter
->num_rx_queues
= adapter
->rss_queues
;
4384 adapter
->num_tx_queues
= adapter
->rss_queues
;
4386 /* start with one vector for every Rx queue */
4387 numvecs
= adapter
->num_rx_queues
;
4389 /* if Tx handler is separate add 1 for every Tx queue */
4390 if (!(adapter
->flags
& IGC_FLAG_QUEUE_PAIRS
))
4391 numvecs
+= adapter
->num_tx_queues
;
4393 /* store the number of vectors reserved for queues */
4394 adapter
->num_q_vectors
= numvecs
;
4396 /* add 1 vector for link status interrupts */
4399 adapter
->msix_entries
= kcalloc(numvecs
, sizeof(struct msix_entry
),
4402 if (!adapter
->msix_entries
)
4405 /* populate entry values */
4406 for (i
= 0; i
< numvecs
; i
++)
4407 adapter
->msix_entries
[i
].entry
= i
;
4409 err
= pci_enable_msix_range(adapter
->pdev
,
4410 adapter
->msix_entries
,
4416 kfree(adapter
->msix_entries
);
4417 adapter
->msix_entries
= NULL
;
4419 igc_reset_interrupt_capability(adapter
);
4422 adapter
->flags
&= ~IGC_FLAG_HAS_MSIX
;
4424 adapter
->rss_queues
= 1;
4425 adapter
->flags
|= IGC_FLAG_QUEUE_PAIRS
;
4426 adapter
->num_rx_queues
= 1;
4427 adapter
->num_tx_queues
= 1;
4428 adapter
->num_q_vectors
= 1;
4429 if (!pci_enable_msi(adapter
->pdev
))
4430 adapter
->flags
|= IGC_FLAG_HAS_MSI
;
4434 * igc_update_ring_itr - update the dynamic ITR value based on packet size
4435 * @q_vector: pointer to q_vector
4437 * Stores a new ITR value based on strictly on packet size. This
4438 * algorithm is less sophisticated than that used in igc_update_itr,
4439 * due to the difficulty of synchronizing statistics across multiple
4440 * receive rings. The divisors and thresholds used by this function
4441 * were determined based on theoretical maximum wire speed and testing
4442 * data, in order to minimize response time while increasing bulk
4444 * NOTE: This function is called only when operating in a multiqueue
4445 * receive environment.
4447 static void igc_update_ring_itr(struct igc_q_vector
*q_vector
)
4449 struct igc_adapter
*adapter
= q_vector
->adapter
;
4450 int new_val
= q_vector
->itr_val
;
4451 int avg_wire_size
= 0;
4452 unsigned int packets
;
4454 /* For non-gigabit speeds, just fix the interrupt rate at 4000
4455 * ints/sec - ITR timer value of 120 ticks.
4457 switch (adapter
->link_speed
) {
4460 new_val
= IGC_4K_ITR
;
4466 packets
= q_vector
->rx
.total_packets
;
4468 avg_wire_size
= q_vector
->rx
.total_bytes
/ packets
;
4470 packets
= q_vector
->tx
.total_packets
;
4472 avg_wire_size
= max_t(u32
, avg_wire_size
,
4473 q_vector
->tx
.total_bytes
/ packets
);
4475 /* if avg_wire_size isn't set no work was done */
4479 /* Add 24 bytes to size to account for CRC, preamble, and gap */
4480 avg_wire_size
+= 24;
4482 /* Don't starve jumbo frames */
4483 avg_wire_size
= min(avg_wire_size
, 3000);
4485 /* Give a little boost to mid-size frames */
4486 if (avg_wire_size
> 300 && avg_wire_size
< 1200)
4487 new_val
= avg_wire_size
/ 3;
4489 new_val
= avg_wire_size
/ 2;
4491 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4492 if (new_val
< IGC_20K_ITR
&&
4493 ((q_vector
->rx
.ring
&& adapter
->rx_itr_setting
== 3) ||
4494 (!q_vector
->rx
.ring
&& adapter
->tx_itr_setting
== 3)))
4495 new_val
= IGC_20K_ITR
;
4498 if (new_val
!= q_vector
->itr_val
) {
4499 q_vector
->itr_val
= new_val
;
4500 q_vector
->set_itr
= 1;
4503 q_vector
->rx
.total_bytes
= 0;
4504 q_vector
->rx
.total_packets
= 0;
4505 q_vector
->tx
.total_bytes
= 0;
4506 q_vector
->tx
.total_packets
= 0;
4509 static void igc_ring_irq_enable(struct igc_q_vector
*q_vector
)
4511 struct igc_adapter
*adapter
= q_vector
->adapter
;
4512 struct igc_hw
*hw
= &adapter
->hw
;
4514 if ((q_vector
->rx
.ring
&& (adapter
->rx_itr_setting
& 3)) ||
4515 (!q_vector
->rx
.ring
&& (adapter
->tx_itr_setting
& 3))) {
4516 if (adapter
->num_q_vectors
== 1)
4517 igc_set_itr(q_vector
);
4519 igc_update_ring_itr(q_vector
);
4522 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
4523 if (adapter
->msix_entries
)
4524 wr32(IGC_EIMS
, q_vector
->eims_value
);
4526 igc_irq_enable(adapter
);
4530 static void igc_add_ring(struct igc_ring
*ring
,
4531 struct igc_ring_container
*head
)
4538 * igc_cache_ring_register - Descriptor ring to register mapping
4539 * @adapter: board private structure to initialize
4541 * Once we know the feature-set enabled for the device, we'll cache
4542 * the register offset the descriptor ring is assigned to.
4544 static void igc_cache_ring_register(struct igc_adapter
*adapter
)
4548 switch (adapter
->hw
.mac
.type
) {
4551 for (; i
< adapter
->num_rx_queues
; i
++)
4552 adapter
->rx_ring
[i
]->reg_idx
= i
;
4553 for (; j
< adapter
->num_tx_queues
; j
++)
4554 adapter
->tx_ring
[j
]->reg_idx
= j
;
4560 * igc_poll - NAPI Rx polling callback
4561 * @napi: napi polling structure
4562 * @budget: count of how many packets we should handle
4564 static int igc_poll(struct napi_struct
*napi
, int budget
)
4566 struct igc_q_vector
*q_vector
= container_of(napi
,
4567 struct igc_q_vector
,
4569 struct igc_ring
*rx_ring
= q_vector
->rx
.ring
;
4570 bool clean_complete
= true;
4573 if (q_vector
->tx
.ring
)
4574 clean_complete
= igc_clean_tx_irq(q_vector
, budget
);
4577 int cleaned
= rx_ring
->xsk_pool
?
4578 igc_clean_rx_irq_zc(q_vector
, budget
) :
4579 igc_clean_rx_irq(q_vector
, budget
);
4581 work_done
+= cleaned
;
4582 if (cleaned
>= budget
)
4583 clean_complete
= false;
4586 /* If all work not completed, return budget and keep polling */
4587 if (!clean_complete
)
4590 /* Exit the polling mode, but don't re-enable interrupts if stack might
4591 * poll us due to busy-polling
4593 if (likely(napi_complete_done(napi
, work_done
)))
4594 igc_ring_irq_enable(q_vector
);
4596 return min(work_done
, budget
- 1);
4600 * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4601 * @adapter: board private structure to initialize
4602 * @v_count: q_vectors allocated on adapter, used for ring interleaving
4603 * @v_idx: index of vector in adapter struct
4604 * @txr_count: total number of Tx rings to allocate
4605 * @txr_idx: index of first Tx ring to allocate
4606 * @rxr_count: total number of Rx rings to allocate
4607 * @rxr_idx: index of first Rx ring to allocate
4609 * We allocate one q_vector. If allocation fails we return -ENOMEM.
4611 static int igc_alloc_q_vector(struct igc_adapter
*adapter
,
4612 unsigned int v_count
, unsigned int v_idx
,
4613 unsigned int txr_count
, unsigned int txr_idx
,
4614 unsigned int rxr_count
, unsigned int rxr_idx
)
4616 struct igc_q_vector
*q_vector
;
4617 struct igc_ring
*ring
;
4620 /* igc only supports 1 Tx and/or 1 Rx queue per vector */
4621 if (txr_count
> 1 || rxr_count
> 1)
4624 ring_count
= txr_count
+ rxr_count
;
4626 /* allocate q_vector and rings */
4627 q_vector
= adapter
->q_vector
[v_idx
];
4629 q_vector
= kzalloc(struct_size(q_vector
, ring
, ring_count
),
4632 memset(q_vector
, 0, struct_size(q_vector
, ring
, ring_count
));
4636 /* initialize NAPI */
4637 netif_napi_add(adapter
->netdev
, &q_vector
->napi
, igc_poll
);
4639 /* tie q_vector and adapter together */
4640 adapter
->q_vector
[v_idx
] = q_vector
;
4641 q_vector
->adapter
= adapter
;
4643 /* initialize work limits */
4644 q_vector
->tx
.work_limit
= adapter
->tx_work_limit
;
4646 /* initialize ITR configuration */
4647 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(0);
4648 q_vector
->itr_val
= IGC_START_ITR
;
4650 /* initialize pointer to rings */
4651 ring
= q_vector
->ring
;
4653 /* initialize ITR */
4655 /* rx or rx/tx vector */
4656 if (!adapter
->rx_itr_setting
|| adapter
->rx_itr_setting
> 3)
4657 q_vector
->itr_val
= adapter
->rx_itr_setting
;
4659 /* tx only vector */
4660 if (!adapter
->tx_itr_setting
|| adapter
->tx_itr_setting
> 3)
4661 q_vector
->itr_val
= adapter
->tx_itr_setting
;
4665 /* assign generic ring traits */
4666 ring
->dev
= &adapter
->pdev
->dev
;
4667 ring
->netdev
= adapter
->netdev
;
4669 /* configure backlink on ring */
4670 ring
->q_vector
= q_vector
;
4672 /* update q_vector Tx values */
4673 igc_add_ring(ring
, &q_vector
->tx
);
4675 /* apply Tx specific ring traits */
4676 ring
->count
= adapter
->tx_ring_count
;
4677 ring
->queue_index
= txr_idx
;
4679 /* assign ring to adapter */
4680 adapter
->tx_ring
[txr_idx
] = ring
;
4682 /* push pointer to next ring */
4687 /* assign generic ring traits */
4688 ring
->dev
= &adapter
->pdev
->dev
;
4689 ring
->netdev
= adapter
->netdev
;
4691 /* configure backlink on ring */
4692 ring
->q_vector
= q_vector
;
4694 /* update q_vector Rx values */
4695 igc_add_ring(ring
, &q_vector
->rx
);
4697 /* apply Rx specific ring traits */
4698 ring
->count
= adapter
->rx_ring_count
;
4699 ring
->queue_index
= rxr_idx
;
4701 /* assign ring to adapter */
4702 adapter
->rx_ring
[rxr_idx
] = ring
;
4709 * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4710 * @adapter: board private structure to initialize
4712 * We allocate one q_vector per queue interrupt. If allocation fails we
4715 static int igc_alloc_q_vectors(struct igc_adapter
*adapter
)
4717 int rxr_remaining
= adapter
->num_rx_queues
;
4718 int txr_remaining
= adapter
->num_tx_queues
;
4719 int rxr_idx
= 0, txr_idx
= 0, v_idx
= 0;
4720 int q_vectors
= adapter
->num_q_vectors
;
4723 if (q_vectors
>= (rxr_remaining
+ txr_remaining
)) {
4724 for (; rxr_remaining
; v_idx
++) {
4725 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
4731 /* update counts and index */
4737 for (; v_idx
< q_vectors
; v_idx
++) {
4738 int rqpv
= DIV_ROUND_UP(rxr_remaining
, q_vectors
- v_idx
);
4739 int tqpv
= DIV_ROUND_UP(txr_remaining
, q_vectors
- v_idx
);
4741 err
= igc_alloc_q_vector(adapter
, q_vectors
, v_idx
,
4742 tqpv
, txr_idx
, rqpv
, rxr_idx
);
4747 /* update counts and index */
4748 rxr_remaining
-= rqpv
;
4749 txr_remaining
-= tqpv
;
4757 adapter
->num_tx_queues
= 0;
4758 adapter
->num_rx_queues
= 0;
4759 adapter
->num_q_vectors
= 0;
4762 igc_free_q_vector(adapter
, v_idx
);
4768 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4769 * @adapter: Pointer to adapter structure
4770 * @msix: boolean for MSI-X capability
4772 * This function initializes the interrupts and allocates all of the queues.
4774 static int igc_init_interrupt_scheme(struct igc_adapter
*adapter
, bool msix
)
4776 struct net_device
*dev
= adapter
->netdev
;
4779 igc_set_interrupt_capability(adapter
, msix
);
4781 err
= igc_alloc_q_vectors(adapter
);
4783 netdev_err(dev
, "Unable to allocate memory for vectors\n");
4784 goto err_alloc_q_vectors
;
4787 igc_cache_ring_register(adapter
);
4791 err_alloc_q_vectors
:
4792 igc_reset_interrupt_capability(adapter
);
4797 * igc_sw_init - Initialize general software structures (struct igc_adapter)
4798 * @adapter: board private structure to initialize
4800 * igc_sw_init initializes the Adapter private data structure.
4801 * Fields are initialized based on PCI device information and
4802 * OS network device settings (MTU size).
4804 static int igc_sw_init(struct igc_adapter
*adapter
)
4806 struct net_device
*netdev
= adapter
->netdev
;
4807 struct pci_dev
*pdev
= adapter
->pdev
;
4808 struct igc_hw
*hw
= &adapter
->hw
;
4810 pci_read_config_word(pdev
, PCI_COMMAND
, &hw
->bus
.pci_cmd_word
);
4812 /* set default ring sizes */
4813 adapter
->tx_ring_count
= IGC_DEFAULT_TXD
;
4814 adapter
->rx_ring_count
= IGC_DEFAULT_RXD
;
4816 /* set default ITR values */
4817 adapter
->rx_itr_setting
= IGC_DEFAULT_ITR
;
4818 adapter
->tx_itr_setting
= IGC_DEFAULT_ITR
;
4820 /* set default work limits */
4821 adapter
->tx_work_limit
= IGC_DEFAULT_TX_WORK
;
4823 /* adjust max frame to be at least the size of a standard frame */
4824 adapter
->max_frame_size
= netdev
->mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+
4826 adapter
->min_frame_size
= ETH_ZLEN
+ ETH_FCS_LEN
;
4828 mutex_init(&adapter
->nfc_rule_lock
);
4829 INIT_LIST_HEAD(&adapter
->nfc_rule_list
);
4830 adapter
->nfc_rule_count
= 0;
4832 spin_lock_init(&adapter
->stats64_lock
);
4833 spin_lock_init(&adapter
->qbv_tx_lock
);
4834 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4835 adapter
->flags
|= IGC_FLAG_HAS_MSIX
;
4837 igc_init_queue_configuration(adapter
);
4839 /* This call may decrease the number of queues */
4840 if (igc_init_interrupt_scheme(adapter
, true)) {
4841 netdev_err(netdev
, "Unable to allocate memory for queues\n");
4845 /* Explicitly disable IRQ since the NIC can be in any state. */
4846 igc_irq_disable(adapter
);
4848 set_bit(__IGC_DOWN
, &adapter
->state
);
4854 * igc_up - Open the interface and prepare it to handle traffic
4855 * @adapter: board private structure
4857 void igc_up(struct igc_adapter
*adapter
)
4859 struct igc_hw
*hw
= &adapter
->hw
;
4862 /* hardware has been reset, we need to reload some things */
4863 igc_configure(adapter
);
4865 clear_bit(__IGC_DOWN
, &adapter
->state
);
4867 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
4868 napi_enable(&adapter
->q_vector
[i
]->napi
);
4870 if (adapter
->msix_entries
)
4871 igc_configure_msix(adapter
);
4873 igc_assign_vector(adapter
->q_vector
[0], 0);
4875 /* Clear any pending interrupts. */
4877 igc_irq_enable(adapter
);
4879 netif_tx_start_all_queues(adapter
->netdev
);
4881 /* start the watchdog. */
4882 hw
->mac
.get_link_status
= true;
4883 schedule_work(&adapter
->watchdog_task
);
4887 * igc_update_stats - Update the board statistics counters
4888 * @adapter: board private structure
4890 void igc_update_stats(struct igc_adapter
*adapter
)
4892 struct rtnl_link_stats64
*net_stats
= &adapter
->stats64
;
4893 struct pci_dev
*pdev
= adapter
->pdev
;
4894 struct igc_hw
*hw
= &adapter
->hw
;
4895 u64 _bytes
, _packets
;
4901 /* Prevent stats update while adapter is being reset, or if the pci
4902 * connection is down.
4904 if (adapter
->link_speed
== 0)
4906 if (pci_channel_offline(pdev
))
4913 for (i
= 0; i
< adapter
->num_rx_queues
; i
++) {
4914 struct igc_ring
*ring
= adapter
->rx_ring
[i
];
4915 u32 rqdpc
= rd32(IGC_RQDPC(i
));
4917 if (hw
->mac
.type
>= igc_i225
)
4918 wr32(IGC_RQDPC(i
), 0);
4921 ring
->rx_stats
.drops
+= rqdpc
;
4922 net_stats
->rx_fifo_errors
+= rqdpc
;
4926 start
= u64_stats_fetch_begin(&ring
->rx_syncp
);
4927 _bytes
= ring
->rx_stats
.bytes
;
4928 _packets
= ring
->rx_stats
.packets
;
4929 } while (u64_stats_fetch_retry(&ring
->rx_syncp
, start
));
4931 packets
+= _packets
;
4934 net_stats
->rx_bytes
= bytes
;
4935 net_stats
->rx_packets
= packets
;
4939 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
4940 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
4943 start
= u64_stats_fetch_begin(&ring
->tx_syncp
);
4944 _bytes
= ring
->tx_stats
.bytes
;
4945 _packets
= ring
->tx_stats
.packets
;
4946 } while (u64_stats_fetch_retry(&ring
->tx_syncp
, start
));
4948 packets
+= _packets
;
4950 net_stats
->tx_bytes
= bytes
;
4951 net_stats
->tx_packets
= packets
;
4954 /* read stats registers */
4955 adapter
->stats
.crcerrs
+= rd32(IGC_CRCERRS
);
4956 adapter
->stats
.gprc
+= rd32(IGC_GPRC
);
4957 adapter
->stats
.gorc
+= rd32(IGC_GORCL
);
4958 rd32(IGC_GORCH
); /* clear GORCL */
4959 adapter
->stats
.bprc
+= rd32(IGC_BPRC
);
4960 adapter
->stats
.mprc
+= rd32(IGC_MPRC
);
4961 adapter
->stats
.roc
+= rd32(IGC_ROC
);
4963 adapter
->stats
.prc64
+= rd32(IGC_PRC64
);
4964 adapter
->stats
.prc127
+= rd32(IGC_PRC127
);
4965 adapter
->stats
.prc255
+= rd32(IGC_PRC255
);
4966 adapter
->stats
.prc511
+= rd32(IGC_PRC511
);
4967 adapter
->stats
.prc1023
+= rd32(IGC_PRC1023
);
4968 adapter
->stats
.prc1522
+= rd32(IGC_PRC1522
);
4969 adapter
->stats
.tlpic
+= rd32(IGC_TLPIC
);
4970 adapter
->stats
.rlpic
+= rd32(IGC_RLPIC
);
4971 adapter
->stats
.hgptc
+= rd32(IGC_HGPTC
);
4973 mpc
= rd32(IGC_MPC
);
4974 adapter
->stats
.mpc
+= mpc
;
4975 net_stats
->rx_fifo_errors
+= mpc
;
4976 adapter
->stats
.scc
+= rd32(IGC_SCC
);
4977 adapter
->stats
.ecol
+= rd32(IGC_ECOL
);
4978 adapter
->stats
.mcc
+= rd32(IGC_MCC
);
4979 adapter
->stats
.latecol
+= rd32(IGC_LATECOL
);
4980 adapter
->stats
.dc
+= rd32(IGC_DC
);
4981 adapter
->stats
.rlec
+= rd32(IGC_RLEC
);
4982 adapter
->stats
.xonrxc
+= rd32(IGC_XONRXC
);
4983 adapter
->stats
.xontxc
+= rd32(IGC_XONTXC
);
4984 adapter
->stats
.xoffrxc
+= rd32(IGC_XOFFRXC
);
4985 adapter
->stats
.xofftxc
+= rd32(IGC_XOFFTXC
);
4986 adapter
->stats
.fcruc
+= rd32(IGC_FCRUC
);
4987 adapter
->stats
.gptc
+= rd32(IGC_GPTC
);
4988 adapter
->stats
.gotc
+= rd32(IGC_GOTCL
);
4989 rd32(IGC_GOTCH
); /* clear GOTCL */
4990 adapter
->stats
.rnbc
+= rd32(IGC_RNBC
);
4991 adapter
->stats
.ruc
+= rd32(IGC_RUC
);
4992 adapter
->stats
.rfc
+= rd32(IGC_RFC
);
4993 adapter
->stats
.rjc
+= rd32(IGC_RJC
);
4994 adapter
->stats
.tor
+= rd32(IGC_TORH
);
4995 adapter
->stats
.tot
+= rd32(IGC_TOTH
);
4996 adapter
->stats
.tpr
+= rd32(IGC_TPR
);
4998 adapter
->stats
.ptc64
+= rd32(IGC_PTC64
);
4999 adapter
->stats
.ptc127
+= rd32(IGC_PTC127
);
5000 adapter
->stats
.ptc255
+= rd32(IGC_PTC255
);
5001 adapter
->stats
.ptc511
+= rd32(IGC_PTC511
);
5002 adapter
->stats
.ptc1023
+= rd32(IGC_PTC1023
);
5003 adapter
->stats
.ptc1522
+= rd32(IGC_PTC1522
);
5005 adapter
->stats
.mptc
+= rd32(IGC_MPTC
);
5006 adapter
->stats
.bptc
+= rd32(IGC_BPTC
);
5008 adapter
->stats
.tpt
+= rd32(IGC_TPT
);
5009 adapter
->stats
.colc
+= rd32(IGC_COLC
);
5010 adapter
->stats
.colc
+= rd32(IGC_RERC
);
5012 adapter
->stats
.algnerrc
+= rd32(IGC_ALGNERRC
);
5014 adapter
->stats
.tsctc
+= rd32(IGC_TSCTC
);
5016 adapter
->stats
.iac
+= rd32(IGC_IAC
);
5018 /* Fill out the OS statistics structure */
5019 net_stats
->multicast
= adapter
->stats
.mprc
;
5020 net_stats
->collisions
= adapter
->stats
.colc
;
5024 /* RLEC on some newer hardware can be incorrect so build
5025 * our own version based on RUC and ROC
5027 net_stats
->rx_errors
= adapter
->stats
.rxerrc
+
5028 adapter
->stats
.crcerrs
+ adapter
->stats
.algnerrc
+
5029 adapter
->stats
.ruc
+ adapter
->stats
.roc
+
5030 adapter
->stats
.cexterr
;
5031 net_stats
->rx_length_errors
= adapter
->stats
.ruc
+
5033 net_stats
->rx_crc_errors
= adapter
->stats
.crcerrs
;
5034 net_stats
->rx_frame_errors
= adapter
->stats
.algnerrc
;
5035 net_stats
->rx_missed_errors
= adapter
->stats
.mpc
;
5038 net_stats
->tx_errors
= adapter
->stats
.ecol
+
5039 adapter
->stats
.latecol
;
5040 net_stats
->tx_aborted_errors
= adapter
->stats
.ecol
;
5041 net_stats
->tx_window_errors
= adapter
->stats
.latecol
;
5042 net_stats
->tx_carrier_errors
= adapter
->stats
.tncrs
;
5045 net_stats
->tx_dropped
= adapter
->stats
.txdrop
;
5047 /* Management Stats */
5048 adapter
->stats
.mgptc
+= rd32(IGC_MGTPTC
);
5049 adapter
->stats
.mgprc
+= rd32(IGC_MGTPRC
);
5050 adapter
->stats
.mgpdc
+= rd32(IGC_MGTPDC
);
5054 * igc_down - Close the interface
5055 * @adapter: board private structure
5057 void igc_down(struct igc_adapter
*adapter
)
5059 struct net_device
*netdev
= adapter
->netdev
;
5060 struct igc_hw
*hw
= &adapter
->hw
;
5064 set_bit(__IGC_DOWN
, &adapter
->state
);
5066 igc_ptp_suspend(adapter
);
5068 if (pci_device_is_present(adapter
->pdev
)) {
5069 /* disable receives in the hardware */
5070 rctl
= rd32(IGC_RCTL
);
5071 wr32(IGC_RCTL
, rctl
& ~IGC_RCTL_EN
);
5072 /* flush and sleep below */
5074 /* set trans_start so we don't get spurious watchdogs during reset */
5075 netif_trans_update(netdev
);
5077 netif_carrier_off(netdev
);
5078 netif_tx_stop_all_queues(netdev
);
5080 if (pci_device_is_present(adapter
->pdev
)) {
5081 /* disable transmits in the hardware */
5082 tctl
= rd32(IGC_TCTL
);
5083 tctl
&= ~IGC_TCTL_EN
;
5084 wr32(IGC_TCTL
, tctl
);
5085 /* flush both disables and wait for them to finish */
5087 usleep_range(10000, 20000);
5089 igc_irq_disable(adapter
);
5092 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
5094 for (i
= 0; i
< adapter
->num_q_vectors
; i
++) {
5095 if (adapter
->q_vector
[i
]) {
5096 napi_synchronize(&adapter
->q_vector
[i
]->napi
);
5097 napi_disable(&adapter
->q_vector
[i
]->napi
);
5101 del_timer_sync(&adapter
->watchdog_timer
);
5102 del_timer_sync(&adapter
->phy_info_timer
);
5104 /* record the stats before reset*/
5105 spin_lock(&adapter
->stats64_lock
);
5106 igc_update_stats(adapter
);
5107 spin_unlock(&adapter
->stats64_lock
);
5109 adapter
->link_speed
= 0;
5110 adapter
->link_duplex
= 0;
5112 if (!pci_channel_offline(adapter
->pdev
))
5115 /* clear VLAN promisc flag so VFTA will be updated if necessary */
5116 adapter
->flags
&= ~IGC_FLAG_VLAN_PROMISC
;
5118 igc_disable_all_tx_rings_hw(adapter
);
5119 igc_clean_all_tx_rings(adapter
);
5120 igc_clean_all_rx_rings(adapter
);
5123 void igc_reinit_locked(struct igc_adapter
*adapter
)
5125 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
5126 usleep_range(1000, 2000);
5129 clear_bit(__IGC_RESETTING
, &adapter
->state
);
5132 static void igc_reset_task(struct work_struct
*work
)
5134 struct igc_adapter
*adapter
;
5136 adapter
= container_of(work
, struct igc_adapter
, reset_task
);
5139 /* If we're already down or resetting, just bail */
5140 if (test_bit(__IGC_DOWN
, &adapter
->state
) ||
5141 test_bit(__IGC_RESETTING
, &adapter
->state
)) {
5146 igc_rings_dump(adapter
);
5147 igc_regs_dump(adapter
);
5148 netdev_err(adapter
->netdev
, "Reset adapter\n");
5149 igc_reinit_locked(adapter
);
5154 * igc_change_mtu - Change the Maximum Transfer Unit
5155 * @netdev: network interface device structure
5156 * @new_mtu: new value for maximum frame size
5158 * Returns 0 on success, negative on failure
5160 static int igc_change_mtu(struct net_device
*netdev
, int new_mtu
)
5162 int max_frame
= new_mtu
+ ETH_HLEN
+ ETH_FCS_LEN
+ VLAN_HLEN
;
5163 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5165 if (igc_xdp_is_enabled(adapter
) && new_mtu
> ETH_DATA_LEN
) {
5166 netdev_dbg(netdev
, "Jumbo frames not supported with XDP");
5170 /* adjust max frame to be at least the size of a standard frame */
5171 if (max_frame
< (ETH_FRAME_LEN
+ ETH_FCS_LEN
))
5172 max_frame
= ETH_FRAME_LEN
+ ETH_FCS_LEN
;
5174 while (test_and_set_bit(__IGC_RESETTING
, &adapter
->state
))
5175 usleep_range(1000, 2000);
5177 /* igc_down has a dependency on max_frame_size */
5178 adapter
->max_frame_size
= max_frame
;
5180 if (netif_running(netdev
))
5183 netdev_dbg(netdev
, "changing MTU from %d to %d\n", netdev
->mtu
, new_mtu
);
5184 netdev
->mtu
= new_mtu
;
5186 if (netif_running(netdev
))
5191 clear_bit(__IGC_RESETTING
, &adapter
->state
);
5197 * igc_tx_timeout - Respond to a Tx Hang
5198 * @netdev: network interface device structure
5199 * @txqueue: queue number that timed out
5201 static void igc_tx_timeout(struct net_device
*netdev
,
5202 unsigned int __always_unused txqueue
)
5204 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5205 struct igc_hw
*hw
= &adapter
->hw
;
5207 /* Do the reset outside of interrupt context */
5208 adapter
->tx_timeout_count
++;
5209 schedule_work(&adapter
->reset_task
);
5211 (adapter
->eims_enable_mask
& ~adapter
->eims_other
));
5215 * igc_get_stats64 - Get System Network Statistics
5216 * @netdev: network interface device structure
5217 * @stats: rtnl_link_stats64 pointer
5219 * Returns the address of the device statistics structure.
5220 * The statistics are updated here and also from the timer callback.
5222 static void igc_get_stats64(struct net_device
*netdev
,
5223 struct rtnl_link_stats64
*stats
)
5225 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5227 spin_lock(&adapter
->stats64_lock
);
5228 if (!test_bit(__IGC_RESETTING
, &adapter
->state
))
5229 igc_update_stats(adapter
);
5230 memcpy(stats
, &adapter
->stats64
, sizeof(*stats
));
5231 spin_unlock(&adapter
->stats64_lock
);
5234 static netdev_features_t
igc_fix_features(struct net_device
*netdev
,
5235 netdev_features_t features
)
5237 /* Since there is no support for separate Rx/Tx vlan accel
5238 * enable/disable make sure Tx flag is always in same state as Rx.
5240 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
5241 features
|= NETIF_F_HW_VLAN_CTAG_TX
;
5243 features
&= ~NETIF_F_HW_VLAN_CTAG_TX
;
5248 static int igc_set_features(struct net_device
*netdev
,
5249 netdev_features_t features
)
5251 netdev_features_t changed
= netdev
->features
^ features
;
5252 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5254 if (changed
& NETIF_F_HW_VLAN_CTAG_RX
)
5255 igc_vlan_mode(netdev
, features
);
5257 /* Add VLAN support */
5258 if (!(changed
& (NETIF_F_RXALL
| NETIF_F_NTUPLE
)))
5261 if (!(features
& NETIF_F_NTUPLE
))
5262 igc_flush_nfc_rules(adapter
);
5264 netdev
->features
= features
;
5266 if (netif_running(netdev
))
5267 igc_reinit_locked(adapter
);
5274 static netdev_features_t
5275 igc_features_check(struct sk_buff
*skb
, struct net_device
*dev
,
5276 netdev_features_t features
)
5278 unsigned int network_hdr_len
, mac_hdr_len
;
5280 /* Make certain the headers can be described by a context descriptor */
5281 mac_hdr_len
= skb_network_header(skb
) - skb
->data
;
5282 if (unlikely(mac_hdr_len
> IGC_MAX_MAC_HDR_LEN
))
5283 return features
& ~(NETIF_F_HW_CSUM
|
5285 NETIF_F_HW_VLAN_CTAG_TX
|
5289 network_hdr_len
= skb_checksum_start(skb
) - skb_network_header(skb
);
5290 if (unlikely(network_hdr_len
> IGC_MAX_NETWORK_HDR_LEN
))
5291 return features
& ~(NETIF_F_HW_CSUM
|
5296 /* We can only support IPv4 TSO in tunnels if we can mangle the
5297 * inner IP ID field, so strip TSO if MANGLEID is not supported.
5299 if (skb
->encapsulation
&& !(features
& NETIF_F_TSO_MANGLEID
))
5300 features
&= ~NETIF_F_TSO
;
5305 static void igc_tsync_interrupt(struct igc_adapter
*adapter
)
5307 u32 ack
, tsauxc
, sec
, nsec
, tsicr
;
5308 struct igc_hw
*hw
= &adapter
->hw
;
5309 struct ptp_clock_event event
;
5310 struct timespec64 ts
;
5312 tsicr
= rd32(IGC_TSICR
);
5315 if (tsicr
& IGC_TSICR_SYS_WRAP
) {
5316 event
.type
= PTP_CLOCK_PPS
;
5317 if (adapter
->ptp_caps
.pps
)
5318 ptp_clock_event(adapter
->ptp_clock
, &event
);
5319 ack
|= IGC_TSICR_SYS_WRAP
;
5322 if (tsicr
& IGC_TSICR_TXTS
) {
5323 /* retrieve hardware timestamp */
5324 igc_ptp_tx_tstamp_event(adapter
);
5325 ack
|= IGC_TSICR_TXTS
;
5328 if (tsicr
& IGC_TSICR_TT0
) {
5329 spin_lock(&adapter
->tmreg_lock
);
5330 ts
= timespec64_add(adapter
->perout
[0].start
,
5331 adapter
->perout
[0].period
);
5332 wr32(IGC_TRGTTIML0
, ts
.tv_nsec
| IGC_TT_IO_TIMER_SEL_SYSTIM0
);
5333 wr32(IGC_TRGTTIMH0
, (u32
)ts
.tv_sec
);
5334 tsauxc
= rd32(IGC_TSAUXC
);
5335 tsauxc
|= IGC_TSAUXC_EN_TT0
;
5336 wr32(IGC_TSAUXC
, tsauxc
);
5337 adapter
->perout
[0].start
= ts
;
5338 spin_unlock(&adapter
->tmreg_lock
);
5339 ack
|= IGC_TSICR_TT0
;
5342 if (tsicr
& IGC_TSICR_TT1
) {
5343 spin_lock(&adapter
->tmreg_lock
);
5344 ts
= timespec64_add(adapter
->perout
[1].start
,
5345 adapter
->perout
[1].period
);
5346 wr32(IGC_TRGTTIML1
, ts
.tv_nsec
| IGC_TT_IO_TIMER_SEL_SYSTIM0
);
5347 wr32(IGC_TRGTTIMH1
, (u32
)ts
.tv_sec
);
5348 tsauxc
= rd32(IGC_TSAUXC
);
5349 tsauxc
|= IGC_TSAUXC_EN_TT1
;
5350 wr32(IGC_TSAUXC
, tsauxc
);
5351 adapter
->perout
[1].start
= ts
;
5352 spin_unlock(&adapter
->tmreg_lock
);
5353 ack
|= IGC_TSICR_TT1
;
5356 if (tsicr
& IGC_TSICR_AUTT0
) {
5357 nsec
= rd32(IGC_AUXSTMPL0
);
5358 sec
= rd32(IGC_AUXSTMPH0
);
5359 event
.type
= PTP_CLOCK_EXTTS
;
5361 event
.timestamp
= sec
* NSEC_PER_SEC
+ nsec
;
5362 ptp_clock_event(adapter
->ptp_clock
, &event
);
5363 ack
|= IGC_TSICR_AUTT0
;
5366 if (tsicr
& IGC_TSICR_AUTT1
) {
5367 nsec
= rd32(IGC_AUXSTMPL1
);
5368 sec
= rd32(IGC_AUXSTMPH1
);
5369 event
.type
= PTP_CLOCK_EXTTS
;
5371 event
.timestamp
= sec
* NSEC_PER_SEC
+ nsec
;
5372 ptp_clock_event(adapter
->ptp_clock
, &event
);
5373 ack
|= IGC_TSICR_AUTT1
;
5376 /* acknowledge the interrupts */
5377 wr32(IGC_TSICR
, ack
);
5381 * igc_msix_other - msix other interrupt handler
5382 * @irq: interrupt number
5383 * @data: pointer to a q_vector
5385 static irqreturn_t
igc_msix_other(int irq
, void *data
)
5387 struct igc_adapter
*adapter
= data
;
5388 struct igc_hw
*hw
= &adapter
->hw
;
5389 u32 icr
= rd32(IGC_ICR
);
5391 /* reading ICR causes bit 31 of EICR to be cleared */
5392 if (icr
& IGC_ICR_DRSTA
)
5393 schedule_work(&adapter
->reset_task
);
5395 if (icr
& IGC_ICR_DOUTSYNC
) {
5396 /* HW is reporting DMA is out of sync */
5397 adapter
->stats
.doosync
++;
5400 if (icr
& IGC_ICR_LSC
) {
5401 hw
->mac
.get_link_status
= true;
5402 /* guard against interrupt when we're going down */
5403 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
5404 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
5407 if (icr
& IGC_ICR_TS
)
5408 igc_tsync_interrupt(adapter
);
5410 wr32(IGC_EIMS
, adapter
->eims_other
);
5415 static void igc_write_itr(struct igc_q_vector
*q_vector
)
5417 u32 itr_val
= q_vector
->itr_val
& IGC_QVECTOR_MASK
;
5419 if (!q_vector
->set_itr
)
5423 itr_val
= IGC_ITR_VAL_MASK
;
5425 itr_val
|= IGC_EITR_CNT_IGNR
;
5427 writel(itr_val
, q_vector
->itr_register
);
5428 q_vector
->set_itr
= 0;
5431 static irqreturn_t
igc_msix_ring(int irq
, void *data
)
5433 struct igc_q_vector
*q_vector
= data
;
5435 /* Write the ITR value calculated from the previous interrupt. */
5436 igc_write_itr(q_vector
);
5438 napi_schedule(&q_vector
->napi
);
5444 * igc_request_msix - Initialize MSI-X interrupts
5445 * @adapter: Pointer to adapter structure
5447 * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5450 static int igc_request_msix(struct igc_adapter
*adapter
)
5452 unsigned int num_q_vectors
= adapter
->num_q_vectors
;
5453 int i
= 0, err
= 0, vector
= 0, free_vector
= 0;
5454 struct net_device
*netdev
= adapter
->netdev
;
5456 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
5457 &igc_msix_other
, 0, netdev
->name
, adapter
);
5461 if (num_q_vectors
> MAX_Q_VECTORS
) {
5462 num_q_vectors
= MAX_Q_VECTORS
;
5463 dev_warn(&adapter
->pdev
->dev
,
5464 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5465 adapter
->num_q_vectors
, MAX_Q_VECTORS
);
5467 for (i
= 0; i
< num_q_vectors
; i
++) {
5468 struct igc_q_vector
*q_vector
= adapter
->q_vector
[i
];
5472 q_vector
->itr_register
= adapter
->io_addr
+ IGC_EITR(vector
);
5474 if (q_vector
->rx
.ring
&& q_vector
->tx
.ring
)
5475 sprintf(q_vector
->name
, "%s-TxRx-%u", netdev
->name
,
5476 q_vector
->rx
.ring
->queue_index
);
5477 else if (q_vector
->tx
.ring
)
5478 sprintf(q_vector
->name
, "%s-tx-%u", netdev
->name
,
5479 q_vector
->tx
.ring
->queue_index
);
5480 else if (q_vector
->rx
.ring
)
5481 sprintf(q_vector
->name
, "%s-rx-%u", netdev
->name
,
5482 q_vector
->rx
.ring
->queue_index
);
5484 sprintf(q_vector
->name
, "%s-unused", netdev
->name
);
5486 err
= request_irq(adapter
->msix_entries
[vector
].vector
,
5487 igc_msix_ring
, 0, q_vector
->name
,
5493 igc_configure_msix(adapter
);
5497 /* free already assigned IRQs */
5498 free_irq(adapter
->msix_entries
[free_vector
++].vector
, adapter
);
5501 for (i
= 0; i
< vector
; i
++) {
5502 free_irq(adapter
->msix_entries
[free_vector
++].vector
,
5503 adapter
->q_vector
[i
]);
5510 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5511 * @adapter: Pointer to adapter structure
5513 * This function resets the device so that it has 0 rx queues, tx queues, and
5514 * MSI-X interrupts allocated.
5516 static void igc_clear_interrupt_scheme(struct igc_adapter
*adapter
)
5518 igc_free_q_vectors(adapter
);
5519 igc_reset_interrupt_capability(adapter
);
5522 /* Need to wait a few seconds after link up to get diagnostic information from
5525 static void igc_update_phy_info(struct timer_list
*t
)
5527 struct igc_adapter
*adapter
= from_timer(adapter
, t
, phy_info_timer
);
5529 igc_get_phy_info(&adapter
->hw
);
5533 * igc_has_link - check shared code for link and determine up/down
5534 * @adapter: pointer to driver private info
5536 bool igc_has_link(struct igc_adapter
*adapter
)
5538 struct igc_hw
*hw
= &adapter
->hw
;
5539 bool link_active
= false;
5541 /* get_link_status is set on LSC (link status) interrupt or
5542 * rx sequence error interrupt. get_link_status will stay
5543 * false until the igc_check_for_link establishes link
5544 * for copper adapters ONLY
5546 if (!hw
->mac
.get_link_status
)
5548 hw
->mac
.ops
.check_for_link(hw
);
5549 link_active
= !hw
->mac
.get_link_status
;
5551 if (hw
->mac
.type
== igc_i225
) {
5552 if (!netif_carrier_ok(adapter
->netdev
)) {
5553 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
5554 } else if (!(adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)) {
5555 adapter
->flags
|= IGC_FLAG_NEED_LINK_UPDATE
;
5556 adapter
->link_check_timeout
= jiffies
;
5564 * igc_watchdog - Timer Call-back
5565 * @t: timer for the watchdog
5567 static void igc_watchdog(struct timer_list
*t
)
5569 struct igc_adapter
*adapter
= from_timer(adapter
, t
, watchdog_timer
);
5570 /* Do the rest outside of interrupt context */
5571 schedule_work(&adapter
->watchdog_task
);
5574 static void igc_watchdog_task(struct work_struct
*work
)
5576 struct igc_adapter
*adapter
= container_of(work
,
5579 struct net_device
*netdev
= adapter
->netdev
;
5580 struct igc_hw
*hw
= &adapter
->hw
;
5581 struct igc_phy_info
*phy
= &hw
->phy
;
5582 u16 phy_data
, retry_count
= 20;
5586 link
= igc_has_link(adapter
);
5588 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
) {
5589 if (time_after(jiffies
, (adapter
->link_check_timeout
+ HZ
)))
5590 adapter
->flags
&= ~IGC_FLAG_NEED_LINK_UPDATE
;
5596 /* Cancel scheduled suspend requests. */
5597 pm_runtime_resume(netdev
->dev
.parent
);
5599 if (!netif_carrier_ok(netdev
)) {
5602 hw
->mac
.ops
.get_speed_and_duplex(hw
,
5603 &adapter
->link_speed
,
5604 &adapter
->link_duplex
);
5606 ctrl
= rd32(IGC_CTRL
);
5607 /* Link status message must follow this format */
5609 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5610 adapter
->link_speed
,
5611 adapter
->link_duplex
== FULL_DUPLEX
?
5613 (ctrl
& IGC_CTRL_TFCE
) &&
5614 (ctrl
& IGC_CTRL_RFCE
) ? "RX/TX" :
5615 (ctrl
& IGC_CTRL_RFCE
) ? "RX" :
5616 (ctrl
& IGC_CTRL_TFCE
) ? "TX" : "None");
5618 /* disable EEE if enabled */
5619 if ((adapter
->flags
& IGC_FLAG_EEE
) &&
5620 adapter
->link_duplex
== HALF_DUPLEX
) {
5622 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5623 adapter
->hw
.dev_spec
._base
.eee_enable
= false;
5624 adapter
->flags
&= ~IGC_FLAG_EEE
;
5627 /* check if SmartSpeed worked */
5628 igc_check_downshift(hw
);
5629 if (phy
->speed_downgraded
)
5630 netdev_warn(netdev
, "Link Speed was downgraded by SmartSpeed\n");
5632 /* adjust timeout factor according to speed/duplex */
5633 adapter
->tx_timeout_factor
= 1;
5634 switch (adapter
->link_speed
) {
5636 adapter
->tx_timeout_factor
= 14;
5641 adapter
->tx_timeout_factor
= 1;
5645 /* Once the launch time has been set on the wire, there
5646 * is a delay before the link speed can be determined
5647 * based on link-up activity. Write into the register
5648 * as soon as we know the correct link speed.
5650 igc_tsn_adjust_txtime_offset(adapter
);
5652 if (adapter
->link_speed
!= SPEED_1000
)
5655 /* wait for Remote receiver status OK */
5657 if (!igc_read_phy_reg(hw
, PHY_1000T_STATUS
,
5659 if (!(phy_data
& SR_1000T_REMOTE_RX_STATUS
) &&
5663 goto retry_read_status
;
5664 } else if (!retry_count
) {
5665 netdev_err(netdev
, "exceed max 2 second\n");
5668 netdev_err(netdev
, "read 1000Base-T Status Reg\n");
5671 netif_carrier_on(netdev
);
5673 /* link state has changed, schedule phy info update */
5674 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
5675 mod_timer(&adapter
->phy_info_timer
,
5676 round_jiffies(jiffies
+ 2 * HZ
));
5679 if (netif_carrier_ok(netdev
)) {
5680 adapter
->link_speed
= 0;
5681 adapter
->link_duplex
= 0;
5683 /* Links status message must follow this format */
5684 netdev_info(netdev
, "NIC Link is Down\n");
5685 netif_carrier_off(netdev
);
5687 /* link state has changed, schedule phy info update */
5688 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
5689 mod_timer(&adapter
->phy_info_timer
,
5690 round_jiffies(jiffies
+ 2 * HZ
));
5692 pm_schedule_suspend(netdev
->dev
.parent
,
5697 spin_lock(&adapter
->stats64_lock
);
5698 igc_update_stats(adapter
);
5699 spin_unlock(&adapter
->stats64_lock
);
5701 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
5702 struct igc_ring
*tx_ring
= adapter
->tx_ring
[i
];
5704 if (!netif_carrier_ok(netdev
)) {
5705 /* We've lost link, so the controller stops DMA,
5706 * but we've got queued Tx work that's never going
5707 * to get done, so reset controller to flush Tx.
5708 * (Do the reset outside of interrupt context).
5710 if (igc_desc_unused(tx_ring
) + 1 < tx_ring
->count
) {
5711 adapter
->tx_timeout_count
++;
5712 schedule_work(&adapter
->reset_task
);
5713 /* return immediately since reset is imminent */
5718 /* Force detection of hung controller every watchdog period */
5719 set_bit(IGC_RING_FLAG_TX_DETECT_HANG
, &tx_ring
->flags
);
5722 /* Cause software interrupt to ensure Rx ring is cleaned */
5723 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
5726 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
5727 eics
|= adapter
->q_vector
[i
]->eims_value
;
5728 wr32(IGC_EICS
, eics
);
5730 wr32(IGC_ICS
, IGC_ICS_RXDMT0
);
5733 igc_ptp_tx_hang(adapter
);
5735 /* Reset the timer */
5736 if (!test_bit(__IGC_DOWN
, &adapter
->state
)) {
5737 if (adapter
->flags
& IGC_FLAG_NEED_LINK_UPDATE
)
5738 mod_timer(&adapter
->watchdog_timer
,
5739 round_jiffies(jiffies
+ HZ
));
5741 mod_timer(&adapter
->watchdog_timer
,
5742 round_jiffies(jiffies
+ 2 * HZ
));
5747 * igc_intr_msi - Interrupt Handler
5748 * @irq: interrupt number
5749 * @data: pointer to a network interface device structure
5751 static irqreturn_t
igc_intr_msi(int irq
, void *data
)
5753 struct igc_adapter
*adapter
= data
;
5754 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
5755 struct igc_hw
*hw
= &adapter
->hw
;
5756 /* read ICR disables interrupts using IAM */
5757 u32 icr
= rd32(IGC_ICR
);
5759 igc_write_itr(q_vector
);
5761 if (icr
& IGC_ICR_DRSTA
)
5762 schedule_work(&adapter
->reset_task
);
5764 if (icr
& IGC_ICR_DOUTSYNC
) {
5765 /* HW is reporting DMA is out of sync */
5766 adapter
->stats
.doosync
++;
5769 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
5770 hw
->mac
.get_link_status
= true;
5771 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
5772 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
5775 if (icr
& IGC_ICR_TS
)
5776 igc_tsync_interrupt(adapter
);
5778 napi_schedule(&q_vector
->napi
);
5784 * igc_intr - Legacy Interrupt Handler
5785 * @irq: interrupt number
5786 * @data: pointer to a network interface device structure
5788 static irqreturn_t
igc_intr(int irq
, void *data
)
5790 struct igc_adapter
*adapter
= data
;
5791 struct igc_q_vector
*q_vector
= adapter
->q_vector
[0];
5792 struct igc_hw
*hw
= &adapter
->hw
;
5793 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5794 * need for the IMC write
5796 u32 icr
= rd32(IGC_ICR
);
5798 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5799 * not set, then the adapter didn't send an interrupt
5801 if (!(icr
& IGC_ICR_INT_ASSERTED
))
5804 igc_write_itr(q_vector
);
5806 if (icr
& IGC_ICR_DRSTA
)
5807 schedule_work(&adapter
->reset_task
);
5809 if (icr
& IGC_ICR_DOUTSYNC
) {
5810 /* HW is reporting DMA is out of sync */
5811 adapter
->stats
.doosync
++;
5814 if (icr
& (IGC_ICR_RXSEQ
| IGC_ICR_LSC
)) {
5815 hw
->mac
.get_link_status
= true;
5816 /* guard against interrupt when we're going down */
5817 if (!test_bit(__IGC_DOWN
, &adapter
->state
))
5818 mod_timer(&adapter
->watchdog_timer
, jiffies
+ 1);
5821 if (icr
& IGC_ICR_TS
)
5822 igc_tsync_interrupt(adapter
);
5824 napi_schedule(&q_vector
->napi
);
5829 static void igc_free_irq(struct igc_adapter
*adapter
)
5831 if (adapter
->msix_entries
) {
5834 free_irq(adapter
->msix_entries
[vector
++].vector
, adapter
);
5836 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
5837 free_irq(adapter
->msix_entries
[vector
++].vector
,
5838 adapter
->q_vector
[i
]);
5840 free_irq(adapter
->pdev
->irq
, adapter
);
5845 * igc_request_irq - initialize interrupts
5846 * @adapter: Pointer to adapter structure
5848 * Attempts to configure interrupts using the best available
5849 * capabilities of the hardware and kernel.
5851 static int igc_request_irq(struct igc_adapter
*adapter
)
5853 struct net_device
*netdev
= adapter
->netdev
;
5854 struct pci_dev
*pdev
= adapter
->pdev
;
5857 if (adapter
->flags
& IGC_FLAG_HAS_MSIX
) {
5858 err
= igc_request_msix(adapter
);
5861 /* fall back to MSI */
5862 igc_free_all_tx_resources(adapter
);
5863 igc_free_all_rx_resources(adapter
);
5865 igc_clear_interrupt_scheme(adapter
);
5866 err
= igc_init_interrupt_scheme(adapter
, false);
5869 igc_setup_all_tx_resources(adapter
);
5870 igc_setup_all_rx_resources(adapter
);
5871 igc_configure(adapter
);
5874 igc_assign_vector(adapter
->q_vector
[0], 0);
5876 if (adapter
->flags
& IGC_FLAG_HAS_MSI
) {
5877 err
= request_irq(pdev
->irq
, &igc_intr_msi
, 0,
5878 netdev
->name
, adapter
);
5882 /* fall back to legacy interrupts */
5883 igc_reset_interrupt_capability(adapter
);
5884 adapter
->flags
&= ~IGC_FLAG_HAS_MSI
;
5887 err
= request_irq(pdev
->irq
, &igc_intr
, IRQF_SHARED
,
5888 netdev
->name
, adapter
);
5891 netdev_err(netdev
, "Error %d getting interrupt\n", err
);
5898 * __igc_open - Called when a network interface is made active
5899 * @netdev: network interface device structure
5900 * @resuming: boolean indicating if the device is resuming
5902 * Returns 0 on success, negative value on failure
5904 * The open entry point is called when a network interface is made
5905 * active by the system (IFF_UP). At this point all resources needed
5906 * for transmit and receive operations are allocated, the interrupt
5907 * handler is registered with the OS, the watchdog timer is started,
5908 * and the stack is notified that the interface is ready.
5910 static int __igc_open(struct net_device
*netdev
, bool resuming
)
5912 struct igc_adapter
*adapter
= netdev_priv(netdev
);
5913 struct pci_dev
*pdev
= adapter
->pdev
;
5914 struct igc_hw
*hw
= &adapter
->hw
;
5918 /* disallow open during test */
5920 if (test_bit(__IGC_TESTING
, &adapter
->state
)) {
5926 pm_runtime_get_sync(&pdev
->dev
);
5928 netif_carrier_off(netdev
);
5930 /* allocate transmit descriptors */
5931 err
= igc_setup_all_tx_resources(adapter
);
5935 /* allocate receive descriptors */
5936 err
= igc_setup_all_rx_resources(adapter
);
5940 igc_power_up_link(adapter
);
5942 igc_configure(adapter
);
5944 err
= igc_request_irq(adapter
);
5948 /* Notify the stack of the actual queue counts. */
5949 err
= netif_set_real_num_tx_queues(netdev
, adapter
->num_tx_queues
);
5951 goto err_set_queues
;
5953 err
= netif_set_real_num_rx_queues(netdev
, adapter
->num_rx_queues
);
5955 goto err_set_queues
;
5957 clear_bit(__IGC_DOWN
, &adapter
->state
);
5959 for (i
= 0; i
< adapter
->num_q_vectors
; i
++)
5960 napi_enable(&adapter
->q_vector
[i
]->napi
);
5962 /* Clear any pending interrupts. */
5964 igc_irq_enable(adapter
);
5967 pm_runtime_put(&pdev
->dev
);
5969 netif_tx_start_all_queues(netdev
);
5971 /* start the watchdog. */
5972 hw
->mac
.get_link_status
= true;
5973 schedule_work(&adapter
->watchdog_task
);
5978 igc_free_irq(adapter
);
5980 igc_release_hw_control(adapter
);
5981 igc_power_down_phy_copper_base(&adapter
->hw
);
5982 igc_free_all_rx_resources(adapter
);
5984 igc_free_all_tx_resources(adapter
);
5988 pm_runtime_put(&pdev
->dev
);
5993 int igc_open(struct net_device
*netdev
)
5995 return __igc_open(netdev
, false);
5999 * __igc_close - Disables a network interface
6000 * @netdev: network interface device structure
6001 * @suspending: boolean indicating the device is suspending
6003 * Returns 0, this is not allowed to fail
6005 * The close entry point is called when an interface is de-activated
6006 * by the OS. The hardware is still under the driver's control, but
6007 * needs to be disabled. A global MAC reset is issued to stop the
6008 * hardware, and all transmit and receive resources are freed.
6010 static int __igc_close(struct net_device
*netdev
, bool suspending
)
6012 struct igc_adapter
*adapter
= netdev_priv(netdev
);
6013 struct pci_dev
*pdev
= adapter
->pdev
;
6015 WARN_ON(test_bit(__IGC_RESETTING
, &adapter
->state
));
6018 pm_runtime_get_sync(&pdev
->dev
);
6022 igc_release_hw_control(adapter
);
6024 igc_free_irq(adapter
);
6026 igc_free_all_tx_resources(adapter
);
6027 igc_free_all_rx_resources(adapter
);
6030 pm_runtime_put_sync(&pdev
->dev
);
6035 int igc_close(struct net_device
*netdev
)
6037 if (netif_device_present(netdev
) || netdev
->dismantle
)
6038 return __igc_close(netdev
, false);
6043 * igc_ioctl - Access the hwtstamp interface
6044 * @netdev: network interface device structure
6045 * @ifr: interface request data
6046 * @cmd: ioctl command
6048 static int igc_ioctl(struct net_device
*netdev
, struct ifreq
*ifr
, int cmd
)
6052 return igc_ptp_get_ts_config(netdev
, ifr
);
6054 return igc_ptp_set_ts_config(netdev
, ifr
);
6060 static int igc_save_launchtime_params(struct igc_adapter
*adapter
, int queue
,
6063 struct igc_ring
*ring
;
6065 if (queue
< 0 || queue
>= adapter
->num_tx_queues
)
6068 ring
= adapter
->tx_ring
[queue
];
6069 ring
->launchtime_enable
= enable
;
6074 static bool is_base_time_past(ktime_t base_time
, const struct timespec64
*now
)
6076 struct timespec64 b
;
6078 b
= ktime_to_timespec64(base_time
);
6080 return timespec64_compare(now
, &b
) > 0;
6083 static bool validate_schedule(struct igc_adapter
*adapter
,
6084 const struct tc_taprio_qopt_offload
*qopt
)
6086 int queue_uses
[IGC_MAX_TX_QUEUES
] = { };
6087 struct igc_hw
*hw
= &adapter
->hw
;
6088 struct timespec64 now
;
6091 if (qopt
->cycle_time_extension
)
6094 igc_ptp_read(adapter
, &now
);
6096 /* If we program the controller's BASET registers with a time
6097 * in the future, it will hold all the packets until that
6098 * time, causing a lot of TX Hangs, so to avoid that, we
6099 * reject schedules that would start in the future.
6100 * Note: Limitation above is no longer in i226.
6102 if (!is_base_time_past(qopt
->base_time
, &now
) &&
6103 igc_is_device_id_i225(hw
))
6106 for (n
= 0; n
< qopt
->num_entries
; n
++) {
6107 const struct tc_taprio_sched_entry
*e
, *prev
;
6110 prev
= n
? &qopt
->entries
[n
- 1] : NULL
;
6111 e
= &qopt
->entries
[n
];
6113 /* i225 only supports "global" frame preemption
6116 if (e
->command
!= TC_TAPRIO_CMD_SET_GATES
)
6119 for (i
= 0; i
< adapter
->num_tx_queues
; i
++)
6120 if (e
->gate_mask
& BIT(i
)) {
6123 /* There are limitations: A single queue cannot
6124 * be opened and closed multiple times per cycle
6125 * unless the gate stays open. Check for it.
6127 if (queue_uses
[i
] > 1 &&
6128 !(prev
->gate_mask
& BIT(i
)))
6136 static int igc_tsn_enable_launchtime(struct igc_adapter
*adapter
,
6137 struct tc_etf_qopt_offload
*qopt
)
6139 struct igc_hw
*hw
= &adapter
->hw
;
6142 if (hw
->mac
.type
!= igc_i225
)
6145 err
= igc_save_launchtime_params(adapter
, qopt
->queue
, qopt
->enable
);
6149 return igc_tsn_offload_apply(adapter
);
6152 static int igc_qbv_clear_schedule(struct igc_adapter
*adapter
)
6154 unsigned long flags
;
6157 adapter
->base_time
= 0;
6158 adapter
->cycle_time
= NSEC_PER_SEC
;
6159 adapter
->taprio_offload_enable
= false;
6160 adapter
->qbv_config_change_errors
= 0;
6161 adapter
->qbv_count
= 0;
6163 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6164 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
6166 ring
->start_time
= 0;
6167 ring
->end_time
= NSEC_PER_SEC
;
6171 spin_lock_irqsave(&adapter
->qbv_tx_lock
, flags
);
6173 adapter
->qbv_transition
= false;
6175 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6176 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
6178 ring
->oper_gate_closed
= false;
6179 ring
->admin_gate_closed
= false;
6182 spin_unlock_irqrestore(&adapter
->qbv_tx_lock
, flags
);
6187 static int igc_tsn_clear_schedule(struct igc_adapter
*adapter
)
6189 igc_qbv_clear_schedule(adapter
);
6194 static void igc_taprio_stats(struct net_device
*dev
,
6195 struct tc_taprio_qopt_stats
*stats
)
6197 /* When Strict_End is enabled, the tx_overruns counter
6198 * will always be zero.
6200 stats
->tx_overruns
= 0;
6203 static void igc_taprio_queue_stats(struct net_device
*dev
,
6204 struct tc_taprio_qopt_queue_stats
*queue_stats
)
6206 struct tc_taprio_qopt_stats
*stats
= &queue_stats
->stats
;
6208 /* When Strict_End is enabled, the tx_overruns counter
6209 * will always be zero.
6211 stats
->tx_overruns
= 0;
6214 static int igc_save_qbv_schedule(struct igc_adapter
*adapter
,
6215 struct tc_taprio_qopt_offload
*qopt
)
6217 bool queue_configured
[IGC_MAX_TX_QUEUES
] = { };
6218 struct igc_hw
*hw
= &adapter
->hw
;
6219 u32 start_time
= 0, end_time
= 0;
6220 struct timespec64 now
;
6221 unsigned long flags
;
6225 switch (qopt
->cmd
) {
6226 case TAPRIO_CMD_REPLACE
:
6228 case TAPRIO_CMD_DESTROY
:
6229 return igc_tsn_clear_schedule(adapter
);
6230 case TAPRIO_CMD_STATS
:
6231 igc_taprio_stats(adapter
->netdev
, &qopt
->stats
);
6233 case TAPRIO_CMD_QUEUE_STATS
:
6234 igc_taprio_queue_stats(adapter
->netdev
, &qopt
->queue_stats
);
6240 if (qopt
->base_time
< 0)
6243 if (igc_is_device_id_i225(hw
) && adapter
->taprio_offload_enable
)
6246 if (!validate_schedule(adapter
, qopt
))
6249 adapter
->cycle_time
= qopt
->cycle_time
;
6250 adapter
->base_time
= qopt
->base_time
;
6251 adapter
->taprio_offload_enable
= true;
6253 igc_ptp_read(adapter
, &now
);
6255 for (n
= 0; n
< qopt
->num_entries
; n
++) {
6256 struct tc_taprio_sched_entry
*e
= &qopt
->entries
[n
];
6258 end_time
+= e
->interval
;
6260 /* If any of the conditions below are true, we need to manually
6261 * control the end time of the cycle.
6262 * 1. Qbv users can specify a cycle time that is not equal
6263 * to the total GCL intervals. Hence, recalculation is
6264 * necessary here to exclude the time interval that
6265 * exceeds the cycle time.
6266 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
6267 * once the end of the list is reached, it will switch
6268 * to the END_OF_CYCLE state and leave the gates in the
6269 * same state until the next cycle is started.
6271 if (end_time
> adapter
->cycle_time
||
6272 n
+ 1 == qopt
->num_entries
)
6273 end_time
= adapter
->cycle_time
;
6275 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6276 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
6278 if (!(e
->gate_mask
& BIT(i
)))
6281 /* Check whether a queue stays open for more than one
6282 * entry. If so, keep the start and advance the end
6285 if (!queue_configured
[i
])
6286 ring
->start_time
= start_time
;
6287 ring
->end_time
= end_time
;
6289 if (ring
->start_time
>= adapter
->cycle_time
)
6290 queue_configured
[i
] = false;
6292 queue_configured
[i
] = true;
6295 start_time
+= e
->interval
;
6298 spin_lock_irqsave(&adapter
->qbv_tx_lock
, flags
);
6300 /* Check whether a queue gets configured.
6301 * If not, set the start and end time to be end time.
6303 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6304 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
6306 if (!is_base_time_past(qopt
->base_time
, &now
)) {
6307 ring
->admin_gate_closed
= false;
6309 ring
->oper_gate_closed
= false;
6310 ring
->admin_gate_closed
= false;
6313 if (!queue_configured
[i
]) {
6314 if (!is_base_time_past(qopt
->base_time
, &now
))
6315 ring
->admin_gate_closed
= true;
6317 ring
->oper_gate_closed
= true;
6319 ring
->start_time
= end_time
;
6320 ring
->end_time
= end_time
;
6324 spin_unlock_irqrestore(&adapter
->qbv_tx_lock
, flags
);
6326 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6327 struct igc_ring
*ring
= adapter
->tx_ring
[i
];
6328 struct net_device
*dev
= adapter
->netdev
;
6330 if (qopt
->max_sdu
[i
])
6331 ring
->max_sdu
= qopt
->max_sdu
[i
] + dev
->hard_header_len
- ETH_TLEN
;
6339 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter
*adapter
,
6340 struct tc_taprio_qopt_offload
*qopt
)
6342 struct igc_hw
*hw
= &adapter
->hw
;
6345 if (hw
->mac
.type
!= igc_i225
)
6348 err
= igc_save_qbv_schedule(adapter
, qopt
);
6352 return igc_tsn_offload_apply(adapter
);
6355 static int igc_save_cbs_params(struct igc_adapter
*adapter
, int queue
,
6356 bool enable
, int idleslope
, int sendslope
,
6357 int hicredit
, int locredit
)
6359 bool cbs_status
[IGC_MAX_SR_QUEUES
] = { false };
6360 struct net_device
*netdev
= adapter
->netdev
;
6361 struct igc_ring
*ring
;
6364 /* i225 has two sets of credit-based shaper logic.
6365 * Supporting it only on the top two priority queues
6367 if (queue
< 0 || queue
> 1)
6370 ring
= adapter
->tx_ring
[queue
];
6372 for (i
= 0; i
< IGC_MAX_SR_QUEUES
; i
++)
6373 if (adapter
->tx_ring
[i
])
6374 cbs_status
[i
] = adapter
->tx_ring
[i
]->cbs_enable
;
6376 /* CBS should be enabled on the highest priority queue first in order
6377 * for the CBS algorithm to operate as intended.
6380 if (queue
== 1 && !cbs_status
[0]) {
6382 "Enabling CBS on queue1 before queue0\n");
6386 if (queue
== 0 && cbs_status
[1]) {
6388 "Disabling CBS on queue0 before queue1\n");
6393 ring
->cbs_enable
= enable
;
6394 ring
->idleslope
= idleslope
;
6395 ring
->sendslope
= sendslope
;
6396 ring
->hicredit
= hicredit
;
6397 ring
->locredit
= locredit
;
6402 static int igc_tsn_enable_cbs(struct igc_adapter
*adapter
,
6403 struct tc_cbs_qopt_offload
*qopt
)
6405 struct igc_hw
*hw
= &adapter
->hw
;
6408 if (hw
->mac
.type
!= igc_i225
)
6411 if (qopt
->queue
< 0 || qopt
->queue
> 1)
6414 err
= igc_save_cbs_params(adapter
, qopt
->queue
, qopt
->enable
,
6415 qopt
->idleslope
, qopt
->sendslope
,
6416 qopt
->hicredit
, qopt
->locredit
);
6420 return igc_tsn_offload_apply(adapter
);
6423 static int igc_tc_query_caps(struct igc_adapter
*adapter
,
6424 struct tc_query_caps_base
*base
)
6426 struct igc_hw
*hw
= &adapter
->hw
;
6428 switch (base
->type
) {
6429 case TC_SETUP_QDISC_TAPRIO
: {
6430 struct tc_taprio_caps
*caps
= base
->caps
;
6432 caps
->broken_mqprio
= true;
6434 if (hw
->mac
.type
== igc_i225
) {
6435 caps
->supports_queue_max_sdu
= true;
6436 caps
->gate_mask_per_txq
= true;
6446 static int igc_setup_tc(struct net_device
*dev
, enum tc_setup_type type
,
6449 struct igc_adapter
*adapter
= netdev_priv(dev
);
6451 adapter
->tc_setup_type
= type
;
6455 return igc_tc_query_caps(adapter
, type_data
);
6456 case TC_SETUP_QDISC_TAPRIO
:
6457 return igc_tsn_enable_qbv_scheduling(adapter
, type_data
);
6459 case TC_SETUP_QDISC_ETF
:
6460 return igc_tsn_enable_launchtime(adapter
, type_data
);
6462 case TC_SETUP_QDISC_CBS
:
6463 return igc_tsn_enable_cbs(adapter
, type_data
);
6470 static int igc_bpf(struct net_device
*dev
, struct netdev_bpf
*bpf
)
6472 struct igc_adapter
*adapter
= netdev_priv(dev
);
6474 switch (bpf
->command
) {
6475 case XDP_SETUP_PROG
:
6476 return igc_xdp_set_prog(adapter
, bpf
->prog
, bpf
->extack
);
6477 case XDP_SETUP_XSK_POOL
:
6478 return igc_xdp_setup_pool(adapter
, bpf
->xsk
.pool
,
6485 static int igc_xdp_xmit(struct net_device
*dev
, int num_frames
,
6486 struct xdp_frame
**frames
, u32 flags
)
6488 struct igc_adapter
*adapter
= netdev_priv(dev
);
6489 int cpu
= smp_processor_id();
6490 struct netdev_queue
*nq
;
6491 struct igc_ring
*ring
;
6494 if (unlikely(!netif_carrier_ok(dev
)))
6497 if (unlikely(flags
& ~XDP_XMIT_FLAGS_MASK
))
6500 ring
= igc_xdp_get_tx_ring(adapter
, cpu
);
6501 nq
= txring_txq(ring
);
6503 __netif_tx_lock(nq
, cpu
);
6505 /* Avoid transmit queue timeout since we share it with the slow path */
6506 txq_trans_cond_update(nq
);
6509 for (i
= 0; i
< num_frames
; i
++) {
6511 struct xdp_frame
*xdpf
= frames
[i
];
6513 err
= igc_xdp_init_tx_descriptor(ring
, xdpf
);
6515 xdp_return_frame_rx_napi(xdpf
);
6520 if (flags
& XDP_XMIT_FLUSH
)
6521 igc_flush_tx_descriptors(ring
);
6523 __netif_tx_unlock(nq
);
6525 return num_frames
- drops
;
6528 static void igc_trigger_rxtxq_interrupt(struct igc_adapter
*adapter
,
6529 struct igc_q_vector
*q_vector
)
6531 struct igc_hw
*hw
= &adapter
->hw
;
6534 eics
|= q_vector
->eims_value
;
6535 wr32(IGC_EICS
, eics
);
6538 int igc_xsk_wakeup(struct net_device
*dev
, u32 queue_id
, u32 flags
)
6540 struct igc_adapter
*adapter
= netdev_priv(dev
);
6541 struct igc_q_vector
*q_vector
;
6542 struct igc_ring
*ring
;
6544 if (test_bit(__IGC_DOWN
, &adapter
->state
))
6547 if (!igc_xdp_is_enabled(adapter
))
6550 if (queue_id
>= adapter
->num_rx_queues
)
6553 ring
= adapter
->rx_ring
[queue_id
];
6555 if (!ring
->xsk_pool
)
6558 q_vector
= adapter
->q_vector
[queue_id
];
6559 if (!napi_if_scheduled_mark_missed(&q_vector
->napi
))
6560 igc_trigger_rxtxq_interrupt(adapter
, q_vector
);
6565 static const struct net_device_ops igc_netdev_ops
= {
6566 .ndo_open
= igc_open
,
6567 .ndo_stop
= igc_close
,
6568 .ndo_start_xmit
= igc_xmit_frame
,
6569 .ndo_set_rx_mode
= igc_set_rx_mode
,
6570 .ndo_set_mac_address
= igc_set_mac
,
6571 .ndo_change_mtu
= igc_change_mtu
,
6572 .ndo_tx_timeout
= igc_tx_timeout
,
6573 .ndo_get_stats64
= igc_get_stats64
,
6574 .ndo_fix_features
= igc_fix_features
,
6575 .ndo_set_features
= igc_set_features
,
6576 .ndo_features_check
= igc_features_check
,
6577 .ndo_eth_ioctl
= igc_ioctl
,
6578 .ndo_setup_tc
= igc_setup_tc
,
6580 .ndo_xdp_xmit
= igc_xdp_xmit
,
6581 .ndo_xsk_wakeup
= igc_xsk_wakeup
,
6584 /* PCIe configuration access */
6585 void igc_read_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
6587 struct igc_adapter
*adapter
= hw
->back
;
6589 pci_read_config_word(adapter
->pdev
, reg
, value
);
6592 void igc_write_pci_cfg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
6594 struct igc_adapter
*adapter
= hw
->back
;
6596 pci_write_config_word(adapter
->pdev
, reg
, *value
);
6599 s32
igc_read_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
6601 struct igc_adapter
*adapter
= hw
->back
;
6603 if (!pci_is_pcie(adapter
->pdev
))
6604 return -IGC_ERR_CONFIG
;
6606 pcie_capability_read_word(adapter
->pdev
, reg
, value
);
6611 s32
igc_write_pcie_cap_reg(struct igc_hw
*hw
, u32 reg
, u16
*value
)
6613 struct igc_adapter
*adapter
= hw
->back
;
6615 if (!pci_is_pcie(adapter
->pdev
))
6616 return -IGC_ERR_CONFIG
;
6618 pcie_capability_write_word(adapter
->pdev
, reg
, *value
);
6623 u32
igc_rd32(struct igc_hw
*hw
, u32 reg
)
6625 struct igc_adapter
*igc
= container_of(hw
, struct igc_adapter
, hw
);
6626 u8 __iomem
*hw_addr
= READ_ONCE(hw
->hw_addr
);
6629 if (IGC_REMOVED(hw_addr
))
6632 value
= readl(&hw_addr
[reg
]);
6634 /* reads should not return all F's */
6635 if (!(~value
) && (!reg
|| !(~readl(hw_addr
)))) {
6636 struct net_device
*netdev
= igc
->netdev
;
6639 netif_device_detach(netdev
);
6640 netdev_err(netdev
, "PCIe link lost, device now detached\n");
6641 WARN(pci_device_is_present(igc
->pdev
),
6642 "igc: Failed to read reg 0x%x!\n", reg
);
6648 /* Mapping HW RSS Type to enum xdp_rss_hash_type */
6649 static enum xdp_rss_hash_type igc_xdp_rss_type
[IGC_RSS_TYPE_MAX_TABLE
] = {
6650 [IGC_RSS_TYPE_NO_HASH
] = XDP_RSS_TYPE_L2
,
6651 [IGC_RSS_TYPE_HASH_TCP_IPV4
] = XDP_RSS_TYPE_L4_IPV4_TCP
,
6652 [IGC_RSS_TYPE_HASH_IPV4
] = XDP_RSS_TYPE_L3_IPV4
,
6653 [IGC_RSS_TYPE_HASH_TCP_IPV6
] = XDP_RSS_TYPE_L4_IPV6_TCP
,
6654 [IGC_RSS_TYPE_HASH_IPV6_EX
] = XDP_RSS_TYPE_L3_IPV6_EX
,
6655 [IGC_RSS_TYPE_HASH_IPV6
] = XDP_RSS_TYPE_L3_IPV6
,
6656 [IGC_RSS_TYPE_HASH_TCP_IPV6_EX
] = XDP_RSS_TYPE_L4_IPV6_TCP_EX
,
6657 [IGC_RSS_TYPE_HASH_UDP_IPV4
] = XDP_RSS_TYPE_L4_IPV4_UDP
,
6658 [IGC_RSS_TYPE_HASH_UDP_IPV6
] = XDP_RSS_TYPE_L4_IPV6_UDP
,
6659 [IGC_RSS_TYPE_HASH_UDP_IPV6_EX
] = XDP_RSS_TYPE_L4_IPV6_UDP_EX
,
6660 [10] = XDP_RSS_TYPE_NONE
, /* RSS Type above 9 "Reserved" by HW */
6661 [11] = XDP_RSS_TYPE_NONE
, /* keep array sized for SW bit-mask */
6662 [12] = XDP_RSS_TYPE_NONE
, /* to handle future HW revisons */
6663 [13] = XDP_RSS_TYPE_NONE
,
6664 [14] = XDP_RSS_TYPE_NONE
,
6665 [15] = XDP_RSS_TYPE_NONE
,
6668 static int igc_xdp_rx_hash(const struct xdp_md
*_ctx
, u32
*hash
,
6669 enum xdp_rss_hash_type
*rss_type
)
6671 const struct igc_xdp_buff
*ctx
= (void *)_ctx
;
6673 if (!(ctx
->xdp
.rxq
->dev
->features
& NETIF_F_RXHASH
))
6676 *hash
= le32_to_cpu(ctx
->rx_desc
->wb
.lower
.hi_dword
.rss
);
6677 *rss_type
= igc_xdp_rss_type
[igc_rss_type(ctx
->rx_desc
)];
6682 static int igc_xdp_rx_timestamp(const struct xdp_md
*_ctx
, u64
*timestamp
)
6684 const struct igc_xdp_buff
*ctx
= (void *)_ctx
;
6686 if (igc_test_staterr(ctx
->rx_desc
, IGC_RXDADV_STAT_TSIP
)) {
6687 *timestamp
= ctx
->rx_ts
;
6695 static const struct xdp_metadata_ops igc_xdp_metadata_ops
= {
6696 .xmo_rx_hash
= igc_xdp_rx_hash
,
6697 .xmo_rx_timestamp
= igc_xdp_rx_timestamp
,
6700 static enum hrtimer_restart
igc_qbv_scheduling_timer(struct hrtimer
*timer
)
6702 struct igc_adapter
*adapter
= container_of(timer
, struct igc_adapter
,
6704 unsigned long flags
;
6707 spin_lock_irqsave(&adapter
->qbv_tx_lock
, flags
);
6709 adapter
->qbv_transition
= true;
6710 for (i
= 0; i
< adapter
->num_tx_queues
; i
++) {
6711 struct igc_ring
*tx_ring
= adapter
->tx_ring
[i
];
6713 if (tx_ring
->admin_gate_closed
) {
6714 tx_ring
->admin_gate_closed
= false;
6715 tx_ring
->oper_gate_closed
= true;
6717 tx_ring
->oper_gate_closed
= false;
6720 adapter
->qbv_transition
= false;
6722 spin_unlock_irqrestore(&adapter
->qbv_tx_lock
, flags
);
6724 return HRTIMER_NORESTART
;
6728 * igc_probe - Device Initialization Routine
6729 * @pdev: PCI device information struct
6730 * @ent: entry in igc_pci_tbl
6732 * Returns 0 on success, negative on failure
6734 * igc_probe initializes an adapter identified by a pci_dev structure.
6735 * The OS initialization, configuring the adapter private structure,
6736 * and a hardware reset occur.
6738 static int igc_probe(struct pci_dev
*pdev
,
6739 const struct pci_device_id
*ent
)
6741 struct igc_adapter
*adapter
;
6742 struct net_device
*netdev
;
6744 const struct igc_info
*ei
= igc_info_tbl
[ent
->driver_data
];
6747 err
= pci_enable_device_mem(pdev
);
6751 err
= dma_set_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
6754 "No usable DMA configuration, aborting\n");
6758 err
= pci_request_mem_regions(pdev
, igc_driver_name
);
6762 err
= pci_enable_ptm(pdev
, NULL
);
6764 dev_info(&pdev
->dev
, "PCIe PTM not supported by PCIe bus/controller\n");
6766 pci_set_master(pdev
);
6769 netdev
= alloc_etherdev_mq(sizeof(struct igc_adapter
),
6773 goto err_alloc_etherdev
;
6775 SET_NETDEV_DEV(netdev
, &pdev
->dev
);
6777 pci_set_drvdata(pdev
, netdev
);
6778 adapter
= netdev_priv(netdev
);
6779 adapter
->netdev
= netdev
;
6780 adapter
->pdev
= pdev
;
6783 adapter
->port_num
= hw
->bus
.func
;
6784 adapter
->msg_enable
= netif_msg_init(debug
, DEFAULT_MSG_ENABLE
);
6786 err
= pci_save_state(pdev
);
6791 adapter
->io_addr
= ioremap(pci_resource_start(pdev
, 0),
6792 pci_resource_len(pdev
, 0));
6793 if (!adapter
->io_addr
)
6796 /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6797 hw
->hw_addr
= adapter
->io_addr
;
6799 netdev
->netdev_ops
= &igc_netdev_ops
;
6800 netdev
->xdp_metadata_ops
= &igc_xdp_metadata_ops
;
6801 igc_ethtool_set_ops(netdev
);
6802 netdev
->watchdog_timeo
= 5 * HZ
;
6804 netdev
->mem_start
= pci_resource_start(pdev
, 0);
6805 netdev
->mem_end
= pci_resource_end(pdev
, 0);
6807 /* PCI config space info */
6808 hw
->vendor_id
= pdev
->vendor
;
6809 hw
->device_id
= pdev
->device
;
6810 hw
->revision_id
= pdev
->revision
;
6811 hw
->subsystem_vendor_id
= pdev
->subsystem_vendor
;
6812 hw
->subsystem_device_id
= pdev
->subsystem_device
;
6814 /* Copy the default MAC and PHY function pointers */
6815 memcpy(&hw
->mac
.ops
, ei
->mac_ops
, sizeof(hw
->mac
.ops
));
6816 memcpy(&hw
->phy
.ops
, ei
->phy_ops
, sizeof(hw
->phy
.ops
));
6818 /* Initialize skew-specific constants */
6819 err
= ei
->get_invariants(hw
);
6823 /* Add supported features to the features list*/
6824 netdev
->features
|= NETIF_F_SG
;
6825 netdev
->features
|= NETIF_F_TSO
;
6826 netdev
->features
|= NETIF_F_TSO6
;
6827 netdev
->features
|= NETIF_F_TSO_ECN
;
6828 netdev
->features
|= NETIF_F_RXHASH
;
6829 netdev
->features
|= NETIF_F_RXCSUM
;
6830 netdev
->features
|= NETIF_F_HW_CSUM
;
6831 netdev
->features
|= NETIF_F_SCTP_CRC
;
6832 netdev
->features
|= NETIF_F_HW_TC
;
6834 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
6835 NETIF_F_GSO_GRE_CSUM | \
6836 NETIF_F_GSO_IPXIP4 | \
6837 NETIF_F_GSO_IPXIP6 | \
6838 NETIF_F_GSO_UDP_TUNNEL | \
6839 NETIF_F_GSO_UDP_TUNNEL_CSUM)
6841 netdev
->gso_partial_features
= IGC_GSO_PARTIAL_FEATURES
;
6842 netdev
->features
|= NETIF_F_GSO_PARTIAL
| IGC_GSO_PARTIAL_FEATURES
;
6844 /* setup the private structure */
6845 err
= igc_sw_init(adapter
);
6849 /* copy netdev features into list of user selectable features */
6850 netdev
->hw_features
|= NETIF_F_NTUPLE
;
6851 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_TX
;
6852 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_RX
;
6853 netdev
->hw_features
|= netdev
->features
;
6855 netdev
->features
|= NETIF_F_HIGHDMA
;
6857 netdev
->vlan_features
|= netdev
->features
| NETIF_F_TSO_MANGLEID
;
6858 netdev
->mpls_features
|= NETIF_F_HW_CSUM
;
6859 netdev
->hw_enc_features
|= netdev
->vlan_features
;
6861 netdev
->xdp_features
= NETDEV_XDP_ACT_BASIC
| NETDEV_XDP_ACT_REDIRECT
|
6862 NETDEV_XDP_ACT_XSK_ZEROCOPY
;
6864 /* MTU range: 68 - 9216 */
6865 netdev
->min_mtu
= ETH_MIN_MTU
;
6866 netdev
->max_mtu
= MAX_STD_JUMBO_FRAME_SIZE
;
6868 /* before reading the NVM, reset the controller to put the device in a
6869 * known good starting state
6871 hw
->mac
.ops
.reset_hw(hw
);
6873 if (igc_get_flash_presence_i225(hw
)) {
6874 if (hw
->nvm
.ops
.validate(hw
) < 0) {
6875 dev_err(&pdev
->dev
, "The NVM Checksum Is Not Valid\n");
6881 if (eth_platform_get_mac_address(&pdev
->dev
, hw
->mac
.addr
)) {
6882 /* copy the MAC address out of the NVM */
6883 if (hw
->mac
.ops
.read_mac_addr(hw
))
6884 dev_err(&pdev
->dev
, "NVM Read Error\n");
6887 eth_hw_addr_set(netdev
, hw
->mac
.addr
);
6889 if (!is_valid_ether_addr(netdev
->dev_addr
)) {
6890 dev_err(&pdev
->dev
, "Invalid MAC Address\n");
6895 /* configure RXPBSIZE and TXPBSIZE */
6896 wr32(IGC_RXPBS
, I225_RXPBSIZE_DEFAULT
);
6897 wr32(IGC_TXPBS
, I225_TXPBSIZE_DEFAULT
);
6899 timer_setup(&adapter
->watchdog_timer
, igc_watchdog
, 0);
6900 timer_setup(&adapter
->phy_info_timer
, igc_update_phy_info
, 0);
6902 INIT_WORK(&adapter
->reset_task
, igc_reset_task
);
6903 INIT_WORK(&adapter
->watchdog_task
, igc_watchdog_task
);
6905 hrtimer_init(&adapter
->hrtimer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
6906 adapter
->hrtimer
.function
= &igc_qbv_scheduling_timer
;
6908 /* Initialize link properties that are user-changeable */
6909 adapter
->fc_autoneg
= true;
6910 hw
->mac
.autoneg
= true;
6911 hw
->phy
.autoneg_advertised
= 0xaf;
6913 hw
->fc
.requested_mode
= igc_fc_default
;
6914 hw
->fc
.current_mode
= igc_fc_default
;
6916 /* By default, support wake on port A */
6917 adapter
->flags
|= IGC_FLAG_WOL_SUPPORTED
;
6919 /* initialize the wol settings based on the eeprom settings */
6920 if (adapter
->flags
& IGC_FLAG_WOL_SUPPORTED
)
6921 adapter
->wol
|= IGC_WUFC_MAG
;
6923 device_set_wakeup_enable(&adapter
->pdev
->dev
,
6924 adapter
->flags
& IGC_FLAG_WOL_SUPPORTED
);
6926 igc_ptp_init(adapter
);
6928 igc_tsn_clear_schedule(adapter
);
6930 /* reset the hardware with the new settings */
6933 /* let the f/w know that the h/w is now under the control of the
6936 igc_get_hw_control(adapter
);
6938 strncpy(netdev
->name
, "eth%d", IFNAMSIZ
);
6939 err
= register_netdev(netdev
);
6943 /* carrier off reporting is important to ethtool even BEFORE open */
6944 netif_carrier_off(netdev
);
6946 /* Check if Media Autosense is enabled */
6949 /* print pcie link status and MAC address */
6950 pcie_print_link_status(pdev
);
6951 netdev_info(netdev
, "MAC: %pM\n", netdev
->dev_addr
);
6953 dev_pm_set_driver_flags(&pdev
->dev
, DPM_FLAG_NO_DIRECT_COMPLETE
);
6954 /* Disable EEE for internal PHY devices */
6955 hw
->dev_spec
._base
.eee_enable
= false;
6956 adapter
->flags
&= ~IGC_FLAG_EEE
;
6957 igc_set_eee_i225(hw
, false, false, false);
6959 pm_runtime_put_noidle(&pdev
->dev
);
6964 igc_release_hw_control(adapter
);
6966 if (!igc_check_reset_block(hw
))
6969 igc_clear_interrupt_scheme(adapter
);
6970 iounmap(adapter
->io_addr
);
6972 free_netdev(netdev
);
6974 pci_release_mem_regions(pdev
);
6977 pci_disable_device(pdev
);
6982 * igc_remove - Device Removal Routine
6983 * @pdev: PCI device information struct
6985 * igc_remove is called by the PCI subsystem to alert the driver
6986 * that it should release a PCI device. This could be caused by a
6987 * Hot-Plug event, or because the driver is going to be removed from
6990 static void igc_remove(struct pci_dev
*pdev
)
6992 struct net_device
*netdev
= pci_get_drvdata(pdev
);
6993 struct igc_adapter
*adapter
= netdev_priv(netdev
);
6995 pm_runtime_get_noresume(&pdev
->dev
);
6997 igc_flush_nfc_rules(adapter
);
6999 igc_ptp_stop(adapter
);
7001 pci_disable_ptm(pdev
);
7002 pci_clear_master(pdev
);
7004 set_bit(__IGC_DOWN
, &adapter
->state
);
7006 del_timer_sync(&adapter
->watchdog_timer
);
7007 del_timer_sync(&adapter
->phy_info_timer
);
7009 cancel_work_sync(&adapter
->reset_task
);
7010 cancel_work_sync(&adapter
->watchdog_task
);
7011 hrtimer_cancel(&adapter
->hrtimer
);
7013 /* Release control of h/w to f/w. If f/w is AMT enabled, this
7014 * would have already happened in close and is redundant.
7016 igc_release_hw_control(adapter
);
7017 unregister_netdev(netdev
);
7019 igc_clear_interrupt_scheme(adapter
);
7020 pci_iounmap(pdev
, adapter
->io_addr
);
7021 pci_release_mem_regions(pdev
);
7023 free_netdev(netdev
);
7025 pci_disable_device(pdev
);
7028 static int __igc_shutdown(struct pci_dev
*pdev
, bool *enable_wake
,
7031 struct net_device
*netdev
= pci_get_drvdata(pdev
);
7032 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7033 u32 wufc
= runtime
? IGC_WUFC_LNKC
: adapter
->wol
;
7034 struct igc_hw
*hw
= &adapter
->hw
;
7035 u32 ctrl
, rctl
, status
;
7039 netif_device_detach(netdev
);
7041 if (netif_running(netdev
))
7042 __igc_close(netdev
, true);
7044 igc_ptp_suspend(adapter
);
7046 igc_clear_interrupt_scheme(adapter
);
7049 status
= rd32(IGC_STATUS
);
7050 if (status
& IGC_STATUS_LU
)
7051 wufc
&= ~IGC_WUFC_LNKC
;
7054 igc_setup_rctl(adapter
);
7055 igc_set_rx_mode(netdev
);
7057 /* turn on all-multi mode if wake on multicast is enabled */
7058 if (wufc
& IGC_WUFC_MC
) {
7059 rctl
= rd32(IGC_RCTL
);
7060 rctl
|= IGC_RCTL_MPE
;
7061 wr32(IGC_RCTL
, rctl
);
7064 ctrl
= rd32(IGC_CTRL
);
7065 ctrl
|= IGC_CTRL_ADVD3WUC
;
7066 wr32(IGC_CTRL
, ctrl
);
7068 /* Allow time for pending master requests to run */
7069 igc_disable_pcie_master(hw
);
7071 wr32(IGC_WUC
, IGC_WUC_PME_EN
);
7072 wr32(IGC_WUFC
, wufc
);
7078 wake
= wufc
|| adapter
->en_mng_pt
;
7080 igc_power_down_phy_copper_base(&adapter
->hw
);
7082 igc_power_up_link(adapter
);
7085 *enable_wake
= wake
;
7087 /* Release control of h/w to f/w. If f/w is AMT enabled, this
7088 * would have already happened in close and is redundant.
7090 igc_release_hw_control(adapter
);
7092 pci_disable_device(pdev
);
7098 static int __maybe_unused
igc_runtime_suspend(struct device
*dev
)
7100 return __igc_shutdown(to_pci_dev(dev
), NULL
, 1);
7103 static void igc_deliver_wake_packet(struct net_device
*netdev
)
7105 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7106 struct igc_hw
*hw
= &adapter
->hw
;
7107 struct sk_buff
*skb
;
7110 wupl
= rd32(IGC_WUPL
) & IGC_WUPL_MASK
;
7112 /* WUPM stores only the first 128 bytes of the wake packet.
7113 * Read the packet only if we have the whole thing.
7115 if (wupl
== 0 || wupl
> IGC_WUPM_BYTES
)
7118 skb
= netdev_alloc_skb_ip_align(netdev
, IGC_WUPM_BYTES
);
7124 /* Ensure reads are 32-bit aligned */
7125 wupl
= roundup(wupl
, 4);
7127 memcpy_fromio(skb
->data
, hw
->hw_addr
+ IGC_WUPM_REG(0), wupl
);
7129 skb
->protocol
= eth_type_trans(skb
, netdev
);
7133 static int __maybe_unused
igc_resume(struct device
*dev
)
7135 struct pci_dev
*pdev
= to_pci_dev(dev
);
7136 struct net_device
*netdev
= pci_get_drvdata(pdev
);
7137 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7138 struct igc_hw
*hw
= &adapter
->hw
;
7141 pci_set_power_state(pdev
, PCI_D0
);
7142 pci_restore_state(pdev
);
7143 pci_save_state(pdev
);
7145 if (!pci_device_is_present(pdev
))
7147 err
= pci_enable_device_mem(pdev
);
7149 netdev_err(netdev
, "Cannot enable PCI device from suspend\n");
7152 pci_set_master(pdev
);
7154 pci_enable_wake(pdev
, PCI_D3hot
, 0);
7155 pci_enable_wake(pdev
, PCI_D3cold
, 0);
7157 if (igc_init_interrupt_scheme(adapter
, true)) {
7158 netdev_err(netdev
, "Unable to allocate memory for queues\n");
7164 /* let the f/w know that the h/w is now under the control of the
7167 igc_get_hw_control(adapter
);
7169 val
= rd32(IGC_WUS
);
7170 if (val
& WAKE_PKT_WUS
)
7171 igc_deliver_wake_packet(netdev
);
7176 if (!err
&& netif_running(netdev
))
7177 err
= __igc_open(netdev
, true);
7180 netif_device_attach(netdev
);
7186 static int __maybe_unused
igc_runtime_resume(struct device
*dev
)
7188 return igc_resume(dev
);
7191 static int __maybe_unused
igc_suspend(struct device
*dev
)
7193 return __igc_shutdown(to_pci_dev(dev
), NULL
, 0);
7196 static int __maybe_unused
igc_runtime_idle(struct device
*dev
)
7198 struct net_device
*netdev
= dev_get_drvdata(dev
);
7199 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7201 if (!igc_has_link(adapter
))
7202 pm_schedule_suspend(dev
, MSEC_PER_SEC
* 5);
7206 #endif /* CONFIG_PM */
7208 static void igc_shutdown(struct pci_dev
*pdev
)
7212 __igc_shutdown(pdev
, &wake
, 0);
7214 if (system_state
== SYSTEM_POWER_OFF
) {
7215 pci_wake_from_d3(pdev
, wake
);
7216 pci_set_power_state(pdev
, PCI_D3hot
);
7221 * igc_io_error_detected - called when PCI error is detected
7222 * @pdev: Pointer to PCI device
7223 * @state: The current PCI connection state
7225 * This function is called after a PCI bus error affecting
7226 * this device has been detected.
7228 static pci_ers_result_t
igc_io_error_detected(struct pci_dev
*pdev
,
7229 pci_channel_state_t state
)
7231 struct net_device
*netdev
= pci_get_drvdata(pdev
);
7232 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7234 netif_device_detach(netdev
);
7236 if (state
== pci_channel_io_perm_failure
)
7237 return PCI_ERS_RESULT_DISCONNECT
;
7239 if (netif_running(netdev
))
7241 pci_disable_device(pdev
);
7243 /* Request a slot reset. */
7244 return PCI_ERS_RESULT_NEED_RESET
;
7248 * igc_io_slot_reset - called after the PCI bus has been reset.
7249 * @pdev: Pointer to PCI device
7251 * Restart the card from scratch, as if from a cold-boot. Implementation
7252 * resembles the first-half of the igc_resume routine.
7254 static pci_ers_result_t
igc_io_slot_reset(struct pci_dev
*pdev
)
7256 struct net_device
*netdev
= pci_get_drvdata(pdev
);
7257 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7258 struct igc_hw
*hw
= &adapter
->hw
;
7259 pci_ers_result_t result
;
7261 if (pci_enable_device_mem(pdev
)) {
7262 netdev_err(netdev
, "Could not re-enable PCI device after reset\n");
7263 result
= PCI_ERS_RESULT_DISCONNECT
;
7265 pci_set_master(pdev
);
7266 pci_restore_state(pdev
);
7267 pci_save_state(pdev
);
7269 pci_enable_wake(pdev
, PCI_D3hot
, 0);
7270 pci_enable_wake(pdev
, PCI_D3cold
, 0);
7272 /* In case of PCI error, adapter loses its HW address
7273 * so we should re-assign it here.
7275 hw
->hw_addr
= adapter
->io_addr
;
7279 result
= PCI_ERS_RESULT_RECOVERED
;
7286 * igc_io_resume - called when traffic can start to flow again.
7287 * @pdev: Pointer to PCI device
7289 * This callback is called when the error recovery driver tells us that
7290 * its OK to resume normal operation. Implementation resembles the
7291 * second-half of the igc_resume routine.
7293 static void igc_io_resume(struct pci_dev
*pdev
)
7295 struct net_device
*netdev
= pci_get_drvdata(pdev
);
7296 struct igc_adapter
*adapter
= netdev_priv(netdev
);
7299 if (netif_running(netdev
)) {
7300 if (igc_open(netdev
)) {
7301 netdev_err(netdev
, "igc_open failed after reset\n");
7306 netif_device_attach(netdev
);
7308 /* let the f/w know that the h/w is now under the control of the
7311 igc_get_hw_control(adapter
);
7315 static const struct pci_error_handlers igc_err_handler
= {
7316 .error_detected
= igc_io_error_detected
,
7317 .slot_reset
= igc_io_slot_reset
,
7318 .resume
= igc_io_resume
,
7322 static const struct dev_pm_ops igc_pm_ops
= {
7323 SET_SYSTEM_SLEEP_PM_OPS(igc_suspend
, igc_resume
)
7324 SET_RUNTIME_PM_OPS(igc_runtime_suspend
, igc_runtime_resume
,
7329 static struct pci_driver igc_driver
= {
7330 .name
= igc_driver_name
,
7331 .id_table
= igc_pci_tbl
,
7333 .remove
= igc_remove
,
7335 .driver
.pm
= &igc_pm_ops
,
7337 .shutdown
= igc_shutdown
,
7338 .err_handler
= &igc_err_handler
,
7342 * igc_reinit_queues - return error
7343 * @adapter: pointer to adapter structure
7345 int igc_reinit_queues(struct igc_adapter
*adapter
)
7347 struct net_device
*netdev
= adapter
->netdev
;
7350 if (netif_running(netdev
))
7353 igc_reset_interrupt_capability(adapter
);
7355 if (igc_init_interrupt_scheme(adapter
, true)) {
7356 netdev_err(netdev
, "Unable to allocate memory for queues\n");
7360 if (netif_running(netdev
))
7361 err
= igc_open(netdev
);
7367 * igc_get_hw_dev - return device
7368 * @hw: pointer to hardware structure
7370 * used by hardware layer to print debugging information
7372 struct net_device
*igc_get_hw_dev(struct igc_hw
*hw
)
7374 struct igc_adapter
*adapter
= hw
->back
;
7376 return adapter
->netdev
;
7379 static void igc_disable_rx_ring_hw(struct igc_ring
*ring
)
7381 struct igc_hw
*hw
= &ring
->q_vector
->adapter
->hw
;
7382 u8 idx
= ring
->reg_idx
;
7385 rxdctl
= rd32(IGC_RXDCTL(idx
));
7386 rxdctl
&= ~IGC_RXDCTL_QUEUE_ENABLE
;
7387 rxdctl
|= IGC_RXDCTL_SWFLUSH
;
7388 wr32(IGC_RXDCTL(idx
), rxdctl
);
7391 void igc_disable_rx_ring(struct igc_ring
*ring
)
7393 igc_disable_rx_ring_hw(ring
);
7394 igc_clean_rx_ring(ring
);
7397 void igc_enable_rx_ring(struct igc_ring
*ring
)
7399 struct igc_adapter
*adapter
= ring
->q_vector
->adapter
;
7401 igc_configure_rx_ring(adapter
, ring
);
7404 igc_alloc_rx_buffers_zc(ring
, igc_desc_unused(ring
));
7406 igc_alloc_rx_buffers(ring
, igc_desc_unused(ring
));
7409 void igc_disable_tx_ring(struct igc_ring
*ring
)
7411 igc_disable_tx_ring_hw(ring
);
7412 igc_clean_tx_ring(ring
);
7415 void igc_enable_tx_ring(struct igc_ring
*ring
)
7417 struct igc_adapter
*adapter
= ring
->q_vector
->adapter
;
7419 igc_configure_tx_ring(adapter
, ring
);
7423 * igc_init_module - Driver Registration Routine
7425 * igc_init_module is the first routine called when the driver is
7426 * loaded. All it does is register with the PCI subsystem.
7428 static int __init
igc_init_module(void)
7432 pr_info("%s\n", igc_driver_string
);
7433 pr_info("%s\n", igc_copyright
);
7435 ret
= pci_register_driver(&igc_driver
);
7439 module_init(igc_init_module
);
7442 * igc_exit_module - Driver Exit Cleanup Routine
7444 * igc_exit_module is called just before the driver is removed
7447 static void __exit
igc_exit_module(void)
7449 pci_unregister_driver(&igc_driver
);
7452 module_exit(igc_exit_module
);