2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/skbuff.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/if_vlan.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/jiffies.h>
42 #include <linux/prefetch.h>
43 #include <linux/export.h>
47 #include <net/busy_poll.h>
48 #ifdef CONFIG_CHELSIO_T4_FCOE
49 #include <scsi/fc/fc_fcoe.h>
50 #endif /* CONFIG_CHELSIO_T4_FCOE */
53 #include "t4_values.h"
56 #include "cxgb4_ptp.h"
57 #include "cxgb4_uld.h"
58 #include "cxgb4_tc_mqprio.h"
62 * Rx buffer size. We use largish buffers if possible but settle for single
63 * pages under memory shortage.
66 # define FL_PG_ORDER 0
68 # define FL_PG_ORDER (16 - PAGE_SHIFT)
71 /* RX_PULL_LEN should be <= RX_COPY_THRES */
72 #define RX_COPY_THRES 256
73 #define RX_PULL_LEN 128
76 * Main body length for sk_buffs used for Rx Ethernet packets with fragments.
77 * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room.
79 #define RX_PKT_SKB_LEN 512
82 * Max number of Tx descriptors we clean up at a time. Should be modest as
83 * freeing skbs isn't cheap and it happens while holding locks. We just need
84 * to free packets faster than they arrive, we eventually catch up and keep
85 * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should
86 * also match the CIDX Flush Threshold.
88 #define MAX_TX_RECLAIM 32
91 * Max number of Rx buffers we replenish at a time. Again keep this modest,
92 * allocating buffers isn't cheap either.
94 #define MAX_RX_REFILL 16U
97 * Period of the Rx queue check timer. This timer is infrequent as it has
98 * something to do only when the system experiences severe memory shortage.
100 #define RX_QCHECK_PERIOD (HZ / 2)
103 * Period of the Tx queue check timer.
105 #define TX_QCHECK_PERIOD (HZ / 2)
108 * Max number of Tx descriptors to be reclaimed by the Tx timer.
110 #define MAX_TIMER_TX_RECLAIM 100
113 * Timer index used when backing off due to memory shortage.
115 #define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
118 * Suspension threshold for non-Ethernet Tx queues. We require enough room
119 * for a full sized WR.
121 #define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
124 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
127 #define MAX_IMM_TX_PKT_LEN 256
130 * Max size of a WR sent through a control Tx queue.
132 #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
134 struct rx_sw_desc
{ /* SW state per Rx descriptor */
140 * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb
141 * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs.
142 * We could easily support more but there doesn't seem to be much need for
145 #define FL_MTU_SMALL 1500
146 #define FL_MTU_LARGE 9000
148 static inline unsigned int fl_mtu_bufsize(struct adapter
*adapter
,
151 struct sge
*s
= &adapter
->sge
;
153 return ALIGN(s
->pktshift
+ ETH_HLEN
+ VLAN_HLEN
+ mtu
, s
->fl_align
);
156 #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
157 #define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
160 * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses
161 * these to specify the buffer size as an index into the SGE Free List Buffer
162 * Size register array. We also use bit 4, when the buffer has been unmapped
163 * for DMA, but this is of course never sent to the hardware and is only used
164 * to prevent double unmappings. All of the above requires that the Free List
165 * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
166 * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal
167 * Free List Buffer alignment is 32 bytes, this works out for us ...
170 RX_BUF_FLAGS
= 0x1f, /* bottom five bits are special */
171 RX_BUF_SIZE
= 0x0f, /* bottom three bits are for buf sizes */
172 RX_UNMAPPED_BUF
= 0x10, /* buffer is not mapped */
175 * XXX We shouldn't depend on being able to use these indices.
176 * XXX Especially when some other Master PF has initialized the
177 * XXX adapter or we use the Firmware Configuration File. We
178 * XXX should really search through the Host Buffer Size register
179 * XXX array for the appropriately sized buffer indices.
181 RX_SMALL_PG_BUF
= 0x0, /* small (PAGE_SIZE) page buffer */
182 RX_LARGE_PG_BUF
= 0x1, /* buffer large (FL_PG_ORDER) page buffer */
184 RX_SMALL_MTU_BUF
= 0x2, /* small MTU buffer */
185 RX_LARGE_MTU_BUF
= 0x3, /* large MTU buffer */
188 static int timer_pkt_quota
[] = {1, 1, 2, 3, 4, 5};
189 #define MIN_NAPI_WORK 1
191 static inline dma_addr_t
get_buf_addr(const struct rx_sw_desc
*d
)
193 return d
->dma_addr
& ~(dma_addr_t
)RX_BUF_FLAGS
;
196 static inline bool is_buf_mapped(const struct rx_sw_desc
*d
)
198 return !(d
->dma_addr
& RX_UNMAPPED_BUF
);
202 * txq_avail - return the number of available slots in a Tx queue
205 * Returns the number of descriptors in a Tx queue available to write new
208 static inline unsigned int txq_avail(const struct sge_txq
*q
)
210 return q
->size
- 1 - q
->in_use
;
214 * fl_cap - return the capacity of a free-buffer list
217 * Returns the capacity of a free-buffer list. The capacity is less than
218 * the size because one descriptor needs to be left unpopulated, otherwise
219 * HW will think the FL is empty.
221 static inline unsigned int fl_cap(const struct sge_fl
*fl
)
223 return fl
->size
- 8; /* 1 descriptor = 8 buffers */
227 * fl_starving - return whether a Free List is starving.
228 * @adapter: pointer to the adapter
231 * Tests specified Free List to see whether the number of buffers
232 * available to the hardware has falled below our "starvation"
235 static inline bool fl_starving(const struct adapter
*adapter
,
236 const struct sge_fl
*fl
)
238 const struct sge
*s
= &adapter
->sge
;
240 return fl
->avail
- fl
->pend_cred
<= s
->fl_starve_thres
;
243 int cxgb4_map_skb(struct device
*dev
, const struct sk_buff
*skb
,
246 const skb_frag_t
*fp
, *end
;
247 const struct skb_shared_info
*si
;
249 *addr
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
250 if (dma_mapping_error(dev
, *addr
))
253 si
= skb_shinfo(skb
);
254 end
= &si
->frags
[si
->nr_frags
];
256 for (fp
= si
->frags
; fp
< end
; fp
++) {
257 *++addr
= skb_frag_dma_map(dev
, fp
, 0, skb_frag_size(fp
),
259 if (dma_mapping_error(dev
, *addr
))
265 while (fp
-- > si
->frags
)
266 dma_unmap_page(dev
, *--addr
, skb_frag_size(fp
), DMA_TO_DEVICE
);
268 dma_unmap_single(dev
, addr
[-1], skb_headlen(skb
), DMA_TO_DEVICE
);
272 EXPORT_SYMBOL(cxgb4_map_skb
);
274 static void unmap_skb(struct device
*dev
, const struct sk_buff
*skb
,
275 const dma_addr_t
*addr
)
277 const skb_frag_t
*fp
, *end
;
278 const struct skb_shared_info
*si
;
280 dma_unmap_single(dev
, *addr
++, skb_headlen(skb
), DMA_TO_DEVICE
);
282 si
= skb_shinfo(skb
);
283 end
= &si
->frags
[si
->nr_frags
];
284 for (fp
= si
->frags
; fp
< end
; fp
++)
285 dma_unmap_page(dev
, *addr
++, skb_frag_size(fp
), DMA_TO_DEVICE
);
288 #ifdef CONFIG_NEED_DMA_MAP_STATE
290 * deferred_unmap_destructor - unmap a packet when it is freed
293 * This is the packet destructor used for Tx packets that need to remain
294 * mapped until they are freed rather than until their Tx descriptors are
297 static void deferred_unmap_destructor(struct sk_buff
*skb
)
299 unmap_skb(skb
->dev
->dev
.parent
, skb
, (dma_addr_t
*)skb
->head
);
304 * free_tx_desc - reclaims Tx descriptors and their buffers
305 * @adapter: the adapter
306 * @q: the Tx queue to reclaim descriptors from
307 * @n: the number of descriptors to reclaim
308 * @unmap: whether the buffers should be unmapped for DMA
310 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
311 * Tx buffers. Called with the Tx queue lock held.
313 void free_tx_desc(struct adapter
*adap
, struct sge_txq
*q
,
314 unsigned int n
, bool unmap
)
316 unsigned int cidx
= q
->cidx
;
317 struct tx_sw_desc
*d
;
321 if (d
->skb
) { /* an SGL is present */
322 if (unmap
&& d
->addr
[0]) {
323 unmap_skb(adap
->pdev_dev
, d
->skb
, d
->addr
);
324 memset(d
->addr
, 0, sizeof(d
->addr
));
326 dev_consume_skb_any(d
->skb
);
330 if (++cidx
== q
->size
) {
339 * Return the number of reclaimable descriptors in a Tx queue.
341 static inline int reclaimable(const struct sge_txq
*q
)
343 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
345 return hw_cidx
< 0 ? hw_cidx
+ q
->size
: hw_cidx
;
349 * reclaim_completed_tx - reclaims completed TX Descriptors
351 * @q: the Tx queue to reclaim completed descriptors from
352 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
353 * @unmap: whether the buffers should be unmapped for DMA
355 * Reclaims Tx Descriptors that the SGE has indicated it has processed,
356 * and frees the associated buffers if possible. If @max == -1, then
357 * we'll use a defaiult maximum. Called with the TX Queue locked.
359 static inline int reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
360 int maxreclaim
, bool unmap
)
362 int reclaim
= reclaimable(q
);
366 * Limit the amount of clean up work we do at a time to keep
367 * the Tx lock hold time O(1).
370 maxreclaim
= MAX_TX_RECLAIM
;
371 if (reclaim
> maxreclaim
)
372 reclaim
= maxreclaim
;
374 free_tx_desc(adap
, q
, reclaim
, unmap
);
375 q
->in_use
-= reclaim
;
382 * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
384 * @q: the Tx queue to reclaim completed descriptors from
385 * @unmap: whether the buffers should be unmapped for DMA
387 * Reclaims Tx descriptors that the SGE has indicated it has processed,
388 * and frees the associated buffers if possible. Called with the Tx
391 void cxgb4_reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
394 (void)reclaim_completed_tx(adap
, q
, -1, unmap
);
396 EXPORT_SYMBOL(cxgb4_reclaim_completed_tx
);
398 static inline int get_buf_size(struct adapter
*adapter
,
399 const struct rx_sw_desc
*d
)
401 struct sge
*s
= &adapter
->sge
;
402 unsigned int rx_buf_size_idx
= d
->dma_addr
& RX_BUF_SIZE
;
405 switch (rx_buf_size_idx
) {
406 case RX_SMALL_PG_BUF
:
407 buf_size
= PAGE_SIZE
;
410 case RX_LARGE_PG_BUF
:
411 buf_size
= PAGE_SIZE
<< s
->fl_pg_order
;
414 case RX_SMALL_MTU_BUF
:
415 buf_size
= FL_MTU_SMALL_BUFSIZE(adapter
);
418 case RX_LARGE_MTU_BUF
:
419 buf_size
= FL_MTU_LARGE_BUFSIZE(adapter
);
430 * free_rx_bufs - free the Rx buffers on an SGE free list
432 * @q: the SGE free list to free buffers from
433 * @n: how many buffers to free
435 * Release the next @n buffers on an SGE free-buffer Rx queue. The
436 * buffers must be made inaccessible to HW before calling this function.
438 static void free_rx_bufs(struct adapter
*adap
, struct sge_fl
*q
, int n
)
441 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
443 if (is_buf_mapped(d
))
444 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
445 get_buf_size(adap
, d
),
449 if (++q
->cidx
== q
->size
)
456 * unmap_rx_buf - unmap the current Rx buffer on an SGE free list
458 * @q: the SGE free list
460 * Unmap the current buffer on an SGE free-buffer Rx queue. The
461 * buffer must be made inaccessible to HW before calling this function.
463 * This is similar to @free_rx_bufs above but does not free the buffer.
464 * Do note that the FL still loses any further access to the buffer.
466 static void unmap_rx_buf(struct adapter
*adap
, struct sge_fl
*q
)
468 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
470 if (is_buf_mapped(d
))
471 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
472 get_buf_size(adap
, d
), PCI_DMA_FROMDEVICE
);
474 if (++q
->cidx
== q
->size
)
479 static inline void ring_fl_db(struct adapter
*adap
, struct sge_fl
*q
)
481 if (q
->pend_cred
>= 8) {
482 u32 val
= adap
->params
.arch
.sge_fl_db
;
484 if (is_t4(adap
->params
.chip
))
485 val
|= PIDX_V(q
->pend_cred
/ 8);
487 val
|= PIDX_T5_V(q
->pend_cred
/ 8);
489 /* Make sure all memory writes to the Free List queue are
490 * committed before we tell the hardware about them.
494 /* If we don't have access to the new User Doorbell (T5+), use
495 * the old doorbell mechanism; otherwise use the new BAR2
498 if (unlikely(q
->bar2_addr
== NULL
)) {
499 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
500 val
| QID_V(q
->cntxt_id
));
502 writel(val
| QID_V(q
->bar2_qid
),
503 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
505 /* This Write memory Barrier will force the write to
506 * the User Doorbell area to be flushed.
514 static inline void set_rx_sw_desc(struct rx_sw_desc
*sd
, struct page
*pg
,
518 sd
->dma_addr
= mapping
; /* includes size low bits */
522 * refill_fl - refill an SGE Rx buffer ring
524 * @q: the ring to refill
525 * @n: the number of new buffers to allocate
526 * @gfp: the gfp flags for the allocations
528 * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
529 * allocated with the supplied gfp flags. The caller must assure that
530 * @n does not exceed the queue's capacity. If afterwards the queue is
531 * found critically low mark it as starving in the bitmap of starving FLs.
533 * Returns the number of buffers allocated.
535 static unsigned int refill_fl(struct adapter
*adap
, struct sge_fl
*q
, int n
,
538 struct sge
*s
= &adap
->sge
;
541 unsigned int cred
= q
->avail
;
542 __be64
*d
= &q
->desc
[q
->pidx
];
543 struct rx_sw_desc
*sd
= &q
->sdesc
[q
->pidx
];
546 #ifdef CONFIG_DEBUG_FS
547 if (test_bit(q
->cntxt_id
- adap
->sge
.egr_start
, adap
->sge
.blocked_fl
))
552 node
= dev_to_node(adap
->pdev_dev
);
554 if (s
->fl_pg_order
== 0)
555 goto alloc_small_pages
;
558 * Prefer large buffers
561 pg
= alloc_pages_node(node
, gfp
| __GFP_COMP
, s
->fl_pg_order
);
563 q
->large_alloc_failed
++;
564 break; /* fall back to single pages */
567 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0,
568 PAGE_SIZE
<< s
->fl_pg_order
,
570 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
571 __free_pages(pg
, s
->fl_pg_order
);
573 goto out
; /* do not try small pages for this error */
575 mapping
|= RX_LARGE_PG_BUF
;
576 *d
++ = cpu_to_be64(mapping
);
578 set_rx_sw_desc(sd
, pg
, mapping
);
582 if (++q
->pidx
== q
->size
) {
592 pg
= alloc_pages_node(node
, gfp
, 0);
598 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0, PAGE_SIZE
,
600 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
605 *d
++ = cpu_to_be64(mapping
);
607 set_rx_sw_desc(sd
, pg
, mapping
);
611 if (++q
->pidx
== q
->size
) {
618 out
: cred
= q
->avail
- cred
;
619 q
->pend_cred
+= cred
;
622 if (unlikely(fl_starving(adap
, q
))) {
625 set_bit(q
->cntxt_id
- adap
->sge
.egr_start
,
626 adap
->sge
.starving_fl
);
632 static inline void __refill_fl(struct adapter
*adap
, struct sge_fl
*fl
)
634 refill_fl(adap
, fl
, min(MAX_RX_REFILL
, fl_cap(fl
) - fl
->avail
),
639 * alloc_ring - allocate resources for an SGE descriptor ring
640 * @dev: the PCI device's core device
641 * @nelem: the number of descriptors
642 * @elem_size: the size of each descriptor
643 * @sw_size: the size of the SW state associated with each ring element
644 * @phys: the physical address of the allocated ring
645 * @metadata: address of the array holding the SW state for the ring
646 * @stat_size: extra space in HW ring for status information
647 * @node: preferred node for memory allocations
649 * Allocates resources for an SGE descriptor ring, such as Tx queues,
650 * free buffer lists, or response queues. Each SGE ring requires
651 * space for its HW descriptors plus, optionally, space for the SW state
652 * associated with each HW entry (the metadata). The function returns
653 * three values: the virtual address for the HW ring (the return value
654 * of the function), the bus address of the HW ring, and the address
657 static void *alloc_ring(struct device
*dev
, size_t nelem
, size_t elem_size
,
658 size_t sw_size
, dma_addr_t
*phys
, void *metadata
,
659 size_t stat_size
, int node
)
661 size_t len
= nelem
* elem_size
+ stat_size
;
663 void *p
= dma_alloc_coherent(dev
, len
, phys
, GFP_KERNEL
);
668 s
= kcalloc_node(sw_size
, nelem
, GFP_KERNEL
, node
);
671 dma_free_coherent(dev
, len
, p
, *phys
);
676 *(void **)metadata
= s
;
681 * sgl_len - calculates the size of an SGL of the given capacity
682 * @n: the number of SGL entries
684 * Calculates the number of flits needed for a scatter/gather list that
685 * can hold the given number of entries.
687 static inline unsigned int sgl_len(unsigned int n
)
689 /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
690 * addresses. The DSGL Work Request starts off with a 32-bit DSGL
691 * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
692 * repeated sequences of { Length[i], Length[i+1], Address[i],
693 * Address[i+1] } (this ensures that all addresses are on 64-bit
694 * boundaries). If N is even, then Length[N+1] should be set to 0 and
695 * Address[N+1] is omitted.
697 * The following calculation incorporates all of the above. It's
698 * somewhat hard to follow but, briefly: the "+2" accounts for the
699 * first two flits which include the DSGL header, Length0 and
700 * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
701 * flits for every pair of the remaining N) +1 if (n-1) is odd; and
702 * finally the "+((n-1)&1)" adds the one remaining flit needed if
706 return (3 * n
) / 2 + (n
& 1) + 2;
710 * flits_to_desc - returns the num of Tx descriptors for the given flits
711 * @n: the number of flits
713 * Returns the number of Tx descriptors needed for the supplied number
716 static inline unsigned int flits_to_desc(unsigned int n
)
718 BUG_ON(n
> SGE_MAX_WR_LEN
/ 8);
719 return DIV_ROUND_UP(n
, 8);
723 * is_eth_imm - can an Ethernet packet be sent as immediate data?
726 * Returns whether an Ethernet packet is small enough to fit as
727 * immediate data. Return value corresponds to headroom required.
729 static inline int is_eth_imm(const struct sk_buff
*skb
, unsigned int chip_ver
)
733 if (skb
->encapsulation
&& skb_shinfo(skb
)->gso_size
&&
734 chip_ver
> CHELSIO_T5
) {
735 hdrlen
= sizeof(struct cpl_tx_tnl_lso
);
736 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
737 } else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
) {
740 hdrlen
= skb_shinfo(skb
)->gso_size
?
741 sizeof(struct cpl_tx_pkt_lso_core
) : 0;
742 hdrlen
+= sizeof(struct cpl_tx_pkt
);
744 if (skb
->len
<= MAX_IMM_TX_PKT_LEN
- hdrlen
)
750 * calc_tx_flits - calculate the number of flits for a packet Tx WR
753 * Returns the number of flits needed for a Tx WR for the given Ethernet
754 * packet, including the needed WR and CPL headers.
756 static inline unsigned int calc_tx_flits(const struct sk_buff
*skb
,
757 unsigned int chip_ver
)
760 int hdrlen
= is_eth_imm(skb
, chip_ver
);
762 /* If the skb is small enough, we can pump it out as a work request
763 * with only immediate data. In that case we just have to have the
764 * TX Packet header plus the skb data in the Work Request.
768 return DIV_ROUND_UP(skb
->len
+ hdrlen
, sizeof(__be64
));
770 /* Otherwise, we're going to have to construct a Scatter gather list
771 * of the skb body and fragments. We also include the flits necessary
772 * for the TX Packet Work Request and CPL. We always have a firmware
773 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
774 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
775 * message or, if we're doing a Large Send Offload, an LSO CPL message
776 * with an embedded TX Packet Write CPL message.
778 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
779 if (skb_shinfo(skb
)->gso_size
) {
780 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
) {
781 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
782 sizeof(struct cpl_tx_tnl_lso
);
783 } else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
) {
786 pkt_hdrlen
= eth_get_headlen(skb
->dev
, skb
->data
,
788 hdrlen
= sizeof(struct fw_eth_tx_eo_wr
) +
789 round_up(pkt_hdrlen
, 16);
791 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
792 sizeof(struct cpl_tx_pkt_lso_core
);
795 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
796 flits
+= (hdrlen
/ sizeof(__be64
));
798 flits
+= (sizeof(struct fw_eth_tx_pkt_wr
) +
799 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
805 * calc_tx_descs - calculate the number of Tx descriptors for a packet
808 * Returns the number of Tx descriptors needed for the given Ethernet
809 * packet, including the needed WR and CPL headers.
811 static inline unsigned int calc_tx_descs(const struct sk_buff
*skb
,
812 unsigned int chip_ver
)
814 return flits_to_desc(calc_tx_flits(skb
, chip_ver
));
818 * cxgb4_write_sgl - populate a scatter/gather list for a packet
820 * @q: the Tx queue we are writing into
821 * @sgl: starting location for writing the SGL
822 * @end: points right after the end of the SGL
823 * @start: start offset into skb main-body data to include in the SGL
824 * @addr: the list of bus addresses for the SGL elements
826 * Generates a gather list for the buffers that make up a packet.
827 * The caller must provide adequate space for the SGL that will be written.
828 * The SGL includes all of the packet's page fragments and the data in its
829 * main body except for the first @start bytes. @sgl must be 16-byte
830 * aligned and within a Tx descriptor with available space. @end points
831 * right after the end of the SGL but does not account for any potential
832 * wrap around, i.e., @end > @sgl.
834 void cxgb4_write_sgl(const struct sk_buff
*skb
, struct sge_txq
*q
,
835 struct ulptx_sgl
*sgl
, u64
*end
, unsigned int start
,
836 const dma_addr_t
*addr
)
839 struct ulptx_sge_pair
*to
;
840 const struct skb_shared_info
*si
= skb_shinfo(skb
);
841 unsigned int nfrags
= si
->nr_frags
;
842 struct ulptx_sge_pair buf
[MAX_SKB_FRAGS
/ 2 + 1];
844 len
= skb_headlen(skb
) - start
;
846 sgl
->len0
= htonl(len
);
847 sgl
->addr0
= cpu_to_be64(addr
[0] + start
);
850 sgl
->len0
= htonl(skb_frag_size(&si
->frags
[0]));
851 sgl
->addr0
= cpu_to_be64(addr
[1]);
854 sgl
->cmd_nsge
= htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
855 ULPTX_NSGE_V(nfrags
));
856 if (likely(--nfrags
== 0))
859 * Most of the complexity below deals with the possibility we hit the
860 * end of the queue in the middle of writing the SGL. For this case
861 * only we create the SGL in a temporary buffer and then copy it.
863 to
= (u8
*)end
> (u8
*)q
->stat
? buf
: sgl
->sge
;
865 for (i
= (nfrags
!= si
->nr_frags
); nfrags
>= 2; nfrags
-= 2, to
++) {
866 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
867 to
->len
[1] = cpu_to_be32(skb_frag_size(&si
->frags
[++i
]));
868 to
->addr
[0] = cpu_to_be64(addr
[i
]);
869 to
->addr
[1] = cpu_to_be64(addr
[++i
]);
872 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
873 to
->len
[1] = cpu_to_be32(0);
874 to
->addr
[0] = cpu_to_be64(addr
[i
+ 1]);
876 if (unlikely((u8
*)end
> (u8
*)q
->stat
)) {
877 unsigned int part0
= (u8
*)q
->stat
- (u8
*)sgl
->sge
, part1
;
880 memcpy(sgl
->sge
, buf
, part0
);
881 part1
= (u8
*)end
- (u8
*)q
->stat
;
882 memcpy(q
->desc
, (u8
*)buf
+ part0
, part1
);
883 end
= (void *)q
->desc
+ part1
;
885 if ((uintptr_t)end
& 8) /* 0-pad to multiple of 16 */
888 EXPORT_SYMBOL(cxgb4_write_sgl
);
890 /* This function copies 64 byte coalesced work request to
891 * memory mapped BAR2 space. For coalesced WR SGE fetches
892 * data from the FIFO instead of from Host.
894 static void cxgb_pio_copy(u64 __iomem
*dst
, u64
*src
)
907 * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell
910 * @n: number of new descriptors to give to HW
912 * Ring the doorbel for a Tx queue.
914 inline void cxgb4_ring_tx_db(struct adapter
*adap
, struct sge_txq
*q
, int n
)
916 /* Make sure that all writes to the TX Descriptors are committed
917 * before we tell the hardware about them.
921 /* If we don't have access to the new User Doorbell (T5+), use the old
922 * doorbell mechanism; otherwise use the new BAR2 mechanism.
924 if (unlikely(q
->bar2_addr
== NULL
)) {
928 /* For T4 we need to participate in the Doorbell Recovery
931 spin_lock_irqsave(&q
->db_lock
, flags
);
933 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
934 QID_V(q
->cntxt_id
) | val
);
937 q
->db_pidx
= q
->pidx
;
938 spin_unlock_irqrestore(&q
->db_lock
, flags
);
940 u32 val
= PIDX_T5_V(n
);
942 /* T4 and later chips share the same PIDX field offset within
943 * the doorbell, but T5 and later shrank the field in order to
944 * gain a bit for Doorbell Priority. The field was absurdly
945 * large in the first place (14 bits) so we just use the T5
946 * and later limits and warn if a Queue ID is too large.
948 WARN_ON(val
& DBPRIO_F
);
950 /* If we're only writing a single TX Descriptor and we can use
951 * Inferred QID registers, we can use the Write Combining
952 * Gather Buffer; otherwise we use the simple doorbell.
954 if (n
== 1 && q
->bar2_qid
== 0) {
958 u64
*wr
= (u64
*)&q
->desc
[index
];
960 cxgb_pio_copy((u64 __iomem
*)
961 (q
->bar2_addr
+ SGE_UDB_WCDOORBELL
),
964 writel(val
| QID_V(q
->bar2_qid
),
965 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
968 /* This Write Memory Barrier will force the write to the User
969 * Doorbell area to be flushed. This is needed to prevent
970 * writes on different CPUs for the same queue from hitting
971 * the adapter out of order. This is required when some Work
972 * Requests take the Write Combine Gather Buffer path (user
973 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
974 * take the traditional path where we simply increment the
975 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
976 * hardware DMA read the actual Work Request.
981 EXPORT_SYMBOL(cxgb4_ring_tx_db
);
984 * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors
986 * @q: the Tx queue where the packet will be inlined
987 * @pos: starting position in the Tx queue where to inline the packet
989 * Inline a packet's contents directly into Tx descriptors, starting at
990 * the given position within the Tx DMA ring.
991 * Most of the complexity of this operation is dealing with wrap arounds
992 * in the middle of the packet we want to inline.
994 void cxgb4_inline_tx_skb(const struct sk_buff
*skb
,
995 const struct sge_txq
*q
, void *pos
)
997 int left
= (void *)q
->stat
- pos
;
1000 if (likely(skb
->len
<= left
)) {
1001 if (likely(!skb
->data_len
))
1002 skb_copy_from_linear_data(skb
, pos
, skb
->len
);
1004 skb_copy_bits(skb
, 0, pos
, skb
->len
);
1007 skb_copy_bits(skb
, 0, pos
, left
);
1008 skb_copy_bits(skb
, left
, q
->desc
, skb
->len
- left
);
1009 pos
= (void *)q
->desc
+ (skb
->len
- left
);
1012 /* 0-pad to multiple of 16 */
1013 p
= PTR_ALIGN(pos
, 8);
1014 if ((uintptr_t)p
& 8)
1017 EXPORT_SYMBOL(cxgb4_inline_tx_skb
);
1019 static void *inline_tx_skb_header(const struct sk_buff
*skb
,
1020 const struct sge_txq
*q
, void *pos
,
1024 int left
= (void *)q
->stat
- pos
;
1026 if (likely(length
<= left
)) {
1027 memcpy(pos
, skb
->data
, length
);
1030 memcpy(pos
, skb
->data
, left
);
1031 memcpy(q
->desc
, skb
->data
+ left
, length
- left
);
1032 pos
= (void *)q
->desc
+ (length
- left
);
1034 /* 0-pad to multiple of 16 */
1035 p
= PTR_ALIGN(pos
, 8);
1036 if ((uintptr_t)p
& 8) {
1044 * Figure out what HW csum a packet wants and return the appropriate control
1047 static u64
hwcsum(enum chip_type chip
, const struct sk_buff
*skb
)
1050 bool inner_hdr_csum
= false;
1053 if (skb
->encapsulation
&&
1054 (CHELSIO_CHIP_VERSION(chip
) > CHELSIO_T5
))
1055 inner_hdr_csum
= true;
1057 if (inner_hdr_csum
) {
1058 ver
= inner_ip_hdr(skb
)->version
;
1059 proto
= (ver
== 4) ? inner_ip_hdr(skb
)->protocol
:
1060 inner_ipv6_hdr(skb
)->nexthdr
;
1062 ver
= ip_hdr(skb
)->version
;
1063 proto
= (ver
== 4) ? ip_hdr(skb
)->protocol
:
1064 ipv6_hdr(skb
)->nexthdr
;
1068 if (proto
== IPPROTO_TCP
)
1069 csum_type
= TX_CSUM_TCPIP
;
1070 else if (proto
== IPPROTO_UDP
)
1071 csum_type
= TX_CSUM_UDPIP
;
1074 * unknown protocol, disable HW csum
1075 * and hope a bad packet is detected
1077 return TXPKT_L4CSUM_DIS_F
;
1081 * this doesn't work with extension headers
1083 if (proto
== IPPROTO_TCP
)
1084 csum_type
= TX_CSUM_TCPIP6
;
1085 else if (proto
== IPPROTO_UDP
)
1086 csum_type
= TX_CSUM_UDPIP6
;
1091 if (likely(csum_type
>= TX_CSUM_TCPIP
)) {
1092 int eth_hdr_len
, l4_len
;
1095 if (inner_hdr_csum
) {
1096 /* This allows checksum offload for all encapsulated
1097 * packets like GRE etc..
1099 l4_len
= skb_inner_network_header_len(skb
);
1100 eth_hdr_len
= skb_inner_network_offset(skb
) - ETH_HLEN
;
1102 l4_len
= skb_network_header_len(skb
);
1103 eth_hdr_len
= skb_network_offset(skb
) - ETH_HLEN
;
1105 hdr_len
= TXPKT_IPHDR_LEN_V(l4_len
);
1107 if (CHELSIO_CHIP_VERSION(chip
) <= CHELSIO_T5
)
1108 hdr_len
|= TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1110 hdr_len
|= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1111 return TXPKT_CSUM_TYPE_V(csum_type
) | hdr_len
;
1113 int start
= skb_transport_offset(skb
);
1115 return TXPKT_CSUM_TYPE_V(csum_type
) |
1116 TXPKT_CSUM_START_V(start
) |
1117 TXPKT_CSUM_LOC_V(start
+ skb
->csum_offset
);
1121 static void eth_txq_stop(struct sge_eth_txq
*q
)
1123 netif_tx_stop_queue(q
->txq
);
1127 static inline void txq_advance(struct sge_txq
*q
, unsigned int n
)
1131 if (q
->pidx
>= q
->size
)
1135 #ifdef CONFIG_CHELSIO_T4_FCOE
1137 cxgb_fcoe_offload(struct sk_buff
*skb
, struct adapter
*adap
,
1138 const struct port_info
*pi
, u64
*cntrl
)
1140 const struct cxgb_fcoe
*fcoe
= &pi
->fcoe
;
1142 if (!(fcoe
->flags
& CXGB_FCOE_ENABLED
))
1145 if (skb
->protocol
!= htons(ETH_P_FCOE
))
1148 skb_reset_mac_header(skb
);
1149 skb
->mac_len
= sizeof(struct ethhdr
);
1151 skb_set_network_header(skb
, skb
->mac_len
);
1152 skb_set_transport_header(skb
, skb
->mac_len
+ sizeof(struct fcoe_hdr
));
1154 if (!cxgb_fcoe_sof_eof_supported(adap
, skb
))
1157 /* FC CRC offload */
1158 *cntrl
= TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE
) |
1159 TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
|
1160 TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START
) |
1161 TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END
) |
1162 TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END
);
1165 #endif /* CONFIG_CHELSIO_T4_FCOE */
1167 /* Returns tunnel type if hardware supports offloading of the same.
1168 * It is called only for T5 and onwards.
1170 enum cpl_tx_tnl_lso_type
cxgb_encap_offload_supported(struct sk_buff
*skb
)
1173 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1174 struct port_info
*pi
= netdev_priv(skb
->dev
);
1175 struct adapter
*adapter
= pi
->adapter
;
1177 if (skb
->inner_protocol_type
!= ENCAP_TYPE_ETHER
||
1178 skb
->inner_protocol
!= htons(ETH_P_TEB
))
1181 switch (vlan_get_protocol(skb
)) {
1182 case htons(ETH_P_IP
):
1183 l4_hdr
= ip_hdr(skb
)->protocol
;
1185 case htons(ETH_P_IPV6
):
1186 l4_hdr
= ipv6_hdr(skb
)->nexthdr
;
1194 if (adapter
->vxlan_port
== udp_hdr(skb
)->dest
)
1195 tnl_type
= TX_TNL_TYPE_VXLAN
;
1196 else if (adapter
->geneve_port
== udp_hdr(skb
)->dest
)
1197 tnl_type
= TX_TNL_TYPE_GENEVE
;
1206 static inline void t6_fill_tnl_lso(struct sk_buff
*skb
,
1207 struct cpl_tx_tnl_lso
*tnl_lso
,
1208 enum cpl_tx_tnl_lso_type tnl_type
)
1211 int in_eth_xtra_len
;
1212 int l3hdr_len
= skb_network_header_len(skb
);
1213 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1214 const struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1215 bool v6
= (ip_hdr(skb
)->version
== 6);
1217 val
= CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO
) |
1218 CPL_TX_TNL_LSO_FIRST_F
|
1219 CPL_TX_TNL_LSO_LAST_F
|
1220 (v6
? CPL_TX_TNL_LSO_IPV6OUT_F
: 0) |
1221 CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len
/ 4) |
1222 CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len
/ 4) |
1223 (v6
? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F
) |
1224 CPL_TX_TNL_LSO_IPLENSETOUT_F
|
1225 (v6
? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F
);
1226 tnl_lso
->op_to_IpIdSplitOut
= htonl(val
);
1228 tnl_lso
->IpIdOffsetOut
= 0;
1230 /* Get the tunnel header length */
1231 val
= skb_inner_mac_header(skb
) - skb_mac_header(skb
);
1232 in_eth_xtra_len
= skb_inner_network_header(skb
) -
1233 skb_inner_mac_header(skb
) - ETH_HLEN
;
1236 case TX_TNL_TYPE_VXLAN
:
1237 case TX_TNL_TYPE_GENEVE
:
1238 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
=
1239 htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F
|
1240 CPL_TX_TNL_LSO_UDPLENSETOUT_F
);
1243 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
= 0;
1247 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
|=
1248 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val
) |
1249 CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type
));
1253 val
= CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len
/ 4) |
1254 CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb
)->version
== 6) |
1255 CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb
) / 4) |
1256 CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb
) / 4);
1257 tnl_lso
->Flow_to_TcpHdrLen
= htonl(val
);
1259 tnl_lso
->IpIdOffset
= htons(0);
1261 tnl_lso
->IpIdSplit_to_Mss
= htons(CPL_TX_TNL_LSO_MSS_V(ssi
->gso_size
));
1262 tnl_lso
->TCPSeqOffset
= htonl(0);
1263 tnl_lso
->EthLenOffset_Size
= htonl(CPL_TX_TNL_LSO_SIZE_V(skb
->len
));
1266 static inline void *write_tso_wr(struct adapter
*adap
, struct sk_buff
*skb
,
1267 struct cpl_tx_pkt_lso_core
*lso
)
1269 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1270 int l3hdr_len
= skb_network_header_len(skb
);
1271 const struct skb_shared_info
*ssi
;
1274 ssi
= skb_shinfo(skb
);
1275 if (ssi
->gso_type
& SKB_GSO_TCPV6
)
1278 lso
->lso_ctrl
= htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1279 LSO_FIRST_SLICE_F
| LSO_LAST_SLICE_F
|
1281 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1282 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1283 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1284 lso
->ipid_ofst
= htons(0);
1285 lso
->mss
= htons(ssi
->gso_size
);
1286 lso
->seqno_offset
= htonl(0);
1287 if (is_t4(adap
->params
.chip
))
1288 lso
->len
= htonl(skb
->len
);
1290 lso
->len
= htonl(LSO_T5_XFER_SIZE_V(skb
->len
));
1292 return (void *)(lso
+ 1);
1296 * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update
1297 * @adap: the adapter
1298 * @eq: the Ethernet TX Queue
1299 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
1301 * We're typically called here to update the state of an Ethernet TX
1302 * Queue with respect to the hardware's progress in consuming the TX
1303 * Work Requests that we've put on that Egress Queue. This happens
1304 * when we get Egress Queue Update messages and also prophylactically
1305 * in regular timer-based Ethernet TX Queue maintenance.
1307 int t4_sge_eth_txq_egress_update(struct adapter
*adap
, struct sge_eth_txq
*eq
,
1310 unsigned int reclaimed
, hw_cidx
;
1311 struct sge_txq
*q
= &eq
->q
;
1314 if (!q
->in_use
|| !__netif_tx_trylock(eq
->txq
))
1317 /* Reclaim pending completed TX Descriptors. */
1318 reclaimed
= reclaim_completed_tx(adap
, &eq
->q
, maxreclaim
, true);
1320 hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
1321 hw_in_use
= q
->pidx
- hw_cidx
;
1323 hw_in_use
+= q
->size
;
1325 /* If the TX Queue is currently stopped and there's now more than half
1326 * the queue available, restart it. Otherwise bail out since the rest
1327 * of what we want do here is with the possibility of shipping any
1328 * currently buffered Coalesced TX Work Request.
1330 if (netif_tx_queue_stopped(eq
->txq
) && hw_in_use
< (q
->size
/ 2)) {
1331 netif_tx_wake_queue(eq
->txq
);
1335 __netif_tx_unlock(eq
->txq
);
1339 static inline int cxgb4_validate_skb(struct sk_buff
*skb
,
1340 struct net_device
*dev
,
1345 /* The chip min packet length is 10 octets but some firmware
1346 * commands have a minimum packet length requirement. So, play
1347 * safe and reject anything shorter than @min_pkt_len.
1349 if (unlikely(skb
->len
< min_pkt_len
))
1352 /* Discard the packet if the length is greater than mtu */
1353 max_pkt_len
= ETH_HLEN
+ dev
->mtu
;
1355 if (skb_vlan_tagged(skb
))
1356 max_pkt_len
+= VLAN_HLEN
;
1358 if (!skb_shinfo(skb
)->gso_size
&& (unlikely(skb
->len
> max_pkt_len
)))
1364 static void *write_eo_udp_wr(struct sk_buff
*skb
, struct fw_eth_tx_eo_wr
*wr
,
1367 wr
->u
.udpseg
.type
= FW_ETH_TX_EO_TYPE_UDPSEG
;
1368 wr
->u
.udpseg
.ethlen
= skb_network_offset(skb
);
1369 wr
->u
.udpseg
.iplen
= cpu_to_be16(skb_network_header_len(skb
));
1370 wr
->u
.udpseg
.udplen
= sizeof(struct udphdr
);
1371 wr
->u
.udpseg
.rtplen
= 0;
1372 wr
->u
.udpseg
.r4
= 0;
1373 if (skb_shinfo(skb
)->gso_size
)
1374 wr
->u
.udpseg
.mss
= cpu_to_be16(skb_shinfo(skb
)->gso_size
);
1376 wr
->u
.udpseg
.mss
= cpu_to_be16(skb
->len
- hdr_len
);
1377 wr
->u
.udpseg
.schedpktsize
= wr
->u
.udpseg
.mss
;
1378 wr
->u
.udpseg
.plen
= cpu_to_be32(skb
->len
- hdr_len
);
1380 return (void *)(wr
+ 1);
1384 * cxgb4_eth_xmit - add a packet to an Ethernet Tx queue
1386 * @dev: the egress net device
1388 * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled.
1390 static netdev_tx_t
cxgb4_eth_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1392 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1393 bool ptp_enabled
= is_ptp_enabled(skb
, dev
);
1394 unsigned int last_desc
, flits
, ndesc
;
1395 u32 wr_mid
, ctrl0
, op
, sgl_off
= 0;
1396 const struct skb_shared_info
*ssi
;
1397 int len
, qidx
, credits
, ret
, left
;
1398 struct tx_sw_desc
*sgl_sdesc
;
1399 struct fw_eth_tx_eo_wr
*eowr
;
1400 struct fw_eth_tx_pkt_wr
*wr
;
1401 struct cpl_tx_pkt_core
*cpl
;
1402 const struct port_info
*pi
;
1403 bool immediate
= false;
1404 u64 cntrl
, *end
, *sgl
;
1405 struct sge_eth_txq
*q
;
1406 unsigned int chip_ver
;
1407 struct adapter
*adap
;
1409 ret
= cxgb4_validate_skb(skb
, dev
, ETH_HLEN
);
1413 pi
= netdev_priv(dev
);
1415 ssi
= skb_shinfo(skb
);
1416 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
1417 if (xfrm_offload(skb
) && !ssi
->gso_size
)
1418 return adap
->uld
[CXGB4_ULD_CRYPTO
].tx_handler(skb
, dev
);
1419 #endif /* CHELSIO_IPSEC_INLINE */
1421 #ifdef CONFIG_CHELSIO_TLS_DEVICE
1423 return adap
->uld
[CXGB4_ULD_CRYPTO
].tx_handler(skb
, dev
);
1424 #endif /* CHELSIO_TLS_DEVICE */
1426 qidx
= skb_get_queue_mapping(skb
);
1428 spin_lock(&adap
->ptp_lock
);
1429 if (!(adap
->ptp_tx_skb
)) {
1430 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
1431 adap
->ptp_tx_skb
= skb_get(skb
);
1433 spin_unlock(&adap
->ptp_lock
);
1436 q
= &adap
->sge
.ptptxq
;
1438 q
= &adap
->sge
.ethtxq
[qidx
+ pi
->first_qset
];
1440 skb_tx_timestamp(skb
);
1442 reclaim_completed_tx(adap
, &q
->q
, -1, true);
1443 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1445 #ifdef CONFIG_CHELSIO_T4_FCOE
1446 ret
= cxgb_fcoe_offload(skb
, adap
, pi
, &cntrl
);
1447 if (unlikely(ret
== -ENOTSUPP
)) {
1449 spin_unlock(&adap
->ptp_lock
);
1452 #endif /* CONFIG_CHELSIO_T4_FCOE */
1454 chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
1455 flits
= calc_tx_flits(skb
, chip_ver
);
1456 ndesc
= flits_to_desc(flits
);
1457 credits
= txq_avail(&q
->q
) - ndesc
;
1459 if (unlikely(credits
< 0)) {
1461 dev_err(adap
->pdev_dev
,
1462 "%s: Tx ring %u full while queue awake!\n",
1465 spin_unlock(&adap
->ptp_lock
);
1466 return NETDEV_TX_BUSY
;
1469 if (is_eth_imm(skb
, chip_ver
))
1472 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
)
1473 tnl_type
= cxgb_encap_offload_supported(skb
);
1475 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1476 if (last_desc
>= q
->q
.size
)
1477 last_desc
-= q
->q
.size
;
1478 sgl_sdesc
= &q
->q
.sdesc
[last_desc
];
1481 unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, sgl_sdesc
->addr
) < 0)) {
1482 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1485 spin_unlock(&adap
->ptp_lock
);
1489 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1490 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1491 /* After we're done injecting the Work Request for this
1492 * packet, we'll be below our "stop threshold" so stop the TX
1493 * Queue now and schedule a request for an SGE Egress Queue
1494 * Update message. The queue will get started later on when
1495 * the firmware processes this Work Request and sends us an
1496 * Egress Queue Status Update message indicating that space
1500 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1503 wr
= (void *)&q
->q
.desc
[q
->q
.pidx
];
1504 eowr
= (void *)&q
->q
.desc
[q
->q
.pidx
];
1505 wr
->equiq_to_len16
= htonl(wr_mid
);
1506 wr
->r3
= cpu_to_be64(0);
1507 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
)
1508 end
= (u64
*)eowr
+ flits
;
1510 end
= (u64
*)wr
+ flits
;
1512 len
= immediate
? skb
->len
: 0;
1513 len
+= sizeof(*cpl
);
1514 if (ssi
->gso_size
&& !(ssi
->gso_type
& SKB_GSO_UDP_L4
)) {
1515 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1516 struct cpl_tx_tnl_lso
*tnl_lso
= (void *)(wr
+ 1);
1519 len
+= sizeof(*tnl_lso
);
1521 len
+= sizeof(*lso
);
1523 wr
->op_immdlen
= htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR
) |
1524 FW_WR_IMMDLEN_V(len
));
1526 struct iphdr
*iph
= ip_hdr(skb
);
1528 t6_fill_tnl_lso(skb
, tnl_lso
, tnl_type
);
1529 cpl
= (void *)(tnl_lso
+ 1);
1530 /* Driver is expected to compute partial checksum that
1531 * does not include the IP Total Length.
1533 if (iph
->version
== 4) {
1536 iph
->check
= (u16
)(~ip_fast_csum((u8
*)iph
,
1539 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1540 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1542 cpl
= write_tso_wr(adap
, skb
, lso
);
1543 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1545 sgl
= (u64
*)(cpl
+ 1); /* sgl start here */
1547 q
->tx_cso
+= ssi
->gso_segs
;
1548 } else if (ssi
->gso_size
) {
1552 hdrlen
= eth_get_headlen(dev
, skb
->data
, skb_headlen(skb
));
1554 wr
->op_immdlen
= cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR
) |
1555 FW_ETH_TX_EO_WR_IMMDLEN_V(len
));
1556 cpl
= write_eo_udp_wr(skb
, eowr
, hdrlen
);
1557 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1559 start
= (u64
*)(cpl
+ 1);
1560 sgl
= (u64
*)inline_tx_skb_header(skb
, &q
->q
, (void *)start
,
1562 if (unlikely(start
> sgl
)) {
1563 left
= (u8
*)end
- (u8
*)q
->q
.stat
;
1564 end
= (void *)q
->q
.desc
+ left
;
1568 q
->tx_cso
+= ssi
->gso_segs
;
1571 op
= FW_PTP_TX_PKT_WR
;
1573 op
= FW_ETH_TX_PKT_WR
;
1574 wr
->op_immdlen
= htonl(FW_WR_OP_V(op
) |
1575 FW_WR_IMMDLEN_V(len
));
1576 cpl
= (void *)(wr
+ 1);
1577 sgl
= (u64
*)(cpl
+ 1);
1578 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1579 cntrl
= hwcsum(adap
->params
.chip
, skb
) |
1585 if (unlikely((u8
*)sgl
>= (u8
*)q
->q
.stat
)) {
1586 /* If current position is already at the end of the
1587 * txq, reset the current to point to start of the queue
1588 * and update the end ptr as well.
1590 left
= (u8
*)end
- (u8
*)q
->q
.stat
;
1591 end
= (void *)q
->q
.desc
+ left
;
1592 sgl
= (void *)q
->q
.desc
;
1595 if (skb_vlan_tag_present(skb
)) {
1597 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1598 #ifdef CONFIG_CHELSIO_T4_FCOE
1599 if (skb
->protocol
== htons(ETH_P_FCOE
))
1600 cntrl
|= TXPKT_VLAN_V(
1601 ((skb
->priority
& 0x7) << VLAN_PRIO_SHIFT
));
1602 #endif /* CONFIG_CHELSIO_T4_FCOE */
1605 ctrl0
= TXPKT_OPCODE_V(CPL_TX_PKT_XT
) | TXPKT_INTF_V(pi
->tx_chan
) |
1606 TXPKT_PF_V(adap
->pf
);
1608 ctrl0
|= TXPKT_TSTAMP_F
;
1609 #ifdef CONFIG_CHELSIO_T4_DCB
1610 if (is_t4(adap
->params
.chip
))
1611 ctrl0
|= TXPKT_OVLAN_IDX_V(q
->dcb_prio
);
1613 ctrl0
|= TXPKT_T5_OVLAN_IDX_V(q
->dcb_prio
);
1615 cpl
->ctrl0
= htonl(ctrl0
);
1616 cpl
->pack
= htons(0);
1617 cpl
->len
= htons(skb
->len
);
1618 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1621 cxgb4_inline_tx_skb(skb
, &q
->q
, sgl
);
1622 dev_consume_skb_any(skb
);
1624 cxgb4_write_sgl(skb
, &q
->q
, (void *)sgl
, end
, sgl_off
,
1627 sgl_sdesc
->skb
= skb
;
1630 txq_advance(&q
->q
, ndesc
);
1632 cxgb4_ring_tx_db(adap
, &q
->q
, ndesc
);
1634 spin_unlock(&adap
->ptp_lock
);
1635 return NETDEV_TX_OK
;
1638 dev_kfree_skb_any(skb
);
1639 return NETDEV_TX_OK
;
1644 /* Egress Queue sizes, producer and consumer indices are all in units
1645 * of Egress Context Units bytes. Note that as far as the hardware is
1646 * concerned, the free list is an Egress Queue (the host produces free
1647 * buffers which the hardware consumes) and free list entries are
1648 * 64-bit PCI DMA addresses.
1650 EQ_UNIT
= SGE_EQ_IDXSIZE
,
1651 FL_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1652 TXD_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1654 T4VF_ETHTXQ_MAX_HDR
= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1655 sizeof(struct cpl_tx_pkt_lso_core
) +
1656 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
),
1660 * t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data?
1663 * Returns whether an Ethernet packet is small enough to fit completely as
1666 static inline int t4vf_is_eth_imm(const struct sk_buff
*skb
)
1668 /* The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
1669 * which does not accommodate immediate data. We could dike out all
1670 * of the support code for immediate data but that would tie our hands
1671 * too much if we ever want to enhace the firmware. It would also
1672 * create more differences between the PF and VF Drivers.
1678 * t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR
1681 * Returns the number of flits needed for a TX Work Request for the
1682 * given Ethernet packet, including the needed WR and CPL headers.
1684 static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff
*skb
)
1688 /* If the skb is small enough, we can pump it out as a work request
1689 * with only immediate data. In that case we just have to have the
1690 * TX Packet header plus the skb data in the Work Request.
1692 if (t4vf_is_eth_imm(skb
))
1693 return DIV_ROUND_UP(skb
->len
+ sizeof(struct cpl_tx_pkt
),
1696 /* Otherwise, we're going to have to construct a Scatter gather list
1697 * of the skb body and fragments. We also include the flits necessary
1698 * for the TX Packet Work Request and CPL. We always have a firmware
1699 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
1700 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
1701 * message or, if we're doing a Large Send Offload, an LSO CPL message
1702 * with an embedded TX Packet Write CPL message.
1704 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
1705 if (skb_shinfo(skb
)->gso_size
)
1706 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1707 sizeof(struct cpl_tx_pkt_lso_core
) +
1708 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1710 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1711 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1716 * cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue
1718 * @dev: the egress net device
1720 * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled.
1722 static netdev_tx_t
cxgb4_vf_eth_xmit(struct sk_buff
*skb
,
1723 struct net_device
*dev
)
1725 unsigned int last_desc
, flits
, ndesc
;
1726 const struct skb_shared_info
*ssi
;
1727 struct fw_eth_tx_pkt_vm_wr
*wr
;
1728 struct tx_sw_desc
*sgl_sdesc
;
1729 struct cpl_tx_pkt_core
*cpl
;
1730 const struct port_info
*pi
;
1731 struct sge_eth_txq
*txq
;
1732 struct adapter
*adapter
;
1733 int qidx
, credits
, ret
;
1734 size_t fw_hdr_copy_len
;
1738 /* The chip minimum packet length is 10 octets but the firmware
1739 * command that we are using requires that we copy the Ethernet header
1740 * (including the VLAN tag) into the header so we reject anything
1741 * smaller than that ...
1743 fw_hdr_copy_len
= sizeof(wr
->ethmacdst
) + sizeof(wr
->ethmacsrc
) +
1744 sizeof(wr
->ethtype
) + sizeof(wr
->vlantci
);
1745 ret
= cxgb4_validate_skb(skb
, dev
, fw_hdr_copy_len
);
1749 /* Figure out which TX Queue we're going to use. */
1750 pi
= netdev_priv(dev
);
1751 adapter
= pi
->adapter
;
1752 qidx
= skb_get_queue_mapping(skb
);
1753 WARN_ON(qidx
>= pi
->nqsets
);
1754 txq
= &adapter
->sge
.ethtxq
[pi
->first_qset
+ qidx
];
1756 /* Take this opportunity to reclaim any TX Descriptors whose DMA
1757 * transfers have completed.
1759 reclaim_completed_tx(adapter
, &txq
->q
, -1, true);
1761 /* Calculate the number of flits and TX Descriptors we're going to
1762 * need along with how many TX Descriptors will be left over after
1763 * we inject our Work Request.
1765 flits
= t4vf_calc_tx_flits(skb
);
1766 ndesc
= flits_to_desc(flits
);
1767 credits
= txq_avail(&txq
->q
) - ndesc
;
1769 if (unlikely(credits
< 0)) {
1770 /* Not enough room for this packet's Work Request. Stop the
1771 * TX Queue and return a "busy" condition. The queue will get
1772 * started later on when the firmware informs us that space
1776 dev_err(adapter
->pdev_dev
,
1777 "%s: TX ring %u full while queue awake!\n",
1779 return NETDEV_TX_BUSY
;
1782 last_desc
= txq
->q
.pidx
+ ndesc
- 1;
1783 if (last_desc
>= txq
->q
.size
)
1784 last_desc
-= txq
->q
.size
;
1785 sgl_sdesc
= &txq
->q
.sdesc
[last_desc
];
1787 if (!t4vf_is_eth_imm(skb
) &&
1788 unlikely(cxgb4_map_skb(adapter
->pdev_dev
, skb
,
1789 sgl_sdesc
->addr
) < 0)) {
1790 /* We need to map the skb into PCI DMA space (because it can't
1791 * be in-lined directly into the Work Request) and the mapping
1792 * operation failed. Record the error and drop the packet.
1794 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1799 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1800 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1801 /* After we're done injecting the Work Request for this
1802 * packet, we'll be below our "stop threshold" so stop the TX
1803 * Queue now and schedule a request for an SGE Egress Queue
1804 * Update message. The queue will get started later on when
1805 * the firmware processes this Work Request and sends us an
1806 * Egress Queue Status Update message indicating that space
1810 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1813 /* Start filling in our Work Request. Note that we do _not_ handle
1814 * the WR Header wrapping around the TX Descriptor Ring. If our
1815 * maximum header size ever exceeds one TX Descriptor, we'll need to
1816 * do something else here.
1818 WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR
, TXD_PER_EQ_UNIT
) > 1);
1819 wr
= (void *)&txq
->q
.desc
[txq
->q
.pidx
];
1820 wr
->equiq_to_len16
= cpu_to_be32(wr_mid
);
1821 wr
->r3
[0] = cpu_to_be32(0);
1822 wr
->r3
[1] = cpu_to_be32(0);
1823 skb_copy_from_linear_data(skb
, (void *)wr
->ethmacdst
, fw_hdr_copy_len
);
1824 end
= (u64
*)wr
+ flits
;
1826 /* If this is a Large Send Offload packet we'll put in an LSO CPL
1827 * message with an encapsulated TX Packet CPL message. Otherwise we
1828 * just use a TX Packet CPL message.
1830 ssi
= skb_shinfo(skb
);
1831 if (ssi
->gso_size
) {
1832 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1833 bool v6
= (ssi
->gso_type
& SKB_GSO_TCPV6
) != 0;
1834 int l3hdr_len
= skb_network_header_len(skb
);
1835 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1838 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1839 FW_WR_IMMDLEN_V(sizeof(*lso
) +
1841 /* Fill in the LSO CPL message. */
1843 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1847 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1848 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1849 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1850 lso
->ipid_ofst
= cpu_to_be16(0);
1851 lso
->mss
= cpu_to_be16(ssi
->gso_size
);
1852 lso
->seqno_offset
= cpu_to_be32(0);
1853 if (is_t4(adapter
->params
.chip
))
1854 lso
->len
= cpu_to_be32(skb
->len
);
1856 lso
->len
= cpu_to_be32(LSO_T5_XFER_SIZE_V(skb
->len
));
1858 /* Set up TX Packet CPL pointer, control word and perform
1861 cpl
= (void *)(lso
+ 1);
1863 if (CHELSIO_CHIP_VERSION(adapter
->params
.chip
) <= CHELSIO_T5
)
1864 cntrl
= TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1866 cntrl
= T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1868 cntrl
|= TXPKT_CSUM_TYPE_V(v6
?
1869 TX_CSUM_TCPIP6
: TX_CSUM_TCPIP
) |
1870 TXPKT_IPHDR_LEN_V(l3hdr_len
);
1872 txq
->tx_cso
+= ssi
->gso_segs
;
1876 len
= (t4vf_is_eth_imm(skb
)
1877 ? skb
->len
+ sizeof(*cpl
)
1880 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1881 FW_WR_IMMDLEN_V(len
));
1883 /* Set up TX Packet CPL pointer, control word and perform
1886 cpl
= (void *)(wr
+ 1);
1887 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1888 cntrl
= hwcsum(adapter
->params
.chip
, skb
) |
1892 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1896 /* If there's a VLAN tag present, add that to the list of things to
1897 * do in this Work Request.
1899 if (skb_vlan_tag_present(skb
)) {
1901 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1904 /* Fill in the TX Packet CPL message header. */
1905 cpl
->ctrl0
= cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT
) |
1906 TXPKT_INTF_V(pi
->port_id
) |
1908 cpl
->pack
= cpu_to_be16(0);
1909 cpl
->len
= cpu_to_be16(skb
->len
);
1910 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1912 /* Fill in the body of the TX Packet CPL message with either in-lined
1913 * data or a Scatter/Gather List.
1915 if (t4vf_is_eth_imm(skb
)) {
1916 /* In-line the packet's data and free the skb since we don't
1917 * need it any longer.
1919 cxgb4_inline_tx_skb(skb
, &txq
->q
, cpl
+ 1);
1920 dev_consume_skb_any(skb
);
1922 /* Write the skb's Scatter/Gather list into the TX Packet CPL
1923 * message and retain a pointer to the skb so we can free it
1924 * later when its DMA completes. (We store the skb pointer
1925 * in the Software Descriptor corresponding to the last TX
1926 * Descriptor used by the Work Request.)
1928 * The retained skb will be freed when the corresponding TX
1929 * Descriptors are reclaimed after their DMAs complete.
1930 * However, this could take quite a while since, in general,
1931 * the hardware is set up to be lazy about sending DMA
1932 * completion notifications to us and we mostly perform TX
1933 * reclaims in the transmit routine.
1935 * This is good for performamce but means that we rely on new
1936 * TX packets arriving to run the destructors of completed
1937 * packets, which open up space in their sockets' send queues.
1938 * Sometimes we do not get such new packets causing TX to
1939 * stall. A single UDP transmitter is a good example of this
1940 * situation. We have a clean up timer that periodically
1941 * reclaims completed packets but it doesn't run often enough
1942 * (nor do we want it to) to prevent lengthy stalls. A
1943 * solution to this problem is to run the destructor early,
1944 * after the packet is queued but before it's DMAd. A con is
1945 * that we lie to socket memory accounting, but the amount of
1946 * extra memory is reasonable (limited by the number of TX
1947 * descriptors), the packets do actually get freed quickly by
1948 * new packets almost always, and for protocols like TCP that
1949 * wait for acks to really free up the data the extra memory
1950 * is even less. On the positive side we run the destructors
1951 * on the sending CPU rather than on a potentially different
1952 * completing CPU, usually a good thing.
1954 * Run the destructor before telling the DMA engine about the
1955 * packet to make sure it doesn't complete and get freed
1958 struct ulptx_sgl
*sgl
= (struct ulptx_sgl
*)(cpl
+ 1);
1959 struct sge_txq
*tq
= &txq
->q
;
1961 /* If the Work Request header was an exact multiple of our TX
1962 * Descriptor length, then it's possible that the starting SGL
1963 * pointer lines up exactly with the end of our TX Descriptor
1964 * ring. If that's the case, wrap around to the beginning
1967 if (unlikely((void *)sgl
== (void *)tq
->stat
)) {
1968 sgl
= (void *)tq
->desc
;
1969 end
= (void *)((void *)tq
->desc
+
1970 ((void *)end
- (void *)tq
->stat
));
1973 cxgb4_write_sgl(skb
, tq
, sgl
, end
, 0, sgl_sdesc
->addr
);
1975 sgl_sdesc
->skb
= skb
;
1978 /* Advance our internal TX Queue state, tell the hardware about
1979 * the new TX descriptors and return success.
1981 txq_advance(&txq
->q
, ndesc
);
1983 cxgb4_ring_tx_db(adapter
, &txq
->q
, ndesc
);
1984 return NETDEV_TX_OK
;
1987 /* An error of some sort happened. Free the TX skb and tell the
1988 * OS that we've "dealt" with the packet ...
1990 dev_kfree_skb_any(skb
);
1991 return NETDEV_TX_OK
;
1995 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1996 * @q: the SGE control Tx queue
1998 * This is a variant of cxgb4_reclaim_completed_tx() that is used
1999 * for Tx queues that send only immediate data (presently just
2000 * the control queues) and thus do not have any sk_buffs to release.
2002 static inline void reclaim_completed_tx_imm(struct sge_txq
*q
)
2004 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
2005 int reclaim
= hw_cidx
- q
->cidx
;
2010 q
->in_use
-= reclaim
;
2014 static inline void eosw_txq_advance_index(u32
*idx
, u32 n
, u32 max
)
2024 void cxgb4_eosw_txq_free_desc(struct adapter
*adap
,
2025 struct sge_eosw_txq
*eosw_txq
, u32 ndesc
)
2027 struct tx_sw_desc
*d
;
2029 d
= &eosw_txq
->desc
[eosw_txq
->last_cidx
];
2033 unmap_skb(adap
->pdev_dev
, d
->skb
, d
->addr
);
2034 memset(d
->addr
, 0, sizeof(d
->addr
));
2036 dev_consume_skb_any(d
->skb
);
2039 eosw_txq_advance_index(&eosw_txq
->last_cidx
, 1,
2041 d
= &eosw_txq
->desc
[eosw_txq
->last_cidx
];
2045 static inline void eosw_txq_advance(struct sge_eosw_txq
*eosw_txq
, u32 n
)
2047 eosw_txq_advance_index(&eosw_txq
->pidx
, n
, eosw_txq
->ndesc
);
2048 eosw_txq
->inuse
+= n
;
2051 static inline int eosw_txq_enqueue(struct sge_eosw_txq
*eosw_txq
,
2052 struct sk_buff
*skb
)
2054 if (eosw_txq
->inuse
== eosw_txq
->ndesc
)
2057 eosw_txq
->desc
[eosw_txq
->pidx
].skb
= skb
;
2061 static inline struct sk_buff
*eosw_txq_peek(struct sge_eosw_txq
*eosw_txq
)
2063 return eosw_txq
->desc
[eosw_txq
->last_pidx
].skb
;
2066 static inline u8
ethofld_calc_tx_flits(struct adapter
*adap
,
2067 struct sk_buff
*skb
, u32 hdr_len
)
2072 wrlen
= sizeof(struct fw_eth_tx_eo_wr
) + sizeof(struct cpl_tx_pkt_core
);
2073 if (skb_shinfo(skb
)->gso_size
&&
2074 !(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
))
2075 wrlen
+= sizeof(struct cpl_tx_pkt_lso_core
);
2077 wrlen
+= roundup(hdr_len
, 16);
2079 /* Packet headers + WR + CPLs */
2080 flits
= DIV_ROUND_UP(wrlen
, 8);
2082 if (skb_shinfo(skb
)->nr_frags
> 0) {
2083 if (skb_headlen(skb
) - hdr_len
)
2084 nsgl
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
2086 nsgl
= sgl_len(skb_shinfo(skb
)->nr_frags
);
2087 } else if (skb
->len
- hdr_len
) {
2091 return flits
+ nsgl
;
2094 static inline void *write_eo_wr(struct adapter
*adap
,
2095 struct sge_eosw_txq
*eosw_txq
,
2096 struct sk_buff
*skb
, struct fw_eth_tx_eo_wr
*wr
,
2097 u32 hdr_len
, u32 wrlen
)
2099 const struct skb_shared_info
*ssi
= skb_shinfo(skb
);
2100 struct cpl_tx_pkt_core
*cpl
;
2101 u32 immd_len
, wrlen16
;
2105 ver
= ip_hdr(skb
)->version
;
2106 proto
= (ver
== 6) ? ipv6_hdr(skb
)->nexthdr
: ip_hdr(skb
)->protocol
;
2108 wrlen16
= DIV_ROUND_UP(wrlen
, 16);
2109 immd_len
= sizeof(struct cpl_tx_pkt_core
);
2110 if (skb_shinfo(skb
)->gso_size
&&
2111 !(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
))
2112 immd_len
+= sizeof(struct cpl_tx_pkt_lso_core
);
2113 immd_len
+= hdr_len
;
2115 if (!eosw_txq
->ncompl
||
2116 eosw_txq
->last_compl
>= adap
->params
.ofldq_wr_cred
/ 2) {
2119 eosw_txq
->last_compl
= 0;
2122 wr
->op_immdlen
= cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR
) |
2123 FW_ETH_TX_EO_WR_IMMDLEN_V(immd_len
) |
2124 FW_WR_COMPL_V(compl));
2125 wr
->equiq_to_len16
= cpu_to_be32(FW_WR_LEN16_V(wrlen16
) |
2126 FW_WR_FLOWID_V(eosw_txq
->hwtid
));
2128 if (proto
== IPPROTO_UDP
) {
2129 cpl
= write_eo_udp_wr(skb
, wr
, hdr_len
);
2131 wr
->u
.tcpseg
.type
= FW_ETH_TX_EO_TYPE_TCPSEG
;
2132 wr
->u
.tcpseg
.ethlen
= skb_network_offset(skb
);
2133 wr
->u
.tcpseg
.iplen
= cpu_to_be16(skb_network_header_len(skb
));
2134 wr
->u
.tcpseg
.tcplen
= tcp_hdrlen(skb
);
2135 wr
->u
.tcpseg
.tsclk_tsoff
= 0;
2136 wr
->u
.tcpseg
.r4
= 0;
2137 wr
->u
.tcpseg
.r5
= 0;
2138 wr
->u
.tcpseg
.plen
= cpu_to_be32(skb
->len
- hdr_len
);
2140 if (ssi
->gso_size
) {
2141 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
2143 wr
->u
.tcpseg
.mss
= cpu_to_be16(ssi
->gso_size
);
2144 cpl
= write_tso_wr(adap
, skb
, lso
);
2146 wr
->u
.tcpseg
.mss
= cpu_to_be16(0xffff);
2147 cpl
= (void *)(wr
+ 1);
2151 eosw_txq
->cred
-= wrlen16
;
2152 eosw_txq
->last_compl
+= wrlen16
;
2156 static void ethofld_hard_xmit(struct net_device
*dev
,
2157 struct sge_eosw_txq
*eosw_txq
)
2159 struct port_info
*pi
= netdev2pinfo(dev
);
2160 struct adapter
*adap
= netdev2adap(dev
);
2161 u32 wrlen
, wrlen16
, hdr_len
, data_len
;
2162 enum sge_eosw_state next_state
;
2163 u64 cntrl
, *start
, *end
, *sgl
;
2164 struct sge_eohw_txq
*eohw_txq
;
2165 struct cpl_tx_pkt_core
*cpl
;
2166 struct fw_eth_tx_eo_wr
*wr
;
2167 bool skip_eotx_wr
= false;
2168 struct tx_sw_desc
*d
;
2169 struct sk_buff
*skb
;
2173 eohw_txq
= &adap
->sge
.eohw_txq
[eosw_txq
->hwqid
];
2174 spin_lock(&eohw_txq
->lock
);
2175 reclaim_completed_tx_imm(&eohw_txq
->q
);
2177 d
= &eosw_txq
->desc
[eosw_txq
->last_pidx
];
2179 skb_tx_timestamp(skb
);
2181 wr
= (struct fw_eth_tx_eo_wr
*)&eohw_txq
->q
.desc
[eohw_txq
->q
.pidx
];
2182 if (unlikely(eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
&&
2183 eosw_txq
->last_pidx
== eosw_txq
->flowc_idx
)) {
2186 flits
= DIV_ROUND_UP(hdr_len
, 8);
2187 if (eosw_txq
->state
== CXGB4_EO_STATE_FLOWC_OPEN_SEND
)
2188 next_state
= CXGB4_EO_STATE_FLOWC_OPEN_REPLY
;
2190 next_state
= CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
;
2191 skip_eotx_wr
= true;
2193 hdr_len
= eth_get_headlen(dev
, skb
->data
, skb_headlen(skb
));
2194 data_len
= skb
->len
- hdr_len
;
2195 flits
= ethofld_calc_tx_flits(adap
, skb
, hdr_len
);
2197 ndesc
= flits_to_desc(flits
);
2199 wrlen16
= DIV_ROUND_UP(wrlen
, 16);
2201 /* If there are no CPL credits, then wait for credits
2202 * to come back and retry again
2204 if (unlikely(wrlen16
> eosw_txq
->cred
))
2207 if (unlikely(skip_eotx_wr
)) {
2209 eosw_txq
->state
= next_state
;
2210 eosw_txq
->cred
-= wrlen16
;
2212 eosw_txq
->last_compl
= 0;
2213 goto write_wr_headers
;
2216 cpl
= write_eo_wr(adap
, eosw_txq
, skb
, wr
, hdr_len
, wrlen
);
2217 cntrl
= hwcsum(adap
->params
.chip
, skb
);
2218 if (skb_vlan_tag_present(skb
))
2219 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
2221 cpl
->ctrl0
= cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT
) |
2222 TXPKT_INTF_V(pi
->tx_chan
) |
2223 TXPKT_PF_V(adap
->pf
));
2225 cpl
->len
= cpu_to_be16(skb
->len
);
2226 cpl
->ctrl1
= cpu_to_be64(cntrl
);
2228 start
= (u64
*)(cpl
+ 1);
2231 sgl
= (u64
*)inline_tx_skb_header(skb
, &eohw_txq
->q
, (void *)start
,
2234 if (unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, d
->addr
))) {
2235 memset(d
->addr
, 0, sizeof(d
->addr
));
2236 eohw_txq
->mapping_err
++;
2240 end
= (u64
*)wr
+ flits
;
2241 if (unlikely(start
> sgl
)) {
2242 left
= (u8
*)end
- (u8
*)eohw_txq
->q
.stat
;
2243 end
= (void *)eohw_txq
->q
.desc
+ left
;
2246 if (unlikely((u8
*)sgl
>= (u8
*)eohw_txq
->q
.stat
)) {
2247 /* If current position is already at the end of the
2248 * txq, reset the current to point to start of the queue
2249 * and update the end ptr as well.
2251 left
= (u8
*)end
- (u8
*)eohw_txq
->q
.stat
;
2253 end
= (void *)eohw_txq
->q
.desc
+ left
;
2254 sgl
= (void *)eohw_txq
->q
.desc
;
2257 cxgb4_write_sgl(skb
, &eohw_txq
->q
, (void *)sgl
, end
, hdr_len
,
2261 if (skb_shinfo(skb
)->gso_size
) {
2262 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
)
2266 eohw_txq
->tx_cso
+= skb_shinfo(skb
)->gso_segs
;
2267 } else if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2271 if (skb_vlan_tag_present(skb
))
2272 eohw_txq
->vlan_ins
++;
2274 txq_advance(&eohw_txq
->q
, ndesc
);
2275 cxgb4_ring_tx_db(adap
, &eohw_txq
->q
, ndesc
);
2276 eosw_txq_advance_index(&eosw_txq
->last_pidx
, 1, eosw_txq
->ndesc
);
2279 spin_unlock(&eohw_txq
->lock
);
2282 static void ethofld_xmit(struct net_device
*dev
, struct sge_eosw_txq
*eosw_txq
)
2284 struct sk_buff
*skb
;
2287 switch (eosw_txq
->state
) {
2288 case CXGB4_EO_STATE_ACTIVE
:
2289 case CXGB4_EO_STATE_FLOWC_OPEN_SEND
:
2290 case CXGB4_EO_STATE_FLOWC_CLOSE_SEND
:
2291 pktcount
= eosw_txq
->pidx
- eosw_txq
->last_pidx
;
2293 pktcount
+= eosw_txq
->ndesc
;
2295 case CXGB4_EO_STATE_FLOWC_OPEN_REPLY
:
2296 case CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
:
2297 case CXGB4_EO_STATE_CLOSED
:
2302 while (pktcount
--) {
2303 skb
= eosw_txq_peek(eosw_txq
);
2305 eosw_txq_advance_index(&eosw_txq
->last_pidx
, 1,
2310 ethofld_hard_xmit(dev
, eosw_txq
);
2314 static netdev_tx_t
cxgb4_ethofld_xmit(struct sk_buff
*skb
,
2315 struct net_device
*dev
)
2317 struct cxgb4_tc_port_mqprio
*tc_port_mqprio
;
2318 struct port_info
*pi
= netdev2pinfo(dev
);
2319 struct adapter
*adap
= netdev2adap(dev
);
2320 struct sge_eosw_txq
*eosw_txq
;
2324 ret
= cxgb4_validate_skb(skb
, dev
, ETH_HLEN
);
2328 tc_port_mqprio
= &adap
->tc_mqprio
->port_mqprio
[pi
->port_id
];
2329 qid
= skb_get_queue_mapping(skb
) - pi
->nqsets
;
2330 eosw_txq
= &tc_port_mqprio
->eosw_txq
[qid
];
2331 spin_lock_bh(&eosw_txq
->lock
);
2332 if (eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
)
2335 ret
= eosw_txq_enqueue(eosw_txq
, skb
);
2339 /* SKB is queued for processing until credits are available.
2340 * So, call the destructor now and we'll free the skb later
2341 * after it has been successfully transmitted.
2345 eosw_txq_advance(eosw_txq
, 1);
2346 ethofld_xmit(dev
, eosw_txq
);
2347 spin_unlock_bh(&eosw_txq
->lock
);
2348 return NETDEV_TX_OK
;
2351 spin_unlock_bh(&eosw_txq
->lock
);
2353 dev_kfree_skb_any(skb
);
2354 return NETDEV_TX_OK
;
2357 netdev_tx_t
t4_start_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
2359 struct port_info
*pi
= netdev_priv(dev
);
2360 u16 qid
= skb_get_queue_mapping(skb
);
2362 if (unlikely(pi
->eth_flags
& PRIV_FLAG_PORT_TX_VM
))
2363 return cxgb4_vf_eth_xmit(skb
, dev
);
2365 if (unlikely(qid
>= pi
->nqsets
))
2366 return cxgb4_ethofld_xmit(skb
, dev
);
2368 return cxgb4_eth_xmit(skb
, dev
);
2371 static void eosw_txq_flush_pending_skbs(struct sge_eosw_txq
*eosw_txq
)
2373 int pktcount
= eosw_txq
->pidx
- eosw_txq
->last_pidx
;
2374 int pidx
= eosw_txq
->pidx
;
2375 struct sk_buff
*skb
;
2381 pktcount
+= eosw_txq
->ndesc
;
2383 while (pktcount
--) {
2386 pidx
+= eosw_txq
->ndesc
;
2388 skb
= eosw_txq
->desc
[pidx
].skb
;
2390 dev_consume_skb_any(skb
);
2391 eosw_txq
->desc
[pidx
].skb
= NULL
;
2396 eosw_txq
->pidx
= eosw_txq
->last_pidx
+ 1;
2400 * cxgb4_ethofld_send_flowc - Send ETHOFLD flowc request to bind eotid to tc.
2402 * @eotid - ETHOFLD tid to bind/unbind
2403 * @tc - traffic class. If set to FW_SCHED_CLS_NONE, then unbinds the @eotid
2405 * Send a FLOWC work request to bind an ETHOFLD TID to a traffic class.
2406 * If @tc is set to FW_SCHED_CLS_NONE, then the @eotid is unbound from
2409 int cxgb4_ethofld_send_flowc(struct net_device
*dev
, u32 eotid
, u32 tc
)
2411 struct port_info
*pi
= netdev2pinfo(dev
);
2412 struct adapter
*adap
= netdev2adap(dev
);
2413 enum sge_eosw_state next_state
;
2414 struct sge_eosw_txq
*eosw_txq
;
2415 u32 len
, len16
, nparams
= 6;
2416 struct fw_flowc_wr
*flowc
;
2417 struct eotid_entry
*entry
;
2418 struct sge_ofld_rxq
*rxq
;
2419 struct sk_buff
*skb
;
2422 len
= sizeof(*flowc
) + sizeof(struct fw_flowc_mnemval
) * nparams
;
2423 len16
= DIV_ROUND_UP(len
, 16);
2425 entry
= cxgb4_lookup_eotid(&adap
->tids
, eotid
);
2429 eosw_txq
= (struct sge_eosw_txq
*)entry
->data
;
2433 skb
= alloc_skb(len
, GFP_KERNEL
);
2437 spin_lock_bh(&eosw_txq
->lock
);
2438 if (tc
!= FW_SCHED_CLS_NONE
) {
2439 if (eosw_txq
->state
!= CXGB4_EO_STATE_CLOSED
)
2442 next_state
= CXGB4_EO_STATE_FLOWC_OPEN_SEND
;
2444 if (eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
)
2447 next_state
= CXGB4_EO_STATE_FLOWC_CLOSE_SEND
;
2450 flowc
= __skb_put(skb
, len
);
2451 memset(flowc
, 0, len
);
2453 rxq
= &adap
->sge
.eohw_rxq
[eosw_txq
->hwqid
];
2454 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(len16
) |
2455 FW_WR_FLOWID_V(eosw_txq
->hwtid
));
2456 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
2457 FW_FLOWC_WR_NPARAMS_V(nparams
) |
2459 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
2460 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V(adap
->pf
));
2461 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
2462 flowc
->mnemval
[1].val
= cpu_to_be32(pi
->tx_chan
);
2463 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
2464 flowc
->mnemval
[2].val
= cpu_to_be32(pi
->tx_chan
);
2465 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
2466 flowc
->mnemval
[3].val
= cpu_to_be32(rxq
->rspq
.abs_id
);
2467 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SCHEDCLASS
;
2468 flowc
->mnemval
[4].val
= cpu_to_be32(tc
);
2469 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_EOSTATE
;
2470 flowc
->mnemval
[5].val
= cpu_to_be32(tc
== FW_SCHED_CLS_NONE
?
2471 FW_FLOWC_MNEM_EOSTATE_CLOSING
:
2472 FW_FLOWC_MNEM_EOSTATE_ESTABLISHED
);
2474 /* Free up any pending skbs to ensure there's room for
2475 * termination FLOWC.
2477 if (tc
== FW_SCHED_CLS_NONE
)
2478 eosw_txq_flush_pending_skbs(eosw_txq
);
2480 ret
= eosw_txq_enqueue(eosw_txq
, skb
);
2482 dev_consume_skb_any(skb
);
2486 eosw_txq
->state
= next_state
;
2487 eosw_txq
->flowc_idx
= eosw_txq
->pidx
;
2488 eosw_txq_advance(eosw_txq
, 1);
2489 ethofld_xmit(dev
, eosw_txq
);
2492 spin_unlock_bh(&eosw_txq
->lock
);
2497 * is_imm - check whether a packet can be sent as immediate data
2500 * Returns true if a packet can be sent as a WR with immediate data.
2502 static inline int is_imm(const struct sk_buff
*skb
)
2504 return skb
->len
<= MAX_CTRL_WR_LEN
;
2508 * ctrlq_check_stop - check if a control queue is full and should stop
2510 * @wr: most recent WR written to the queue
2512 * Check if a control queue has become full and should be stopped.
2513 * We clean up control queue descriptors very lazily, only when we are out.
2514 * If the queue is still full after reclaiming any completed descriptors
2515 * we suspend it and have the last WR wake it up.
2517 static void ctrlq_check_stop(struct sge_ctrl_txq
*q
, struct fw_wr_hdr
*wr
)
2519 reclaim_completed_tx_imm(&q
->q
);
2520 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2521 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2528 * ctrl_xmit - send a packet through an SGE control Tx queue
2529 * @q: the control queue
2532 * Send a packet through an SGE control Tx queue. Packets sent through
2533 * a control queue must fit entirely as immediate data.
2535 static int ctrl_xmit(struct sge_ctrl_txq
*q
, struct sk_buff
*skb
)
2538 struct fw_wr_hdr
*wr
;
2540 if (unlikely(!is_imm(skb
))) {
2543 return NET_XMIT_DROP
;
2546 ndesc
= DIV_ROUND_UP(skb
->len
, sizeof(struct tx_desc
));
2547 spin_lock(&q
->sendq
.lock
);
2549 if (unlikely(q
->full
)) {
2550 skb
->priority
= ndesc
; /* save for restart */
2551 __skb_queue_tail(&q
->sendq
, skb
);
2552 spin_unlock(&q
->sendq
.lock
);
2556 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2557 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2559 txq_advance(&q
->q
, ndesc
);
2560 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
))
2561 ctrlq_check_stop(q
, wr
);
2563 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
2564 spin_unlock(&q
->sendq
.lock
);
2567 return NET_XMIT_SUCCESS
;
2571 * restart_ctrlq - restart a suspended control queue
2572 * @data: the control queue to restart
2574 * Resumes transmission on a suspended Tx control queue.
2576 static void restart_ctrlq(unsigned long data
)
2578 struct sk_buff
*skb
;
2579 unsigned int written
= 0;
2580 struct sge_ctrl_txq
*q
= (struct sge_ctrl_txq
*)data
;
2582 spin_lock(&q
->sendq
.lock
);
2583 reclaim_completed_tx_imm(&q
->q
);
2584 BUG_ON(txq_avail(&q
->q
) < TXQ_STOP_THRES
); /* q should be empty */
2586 while ((skb
= __skb_dequeue(&q
->sendq
)) != NULL
) {
2587 struct fw_wr_hdr
*wr
;
2588 unsigned int ndesc
= skb
->priority
; /* previously saved */
2591 /* Write descriptors and free skbs outside the lock to limit
2592 * wait times. q->full is still set so new skbs will be queued.
2594 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2595 txq_advance(&q
->q
, ndesc
);
2596 spin_unlock(&q
->sendq
.lock
);
2598 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2601 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2602 unsigned long old
= q
->q
.stops
;
2604 ctrlq_check_stop(q
, wr
);
2605 if (q
->q
.stops
!= old
) { /* suspended anew */
2606 spin_lock(&q
->sendq
.lock
);
2611 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2614 spin_lock(&q
->sendq
.lock
);
2619 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2620 spin_unlock(&q
->sendq
.lock
);
2624 * t4_mgmt_tx - send a management message
2625 * @adap: the adapter
2626 * @skb: the packet containing the management message
2628 * Send a management message through control queue 0.
2630 int t4_mgmt_tx(struct adapter
*adap
, struct sk_buff
*skb
)
2635 ret
= ctrl_xmit(&adap
->sge
.ctrlq
[0], skb
);
2641 * is_ofld_imm - check whether a packet can be sent as immediate data
2644 * Returns true if a packet can be sent as an offload WR with immediate
2645 * data. We currently use the same limit as for Ethernet packets.
2647 static inline int is_ofld_imm(const struct sk_buff
*skb
)
2649 struct work_request_hdr
*req
= (struct work_request_hdr
*)skb
->data
;
2650 unsigned long opcode
= FW_WR_OP_G(ntohl(req
->wr_hi
));
2652 if (opcode
== FW_CRYPTO_LOOKASIDE_WR
)
2653 return skb
->len
<= SGE_MAX_WR_LEN
;
2655 return skb
->len
<= MAX_IMM_TX_PKT_LEN
;
2659 * calc_tx_flits_ofld - calculate # of flits for an offload packet
2662 * Returns the number of flits needed for the given offload packet.
2663 * These packets are already fully constructed and no additional headers
2666 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff
*skb
)
2668 unsigned int flits
, cnt
;
2670 if (is_ofld_imm(skb
))
2671 return DIV_ROUND_UP(skb
->len
, 8);
2673 flits
= skb_transport_offset(skb
) / 8U; /* headers */
2674 cnt
= skb_shinfo(skb
)->nr_frags
;
2675 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
2677 return flits
+ sgl_len(cnt
);
2681 * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
2682 * @adap: the adapter
2683 * @q: the queue to stop
2685 * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
2686 * inability to map packets. A periodic timer attempts to restart
2689 static void txq_stop_maperr(struct sge_uld_txq
*q
)
2693 set_bit(q
->q
.cntxt_id
- q
->adap
->sge
.egr_start
,
2694 q
->adap
->sge
.txq_maperr
);
2698 * ofldtxq_stop - stop an offload Tx queue that has become full
2699 * @q: the queue to stop
2700 * @wr: the Work Request causing the queue to become full
2702 * Stops an offload Tx queue that has become full and modifies the packet
2703 * being written to request a wakeup.
2705 static void ofldtxq_stop(struct sge_uld_txq
*q
, struct fw_wr_hdr
*wr
)
2707 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2713 * service_ofldq - service/restart a suspended offload queue
2714 * @q: the offload queue
2716 * Services an offload Tx queue by moving packets from its Pending Send
2717 * Queue to the Hardware TX ring. The function starts and ends with the
2718 * Send Queue locked, but drops the lock while putting the skb at the
2719 * head of the Send Queue onto the Hardware TX Ring. Dropping the lock
2720 * allows more skbs to be added to the Send Queue by other threads.
2721 * The packet being processed at the head of the Pending Send Queue is
2722 * left on the queue in case we experience DMA Mapping errors, etc.
2723 * and need to give up and restart later.
2725 * service_ofldq() can be thought of as a task which opportunistically
2726 * uses other threads execution contexts. We use the Offload Queue
2727 * boolean "service_ofldq_running" to make sure that only one instance
2728 * is ever running at a time ...
2730 static void service_ofldq(struct sge_uld_txq
*q
)
2731 __must_hold(&q
->sendq
.lock
)
2733 u64
*pos
, *before
, *end
;
2735 struct sk_buff
*skb
;
2736 struct sge_txq
*txq
;
2738 unsigned int written
= 0;
2739 unsigned int flits
, ndesc
;
2741 /* If another thread is currently in service_ofldq() processing the
2742 * Pending Send Queue then there's nothing to do. Otherwise, flag
2743 * that we're doing the work and continue. Examining/modifying
2744 * the Offload Queue boolean "service_ofldq_running" must be done
2745 * while holding the Pending Send Queue Lock.
2747 if (q
->service_ofldq_running
)
2749 q
->service_ofldq_running
= true;
2751 while ((skb
= skb_peek(&q
->sendq
)) != NULL
&& !q
->full
) {
2752 /* We drop the lock while we're working with the skb at the
2753 * head of the Pending Send Queue. This allows more skbs to
2754 * be added to the Pending Send Queue while we're working on
2755 * this one. We don't need to lock to guard the TX Ring
2756 * updates because only one thread of execution is ever
2757 * allowed into service_ofldq() at a time.
2759 spin_unlock(&q
->sendq
.lock
);
2761 cxgb4_reclaim_completed_tx(q
->adap
, &q
->q
, false);
2763 flits
= skb
->priority
; /* previously saved */
2764 ndesc
= flits_to_desc(flits
);
2765 credits
= txq_avail(&q
->q
) - ndesc
;
2766 BUG_ON(credits
< 0);
2767 if (unlikely(credits
< TXQ_STOP_THRES
))
2768 ofldtxq_stop(q
, (struct fw_wr_hdr
*)skb
->data
);
2770 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
2771 if (is_ofld_imm(skb
))
2772 cxgb4_inline_tx_skb(skb
, &q
->q
, pos
);
2773 else if (cxgb4_map_skb(q
->adap
->pdev_dev
, skb
,
2774 (dma_addr_t
*)skb
->head
)) {
2776 spin_lock(&q
->sendq
.lock
);
2779 int last_desc
, hdr_len
= skb_transport_offset(skb
);
2781 /* The WR headers may not fit within one descriptor.
2782 * So we need to deal with wrap-around here.
2784 before
= (u64
*)pos
;
2785 end
= (u64
*)pos
+ flits
;
2787 pos
= (void *)inline_tx_skb_header(skb
, &q
->q
,
2790 if (before
> (u64
*)pos
) {
2791 left
= (u8
*)end
- (u8
*)txq
->stat
;
2792 end
= (void *)txq
->desc
+ left
;
2795 /* If current position is already at the end of the
2796 * ofld queue, reset the current to point to
2797 * start of the queue and update the end ptr as well.
2799 if (pos
== (u64
*)txq
->stat
) {
2800 left
= (u8
*)end
- (u8
*)txq
->stat
;
2801 end
= (void *)txq
->desc
+ left
;
2802 pos
= (void *)txq
->desc
;
2805 cxgb4_write_sgl(skb
, &q
->q
, (void *)pos
,
2807 (dma_addr_t
*)skb
->head
);
2808 #ifdef CONFIG_NEED_DMA_MAP_STATE
2809 skb
->dev
= q
->adap
->port
[0];
2810 skb
->destructor
= deferred_unmap_destructor
;
2812 last_desc
= q
->q
.pidx
+ ndesc
- 1;
2813 if (last_desc
>= q
->q
.size
)
2814 last_desc
-= q
->q
.size
;
2815 q
->q
.sdesc
[last_desc
].skb
= skb
;
2818 txq_advance(&q
->q
, ndesc
);
2820 if (unlikely(written
> 32)) {
2821 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2825 /* Reacquire the Pending Send Queue Lock so we can unlink the
2826 * skb we've just successfully transferred to the TX Ring and
2827 * loop for the next skb which may be at the head of the
2828 * Pending Send Queue.
2830 spin_lock(&q
->sendq
.lock
);
2831 __skb_unlink(skb
, &q
->sendq
);
2832 if (is_ofld_imm(skb
))
2835 if (likely(written
))
2836 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2838 /*Indicate that no thread is processing the Pending Send Queue
2841 q
->service_ofldq_running
= false;
2845 * ofld_xmit - send a packet through an offload queue
2846 * @q: the Tx offload queue
2849 * Send an offload packet through an SGE offload queue.
2851 static int ofld_xmit(struct sge_uld_txq
*q
, struct sk_buff
*skb
)
2853 skb
->priority
= calc_tx_flits_ofld(skb
); /* save for restart */
2854 spin_lock(&q
->sendq
.lock
);
2856 /* Queue the new skb onto the Offload Queue's Pending Send Queue. If
2857 * that results in this new skb being the only one on the queue, start
2858 * servicing it. If there are other skbs already on the list, then
2859 * either the queue is currently being processed or it's been stopped
2860 * for some reason and it'll be restarted at a later time. Restart
2861 * paths are triggered by events like experiencing a DMA Mapping Error
2862 * or filling the Hardware TX Ring.
2864 __skb_queue_tail(&q
->sendq
, skb
);
2865 if (q
->sendq
.qlen
== 1)
2868 spin_unlock(&q
->sendq
.lock
);
2869 return NET_XMIT_SUCCESS
;
2873 * restart_ofldq - restart a suspended offload queue
2874 * @data: the offload queue to restart
2876 * Resumes transmission on a suspended Tx offload queue.
2878 static void restart_ofldq(unsigned long data
)
2880 struct sge_uld_txq
*q
= (struct sge_uld_txq
*)data
;
2882 spin_lock(&q
->sendq
.lock
);
2883 q
->full
= 0; /* the queue actually is completely empty now */
2885 spin_unlock(&q
->sendq
.lock
);
2889 * skb_txq - return the Tx queue an offload packet should use
2892 * Returns the Tx queue an offload packet should use as indicated by bits
2893 * 1-15 in the packet's queue_mapping.
2895 static inline unsigned int skb_txq(const struct sk_buff
*skb
)
2897 return skb
->queue_mapping
>> 1;
2901 * is_ctrl_pkt - return whether an offload packet is a control packet
2904 * Returns whether an offload packet should use an OFLD or a CTRL
2905 * Tx queue as indicated by bit 0 in the packet's queue_mapping.
2907 static inline unsigned int is_ctrl_pkt(const struct sk_buff
*skb
)
2909 return skb
->queue_mapping
& 1;
2912 static inline int uld_send(struct adapter
*adap
, struct sk_buff
*skb
,
2913 unsigned int tx_uld_type
)
2915 struct sge_uld_txq_info
*txq_info
;
2916 struct sge_uld_txq
*txq
;
2917 unsigned int idx
= skb_txq(skb
);
2919 if (unlikely(is_ctrl_pkt(skb
))) {
2920 /* Single ctrl queue is a requirement for LE workaround path */
2921 if (adap
->tids
.nsftids
)
2923 return ctrl_xmit(&adap
->sge
.ctrlq
[idx
], skb
);
2926 txq_info
= adap
->sge
.uld_txq_info
[tx_uld_type
];
2927 if (unlikely(!txq_info
)) {
2929 return NET_XMIT_DROP
;
2932 txq
= &txq_info
->uldtxq
[idx
];
2933 return ofld_xmit(txq
, skb
);
2937 * t4_ofld_send - send an offload packet
2938 * @adap: the adapter
2941 * Sends an offload packet. We use the packet queue_mapping to select the
2942 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2943 * should be sent as regular or control, bits 1-15 select the queue.
2945 int t4_ofld_send(struct adapter
*adap
, struct sk_buff
*skb
)
2950 ret
= uld_send(adap
, skb
, CXGB4_TX_OFLD
);
2956 * cxgb4_ofld_send - send an offload packet
2957 * @dev: the net device
2960 * Sends an offload packet. This is an exported version of @t4_ofld_send,
2961 * intended for ULDs.
2963 int cxgb4_ofld_send(struct net_device
*dev
, struct sk_buff
*skb
)
2965 return t4_ofld_send(netdev2adap(dev
), skb
);
2967 EXPORT_SYMBOL(cxgb4_ofld_send
);
2969 static void *inline_tx_header(const void *src
,
2970 const struct sge_txq
*q
,
2971 void *pos
, int length
)
2973 int left
= (void *)q
->stat
- pos
;
2976 if (likely(length
<= left
)) {
2977 memcpy(pos
, src
, length
);
2980 memcpy(pos
, src
, left
);
2981 memcpy(q
->desc
, src
+ left
, length
- left
);
2982 pos
= (void *)q
->desc
+ (length
- left
);
2984 /* 0-pad to multiple of 16 */
2985 p
= PTR_ALIGN(pos
, 8);
2986 if ((uintptr_t)p
& 8) {
2994 * ofld_xmit_direct - copy a WR into offload queue
2995 * @q: the Tx offload queue
2996 * @src: location of WR
2999 * Copy an immediate WR into an uncontended SGE offload queue.
3001 static int ofld_xmit_direct(struct sge_uld_txq
*q
, const void *src
,
3008 /* Use the lower limit as the cut-off */
3009 if (len
> MAX_IMM_OFLD_TX_DATA_WR_LEN
) {
3011 return NET_XMIT_DROP
;
3014 /* Don't return NET_XMIT_CN here as the current
3015 * implementation doesn't queue the request
3016 * using an skb when the following conditions not met
3018 if (!spin_trylock(&q
->sendq
.lock
))
3019 return NET_XMIT_DROP
;
3021 if (q
->full
|| !skb_queue_empty(&q
->sendq
) ||
3022 q
->service_ofldq_running
) {
3023 spin_unlock(&q
->sendq
.lock
);
3024 return NET_XMIT_DROP
;
3026 ndesc
= flits_to_desc(DIV_ROUND_UP(len
, 8));
3027 credits
= txq_avail(&q
->q
) - ndesc
;
3028 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
3030 /* ofldtxq_stop modifies WR header in-situ */
3031 inline_tx_header(src
, &q
->q
, pos
, len
);
3032 if (unlikely(credits
< TXQ_STOP_THRES
))
3033 ofldtxq_stop(q
, (struct fw_wr_hdr
*)pos
);
3034 txq_advance(&q
->q
, ndesc
);
3035 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
3037 spin_unlock(&q
->sendq
.lock
);
3038 return NET_XMIT_SUCCESS
;
3041 int cxgb4_immdata_send(struct net_device
*dev
, unsigned int idx
,
3042 const void *src
, unsigned int len
)
3044 struct sge_uld_txq_info
*txq_info
;
3045 struct sge_uld_txq
*txq
;
3046 struct adapter
*adap
;
3049 adap
= netdev2adap(dev
);
3052 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
3053 if (unlikely(!txq_info
)) {
3056 return NET_XMIT_DROP
;
3058 txq
= &txq_info
->uldtxq
[idx
];
3060 ret
= ofld_xmit_direct(txq
, src
, len
);
3062 return net_xmit_eval(ret
);
3064 EXPORT_SYMBOL(cxgb4_immdata_send
);
3067 * t4_crypto_send - send crypto packet
3068 * @adap: the adapter
3071 * Sends crypto packet. We use the packet queue_mapping to select the
3072 * appropriate Tx queue as follows: bit 0 indicates whether the packet
3073 * should be sent as regular or control, bits 1-15 select the queue.
3075 static int t4_crypto_send(struct adapter
*adap
, struct sk_buff
*skb
)
3080 ret
= uld_send(adap
, skb
, CXGB4_TX_CRYPTO
);
3086 * cxgb4_crypto_send - send crypto packet
3087 * @dev: the net device
3090 * Sends crypto packet. This is an exported version of @t4_crypto_send,
3091 * intended for ULDs.
3093 int cxgb4_crypto_send(struct net_device
*dev
, struct sk_buff
*skb
)
3095 return t4_crypto_send(netdev2adap(dev
), skb
);
3097 EXPORT_SYMBOL(cxgb4_crypto_send
);
3099 static inline void copy_frags(struct sk_buff
*skb
,
3100 const struct pkt_gl
*gl
, unsigned int offset
)
3104 /* usually there's just one frag */
3105 __skb_fill_page_desc(skb
, 0, gl
->frags
[0].page
,
3106 gl
->frags
[0].offset
+ offset
,
3107 gl
->frags
[0].size
- offset
);
3108 skb_shinfo(skb
)->nr_frags
= gl
->nfrags
;
3109 for (i
= 1; i
< gl
->nfrags
; i
++)
3110 __skb_fill_page_desc(skb
, i
, gl
->frags
[i
].page
,
3111 gl
->frags
[i
].offset
,
3114 /* get a reference to the last page, we don't own it */
3115 get_page(gl
->frags
[gl
->nfrags
- 1].page
);
3119 * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list
3120 * @gl: the gather list
3121 * @skb_len: size of sk_buff main body if it carries fragments
3122 * @pull_len: amount of data to move to the sk_buff's main body
3124 * Builds an sk_buff from the given packet gather list. Returns the
3125 * sk_buff or %NULL if sk_buff allocation failed.
3127 struct sk_buff
*cxgb4_pktgl_to_skb(const struct pkt_gl
*gl
,
3128 unsigned int skb_len
, unsigned int pull_len
)
3130 struct sk_buff
*skb
;
3133 * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer
3134 * size, which is expected since buffers are at least PAGE_SIZEd.
3135 * In this case packets up to RX_COPY_THRES have only one fragment.
3137 if (gl
->tot_len
<= RX_COPY_THRES
) {
3138 skb
= dev_alloc_skb(gl
->tot_len
);
3141 __skb_put(skb
, gl
->tot_len
);
3142 skb_copy_to_linear_data(skb
, gl
->va
, gl
->tot_len
);
3144 skb
= dev_alloc_skb(skb_len
);
3147 __skb_put(skb
, pull_len
);
3148 skb_copy_to_linear_data(skb
, gl
->va
, pull_len
);
3150 copy_frags(skb
, gl
, pull_len
);
3151 skb
->len
= gl
->tot_len
;
3152 skb
->data_len
= skb
->len
- pull_len
;
3153 skb
->truesize
+= skb
->data_len
;
3157 EXPORT_SYMBOL(cxgb4_pktgl_to_skb
);
3160 * t4_pktgl_free - free a packet gather list
3161 * @gl: the gather list
3163 * Releases the pages of a packet gather list. We do not own the last
3164 * page on the list and do not free it.
3166 static void t4_pktgl_free(const struct pkt_gl
*gl
)
3169 const struct page_frag
*p
;
3171 for (p
= gl
->frags
, n
= gl
->nfrags
- 1; n
--; p
++)
3176 * Process an MPS trace packet. Give it an unused protocol number so it won't
3177 * be delivered to anyone and send it to the stack for capture.
3179 static noinline
int handle_trace_pkt(struct adapter
*adap
,
3180 const struct pkt_gl
*gl
)
3182 struct sk_buff
*skb
;
3184 skb
= cxgb4_pktgl_to_skb(gl
, RX_PULL_LEN
, RX_PULL_LEN
);
3185 if (unlikely(!skb
)) {
3190 if (is_t4(adap
->params
.chip
))
3191 __skb_pull(skb
, sizeof(struct cpl_trace_pkt
));
3193 __skb_pull(skb
, sizeof(struct cpl_t5_trace_pkt
));
3195 skb_reset_mac_header(skb
);
3196 skb
->protocol
= htons(0xffff);
3197 skb
->dev
= adap
->port
[0];
3198 netif_receive_skb(skb
);
3203 * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp
3204 * @adap: the adapter
3205 * @hwtstamps: time stamp structure to update
3206 * @sgetstamp: 60bit iqe timestamp
3208 * Every ingress queue entry has the 60-bit timestamp, convert that timestamp
3209 * which is in Core Clock ticks into ktime_t and assign it
3211 static void cxgb4_sgetim_to_hwtstamp(struct adapter
*adap
,
3212 struct skb_shared_hwtstamps
*hwtstamps
,
3216 u64 tmp
= (sgetstamp
* 1000 * 1000 + adap
->params
.vpd
.cclk
/ 2);
3218 ns
= div_u64(tmp
, adap
->params
.vpd
.cclk
);
3220 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
3221 hwtstamps
->hwtstamp
= ns_to_ktime(ns
);
3224 static void do_gro(struct sge_eth_rxq
*rxq
, const struct pkt_gl
*gl
,
3225 const struct cpl_rx_pkt
*pkt
, unsigned long tnl_hdr_len
)
3227 struct adapter
*adapter
= rxq
->rspq
.adap
;
3228 struct sge
*s
= &adapter
->sge
;
3229 struct port_info
*pi
;
3231 struct sk_buff
*skb
;
3233 skb
= napi_get_frags(&rxq
->rspq
.napi
);
3234 if (unlikely(!skb
)) {
3236 rxq
->stats
.rx_drops
++;
3240 copy_frags(skb
, gl
, s
->pktshift
);
3242 skb
->csum_level
= 1;
3243 skb
->len
= gl
->tot_len
- s
->pktshift
;
3244 skb
->data_len
= skb
->len
;
3245 skb
->truesize
+= skb
->data_len
;
3246 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3247 skb_record_rx_queue(skb
, rxq
->rspq
.idx
);
3248 pi
= netdev_priv(skb
->dev
);
3250 cxgb4_sgetim_to_hwtstamp(adapter
, skb_hwtstamps(skb
),
3252 if (rxq
->rspq
.netdev
->features
& NETIF_F_RXHASH
)
3253 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
3256 if (unlikely(pkt
->vlan_ex
)) {
3257 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
3258 rxq
->stats
.vlan_ex
++;
3260 ret
= napi_gro_frags(&rxq
->rspq
.napi
);
3261 if (ret
== GRO_HELD
)
3262 rxq
->stats
.lro_pkts
++;
3263 else if (ret
== GRO_MERGED
|| ret
== GRO_MERGED_FREE
)
3264 rxq
->stats
.lro_merged
++;
3266 rxq
->stats
.rx_cso
++;
3276 * t4_systim_to_hwstamp - read hardware time stamp
3277 * @adap: the adapter
3280 * Read Time Stamp from MPS packet and insert in skb which
3281 * is forwarded to PTP application
3283 static noinline
int t4_systim_to_hwstamp(struct adapter
*adapter
,
3284 struct sk_buff
*skb
)
3286 struct skb_shared_hwtstamps
*hwtstamps
;
3287 struct cpl_rx_mps_pkt
*cpl
= NULL
;
3288 unsigned char *data
;
3291 cpl
= (struct cpl_rx_mps_pkt
*)skb
->data
;
3292 if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl
->op_to_r1_hi
)) &
3293 X_CPL_RX_MPS_PKT_TYPE_PTP
))
3294 return RX_PTP_PKT_ERR
;
3296 data
= skb
->data
+ sizeof(*cpl
);
3297 skb_pull(skb
, 2 * sizeof(u64
) + sizeof(struct cpl_rx_mps_pkt
));
3298 offset
= ETH_HLEN
+ IPV4_HLEN(skb
->data
) + UDP_HLEN
;
3299 if (skb
->len
< offset
+ OFF_PTP_SEQUENCE_ID
+ sizeof(short))
3300 return RX_PTP_PKT_ERR
;
3302 hwtstamps
= skb_hwtstamps(skb
);
3303 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
3304 hwtstamps
->hwtstamp
= ns_to_ktime(be64_to_cpu(*((u64
*)data
)));
3306 return RX_PTP_PKT_SUC
;
3310 * t4_rx_hststamp - Recv PTP Event Message
3311 * @adap: the adapter
3312 * @rsp: the response queue descriptor holding the RX_PKT message
3315 * PTP enabled and MPS packet, read HW timestamp
3317 static int t4_rx_hststamp(struct adapter
*adapter
, const __be64
*rsp
,
3318 struct sge_eth_rxq
*rxq
, struct sk_buff
*skb
)
3322 if (unlikely((*(u8
*)rsp
== CPL_RX_MPS_PKT
) &&
3323 !is_t4(adapter
->params
.chip
))) {
3324 ret
= t4_systim_to_hwstamp(adapter
, skb
);
3325 if (ret
== RX_PTP_PKT_ERR
) {
3327 rxq
->stats
.rx_drops
++;
3331 return RX_NON_PTP_PKT
;
3335 * t4_tx_hststamp - Loopback PTP Transmit Event Message
3336 * @adap: the adapter
3338 * @dev: the ingress net device
3340 * Read hardware timestamp for the loopback PTP Tx event message
3342 static int t4_tx_hststamp(struct adapter
*adapter
, struct sk_buff
*skb
,
3343 struct net_device
*dev
)
3345 struct port_info
*pi
= netdev_priv(dev
);
3347 if (!is_t4(adapter
->params
.chip
) && adapter
->ptp_tx_skb
) {
3348 cxgb4_ptp_read_hwstamp(adapter
, pi
);
3356 * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages
3357 * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue
3358 * @rsp: Response Entry pointer into Response Queue
3359 * @gl: Gather List pointer
3361 * For adapters which support the SGE Doorbell Queue Timer facility,
3362 * we configure the Ethernet TX Queues to send CIDX Updates to the
3363 * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE
3364 * messages. This adds a small load to PCIe Link RX bandwidth and,
3365 * potentially, higher CPU Interrupt load, but allows us to respond
3366 * much more quickly to the CIDX Updates. This is important for
3367 * Upper Layer Software which isn't willing to have a large amount
3368 * of TX Data outstanding before receiving DMA Completions.
3370 static void t4_tx_completion_handler(struct sge_rspq
*rspq
,
3372 const struct pkt_gl
*gl
)
3374 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3375 struct port_info
*pi
= netdev_priv(rspq
->netdev
);
3376 struct adapter
*adapter
= rspq
->adap
;
3377 struct sge
*s
= &adapter
->sge
;
3378 struct sge_eth_txq
*txq
;
3380 /* skip RSS header */
3383 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
3385 if (unlikely(opcode
== CPL_FW4_MSG
&&
3386 ((const struct cpl_fw4_msg
*)rsp
)->type
==
3389 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3393 if (unlikely(opcode
!= CPL_SGE_EGR_UPDATE
)) {
3394 pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n",
3399 txq
= &s
->ethtxq
[pi
->first_qset
+ rspq
->idx
];
3400 t4_sge_eth_txq_egress_update(adapter
, txq
, -1);
3404 * t4_ethrx_handler - process an ingress ethernet packet
3405 * @q: the response queue that received the packet
3406 * @rsp: the response queue descriptor holding the RX_PKT message
3407 * @si: the gather list of packet fragments
3409 * Process an ingress ethernet packet and deliver it to the stack.
3411 int t4_ethrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
3412 const struct pkt_gl
*si
)
3415 struct sk_buff
*skb
;
3416 const struct cpl_rx_pkt
*pkt
;
3417 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
3418 struct adapter
*adapter
= q
->adap
;
3419 struct sge
*s
= &q
->adap
->sge
;
3420 int cpl_trace_pkt
= is_t4(q
->adap
->params
.chip
) ?
3421 CPL_TRACE_PKT
: CPL_TRACE_PKT_T5
;
3422 u16 err_vec
, tnl_hdr_len
= 0;
3423 struct port_info
*pi
;
3426 /* If we're looking at TX Queue CIDX Update, handle that separately
3429 if (unlikely((*(u8
*)rsp
== CPL_FW4_MSG
) ||
3430 (*(u8
*)rsp
== CPL_SGE_EGR_UPDATE
))) {
3431 t4_tx_completion_handler(q
, rsp
, si
);
3435 if (unlikely(*(u8
*)rsp
== cpl_trace_pkt
))
3436 return handle_trace_pkt(q
->adap
, si
);
3438 pkt
= (const struct cpl_rx_pkt
*)rsp
;
3439 /* Compressed error vector is enabled for T6 only */
3440 if (q
->adap
->params
.tp
.rx_pkt_encap
) {
3441 err_vec
= T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt
->err_vec
));
3442 tnl_hdr_len
= T6_RX_TNLHDR_LEN_G(ntohs(pkt
->err_vec
));
3444 err_vec
= be16_to_cpu(pkt
->err_vec
);
3447 csum_ok
= pkt
->csum_calc
&& !err_vec
&&
3448 (q
->netdev
->features
& NETIF_F_RXCSUM
);
3451 rxq
->stats
.bad_rx_pkts
++;
3453 if (((pkt
->l2info
& htonl(RXF_TCP_F
)) ||
3455 (q
->netdev
->features
& NETIF_F_GRO
) && csum_ok
&& !pkt
->ip_frag
) {
3456 do_gro(rxq
, si
, pkt
, tnl_hdr_len
);
3460 skb
= cxgb4_pktgl_to_skb(si
, RX_PKT_SKB_LEN
, RX_PULL_LEN
);
3461 if (unlikely(!skb
)) {
3463 rxq
->stats
.rx_drops
++;
3466 pi
= netdev_priv(q
->netdev
);
3468 /* Handle PTP Event Rx packet */
3469 if (unlikely(pi
->ptp_enable
)) {
3470 ret
= t4_rx_hststamp(adapter
, rsp
, rxq
, skb
);
3471 if (ret
== RX_PTP_PKT_ERR
)
3475 __skb_pull(skb
, s
->pktshift
); /* remove ethernet header pad */
3477 /* Handle the PTP Event Tx Loopback packet */
3478 if (unlikely(pi
->ptp_enable
&& !ret
&&
3479 (pkt
->l2info
& htonl(RXF_UDP_F
)) &&
3480 cxgb4_ptp_is_ptp_rx(skb
))) {
3481 if (!t4_tx_hststamp(adapter
, skb
, q
->netdev
))
3485 skb
->protocol
= eth_type_trans(skb
, q
->netdev
);
3486 skb_record_rx_queue(skb
, q
->idx
);
3487 if (skb
->dev
->features
& NETIF_F_RXHASH
)
3488 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
3494 cxgb4_sgetim_to_hwtstamp(q
->adap
, skb_hwtstamps(skb
),
3496 if (csum_ok
&& (pkt
->l2info
& htonl(RXF_UDP_F
| RXF_TCP_F
))) {
3497 if (!pkt
->ip_frag
) {
3498 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3499 rxq
->stats
.rx_cso
++;
3500 } else if (pkt
->l2info
& htonl(RXF_IP_F
)) {
3501 __sum16 c
= (__force __sum16
)pkt
->csum
;
3502 skb
->csum
= csum_unfold(c
);
3505 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3506 skb
->csum_level
= 1;
3508 skb
->ip_summed
= CHECKSUM_COMPLETE
;
3510 rxq
->stats
.rx_cso
++;
3513 skb_checksum_none_assert(skb
);
3514 #ifdef CONFIG_CHELSIO_T4_FCOE
3515 #define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \
3516 RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F)
3518 if (!(pkt
->l2info
& cpu_to_be32(CPL_RX_PKT_FLAGS
))) {
3519 if ((pkt
->l2info
& cpu_to_be32(RXF_FCOE_F
)) &&
3520 (pi
->fcoe
.flags
& CXGB_FCOE_ENABLED
)) {
3521 if (q
->adap
->params
.tp
.rx_pkt_encap
)
3523 T6_COMPR_RXERR_SUM_F
;
3525 csum_ok
= err_vec
& RXERR_CSUM_F
;
3527 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3531 #undef CPL_RX_PKT_FLAGS
3532 #endif /* CONFIG_CHELSIO_T4_FCOE */
3535 if (unlikely(pkt
->vlan_ex
)) {
3536 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
3537 rxq
->stats
.vlan_ex
++;
3539 skb_mark_napi_id(skb
, &q
->napi
);
3540 netif_receive_skb(skb
);
3545 * restore_rx_bufs - put back a packet's Rx buffers
3546 * @si: the packet gather list
3547 * @q: the SGE free list
3548 * @frags: number of FL buffers to restore
3550 * Puts back on an FL the Rx buffers associated with @si. The buffers
3551 * have already been unmapped and are left unmapped, we mark them so to
3552 * prevent further unmapping attempts.
3554 * This function undoes a series of @unmap_rx_buf calls when we find out
3555 * that the current packet can't be processed right away afterall and we
3556 * need to come back to it later. This is a very rare event and there's
3557 * no effort to make this particularly efficient.
3559 static void restore_rx_bufs(const struct pkt_gl
*si
, struct sge_fl
*q
,
3562 struct rx_sw_desc
*d
;
3566 q
->cidx
= q
->size
- 1;
3569 d
= &q
->sdesc
[q
->cidx
];
3570 d
->page
= si
->frags
[frags
].page
;
3571 d
->dma_addr
|= RX_UNMAPPED_BUF
;
3577 * is_new_response - check if a response is newly written
3578 * @r: the response descriptor
3579 * @q: the response queue
3581 * Returns true if a response descriptor contains a yet unprocessed
3584 static inline bool is_new_response(const struct rsp_ctrl
*r
,
3585 const struct sge_rspq
*q
)
3587 return (r
->type_gen
>> RSPD_GEN_S
) == q
->gen
;
3591 * rspq_next - advance to the next entry in a response queue
3594 * Updates the state of a response queue to advance it to the next entry.
3596 static inline void rspq_next(struct sge_rspq
*q
)
3598 q
->cur_desc
= (void *)q
->cur_desc
+ q
->iqe_len
;
3599 if (unlikely(++q
->cidx
== q
->size
)) {
3602 q
->cur_desc
= q
->desc
;
3607 * process_responses - process responses from an SGE response queue
3608 * @q: the ingress queue to process
3609 * @budget: how many responses can be processed in this round
3611 * Process responses from an SGE response queue up to the supplied budget.
3612 * Responses include received packets as well as control messages from FW
3615 * Additionally choose the interrupt holdoff time for the next interrupt
3616 * on this queue. If the system is under memory shortage use a fairly
3617 * long delay to help recovery.
3619 static int process_responses(struct sge_rspq
*q
, int budget
)
3622 int budget_left
= budget
;
3623 const struct rsp_ctrl
*rc
;
3624 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
3625 struct adapter
*adapter
= q
->adap
;
3626 struct sge
*s
= &adapter
->sge
;
3628 while (likely(budget_left
)) {
3629 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3630 if (!is_new_response(rc
, q
)) {
3631 if (q
->flush_handler
)
3632 q
->flush_handler(q
);
3637 rsp_type
= RSPD_TYPE_G(rc
->type_gen
);
3638 if (likely(rsp_type
== RSPD_TYPE_FLBUF_X
)) {
3639 struct page_frag
*fp
;
3641 const struct rx_sw_desc
*rsd
;
3642 u32 len
= ntohl(rc
->pldbuflen_qid
), bufsz
, frags
;
3644 if (len
& RSPD_NEWBUF_F
) {
3645 if (likely(q
->offset
> 0)) {
3646 free_rx_bufs(q
->adap
, &rxq
->fl
, 1);
3649 len
= RSPD_LEN_G(len
);
3653 /* gather packet fragments */
3654 for (frags
= 0, fp
= si
.frags
; ; frags
++, fp
++) {
3655 rsd
= &rxq
->fl
.sdesc
[rxq
->fl
.cidx
];
3656 bufsz
= get_buf_size(adapter
, rsd
);
3657 fp
->page
= rsd
->page
;
3658 fp
->offset
= q
->offset
;
3659 fp
->size
= min(bufsz
, len
);
3663 unmap_rx_buf(q
->adap
, &rxq
->fl
);
3666 si
.sgetstamp
= SGE_TIMESTAMP_G(
3667 be64_to_cpu(rc
->last_flit
));
3669 * Last buffer remains mapped so explicitly make it
3670 * coherent for CPU access.
3672 dma_sync_single_for_cpu(q
->adap
->pdev_dev
,
3674 fp
->size
, DMA_FROM_DEVICE
);
3676 si
.va
= page_address(si
.frags
[0].page
) +
3680 si
.nfrags
= frags
+ 1;
3681 ret
= q
->handler(q
, q
->cur_desc
, &si
);
3682 if (likely(ret
== 0))
3683 q
->offset
+= ALIGN(fp
->size
, s
->fl_align
);
3685 restore_rx_bufs(&si
, &rxq
->fl
, frags
);
3686 } else if (likely(rsp_type
== RSPD_TYPE_CPL_X
)) {
3687 ret
= q
->handler(q
, q
->cur_desc
, NULL
);
3689 ret
= q
->handler(q
, (const __be64
*)rc
, CXGB4_MSG_AN
);
3692 if (unlikely(ret
)) {
3693 /* couldn't process descriptor, back off for recovery */
3694 q
->next_intr_params
= QINTR_TIMER_IDX_V(NOMEM_TMR_IDX
);
3702 if (q
->offset
>= 0 && fl_cap(&rxq
->fl
) - rxq
->fl
.avail
>= 16)
3703 __refill_fl(q
->adap
, &rxq
->fl
);
3704 return budget
- budget_left
;
3708 * napi_rx_handler - the NAPI handler for Rx processing
3709 * @napi: the napi instance
3710 * @budget: how many packets we can process in this round
3712 * Handler for new data events when using NAPI. This does not need any
3713 * locking or protection from interrupts as data interrupts are off at
3714 * this point and other adapter interrupts do not interfere (the latter
3715 * in not a concern at all with MSI-X as non-data interrupts then have
3716 * a separate handler).
3718 static int napi_rx_handler(struct napi_struct
*napi
, int budget
)
3720 unsigned int params
;
3721 struct sge_rspq
*q
= container_of(napi
, struct sge_rspq
, napi
);
3725 work_done
= process_responses(q
, budget
);
3726 if (likely(work_done
< budget
)) {
3729 napi_complete_done(napi
, work_done
);
3730 timer_index
= QINTR_TIMER_IDX_G(q
->next_intr_params
);
3732 if (q
->adaptive_rx
) {
3733 if (work_done
> max(timer_pkt_quota
[timer_index
],
3735 timer_index
= (timer_index
+ 1);
3737 timer_index
= timer_index
- 1;
3739 timer_index
= clamp(timer_index
, 0, SGE_TIMERREGS
- 1);
3740 q
->next_intr_params
=
3741 QINTR_TIMER_IDX_V(timer_index
) |
3743 params
= q
->next_intr_params
;
3745 params
= q
->next_intr_params
;
3746 q
->next_intr_params
= q
->intr_params
;
3749 params
= QINTR_TIMER_IDX_V(7);
3751 val
= CIDXINC_V(work_done
) | SEINTARM_V(params
);
3753 /* If we don't have access to the new User GTS (T5+), use the old
3754 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3756 if (unlikely(q
->bar2_addr
== NULL
)) {
3757 t4_write_reg(q
->adap
, MYPF_REG(SGE_PF_GTS_A
),
3758 val
| INGRESSQID_V((u32
)q
->cntxt_id
));
3760 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3761 q
->bar2_addr
+ SGE_UDB_GTS
);
3767 void cxgb4_ethofld_restart(unsigned long data
)
3769 struct sge_eosw_txq
*eosw_txq
= (struct sge_eosw_txq
*)data
;
3772 spin_lock(&eosw_txq
->lock
);
3773 pktcount
= eosw_txq
->cidx
- eosw_txq
->last_cidx
;
3775 pktcount
+= eosw_txq
->ndesc
;
3778 cxgb4_eosw_txq_free_desc(netdev2adap(eosw_txq
->netdev
),
3779 eosw_txq
, pktcount
);
3780 eosw_txq
->inuse
-= pktcount
;
3783 /* There may be some packets waiting for completions. So,
3784 * attempt to send these packets now.
3786 ethofld_xmit(eosw_txq
->netdev
, eosw_txq
);
3787 spin_unlock(&eosw_txq
->lock
);
3790 /* cxgb4_ethofld_rx_handler - Process ETHOFLD Tx completions
3791 * @q: the response queue that received the packet
3792 * @rsp: the response queue descriptor holding the CPL message
3793 * @si: the gather list of packet fragments
3795 * Process a ETHOFLD Tx completion. Increment the cidx here, but
3796 * free up the descriptors in a tasklet later.
3798 int cxgb4_ethofld_rx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
3799 const struct pkt_gl
*si
)
3801 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3803 /* skip RSS header */
3806 if (opcode
== CPL_FW4_ACK
) {
3807 const struct cpl_fw4_ack
*cpl
;
3808 struct sge_eosw_txq
*eosw_txq
;
3809 struct eotid_entry
*entry
;
3810 struct sk_buff
*skb
;
3815 cpl
= (const struct cpl_fw4_ack
*)rsp
;
3816 eotid
= CPL_FW4_ACK_FLOWID_G(ntohl(OPCODE_TID(cpl
))) -
3817 q
->adap
->tids
.eotid_base
;
3818 entry
= cxgb4_lookup_eotid(&q
->adap
->tids
, eotid
);
3822 eosw_txq
= (struct sge_eosw_txq
*)entry
->data
;
3826 spin_lock(&eosw_txq
->lock
);
3827 credits
= cpl
->credits
;
3828 while (credits
> 0) {
3829 skb
= eosw_txq
->desc
[eosw_txq
->cidx
].skb
;
3833 if (unlikely((eosw_txq
->state
==
3834 CXGB4_EO_STATE_FLOWC_OPEN_REPLY
||
3836 CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
) &&
3837 eosw_txq
->cidx
== eosw_txq
->flowc_idx
)) {
3838 flits
= DIV_ROUND_UP(skb
->len
, 8);
3839 if (eosw_txq
->state
==
3840 CXGB4_EO_STATE_FLOWC_OPEN_REPLY
)
3841 eosw_txq
->state
= CXGB4_EO_STATE_ACTIVE
;
3843 eosw_txq
->state
= CXGB4_EO_STATE_CLOSED
;
3844 complete(&eosw_txq
->completion
);
3846 hdr_len
= eth_get_headlen(eosw_txq
->netdev
,
3849 flits
= ethofld_calc_tx_flits(q
->adap
, skb
,
3852 eosw_txq_advance_index(&eosw_txq
->cidx
, 1,
3854 wrlen16
= DIV_ROUND_UP(flits
* 8, 16);
3858 eosw_txq
->cred
+= cpl
->credits
;
3861 spin_unlock(&eosw_txq
->lock
);
3863 /* Schedule a tasklet to reclaim SKBs and restart ETHOFLD Tx,
3864 * if there were packets waiting for completion.
3866 tasklet_schedule(&eosw_txq
->qresume_tsk
);
3874 * The MSI-X interrupt handler for an SGE response queue.
3876 irqreturn_t
t4_sge_intr_msix(int irq
, void *cookie
)
3878 struct sge_rspq
*q
= cookie
;
3880 napi_schedule(&q
->napi
);
3885 * Process the indirect interrupt entries in the interrupt queue and kick off
3886 * NAPI for each queue that has generated an entry.
3888 static unsigned int process_intrq(struct adapter
*adap
)
3890 unsigned int credits
;
3891 const struct rsp_ctrl
*rc
;
3892 struct sge_rspq
*q
= &adap
->sge
.intrq
;
3895 spin_lock(&adap
->sge
.intrq_lock
);
3896 for (credits
= 0; ; credits
++) {
3897 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3898 if (!is_new_response(rc
, q
))
3902 if (RSPD_TYPE_G(rc
->type_gen
) == RSPD_TYPE_INTR_X
) {
3903 unsigned int qid
= ntohl(rc
->pldbuflen_qid
);
3905 qid
-= adap
->sge
.ingr_start
;
3906 napi_schedule(&adap
->sge
.ingr_map
[qid
]->napi
);
3912 val
= CIDXINC_V(credits
) | SEINTARM_V(q
->intr_params
);
3914 /* If we don't have access to the new User GTS (T5+), use the old
3915 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3917 if (unlikely(q
->bar2_addr
== NULL
)) {
3918 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS_A
),
3919 val
| INGRESSQID_V(q
->cntxt_id
));
3921 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3922 q
->bar2_addr
+ SGE_UDB_GTS
);
3925 spin_unlock(&adap
->sge
.intrq_lock
);
3930 * The MSI interrupt handler, which handles data events from SGE response queues
3931 * as well as error and other async events as they all use the same MSI vector.
3933 static irqreturn_t
t4_intr_msi(int irq
, void *cookie
)
3935 struct adapter
*adap
= cookie
;
3937 if (adap
->flags
& CXGB4_MASTER_PF
)
3938 t4_slow_intr_handler(adap
);
3939 process_intrq(adap
);
3944 * Interrupt handler for legacy INTx interrupts.
3945 * Handles data events from SGE response queues as well as error and other
3946 * async events as they all use the same interrupt line.
3948 static irqreturn_t
t4_intr_intx(int irq
, void *cookie
)
3950 struct adapter
*adap
= cookie
;
3952 t4_write_reg(adap
, MYPF_REG(PCIE_PF_CLI_A
), 0);
3953 if (((adap
->flags
& CXGB4_MASTER_PF
) && t4_slow_intr_handler(adap
)) |
3954 process_intrq(adap
))
3956 return IRQ_NONE
; /* probably shared interrupt */
3960 * t4_intr_handler - select the top-level interrupt handler
3961 * @adap: the adapter
3963 * Selects the top-level interrupt handler based on the type of interrupts
3964 * (MSI-X, MSI, or INTx).
3966 irq_handler_t
t4_intr_handler(struct adapter
*adap
)
3968 if (adap
->flags
& CXGB4_USING_MSIX
)
3969 return t4_sge_intr_msix
;
3970 if (adap
->flags
& CXGB4_USING_MSI
)
3972 return t4_intr_intx
;
3975 static void sge_rx_timer_cb(struct timer_list
*t
)
3979 struct adapter
*adap
= from_timer(adap
, t
, sge
.rx_timer
);
3980 struct sge
*s
= &adap
->sge
;
3982 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
3983 for (m
= s
->starving_fl
[i
]; m
; m
&= m
- 1) {
3984 struct sge_eth_rxq
*rxq
;
3985 unsigned int id
= __ffs(m
) + i
* BITS_PER_LONG
;
3986 struct sge_fl
*fl
= s
->egr_map
[id
];
3988 clear_bit(id
, s
->starving_fl
);
3989 smp_mb__after_atomic();
3991 if (fl_starving(adap
, fl
)) {
3992 rxq
= container_of(fl
, struct sge_eth_rxq
, fl
);
3993 if (napi_reschedule(&rxq
->rspq
.napi
))
3996 set_bit(id
, s
->starving_fl
);
3999 /* The remainder of the SGE RX Timer Callback routine is dedicated to
4000 * global Master PF activities like checking for chip ingress stalls,
4003 if (!(adap
->flags
& CXGB4_MASTER_PF
))
4006 t4_idma_monitor(adap
, &s
->idma_monitor
, HZ
, RX_QCHECK_PERIOD
);
4009 mod_timer(&s
->rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
4012 static void sge_tx_timer_cb(struct timer_list
*t
)
4014 struct adapter
*adap
= from_timer(adap
, t
, sge
.tx_timer
);
4015 struct sge
*s
= &adap
->sge
;
4016 unsigned long m
, period
;
4017 unsigned int i
, budget
;
4019 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
4020 for (m
= s
->txq_maperr
[i
]; m
; m
&= m
- 1) {
4021 unsigned long id
= __ffs(m
) + i
* BITS_PER_LONG
;
4022 struct sge_uld_txq
*txq
= s
->egr_map
[id
];
4024 clear_bit(id
, s
->txq_maperr
);
4025 tasklet_schedule(&txq
->qresume_tsk
);
4028 if (!is_t4(adap
->params
.chip
)) {
4029 struct sge_eth_txq
*q
= &s
->ptptxq
;
4032 spin_lock(&adap
->ptp_lock
);
4033 avail
= reclaimable(&q
->q
);
4036 free_tx_desc(adap
, &q
->q
, avail
, false);
4037 q
->q
.in_use
-= avail
;
4039 spin_unlock(&adap
->ptp_lock
);
4042 budget
= MAX_TIMER_TX_RECLAIM
;
4043 i
= s
->ethtxq_rover
;
4045 budget
-= t4_sge_eth_txq_egress_update(adap
, &s
->ethtxq
[i
],
4050 if (++i
>= s
->ethqsets
)
4052 } while (i
!= s
->ethtxq_rover
);
4053 s
->ethtxq_rover
= i
;
4056 /* If we found too many reclaimable packets schedule a timer
4057 * in the near future to continue where we left off.
4061 /* We reclaimed all reclaimable TX Descriptors, so reschedule
4062 * at the normal period.
4064 period
= TX_QCHECK_PERIOD
;
4067 mod_timer(&s
->tx_timer
, jiffies
+ period
);
4071 * bar2_address - return the BAR2 address for an SGE Queue's Registers
4072 * @adapter: the adapter
4073 * @qid: the SGE Queue ID
4074 * @qtype: the SGE Queue Type (Egress or Ingress)
4075 * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
4077 * Returns the BAR2 address for the SGE Queue Registers associated with
4078 * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
4079 * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
4080 * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
4081 * Registers are supported (e.g. the Write Combining Doorbell Buffer).
4083 static void __iomem
*bar2_address(struct adapter
*adapter
,
4085 enum t4_bar2_qtype qtype
,
4086 unsigned int *pbar2_qid
)
4091 ret
= t4_bar2_sge_qregs(adapter
, qid
, qtype
, 0,
4092 &bar2_qoffset
, pbar2_qid
);
4096 return adapter
->bar2
+ bar2_qoffset
;
4099 /* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
4100 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
4102 int t4_sge_alloc_rxq(struct adapter
*adap
, struct sge_rspq
*iq
, bool fwevtq
,
4103 struct net_device
*dev
, int intr_idx
,
4104 struct sge_fl
*fl
, rspq_handler_t hnd
,
4105 rspq_flush_handler_t flush_hnd
, int cong
)
4109 struct sge
*s
= &adap
->sge
;
4110 struct port_info
*pi
= netdev_priv(dev
);
4111 int relaxed
= !(adap
->flags
& CXGB4_ROOT_NO_RELAXED_ORDERING
);
4113 /* Size needs to be multiple of 16, including status entry. */
4114 iq
->size
= roundup(iq
->size
, 16);
4116 iq
->desc
= alloc_ring(adap
->pdev_dev
, iq
->size
, iq
->iqe_len
, 0,
4117 &iq
->phys_addr
, NULL
, 0,
4118 dev_to_node(adap
->pdev_dev
));
4122 memset(&c
, 0, sizeof(c
));
4123 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_IQ_CMD
) | FW_CMD_REQUEST_F
|
4124 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4125 FW_IQ_CMD_PFN_V(adap
->pf
) | FW_IQ_CMD_VFN_V(0));
4126 c
.alloc_to_len16
= htonl(FW_IQ_CMD_ALLOC_F
| FW_IQ_CMD_IQSTART_F
|
4128 c
.type_to_iqandstindex
= htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP
) |
4129 FW_IQ_CMD_IQASYNCH_V(fwevtq
) | FW_IQ_CMD_VIID_V(pi
->viid
) |
4130 FW_IQ_CMD_IQANDST_V(intr_idx
< 0) |
4131 FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X
) |
4132 FW_IQ_CMD_IQANDSTINDEX_V(intr_idx
>= 0 ? intr_idx
:
4134 c
.iqdroprss_to_iqesize
= htons(FW_IQ_CMD_IQPCIECH_V(pi
->tx_chan
) |
4135 FW_IQ_CMD_IQGTSMODE_F
|
4136 FW_IQ_CMD_IQINTCNTTHRESH_V(iq
->pktcnt_idx
) |
4137 FW_IQ_CMD_IQESIZE_V(ilog2(iq
->iqe_len
) - 4));
4138 c
.iqsize
= htons(iq
->size
);
4139 c
.iqaddr
= cpu_to_be64(iq
->phys_addr
);
4141 c
.iqns_to_fl0congen
= htonl(FW_IQ_CMD_IQFLINTCONGEN_F
|
4142 FW_IQ_CMD_IQTYPE_V(cong
? FW_IQ_IQTYPE_NIC
4143 : FW_IQ_IQTYPE_OFLD
));
4146 unsigned int chip_ver
=
4147 CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4149 /* Allocate the ring for the hardware free list (with space
4150 * for its status page) along with the associated software
4151 * descriptor ring. The free list size needs to be a multiple
4152 * of the Egress Queue Unit and at least 2 Egress Units larger
4153 * than the SGE's Egress Congrestion Threshold
4154 * (fl_starve_thres - 1).
4156 if (fl
->size
< s
->fl_starve_thres
- 1 + 2 * 8)
4157 fl
->size
= s
->fl_starve_thres
- 1 + 2 * 8;
4158 fl
->size
= roundup(fl
->size
, 8);
4159 fl
->desc
= alloc_ring(adap
->pdev_dev
, fl
->size
, sizeof(__be64
),
4160 sizeof(struct rx_sw_desc
), &fl
->addr
,
4161 &fl
->sdesc
, s
->stat_len
,
4162 dev_to_node(adap
->pdev_dev
));
4166 flsz
= fl
->size
/ 8 + s
->stat_len
/ sizeof(struct tx_desc
);
4167 c
.iqns_to_fl0congen
|= htonl(FW_IQ_CMD_FL0PACKEN_F
|
4168 FW_IQ_CMD_FL0FETCHRO_V(relaxed
) |
4169 FW_IQ_CMD_FL0DATARO_V(relaxed
) |
4170 FW_IQ_CMD_FL0PADEN_F
);
4172 c
.iqns_to_fl0congen
|=
4173 htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong
) |
4174 FW_IQ_CMD_FL0CONGCIF_F
|
4175 FW_IQ_CMD_FL0CONGEN_F
);
4176 /* In T6, for egress queue type FL there is internal overhead
4177 * of 16B for header going into FLM module. Hence the maximum
4178 * allowed burst size is 448 bytes. For T4/T5, the hardware
4179 * doesn't coalesce fetch requests if more than 64 bytes of
4180 * Free List pointers are provided, so we use a 128-byte Fetch
4181 * Burst Minimum there (T6 implements coalescing so we can use
4182 * the smaller 64-byte value there).
4184 c
.fl0dcaen_to_fl0cidxfthresh
=
4185 htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver
<= CHELSIO_T5
?
4186 FETCHBURSTMIN_128B_X
:
4187 FETCHBURSTMIN_64B_T6_X
) |
4188 FW_IQ_CMD_FL0FBMAX_V((chip_ver
<= CHELSIO_T5
) ?
4189 FETCHBURSTMAX_512B_X
:
4190 FETCHBURSTMAX_256B_X
));
4191 c
.fl0size
= htons(flsz
);
4192 c
.fl0addr
= cpu_to_be64(fl
->addr
);
4195 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4199 netif_napi_add(dev
, &iq
->napi
, napi_rx_handler
, 64);
4200 iq
->cur_desc
= iq
->desc
;
4203 iq
->next_intr_params
= iq
->intr_params
;
4204 iq
->cntxt_id
= ntohs(c
.iqid
);
4205 iq
->abs_id
= ntohs(c
.physiqid
);
4206 iq
->bar2_addr
= bar2_address(adap
,
4208 T4_BAR2_QTYPE_INGRESS
,
4210 iq
->size
--; /* subtract status entry */
4213 iq
->flush_handler
= flush_hnd
;
4215 memset(&iq
->lro_mgr
, 0, sizeof(struct t4_lro_mgr
));
4216 skb_queue_head_init(&iq
->lro_mgr
.lroq
);
4218 /* set offset to -1 to distinguish ingress queues without FL */
4219 iq
->offset
= fl
? 0 : -1;
4221 adap
->sge
.ingr_map
[iq
->cntxt_id
- adap
->sge
.ingr_start
] = iq
;
4224 fl
->cntxt_id
= ntohs(c
.fl0id
);
4225 fl
->avail
= fl
->pend_cred
= 0;
4226 fl
->pidx
= fl
->cidx
= 0;
4227 fl
->alloc_failed
= fl
->large_alloc_failed
= fl
->starving
= 0;
4228 adap
->sge
.egr_map
[fl
->cntxt_id
- adap
->sge
.egr_start
] = fl
;
4230 /* Note, we must initialize the BAR2 Free List User Doorbell
4231 * information before refilling the Free List!
4233 fl
->bar2_addr
= bar2_address(adap
,
4235 T4_BAR2_QTYPE_EGRESS
,
4237 refill_fl(adap
, fl
, fl_cap(fl
), GFP_KERNEL
);
4240 /* For T5 and later we attempt to set up the Congestion Manager values
4241 * of the new RX Ethernet Queue. This should really be handled by
4242 * firmware because it's more complex than any host driver wants to
4243 * get involved with and it's different per chip and this is almost
4244 * certainly wrong. Firmware would be wrong as well, but it would be
4245 * a lot easier to fix in one place ... For now we do something very
4246 * simple (and hopefully less wrong).
4248 if (!is_t4(adap
->params
.chip
) && cong
>= 0) {
4249 u32 param
, val
, ch_map
= 0;
4251 u16 cng_ch_bits_log
= adap
->params
.arch
.cng_ch_bits_log
;
4253 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
4254 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT
) |
4255 FW_PARAMS_PARAM_YZ_V(iq
->cntxt_id
));
4257 val
= CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X
);
4260 CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X
);
4261 for (i
= 0; i
< 4; i
++) {
4262 if (cong
& (1 << i
))
4263 ch_map
|= 1 << (i
<< cng_ch_bits_log
);
4265 val
|= CONMCTXT_CNGCHMAP_V(ch_map
);
4267 ret
= t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1,
4270 dev_warn(adap
->pdev_dev
, "Failed to set Congestion"
4271 " Manager Context for Ingress Queue %d: %d\n",
4272 iq
->cntxt_id
, -ret
);
4281 dma_free_coherent(adap
->pdev_dev
, iq
->size
* iq
->iqe_len
,
4282 iq
->desc
, iq
->phys_addr
);
4285 if (fl
&& fl
->desc
) {
4288 dma_free_coherent(adap
->pdev_dev
, flsz
* sizeof(struct tx_desc
),
4289 fl
->desc
, fl
->addr
);
4295 static void init_txq(struct adapter
*adap
, struct sge_txq
*q
, unsigned int id
)
4298 q
->bar2_addr
= bar2_address(adap
,
4300 T4_BAR2_QTYPE_EGRESS
,
4303 q
->cidx
= q
->pidx
= 0;
4304 q
->stops
= q
->restarts
= 0;
4305 q
->stat
= (void *)&q
->desc
[q
->size
];
4306 spin_lock_init(&q
->db_lock
);
4307 adap
->sge
.egr_map
[id
- adap
->sge
.egr_start
] = q
;
4311 * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue
4312 * @adap: the adapter
4313 * @txq: the SGE Ethernet TX Queue to initialize
4314 * @dev: the Linux Network Device
4315 * @netdevq: the corresponding Linux TX Queue
4316 * @iqid: the Ingress Queue to which to deliver CIDX Update messages
4317 * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers
4319 int t4_sge_alloc_eth_txq(struct adapter
*adap
, struct sge_eth_txq
*txq
,
4320 struct net_device
*dev
, struct netdev_queue
*netdevq
,
4321 unsigned int iqid
, u8 dbqt
)
4323 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4324 struct port_info
*pi
= netdev_priv(dev
);
4325 struct sge
*s
= &adap
->sge
;
4326 struct fw_eq_eth_cmd c
;
4329 /* Add status entries */
4330 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4332 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, txq
->q
.size
,
4333 sizeof(struct tx_desc
), sizeof(struct tx_sw_desc
),
4334 &txq
->q
.phys_addr
, &txq
->q
.sdesc
, s
->stat_len
,
4335 netdev_queue_numa_node_read(netdevq
));
4339 memset(&c
, 0, sizeof(c
));
4340 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD
) | FW_CMD_REQUEST_F
|
4341 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4342 FW_EQ_ETH_CMD_PFN_V(adap
->pf
) |
4343 FW_EQ_ETH_CMD_VFN_V(0));
4344 c
.alloc_to_len16
= htonl(FW_EQ_ETH_CMD_ALLOC_F
|
4345 FW_EQ_ETH_CMD_EQSTART_F
| FW_LEN16(c
));
4347 /* For TX Ethernet Queues using the SGE Doorbell Queue Timer
4348 * mechanism, we use Ingress Queue messages for Hardware Consumer
4349 * Index Updates on the TX Queue. Otherwise we have the Hardware
4350 * write the CIDX Updates into the Status Page at the end of the
4353 c
.autoequiqe_to_viid
= htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F
|
4354 FW_EQ_ETH_CMD_VIID_V(pi
->viid
));
4356 c
.fetchszm_to_iqid
=
4357 htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4358 FW_EQ_ETH_CMD_PCIECHN_V(pi
->tx_chan
) |
4359 FW_EQ_ETH_CMD_FETCHRO_F
| FW_EQ_ETH_CMD_IQID_V(iqid
));
4361 /* Note that the CIDX Flush Threshold should match MAX_TX_RECLAIM. */
4363 htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
4364 ? FETCHBURSTMIN_64B_X
4365 : FETCHBURSTMIN_64B_T6_X
) |
4366 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4367 FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4368 FW_EQ_ETH_CMD_EQSIZE_V(nentries
));
4370 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
4372 /* If we're using the SGE Doorbell Queue Timer mechanism, pass in the
4373 * currently configured Timer Index. THis can be changed later via an
4374 * ethtool -C tx-usecs {Timer Val} command. Note that the SGE
4375 * Doorbell Queue mode is currently automatically enabled in the
4376 * Firmware by setting either AUTOEQUEQE or AUTOEQUIQE ...
4380 cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F
|
4381 FW_EQ_ETH_CMD_TIMERIX_V(txq
->dbqtimerix
));
4383 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4385 kfree(txq
->q
.sdesc
);
4386 txq
->q
.sdesc
= NULL
;
4387 dma_free_coherent(adap
->pdev_dev
,
4388 nentries
* sizeof(struct tx_desc
),
4389 txq
->q
.desc
, txq
->q
.phys_addr
);
4394 txq
->q
.q_type
= CXGB4_TXQ_ETH
;
4395 init_txq(adap
, &txq
->q
, FW_EQ_ETH_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
4401 txq
->mapping_err
= 0;
4407 int t4_sge_alloc_ctrl_txq(struct adapter
*adap
, struct sge_ctrl_txq
*txq
,
4408 struct net_device
*dev
, unsigned int iqid
,
4409 unsigned int cmplqid
)
4411 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4412 struct port_info
*pi
= netdev_priv(dev
);
4413 struct sge
*s
= &adap
->sge
;
4414 struct fw_eq_ctrl_cmd c
;
4417 /* Add status entries */
4418 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4420 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, nentries
,
4421 sizeof(struct tx_desc
), 0, &txq
->q
.phys_addr
,
4422 NULL
, 0, dev_to_node(adap
->pdev_dev
));
4426 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD
) | FW_CMD_REQUEST_F
|
4427 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4428 FW_EQ_CTRL_CMD_PFN_V(adap
->pf
) |
4429 FW_EQ_CTRL_CMD_VFN_V(0));
4430 c
.alloc_to_len16
= htonl(FW_EQ_CTRL_CMD_ALLOC_F
|
4431 FW_EQ_CTRL_CMD_EQSTART_F
| FW_LEN16(c
));
4432 c
.cmpliqid_eqid
= htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid
));
4433 c
.physeqid_pkd
= htonl(0);
4434 c
.fetchszm_to_iqid
=
4435 htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4436 FW_EQ_CTRL_CMD_PCIECHN_V(pi
->tx_chan
) |
4437 FW_EQ_CTRL_CMD_FETCHRO_F
| FW_EQ_CTRL_CMD_IQID_V(iqid
));
4439 htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
4440 ? FETCHBURSTMIN_64B_X
4441 : FETCHBURSTMIN_64B_T6_X
) |
4442 FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4443 FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4444 FW_EQ_CTRL_CMD_EQSIZE_V(nentries
));
4445 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
4447 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4449 dma_free_coherent(adap
->pdev_dev
,
4450 nentries
* sizeof(struct tx_desc
),
4451 txq
->q
.desc
, txq
->q
.phys_addr
);
4456 txq
->q
.q_type
= CXGB4_TXQ_CTRL
;
4457 init_txq(adap
, &txq
->q
, FW_EQ_CTRL_CMD_EQID_G(ntohl(c
.cmpliqid_eqid
)));
4459 skb_queue_head_init(&txq
->sendq
);
4460 tasklet_init(&txq
->qresume_tsk
, restart_ctrlq
, (unsigned long)txq
);
4465 int t4_sge_mod_ctrl_txq(struct adapter
*adap
, unsigned int eqid
,
4466 unsigned int cmplqid
)
4470 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
4471 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL
) |
4472 FW_PARAMS_PARAM_YZ_V(eqid
));
4474 return t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1, ¶m
, &val
);
4477 static int t4_sge_alloc_ofld_txq(struct adapter
*adap
, struct sge_txq
*q
,
4478 struct net_device
*dev
, u32 cmd
, u32 iqid
)
4480 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4481 struct port_info
*pi
= netdev_priv(dev
);
4482 struct sge
*s
= &adap
->sge
;
4483 struct fw_eq_ofld_cmd c
;
4484 u32 fb_min
, nentries
;
4487 /* Add status entries */
4488 nentries
= q
->size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4489 q
->desc
= alloc_ring(adap
->pdev_dev
, q
->size
, sizeof(struct tx_desc
),
4490 sizeof(struct tx_sw_desc
), &q
->phys_addr
,
4491 &q
->sdesc
, s
->stat_len
, NUMA_NO_NODE
);
4495 if (chip_ver
<= CHELSIO_T5
)
4496 fb_min
= FETCHBURSTMIN_64B_X
;
4498 fb_min
= FETCHBURSTMIN_64B_T6_X
;
4500 memset(&c
, 0, sizeof(c
));
4501 c
.op_to_vfn
= htonl(FW_CMD_OP_V(cmd
) | FW_CMD_REQUEST_F
|
4502 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4503 FW_EQ_OFLD_CMD_PFN_V(adap
->pf
) |
4504 FW_EQ_OFLD_CMD_VFN_V(0));
4505 c
.alloc_to_len16
= htonl(FW_EQ_OFLD_CMD_ALLOC_F
|
4506 FW_EQ_OFLD_CMD_EQSTART_F
| FW_LEN16(c
));
4507 c
.fetchszm_to_iqid
=
4508 htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4509 FW_EQ_OFLD_CMD_PCIECHN_V(pi
->tx_chan
) |
4510 FW_EQ_OFLD_CMD_FETCHRO_F
| FW_EQ_OFLD_CMD_IQID_V(iqid
));
4512 htonl(FW_EQ_OFLD_CMD_FBMIN_V(fb_min
) |
4513 FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4514 FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4515 FW_EQ_OFLD_CMD_EQSIZE_V(nentries
));
4516 c
.eqaddr
= cpu_to_be64(q
->phys_addr
);
4518 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4522 dma_free_coherent(adap
->pdev_dev
,
4523 nentries
* sizeof(struct tx_desc
),
4524 q
->desc
, q
->phys_addr
);
4529 init_txq(adap
, q
, FW_EQ_OFLD_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
4533 int t4_sge_alloc_uld_txq(struct adapter
*adap
, struct sge_uld_txq
*txq
,
4534 struct net_device
*dev
, unsigned int iqid
,
4535 unsigned int uld_type
)
4537 u32 cmd
= FW_EQ_OFLD_CMD
;
4540 if (unlikely(uld_type
== CXGB4_TX_CRYPTO
))
4541 cmd
= FW_EQ_CTRL_CMD
;
4543 ret
= t4_sge_alloc_ofld_txq(adap
, &txq
->q
, dev
, cmd
, iqid
);
4547 txq
->q
.q_type
= CXGB4_TXQ_ULD
;
4549 skb_queue_head_init(&txq
->sendq
);
4550 tasklet_init(&txq
->qresume_tsk
, restart_ofldq
, (unsigned long)txq
);
4552 txq
->mapping_err
= 0;
4556 int t4_sge_alloc_ethofld_txq(struct adapter
*adap
, struct sge_eohw_txq
*txq
,
4557 struct net_device
*dev
, u32 iqid
)
4561 ret
= t4_sge_alloc_ofld_txq(adap
, &txq
->q
, dev
, FW_EQ_OFLD_CMD
, iqid
);
4565 txq
->q
.q_type
= CXGB4_TXQ_ULD
;
4566 spin_lock_init(&txq
->lock
);
4572 txq
->mapping_err
= 0;
4576 void free_txq(struct adapter
*adap
, struct sge_txq
*q
)
4578 struct sge
*s
= &adap
->sge
;
4580 dma_free_coherent(adap
->pdev_dev
,
4581 q
->size
* sizeof(struct tx_desc
) + s
->stat_len
,
4582 q
->desc
, q
->phys_addr
);
4588 void free_rspq_fl(struct adapter
*adap
, struct sge_rspq
*rq
,
4591 struct sge
*s
= &adap
->sge
;
4592 unsigned int fl_id
= fl
? fl
->cntxt_id
: 0xffff;
4594 adap
->sge
.ingr_map
[rq
->cntxt_id
- adap
->sge
.ingr_start
] = NULL
;
4595 t4_iq_free(adap
, adap
->mbox
, adap
->pf
, 0, FW_IQ_TYPE_FL_INT_CAP
,
4596 rq
->cntxt_id
, fl_id
, 0xffff);
4597 dma_free_coherent(adap
->pdev_dev
, (rq
->size
+ 1) * rq
->iqe_len
,
4598 rq
->desc
, rq
->phys_addr
);
4599 netif_napi_del(&rq
->napi
);
4601 rq
->cntxt_id
= rq
->abs_id
= 0;
4605 free_rx_bufs(adap
, fl
, fl
->avail
);
4606 dma_free_coherent(adap
->pdev_dev
, fl
->size
* 8 + s
->stat_len
,
4607 fl
->desc
, fl
->addr
);
4616 * t4_free_ofld_rxqs - free a block of consecutive Rx queues
4617 * @adap: the adapter
4618 * @n: number of queues
4619 * @q: pointer to first queue
4621 * Release the resources of a consecutive block of offload Rx queues.
4623 void t4_free_ofld_rxqs(struct adapter
*adap
, int n
, struct sge_ofld_rxq
*q
)
4625 for ( ; n
; n
--, q
++)
4627 free_rspq_fl(adap
, &q
->rspq
,
4628 q
->fl
.size
? &q
->fl
: NULL
);
4631 void t4_sge_free_ethofld_txq(struct adapter
*adap
, struct sge_eohw_txq
*txq
)
4634 t4_ofld_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4636 free_tx_desc(adap
, &txq
->q
, txq
->q
.in_use
, false);
4637 kfree(txq
->q
.sdesc
);
4638 free_txq(adap
, &txq
->q
);
4643 * t4_free_sge_resources - free SGE resources
4644 * @adap: the adapter
4646 * Frees resources used by the SGE queue sets.
4648 void t4_free_sge_resources(struct adapter
*adap
)
4651 struct sge_eth_rxq
*eq
;
4652 struct sge_eth_txq
*etq
;
4654 /* stop all Rx queues in order to start them draining */
4655 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4656 eq
= &adap
->sge
.ethrxq
[i
];
4658 t4_iq_stop(adap
, adap
->mbox
, adap
->pf
, 0,
4659 FW_IQ_TYPE_FL_INT_CAP
,
4661 eq
->fl
.size
? eq
->fl
.cntxt_id
: 0xffff,
4665 /* clean up Ethernet Tx/Rx queues */
4666 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4667 eq
= &adap
->sge
.ethrxq
[i
];
4669 free_rspq_fl(adap
, &eq
->rspq
,
4670 eq
->fl
.size
? &eq
->fl
: NULL
);
4672 cxgb4_free_msix_idx_in_bmap(adap
, eq
->msix
->idx
);
4676 etq
= &adap
->sge
.ethtxq
[i
];
4678 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4680 __netif_tx_lock_bh(etq
->txq
);
4681 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4682 __netif_tx_unlock_bh(etq
->txq
);
4683 kfree(etq
->q
.sdesc
);
4684 free_txq(adap
, &etq
->q
);
4688 /* clean up control Tx queues */
4689 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ctrlq
); i
++) {
4690 struct sge_ctrl_txq
*cq
= &adap
->sge
.ctrlq
[i
];
4693 tasklet_kill(&cq
->qresume_tsk
);
4694 t4_ctrl_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4696 __skb_queue_purge(&cq
->sendq
);
4697 free_txq(adap
, &cq
->q
);
4701 if (adap
->sge
.fw_evtq
.desc
) {
4702 free_rspq_fl(adap
, &adap
->sge
.fw_evtq
, NULL
);
4703 if (adap
->sge
.fwevtq_msix_idx
>= 0)
4704 cxgb4_free_msix_idx_in_bmap(adap
,
4705 adap
->sge
.fwevtq_msix_idx
);
4708 if (adap
->sge
.nd_msix_idx
>= 0)
4709 cxgb4_free_msix_idx_in_bmap(adap
, adap
->sge
.nd_msix_idx
);
4711 if (adap
->sge
.intrq
.desc
)
4712 free_rspq_fl(adap
, &adap
->sge
.intrq
, NULL
);
4714 if (!is_t4(adap
->params
.chip
)) {
4715 etq
= &adap
->sge
.ptptxq
;
4717 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4719 spin_lock_bh(&adap
->ptp_lock
);
4720 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4721 spin_unlock_bh(&adap
->ptp_lock
);
4722 kfree(etq
->q
.sdesc
);
4723 free_txq(adap
, &etq
->q
);
4727 /* clear the reverse egress queue map */
4728 memset(adap
->sge
.egr_map
, 0,
4729 adap
->sge
.egr_sz
* sizeof(*adap
->sge
.egr_map
));
4732 void t4_sge_start(struct adapter
*adap
)
4734 adap
->sge
.ethtxq_rover
= 0;
4735 mod_timer(&adap
->sge
.rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
4736 mod_timer(&adap
->sge
.tx_timer
, jiffies
+ TX_QCHECK_PERIOD
);
4740 * t4_sge_stop - disable SGE operation
4741 * @adap: the adapter
4743 * Stop tasklets and timers associated with the DMA engine. Note that
4744 * this is effective only if measures have been taken to disable any HW
4745 * events that may restart them.
4747 void t4_sge_stop(struct adapter
*adap
)
4750 struct sge
*s
= &adap
->sge
;
4752 if (in_interrupt()) /* actions below require waiting */
4755 if (s
->rx_timer
.function
)
4756 del_timer_sync(&s
->rx_timer
);
4757 if (s
->tx_timer
.function
)
4758 del_timer_sync(&s
->tx_timer
);
4760 if (is_offload(adap
)) {
4761 struct sge_uld_txq_info
*txq_info
;
4763 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
4765 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4767 for_each_ofldtxq(&adap
->sge
, i
) {
4769 tasklet_kill(&txq
->qresume_tsk
);
4774 if (is_pci_uld(adap
)) {
4775 struct sge_uld_txq_info
*txq_info
;
4777 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_CRYPTO
];
4779 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4781 for_each_ofldtxq(&adap
->sge
, i
) {
4783 tasklet_kill(&txq
->qresume_tsk
);
4788 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++) {
4789 struct sge_ctrl_txq
*cq
= &s
->ctrlq
[i
];
4792 tasklet_kill(&cq
->qresume_tsk
);
4797 * t4_sge_init_soft - grab core SGE values needed by SGE code
4798 * @adap: the adapter
4800 * We need to grab the SGE operating parameters that we need to have
4801 * in order to do our job and make sure we can live with them.
4804 static int t4_sge_init_soft(struct adapter
*adap
)
4806 struct sge
*s
= &adap
->sge
;
4807 u32 fl_small_pg
, fl_large_pg
, fl_small_mtu
, fl_large_mtu
;
4808 u32 timer_value_0_and_1
, timer_value_2_and_3
, timer_value_4_and_5
;
4809 u32 ingress_rx_threshold
;
4812 * Verify that CPL messages are going to the Ingress Queue for
4813 * process_responses() and that only packet data is going to the
4816 if ((t4_read_reg(adap
, SGE_CONTROL_A
) & RXPKTCPLMODE_F
) !=
4817 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X
)) {
4818 dev_err(adap
->pdev_dev
, "bad SGE CPL MODE\n");
4823 * Validate the Host Buffer Register Array indices that we want to
4826 * XXX Note that we should really read through the Host Buffer Size
4827 * XXX register array and find the indices of the Buffer Sizes which
4828 * XXX meet our needs!
4830 #define READ_FL_BUF(x) \
4831 t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
4833 fl_small_pg
= READ_FL_BUF(RX_SMALL_PG_BUF
);
4834 fl_large_pg
= READ_FL_BUF(RX_LARGE_PG_BUF
);
4835 fl_small_mtu
= READ_FL_BUF(RX_SMALL_MTU_BUF
);
4836 fl_large_mtu
= READ_FL_BUF(RX_LARGE_MTU_BUF
);
4838 /* We only bother using the Large Page logic if the Large Page Buffer
4839 * is larger than our Page Size Buffer.
4841 if (fl_large_pg
<= fl_small_pg
)
4846 /* The Page Size Buffer must be exactly equal to our Page Size and the
4847 * Large Page Size Buffer should be 0 (per above) or a power of 2.
4849 if (fl_small_pg
!= PAGE_SIZE
||
4850 (fl_large_pg
& (fl_large_pg
-1)) != 0) {
4851 dev_err(adap
->pdev_dev
, "bad SGE FL page buffer sizes [%d, %d]\n",
4852 fl_small_pg
, fl_large_pg
);
4856 s
->fl_pg_order
= ilog2(fl_large_pg
) - PAGE_SHIFT
;
4858 if (fl_small_mtu
< FL_MTU_SMALL_BUFSIZE(adap
) ||
4859 fl_large_mtu
< FL_MTU_LARGE_BUFSIZE(adap
)) {
4860 dev_err(adap
->pdev_dev
, "bad SGE FL MTU sizes [%d, %d]\n",
4861 fl_small_mtu
, fl_large_mtu
);
4866 * Retrieve our RX interrupt holdoff timer values and counter
4867 * threshold values from the SGE parameters.
4869 timer_value_0_and_1
= t4_read_reg(adap
, SGE_TIMER_VALUE_0_AND_1_A
);
4870 timer_value_2_and_3
= t4_read_reg(adap
, SGE_TIMER_VALUE_2_AND_3_A
);
4871 timer_value_4_and_5
= t4_read_reg(adap
, SGE_TIMER_VALUE_4_AND_5_A
);
4872 s
->timer_val
[0] = core_ticks_to_us(adap
,
4873 TIMERVALUE0_G(timer_value_0_and_1
));
4874 s
->timer_val
[1] = core_ticks_to_us(adap
,
4875 TIMERVALUE1_G(timer_value_0_and_1
));
4876 s
->timer_val
[2] = core_ticks_to_us(adap
,
4877 TIMERVALUE2_G(timer_value_2_and_3
));
4878 s
->timer_val
[3] = core_ticks_to_us(adap
,
4879 TIMERVALUE3_G(timer_value_2_and_3
));
4880 s
->timer_val
[4] = core_ticks_to_us(adap
,
4881 TIMERVALUE4_G(timer_value_4_and_5
));
4882 s
->timer_val
[5] = core_ticks_to_us(adap
,
4883 TIMERVALUE5_G(timer_value_4_and_5
));
4885 ingress_rx_threshold
= t4_read_reg(adap
, SGE_INGRESS_RX_THRESHOLD_A
);
4886 s
->counter_val
[0] = THRESHOLD_0_G(ingress_rx_threshold
);
4887 s
->counter_val
[1] = THRESHOLD_1_G(ingress_rx_threshold
);
4888 s
->counter_val
[2] = THRESHOLD_2_G(ingress_rx_threshold
);
4889 s
->counter_val
[3] = THRESHOLD_3_G(ingress_rx_threshold
);
4895 * t4_sge_init - initialize SGE
4896 * @adap: the adapter
4898 * Perform low-level SGE code initialization needed every time after a
4901 int t4_sge_init(struct adapter
*adap
)
4903 struct sge
*s
= &adap
->sge
;
4904 u32 sge_control
, sge_conm_ctrl
;
4905 int ret
, egress_threshold
;
4908 * Ingress Padding Boundary and Egress Status Page Size are set up by
4909 * t4_fixup_host_params().
4911 sge_control
= t4_read_reg(adap
, SGE_CONTROL_A
);
4912 s
->pktshift
= PKTSHIFT_G(sge_control
);
4913 s
->stat_len
= (sge_control
& EGRSTATUSPAGESIZE_F
) ? 128 : 64;
4915 s
->fl_align
= t4_fl_pkt_align(adap
);
4916 ret
= t4_sge_init_soft(adap
);
4921 * A FL with <= fl_starve_thres buffers is starving and a periodic
4922 * timer will attempt to refill it. This needs to be larger than the
4923 * SGE's Egress Congestion Threshold. If it isn't, then we can get
4924 * stuck waiting for new packets while the SGE is waiting for us to
4925 * give it more Free List entries. (Note that the SGE's Egress
4926 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
4927 * there was only a single field to control this. For T5 there's the
4928 * original field which now only applies to Unpacked Mode Free List
4929 * buffers and a new field which only applies to Packed Mode Free List
4932 sge_conm_ctrl
= t4_read_reg(adap
, SGE_CONM_CTRL_A
);
4933 switch (CHELSIO_CHIP_VERSION(adap
->params
.chip
)) {
4935 egress_threshold
= EGRTHRESHOLD_G(sge_conm_ctrl
);
4938 egress_threshold
= EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4941 egress_threshold
= T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4944 dev_err(adap
->pdev_dev
, "Unsupported Chip version %d\n",
4945 CHELSIO_CHIP_VERSION(adap
->params
.chip
));
4948 s
->fl_starve_thres
= 2*egress_threshold
+ 1;
4950 t4_idma_monitor_init(adap
, &s
->idma_monitor
);
4952 /* Set up timers used for recuring callbacks to process RX and TX
4953 * administrative tasks.
4955 timer_setup(&s
->rx_timer
, sge_rx_timer_cb
, 0);
4956 timer_setup(&s
->tx_timer
, sge_tx_timer_cb
, 0);
4958 spin_lock_init(&s
->intrq_lock
);