2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/skbuff.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/if_vlan.h>
40 #include <linux/dma-mapping.h>
41 #include <linux/jiffies.h>
42 #include <linux/prefetch.h>
43 #include <linux/export.h>
47 #include <net/busy_poll.h>
48 #ifdef CONFIG_CHELSIO_T4_FCOE
49 #include <scsi/fc/fc_fcoe.h>
50 #endif /* CONFIG_CHELSIO_T4_FCOE */
53 #include "t4_values.h"
56 #include "cxgb4_ptp.h"
57 #include "cxgb4_uld.h"
58 #include "cxgb4_tc_mqprio.h"
62 * Rx buffer size. We use largish buffers if possible but settle for single
63 * pages under memory shortage.
66 # define FL_PG_ORDER 0
68 # define FL_PG_ORDER (16 - PAGE_SHIFT)
71 /* RX_PULL_LEN should be <= RX_COPY_THRES */
72 #define RX_COPY_THRES 256
73 #define RX_PULL_LEN 128
76 * Main body length for sk_buffs used for Rx Ethernet packets with fragments.
77 * Should be >= RX_PULL_LEN but possibly bigger to give pskb_may_pull some room.
79 #define RX_PKT_SKB_LEN 512
82 * Max number of Tx descriptors we clean up at a time. Should be modest as
83 * freeing skbs isn't cheap and it happens while holding locks. We just need
84 * to free packets faster than they arrive, we eventually catch up and keep
85 * the amortized cost reasonable. Must be >= 2 * TXQ_STOP_THRES. It should
86 * also match the CIDX Flush Threshold.
88 #define MAX_TX_RECLAIM 32
91 * Max number of Rx buffers we replenish at a time. Again keep this modest,
92 * allocating buffers isn't cheap either.
94 #define MAX_RX_REFILL 16U
97 * Period of the Rx queue check timer. This timer is infrequent as it has
98 * something to do only when the system experiences severe memory shortage.
100 #define RX_QCHECK_PERIOD (HZ / 2)
103 * Period of the Tx queue check timer.
105 #define TX_QCHECK_PERIOD (HZ / 2)
108 * Max number of Tx descriptors to be reclaimed by the Tx timer.
110 #define MAX_TIMER_TX_RECLAIM 100
113 * Timer index used when backing off due to memory shortage.
115 #define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
118 * Suspension threshold for non-Ethernet Tx queues. We require enough room
119 * for a full sized WR.
121 #define TXQ_STOP_THRES (SGE_MAX_WR_LEN / sizeof(struct tx_desc))
124 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
127 #define MAX_IMM_TX_PKT_LEN 256
130 * Max size of a WR sent through a control Tx queue.
132 #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
134 struct rx_sw_desc
{ /* SW state per Rx descriptor */
140 * Rx buffer sizes for "useskbs" Free List buffers (one ingress packet pe skb
141 * buffer). We currently only support two sizes for 1500- and 9000-byte MTUs.
142 * We could easily support more but there doesn't seem to be much need for
145 #define FL_MTU_SMALL 1500
146 #define FL_MTU_LARGE 9000
148 static inline unsigned int fl_mtu_bufsize(struct adapter
*adapter
,
151 struct sge
*s
= &adapter
->sge
;
153 return ALIGN(s
->pktshift
+ ETH_HLEN
+ VLAN_HLEN
+ mtu
, s
->fl_align
);
156 #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
157 #define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
160 * Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses
161 * these to specify the buffer size as an index into the SGE Free List Buffer
162 * Size register array. We also use bit 4, when the buffer has been unmapped
163 * for DMA, but this is of course never sent to the hardware and is only used
164 * to prevent double unmappings. All of the above requires that the Free List
165 * Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
166 * 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal
167 * Free List Buffer alignment is 32 bytes, this works out for us ...
170 RX_BUF_FLAGS
= 0x1f, /* bottom five bits are special */
171 RX_BUF_SIZE
= 0x0f, /* bottom three bits are for buf sizes */
172 RX_UNMAPPED_BUF
= 0x10, /* buffer is not mapped */
175 * XXX We shouldn't depend on being able to use these indices.
176 * XXX Especially when some other Master PF has initialized the
177 * XXX adapter or we use the Firmware Configuration File. We
178 * XXX should really search through the Host Buffer Size register
179 * XXX array for the appropriately sized buffer indices.
181 RX_SMALL_PG_BUF
= 0x0, /* small (PAGE_SIZE) page buffer */
182 RX_LARGE_PG_BUF
= 0x1, /* buffer large (FL_PG_ORDER) page buffer */
184 RX_SMALL_MTU_BUF
= 0x2, /* small MTU buffer */
185 RX_LARGE_MTU_BUF
= 0x3, /* large MTU buffer */
188 static int timer_pkt_quota
[] = {1, 1, 2, 3, 4, 5};
189 #define MIN_NAPI_WORK 1
191 static inline dma_addr_t
get_buf_addr(const struct rx_sw_desc
*d
)
193 return d
->dma_addr
& ~(dma_addr_t
)RX_BUF_FLAGS
;
196 static inline bool is_buf_mapped(const struct rx_sw_desc
*d
)
198 return !(d
->dma_addr
& RX_UNMAPPED_BUF
);
202 * txq_avail - return the number of available slots in a Tx queue
205 * Returns the number of descriptors in a Tx queue available to write new
208 static inline unsigned int txq_avail(const struct sge_txq
*q
)
210 return q
->size
- 1 - q
->in_use
;
214 * fl_cap - return the capacity of a free-buffer list
217 * Returns the capacity of a free-buffer list. The capacity is less than
218 * the size because one descriptor needs to be left unpopulated, otherwise
219 * HW will think the FL is empty.
221 static inline unsigned int fl_cap(const struct sge_fl
*fl
)
223 return fl
->size
- 8; /* 1 descriptor = 8 buffers */
227 * fl_starving - return whether a Free List is starving.
228 * @adapter: pointer to the adapter
231 * Tests specified Free List to see whether the number of buffers
232 * available to the hardware has falled below our "starvation"
235 static inline bool fl_starving(const struct adapter
*adapter
,
236 const struct sge_fl
*fl
)
238 const struct sge
*s
= &adapter
->sge
;
240 return fl
->avail
- fl
->pend_cred
<= s
->fl_starve_thres
;
243 int cxgb4_map_skb(struct device
*dev
, const struct sk_buff
*skb
,
246 const skb_frag_t
*fp
, *end
;
247 const struct skb_shared_info
*si
;
249 *addr
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
250 if (dma_mapping_error(dev
, *addr
))
253 si
= skb_shinfo(skb
);
254 end
= &si
->frags
[si
->nr_frags
];
256 for (fp
= si
->frags
; fp
< end
; fp
++) {
257 *++addr
= skb_frag_dma_map(dev
, fp
, 0, skb_frag_size(fp
),
259 if (dma_mapping_error(dev
, *addr
))
265 while (fp
-- > si
->frags
)
266 dma_unmap_page(dev
, *--addr
, skb_frag_size(fp
), DMA_TO_DEVICE
);
268 dma_unmap_single(dev
, addr
[-1], skb_headlen(skb
), DMA_TO_DEVICE
);
272 EXPORT_SYMBOL(cxgb4_map_skb
);
274 static void unmap_skb(struct device
*dev
, const struct sk_buff
*skb
,
275 const dma_addr_t
*addr
)
277 const skb_frag_t
*fp
, *end
;
278 const struct skb_shared_info
*si
;
280 dma_unmap_single(dev
, *addr
++, skb_headlen(skb
), DMA_TO_DEVICE
);
282 si
= skb_shinfo(skb
);
283 end
= &si
->frags
[si
->nr_frags
];
284 for (fp
= si
->frags
; fp
< end
; fp
++)
285 dma_unmap_page(dev
, *addr
++, skb_frag_size(fp
), DMA_TO_DEVICE
);
288 #ifdef CONFIG_NEED_DMA_MAP_STATE
290 * deferred_unmap_destructor - unmap a packet when it is freed
293 * This is the packet destructor used for Tx packets that need to remain
294 * mapped until they are freed rather than until their Tx descriptors are
297 static void deferred_unmap_destructor(struct sk_buff
*skb
)
299 unmap_skb(skb
->dev
->dev
.parent
, skb
, (dma_addr_t
*)skb
->head
);
304 * free_tx_desc - reclaims Tx descriptors and their buffers
305 * @adapter: the adapter
306 * @q: the Tx queue to reclaim descriptors from
307 * @n: the number of descriptors to reclaim
308 * @unmap: whether the buffers should be unmapped for DMA
310 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
311 * Tx buffers. Called with the Tx queue lock held.
313 void free_tx_desc(struct adapter
*adap
, struct sge_txq
*q
,
314 unsigned int n
, bool unmap
)
316 unsigned int cidx
= q
->cidx
;
317 struct tx_sw_desc
*d
;
321 if (d
->skb
) { /* an SGL is present */
322 if (unmap
&& d
->addr
[0]) {
323 unmap_skb(adap
->pdev_dev
, d
->skb
, d
->addr
);
324 memset(d
->addr
, 0, sizeof(d
->addr
));
326 dev_consume_skb_any(d
->skb
);
330 if (++cidx
== q
->size
) {
339 * Return the number of reclaimable descriptors in a Tx queue.
341 static inline int reclaimable(const struct sge_txq
*q
)
343 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
345 return hw_cidx
< 0 ? hw_cidx
+ q
->size
: hw_cidx
;
349 * reclaim_completed_tx - reclaims completed TX Descriptors
351 * @q: the Tx queue to reclaim completed descriptors from
352 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
353 * @unmap: whether the buffers should be unmapped for DMA
355 * Reclaims Tx Descriptors that the SGE has indicated it has processed,
356 * and frees the associated buffers if possible. If @max == -1, then
357 * we'll use a defaiult maximum. Called with the TX Queue locked.
359 static inline int reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
360 int maxreclaim
, bool unmap
)
362 int reclaim
= reclaimable(q
);
366 * Limit the amount of clean up work we do at a time to keep
367 * the Tx lock hold time O(1).
370 maxreclaim
= MAX_TX_RECLAIM
;
371 if (reclaim
> maxreclaim
)
372 reclaim
= maxreclaim
;
374 free_tx_desc(adap
, q
, reclaim
, unmap
);
375 q
->in_use
-= reclaim
;
382 * cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
384 * @q: the Tx queue to reclaim completed descriptors from
385 * @unmap: whether the buffers should be unmapped for DMA
387 * Reclaims Tx descriptors that the SGE has indicated it has processed,
388 * and frees the associated buffers if possible. Called with the Tx
391 void cxgb4_reclaim_completed_tx(struct adapter
*adap
, struct sge_txq
*q
,
394 (void)reclaim_completed_tx(adap
, q
, -1, unmap
);
396 EXPORT_SYMBOL(cxgb4_reclaim_completed_tx
);
398 static inline int get_buf_size(struct adapter
*adapter
,
399 const struct rx_sw_desc
*d
)
401 struct sge
*s
= &adapter
->sge
;
402 unsigned int rx_buf_size_idx
= d
->dma_addr
& RX_BUF_SIZE
;
405 switch (rx_buf_size_idx
) {
406 case RX_SMALL_PG_BUF
:
407 buf_size
= PAGE_SIZE
;
410 case RX_LARGE_PG_BUF
:
411 buf_size
= PAGE_SIZE
<< s
->fl_pg_order
;
414 case RX_SMALL_MTU_BUF
:
415 buf_size
= FL_MTU_SMALL_BUFSIZE(adapter
);
418 case RX_LARGE_MTU_BUF
:
419 buf_size
= FL_MTU_LARGE_BUFSIZE(adapter
);
430 * free_rx_bufs - free the Rx buffers on an SGE free list
432 * @q: the SGE free list to free buffers from
433 * @n: how many buffers to free
435 * Release the next @n buffers on an SGE free-buffer Rx queue. The
436 * buffers must be made inaccessible to HW before calling this function.
438 static void free_rx_bufs(struct adapter
*adap
, struct sge_fl
*q
, int n
)
441 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
443 if (is_buf_mapped(d
))
444 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
445 get_buf_size(adap
, d
),
449 if (++q
->cidx
== q
->size
)
456 * unmap_rx_buf - unmap the current Rx buffer on an SGE free list
458 * @q: the SGE free list
460 * Unmap the current buffer on an SGE free-buffer Rx queue. The
461 * buffer must be made inaccessible to HW before calling this function.
463 * This is similar to @free_rx_bufs above but does not free the buffer.
464 * Do note that the FL still loses any further access to the buffer.
466 static void unmap_rx_buf(struct adapter
*adap
, struct sge_fl
*q
)
468 struct rx_sw_desc
*d
= &q
->sdesc
[q
->cidx
];
470 if (is_buf_mapped(d
))
471 dma_unmap_page(adap
->pdev_dev
, get_buf_addr(d
),
472 get_buf_size(adap
, d
), PCI_DMA_FROMDEVICE
);
474 if (++q
->cidx
== q
->size
)
479 static inline void ring_fl_db(struct adapter
*adap
, struct sge_fl
*q
)
481 if (q
->pend_cred
>= 8) {
482 u32 val
= adap
->params
.arch
.sge_fl_db
;
484 if (is_t4(adap
->params
.chip
))
485 val
|= PIDX_V(q
->pend_cred
/ 8);
487 val
|= PIDX_T5_V(q
->pend_cred
/ 8);
489 /* Make sure all memory writes to the Free List queue are
490 * committed before we tell the hardware about them.
494 /* If we don't have access to the new User Doorbell (T5+), use
495 * the old doorbell mechanism; otherwise use the new BAR2
498 if (unlikely(q
->bar2_addr
== NULL
)) {
499 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
500 val
| QID_V(q
->cntxt_id
));
502 writel(val
| QID_V(q
->bar2_qid
),
503 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
505 /* This Write memory Barrier will force the write to
506 * the User Doorbell area to be flushed.
514 static inline void set_rx_sw_desc(struct rx_sw_desc
*sd
, struct page
*pg
,
518 sd
->dma_addr
= mapping
; /* includes size low bits */
522 * refill_fl - refill an SGE Rx buffer ring
524 * @q: the ring to refill
525 * @n: the number of new buffers to allocate
526 * @gfp: the gfp flags for the allocations
528 * (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
529 * allocated with the supplied gfp flags. The caller must assure that
530 * @n does not exceed the queue's capacity. If afterwards the queue is
531 * found critically low mark it as starving in the bitmap of starving FLs.
533 * Returns the number of buffers allocated.
535 static unsigned int refill_fl(struct adapter
*adap
, struct sge_fl
*q
, int n
,
538 struct sge
*s
= &adap
->sge
;
541 unsigned int cred
= q
->avail
;
542 __be64
*d
= &q
->desc
[q
->pidx
];
543 struct rx_sw_desc
*sd
= &q
->sdesc
[q
->pidx
];
546 #ifdef CONFIG_DEBUG_FS
547 if (test_bit(q
->cntxt_id
- adap
->sge
.egr_start
, adap
->sge
.blocked_fl
))
552 node
= dev_to_node(adap
->pdev_dev
);
554 if (s
->fl_pg_order
== 0)
555 goto alloc_small_pages
;
558 * Prefer large buffers
561 pg
= alloc_pages_node(node
, gfp
| __GFP_COMP
, s
->fl_pg_order
);
563 q
->large_alloc_failed
++;
564 break; /* fall back to single pages */
567 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0,
568 PAGE_SIZE
<< s
->fl_pg_order
,
570 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
571 __free_pages(pg
, s
->fl_pg_order
);
573 goto out
; /* do not try small pages for this error */
575 mapping
|= RX_LARGE_PG_BUF
;
576 *d
++ = cpu_to_be64(mapping
);
578 set_rx_sw_desc(sd
, pg
, mapping
);
582 if (++q
->pidx
== q
->size
) {
592 pg
= alloc_pages_node(node
, gfp
, 0);
598 mapping
= dma_map_page(adap
->pdev_dev
, pg
, 0, PAGE_SIZE
,
600 if (unlikely(dma_mapping_error(adap
->pdev_dev
, mapping
))) {
605 *d
++ = cpu_to_be64(mapping
);
607 set_rx_sw_desc(sd
, pg
, mapping
);
611 if (++q
->pidx
== q
->size
) {
618 out
: cred
= q
->avail
- cred
;
619 q
->pend_cred
+= cred
;
622 if (unlikely(fl_starving(adap
, q
))) {
625 set_bit(q
->cntxt_id
- adap
->sge
.egr_start
,
626 adap
->sge
.starving_fl
);
632 static inline void __refill_fl(struct adapter
*adap
, struct sge_fl
*fl
)
634 refill_fl(adap
, fl
, min(MAX_RX_REFILL
, fl_cap(fl
) - fl
->avail
),
639 * alloc_ring - allocate resources for an SGE descriptor ring
640 * @dev: the PCI device's core device
641 * @nelem: the number of descriptors
642 * @elem_size: the size of each descriptor
643 * @sw_size: the size of the SW state associated with each ring element
644 * @phys: the physical address of the allocated ring
645 * @metadata: address of the array holding the SW state for the ring
646 * @stat_size: extra space in HW ring for status information
647 * @node: preferred node for memory allocations
649 * Allocates resources for an SGE descriptor ring, such as Tx queues,
650 * free buffer lists, or response queues. Each SGE ring requires
651 * space for its HW descriptors plus, optionally, space for the SW state
652 * associated with each HW entry (the metadata). The function returns
653 * three values: the virtual address for the HW ring (the return value
654 * of the function), the bus address of the HW ring, and the address
657 static void *alloc_ring(struct device
*dev
, size_t nelem
, size_t elem_size
,
658 size_t sw_size
, dma_addr_t
*phys
, void *metadata
,
659 size_t stat_size
, int node
)
661 size_t len
= nelem
* elem_size
+ stat_size
;
663 void *p
= dma_alloc_coherent(dev
, len
, phys
, GFP_KERNEL
);
668 s
= kcalloc_node(sw_size
, nelem
, GFP_KERNEL
, node
);
671 dma_free_coherent(dev
, len
, p
, *phys
);
676 *(void **)metadata
= s
;
681 * sgl_len - calculates the size of an SGL of the given capacity
682 * @n: the number of SGL entries
684 * Calculates the number of flits needed for a scatter/gather list that
685 * can hold the given number of entries.
687 static inline unsigned int sgl_len(unsigned int n
)
689 /* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
690 * addresses. The DSGL Work Request starts off with a 32-bit DSGL
691 * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
692 * repeated sequences of { Length[i], Length[i+1], Address[i],
693 * Address[i+1] } (this ensures that all addresses are on 64-bit
694 * boundaries). If N is even, then Length[N+1] should be set to 0 and
695 * Address[N+1] is omitted.
697 * The following calculation incorporates all of the above. It's
698 * somewhat hard to follow but, briefly: the "+2" accounts for the
699 * first two flits which include the DSGL header, Length0 and
700 * Address0; the "(3*(n-1))/2" covers the main body of list entries (3
701 * flits for every pair of the remaining N) +1 if (n-1) is odd; and
702 * finally the "+((n-1)&1)" adds the one remaining flit needed if
706 return (3 * n
) / 2 + (n
& 1) + 2;
710 * flits_to_desc - returns the num of Tx descriptors for the given flits
711 * @n: the number of flits
713 * Returns the number of Tx descriptors needed for the supplied number
716 static inline unsigned int flits_to_desc(unsigned int n
)
718 BUG_ON(n
> SGE_MAX_WR_LEN
/ 8);
719 return DIV_ROUND_UP(n
, 8);
723 * is_eth_imm - can an Ethernet packet be sent as immediate data?
726 * Returns whether an Ethernet packet is small enough to fit as
727 * immediate data. Return value corresponds to headroom required.
729 static inline int is_eth_imm(const struct sk_buff
*skb
, unsigned int chip_ver
)
733 if (skb
->encapsulation
&& skb_shinfo(skb
)->gso_size
&&
734 chip_ver
> CHELSIO_T5
) {
735 hdrlen
= sizeof(struct cpl_tx_tnl_lso
);
736 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
737 } else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
) {
740 hdrlen
= skb_shinfo(skb
)->gso_size
?
741 sizeof(struct cpl_tx_pkt_lso_core
) : 0;
742 hdrlen
+= sizeof(struct cpl_tx_pkt
);
744 if (skb
->len
<= MAX_IMM_TX_PKT_LEN
- hdrlen
)
750 * calc_tx_flits - calculate the number of flits for a packet Tx WR
753 * Returns the number of flits needed for a Tx WR for the given Ethernet
754 * packet, including the needed WR and CPL headers.
756 static inline unsigned int calc_tx_flits(const struct sk_buff
*skb
,
757 unsigned int chip_ver
)
760 int hdrlen
= is_eth_imm(skb
, chip_ver
);
762 /* If the skb is small enough, we can pump it out as a work request
763 * with only immediate data. In that case we just have to have the
764 * TX Packet header plus the skb data in the Work Request.
768 return DIV_ROUND_UP(skb
->len
+ hdrlen
, sizeof(__be64
));
770 /* Otherwise, we're going to have to construct a Scatter gather list
771 * of the skb body and fragments. We also include the flits necessary
772 * for the TX Packet Work Request and CPL. We always have a firmware
773 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
774 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
775 * message or, if we're doing a Large Send Offload, an LSO CPL message
776 * with an embedded TX Packet Write CPL message.
778 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
779 if (skb_shinfo(skb
)->gso_size
) {
780 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
) {
781 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
782 sizeof(struct cpl_tx_tnl_lso
);
783 } else if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
) {
786 pkt_hdrlen
= eth_get_headlen(skb
->dev
, skb
->data
,
788 hdrlen
= sizeof(struct fw_eth_tx_eo_wr
) +
789 round_up(pkt_hdrlen
, 16);
791 hdrlen
= sizeof(struct fw_eth_tx_pkt_wr
) +
792 sizeof(struct cpl_tx_pkt_lso_core
);
795 hdrlen
+= sizeof(struct cpl_tx_pkt_core
);
796 flits
+= (hdrlen
/ sizeof(__be64
));
798 flits
+= (sizeof(struct fw_eth_tx_pkt_wr
) +
799 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
805 * calc_tx_descs - calculate the number of Tx descriptors for a packet
808 * Returns the number of Tx descriptors needed for the given Ethernet
809 * packet, including the needed WR and CPL headers.
811 static inline unsigned int calc_tx_descs(const struct sk_buff
*skb
,
812 unsigned int chip_ver
)
814 return flits_to_desc(calc_tx_flits(skb
, chip_ver
));
818 * cxgb4_write_sgl - populate a scatter/gather list for a packet
820 * @q: the Tx queue we are writing into
821 * @sgl: starting location for writing the SGL
822 * @end: points right after the end of the SGL
823 * @start: start offset into skb main-body data to include in the SGL
824 * @addr: the list of bus addresses for the SGL elements
826 * Generates a gather list for the buffers that make up a packet.
827 * The caller must provide adequate space for the SGL that will be written.
828 * The SGL includes all of the packet's page fragments and the data in its
829 * main body except for the first @start bytes. @sgl must be 16-byte
830 * aligned and within a Tx descriptor with available space. @end points
831 * right after the end of the SGL but does not account for any potential
832 * wrap around, i.e., @end > @sgl.
834 void cxgb4_write_sgl(const struct sk_buff
*skb
, struct sge_txq
*q
,
835 struct ulptx_sgl
*sgl
, u64
*end
, unsigned int start
,
836 const dma_addr_t
*addr
)
839 struct ulptx_sge_pair
*to
;
840 const struct skb_shared_info
*si
= skb_shinfo(skb
);
841 unsigned int nfrags
= si
->nr_frags
;
842 struct ulptx_sge_pair buf
[MAX_SKB_FRAGS
/ 2 + 1];
844 len
= skb_headlen(skb
) - start
;
846 sgl
->len0
= htonl(len
);
847 sgl
->addr0
= cpu_to_be64(addr
[0] + start
);
850 sgl
->len0
= htonl(skb_frag_size(&si
->frags
[0]));
851 sgl
->addr0
= cpu_to_be64(addr
[1]);
854 sgl
->cmd_nsge
= htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
855 ULPTX_NSGE_V(nfrags
));
856 if (likely(--nfrags
== 0))
859 * Most of the complexity below deals with the possibility we hit the
860 * end of the queue in the middle of writing the SGL. For this case
861 * only we create the SGL in a temporary buffer and then copy it.
863 to
= (u8
*)end
> (u8
*)q
->stat
? buf
: sgl
->sge
;
865 for (i
= (nfrags
!= si
->nr_frags
); nfrags
>= 2; nfrags
-= 2, to
++) {
866 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
867 to
->len
[1] = cpu_to_be32(skb_frag_size(&si
->frags
[++i
]));
868 to
->addr
[0] = cpu_to_be64(addr
[i
]);
869 to
->addr
[1] = cpu_to_be64(addr
[++i
]);
872 to
->len
[0] = cpu_to_be32(skb_frag_size(&si
->frags
[i
]));
873 to
->len
[1] = cpu_to_be32(0);
874 to
->addr
[0] = cpu_to_be64(addr
[i
+ 1]);
876 if (unlikely((u8
*)end
> (u8
*)q
->stat
)) {
877 unsigned int part0
= (u8
*)q
->stat
- (u8
*)sgl
->sge
, part1
;
880 memcpy(sgl
->sge
, buf
, part0
);
881 part1
= (u8
*)end
- (u8
*)q
->stat
;
882 memcpy(q
->desc
, (u8
*)buf
+ part0
, part1
);
883 end
= (void *)q
->desc
+ part1
;
885 if ((uintptr_t)end
& 8) /* 0-pad to multiple of 16 */
888 EXPORT_SYMBOL(cxgb4_write_sgl
);
890 /* This function copies 64 byte coalesced work request to
891 * memory mapped BAR2 space. For coalesced WR SGE fetches
892 * data from the FIFO instead of from Host.
894 static void cxgb_pio_copy(u64 __iomem
*dst
, u64
*src
)
907 * cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell
910 * @n: number of new descriptors to give to HW
912 * Ring the doorbel for a Tx queue.
914 inline void cxgb4_ring_tx_db(struct adapter
*adap
, struct sge_txq
*q
, int n
)
916 /* Make sure that all writes to the TX Descriptors are committed
917 * before we tell the hardware about them.
921 /* If we don't have access to the new User Doorbell (T5+), use the old
922 * doorbell mechanism; otherwise use the new BAR2 mechanism.
924 if (unlikely(q
->bar2_addr
== NULL
)) {
928 /* For T4 we need to participate in the Doorbell Recovery
931 spin_lock_irqsave(&q
->db_lock
, flags
);
933 t4_write_reg(adap
, MYPF_REG(SGE_PF_KDOORBELL_A
),
934 QID_V(q
->cntxt_id
) | val
);
937 q
->db_pidx
= q
->pidx
;
938 spin_unlock_irqrestore(&q
->db_lock
, flags
);
940 u32 val
= PIDX_T5_V(n
);
942 /* T4 and later chips share the same PIDX field offset within
943 * the doorbell, but T5 and later shrank the field in order to
944 * gain a bit for Doorbell Priority. The field was absurdly
945 * large in the first place (14 bits) so we just use the T5
946 * and later limits and warn if a Queue ID is too large.
948 WARN_ON(val
& DBPRIO_F
);
950 /* If we're only writing a single TX Descriptor and we can use
951 * Inferred QID registers, we can use the Write Combining
952 * Gather Buffer; otherwise we use the simple doorbell.
954 if (n
== 1 && q
->bar2_qid
== 0) {
958 u64
*wr
= (u64
*)&q
->desc
[index
];
960 cxgb_pio_copy((u64 __iomem
*)
961 (q
->bar2_addr
+ SGE_UDB_WCDOORBELL
),
964 writel(val
| QID_V(q
->bar2_qid
),
965 q
->bar2_addr
+ SGE_UDB_KDOORBELL
);
968 /* This Write Memory Barrier will force the write to the User
969 * Doorbell area to be flushed. This is needed to prevent
970 * writes on different CPUs for the same queue from hitting
971 * the adapter out of order. This is required when some Work
972 * Requests take the Write Combine Gather Buffer path (user
973 * doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
974 * take the traditional path where we simply increment the
975 * PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
976 * hardware DMA read the actual Work Request.
981 EXPORT_SYMBOL(cxgb4_ring_tx_db
);
984 * cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors
986 * @q: the Tx queue where the packet will be inlined
987 * @pos: starting position in the Tx queue where to inline the packet
989 * Inline a packet's contents directly into Tx descriptors, starting at
990 * the given position within the Tx DMA ring.
991 * Most of the complexity of this operation is dealing with wrap arounds
992 * in the middle of the packet we want to inline.
994 void cxgb4_inline_tx_skb(const struct sk_buff
*skb
,
995 const struct sge_txq
*q
, void *pos
)
997 int left
= (void *)q
->stat
- pos
;
1000 if (likely(skb
->len
<= left
)) {
1001 if (likely(!skb
->data_len
))
1002 skb_copy_from_linear_data(skb
, pos
, skb
->len
);
1004 skb_copy_bits(skb
, 0, pos
, skb
->len
);
1007 skb_copy_bits(skb
, 0, pos
, left
);
1008 skb_copy_bits(skb
, left
, q
->desc
, skb
->len
- left
);
1009 pos
= (void *)q
->desc
+ (skb
->len
- left
);
1012 /* 0-pad to multiple of 16 */
1013 p
= PTR_ALIGN(pos
, 8);
1014 if ((uintptr_t)p
& 8)
1017 EXPORT_SYMBOL(cxgb4_inline_tx_skb
);
1019 static void *inline_tx_skb_header(const struct sk_buff
*skb
,
1020 const struct sge_txq
*q
, void *pos
,
1024 int left
= (void *)q
->stat
- pos
;
1026 if (likely(length
<= left
)) {
1027 memcpy(pos
, skb
->data
, length
);
1030 memcpy(pos
, skb
->data
, left
);
1031 memcpy(q
->desc
, skb
->data
+ left
, length
- left
);
1032 pos
= (void *)q
->desc
+ (length
- left
);
1034 /* 0-pad to multiple of 16 */
1035 p
= PTR_ALIGN(pos
, 8);
1036 if ((uintptr_t)p
& 8) {
1044 * Figure out what HW csum a packet wants and return the appropriate control
1047 static u64
hwcsum(enum chip_type chip
, const struct sk_buff
*skb
)
1050 bool inner_hdr_csum
= false;
1053 if (skb
->encapsulation
&&
1054 (CHELSIO_CHIP_VERSION(chip
) > CHELSIO_T5
))
1055 inner_hdr_csum
= true;
1057 if (inner_hdr_csum
) {
1058 ver
= inner_ip_hdr(skb
)->version
;
1059 proto
= (ver
== 4) ? inner_ip_hdr(skb
)->protocol
:
1060 inner_ipv6_hdr(skb
)->nexthdr
;
1062 ver
= ip_hdr(skb
)->version
;
1063 proto
= (ver
== 4) ? ip_hdr(skb
)->protocol
:
1064 ipv6_hdr(skb
)->nexthdr
;
1068 if (proto
== IPPROTO_TCP
)
1069 csum_type
= TX_CSUM_TCPIP
;
1070 else if (proto
== IPPROTO_UDP
)
1071 csum_type
= TX_CSUM_UDPIP
;
1074 * unknown protocol, disable HW csum
1075 * and hope a bad packet is detected
1077 return TXPKT_L4CSUM_DIS_F
;
1081 * this doesn't work with extension headers
1083 if (proto
== IPPROTO_TCP
)
1084 csum_type
= TX_CSUM_TCPIP6
;
1085 else if (proto
== IPPROTO_UDP
)
1086 csum_type
= TX_CSUM_UDPIP6
;
1091 if (likely(csum_type
>= TX_CSUM_TCPIP
)) {
1092 int eth_hdr_len
, l4_len
;
1095 if (inner_hdr_csum
) {
1096 /* This allows checksum offload for all encapsulated
1097 * packets like GRE etc..
1099 l4_len
= skb_inner_network_header_len(skb
);
1100 eth_hdr_len
= skb_inner_network_offset(skb
) - ETH_HLEN
;
1102 l4_len
= skb_network_header_len(skb
);
1103 eth_hdr_len
= skb_network_offset(skb
) - ETH_HLEN
;
1105 hdr_len
= TXPKT_IPHDR_LEN_V(l4_len
);
1107 if (CHELSIO_CHIP_VERSION(chip
) <= CHELSIO_T5
)
1108 hdr_len
|= TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1110 hdr_len
|= T6_TXPKT_ETHHDR_LEN_V(eth_hdr_len
);
1111 return TXPKT_CSUM_TYPE_V(csum_type
) | hdr_len
;
1113 int start
= skb_transport_offset(skb
);
1115 return TXPKT_CSUM_TYPE_V(csum_type
) |
1116 TXPKT_CSUM_START_V(start
) |
1117 TXPKT_CSUM_LOC_V(start
+ skb
->csum_offset
);
1121 static void eth_txq_stop(struct sge_eth_txq
*q
)
1123 netif_tx_stop_queue(q
->txq
);
1127 static inline void txq_advance(struct sge_txq
*q
, unsigned int n
)
1131 if (q
->pidx
>= q
->size
)
1135 #ifdef CONFIG_CHELSIO_T4_FCOE
1137 cxgb_fcoe_offload(struct sk_buff
*skb
, struct adapter
*adap
,
1138 const struct port_info
*pi
, u64
*cntrl
)
1140 const struct cxgb_fcoe
*fcoe
= &pi
->fcoe
;
1142 if (!(fcoe
->flags
& CXGB_FCOE_ENABLED
))
1145 if (skb
->protocol
!= htons(ETH_P_FCOE
))
1148 skb_reset_mac_header(skb
);
1149 skb
->mac_len
= sizeof(struct ethhdr
);
1151 skb_set_network_header(skb
, skb
->mac_len
);
1152 skb_set_transport_header(skb
, skb
->mac_len
+ sizeof(struct fcoe_hdr
));
1154 if (!cxgb_fcoe_sof_eof_supported(adap
, skb
))
1157 /* FC CRC offload */
1158 *cntrl
= TXPKT_CSUM_TYPE_V(TX_CSUM_FCOE
) |
1159 TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
|
1160 TXPKT_CSUM_START_V(CXGB_FCOE_TXPKT_CSUM_START
) |
1161 TXPKT_CSUM_END_V(CXGB_FCOE_TXPKT_CSUM_END
) |
1162 TXPKT_CSUM_LOC_V(CXGB_FCOE_TXPKT_CSUM_END
);
1165 #endif /* CONFIG_CHELSIO_T4_FCOE */
1167 /* Returns tunnel type if hardware supports offloading of the same.
1168 * It is called only for T5 and onwards.
1170 enum cpl_tx_tnl_lso_type
cxgb_encap_offload_supported(struct sk_buff
*skb
)
1173 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1174 struct port_info
*pi
= netdev_priv(skb
->dev
);
1175 struct adapter
*adapter
= pi
->adapter
;
1177 if (skb
->inner_protocol_type
!= ENCAP_TYPE_ETHER
||
1178 skb
->inner_protocol
!= htons(ETH_P_TEB
))
1181 switch (vlan_get_protocol(skb
)) {
1182 case htons(ETH_P_IP
):
1183 l4_hdr
= ip_hdr(skb
)->protocol
;
1185 case htons(ETH_P_IPV6
):
1186 l4_hdr
= ipv6_hdr(skb
)->nexthdr
;
1194 if (adapter
->vxlan_port
== udp_hdr(skb
)->dest
)
1195 tnl_type
= TX_TNL_TYPE_VXLAN
;
1196 else if (adapter
->geneve_port
== udp_hdr(skb
)->dest
)
1197 tnl_type
= TX_TNL_TYPE_GENEVE
;
1206 static inline void t6_fill_tnl_lso(struct sk_buff
*skb
,
1207 struct cpl_tx_tnl_lso
*tnl_lso
,
1208 enum cpl_tx_tnl_lso_type tnl_type
)
1211 int in_eth_xtra_len
;
1212 int l3hdr_len
= skb_network_header_len(skb
);
1213 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1214 const struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1215 bool v6
= (ip_hdr(skb
)->version
== 6);
1217 val
= CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO
) |
1218 CPL_TX_TNL_LSO_FIRST_F
|
1219 CPL_TX_TNL_LSO_LAST_F
|
1220 (v6
? CPL_TX_TNL_LSO_IPV6OUT_F
: 0) |
1221 CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len
/ 4) |
1222 CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len
/ 4) |
1223 (v6
? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F
) |
1224 CPL_TX_TNL_LSO_IPLENSETOUT_F
|
1225 (v6
? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F
);
1226 tnl_lso
->op_to_IpIdSplitOut
= htonl(val
);
1228 tnl_lso
->IpIdOffsetOut
= 0;
1230 /* Get the tunnel header length */
1231 val
= skb_inner_mac_header(skb
) - skb_mac_header(skb
);
1232 in_eth_xtra_len
= skb_inner_network_header(skb
) -
1233 skb_inner_mac_header(skb
) - ETH_HLEN
;
1236 case TX_TNL_TYPE_VXLAN
:
1237 case TX_TNL_TYPE_GENEVE
:
1238 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
=
1239 htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F
|
1240 CPL_TX_TNL_LSO_UDPLENSETOUT_F
);
1243 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
= 0;
1247 tnl_lso
->UdpLenSetOut_to_TnlHdrLen
|=
1248 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val
) |
1249 CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type
));
1253 val
= CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len
/ 4) |
1254 CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb
)->version
== 6) |
1255 CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb
) / 4) |
1256 CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb
) / 4);
1257 tnl_lso
->Flow_to_TcpHdrLen
= htonl(val
);
1259 tnl_lso
->IpIdOffset
= htons(0);
1261 tnl_lso
->IpIdSplit_to_Mss
= htons(CPL_TX_TNL_LSO_MSS_V(ssi
->gso_size
));
1262 tnl_lso
->TCPSeqOffset
= htonl(0);
1263 tnl_lso
->EthLenOffset_Size
= htonl(CPL_TX_TNL_LSO_SIZE_V(skb
->len
));
1266 static inline void *write_tso_wr(struct adapter
*adap
, struct sk_buff
*skb
,
1267 struct cpl_tx_pkt_lso_core
*lso
)
1269 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1270 int l3hdr_len
= skb_network_header_len(skb
);
1271 const struct skb_shared_info
*ssi
;
1274 ssi
= skb_shinfo(skb
);
1275 if (ssi
->gso_type
& SKB_GSO_TCPV6
)
1278 lso
->lso_ctrl
= htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1279 LSO_FIRST_SLICE_F
| LSO_LAST_SLICE_F
|
1281 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1282 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1283 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1284 lso
->ipid_ofst
= htons(0);
1285 lso
->mss
= htons(ssi
->gso_size
);
1286 lso
->seqno_offset
= htonl(0);
1287 if (is_t4(adap
->params
.chip
))
1288 lso
->len
= htonl(skb
->len
);
1290 lso
->len
= htonl(LSO_T5_XFER_SIZE_V(skb
->len
));
1292 return (void *)(lso
+ 1);
1296 * t4_sge_eth_txq_egress_update - handle Ethernet TX Queue update
1297 * @adap: the adapter
1298 * @eq: the Ethernet TX Queue
1299 * @maxreclaim: the maximum number of TX Descriptors to reclaim or -1
1301 * We're typically called here to update the state of an Ethernet TX
1302 * Queue with respect to the hardware's progress in consuming the TX
1303 * Work Requests that we've put on that Egress Queue. This happens
1304 * when we get Egress Queue Update messages and also prophylactically
1305 * in regular timer-based Ethernet TX Queue maintenance.
1307 int t4_sge_eth_txq_egress_update(struct adapter
*adap
, struct sge_eth_txq
*eq
,
1310 unsigned int reclaimed
, hw_cidx
;
1311 struct sge_txq
*q
= &eq
->q
;
1314 if (!q
->in_use
|| !__netif_tx_trylock(eq
->txq
))
1317 /* Reclaim pending completed TX Descriptors. */
1318 reclaimed
= reclaim_completed_tx(adap
, &eq
->q
, maxreclaim
, true);
1320 hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
1321 hw_in_use
= q
->pidx
- hw_cidx
;
1323 hw_in_use
+= q
->size
;
1325 /* If the TX Queue is currently stopped and there's now more than half
1326 * the queue available, restart it. Otherwise bail out since the rest
1327 * of what we want do here is with the possibility of shipping any
1328 * currently buffered Coalesced TX Work Request.
1330 if (netif_tx_queue_stopped(eq
->txq
) && hw_in_use
< (q
->size
/ 2)) {
1331 netif_tx_wake_queue(eq
->txq
);
1335 __netif_tx_unlock(eq
->txq
);
1339 static inline int cxgb4_validate_skb(struct sk_buff
*skb
,
1340 struct net_device
*dev
,
1345 /* The chip min packet length is 10 octets but some firmware
1346 * commands have a minimum packet length requirement. So, play
1347 * safe and reject anything shorter than @min_pkt_len.
1349 if (unlikely(skb
->len
< min_pkt_len
))
1352 /* Discard the packet if the length is greater than mtu */
1353 max_pkt_len
= ETH_HLEN
+ dev
->mtu
;
1355 if (skb_vlan_tagged(skb
))
1356 max_pkt_len
+= VLAN_HLEN
;
1358 if (!skb_shinfo(skb
)->gso_size
&& (unlikely(skb
->len
> max_pkt_len
)))
1364 static void *write_eo_udp_wr(struct sk_buff
*skb
, struct fw_eth_tx_eo_wr
*wr
,
1367 wr
->u
.udpseg
.type
= FW_ETH_TX_EO_TYPE_UDPSEG
;
1368 wr
->u
.udpseg
.ethlen
= skb_network_offset(skb
);
1369 wr
->u
.udpseg
.iplen
= cpu_to_be16(skb_network_header_len(skb
));
1370 wr
->u
.udpseg
.udplen
= sizeof(struct udphdr
);
1371 wr
->u
.udpseg
.rtplen
= 0;
1372 wr
->u
.udpseg
.r4
= 0;
1373 if (skb_shinfo(skb
)->gso_size
)
1374 wr
->u
.udpseg
.mss
= cpu_to_be16(skb_shinfo(skb
)->gso_size
);
1376 wr
->u
.udpseg
.mss
= cpu_to_be16(skb
->len
- hdr_len
);
1377 wr
->u
.udpseg
.schedpktsize
= wr
->u
.udpseg
.mss
;
1378 wr
->u
.udpseg
.plen
= cpu_to_be32(skb
->len
- hdr_len
);
1380 return (void *)(wr
+ 1);
1384 * cxgb4_eth_xmit - add a packet to an Ethernet Tx queue
1386 * @dev: the egress net device
1388 * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled.
1390 static netdev_tx_t
cxgb4_eth_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1392 enum cpl_tx_tnl_lso_type tnl_type
= TX_TNL_TYPE_OPAQUE
;
1393 bool ptp_enabled
= is_ptp_enabled(skb
, dev
);
1394 unsigned int last_desc
, flits
, ndesc
;
1395 u32 wr_mid
, ctrl0
, op
, sgl_off
= 0;
1396 const struct skb_shared_info
*ssi
;
1397 int len
, qidx
, credits
, ret
, left
;
1398 struct tx_sw_desc
*sgl_sdesc
;
1399 struct fw_eth_tx_eo_wr
*eowr
;
1400 struct fw_eth_tx_pkt_wr
*wr
;
1401 struct cpl_tx_pkt_core
*cpl
;
1402 const struct port_info
*pi
;
1403 bool immediate
= false;
1404 u64 cntrl
, *end
, *sgl
;
1405 struct sge_eth_txq
*q
;
1406 unsigned int chip_ver
;
1407 struct adapter
*adap
;
1409 ret
= cxgb4_validate_skb(skb
, dev
, ETH_HLEN
);
1413 pi
= netdev_priv(dev
);
1415 ssi
= skb_shinfo(skb
);
1416 #ifdef CONFIG_CHELSIO_IPSEC_INLINE
1417 if (xfrm_offload(skb
) && !ssi
->gso_size
)
1418 return adap
->uld
[CXGB4_ULD_CRYPTO
].tx_handler(skb
, dev
);
1419 #endif /* CHELSIO_IPSEC_INLINE */
1421 #ifdef CONFIG_CHELSIO_TLS_DEVICE
1423 return adap
->uld
[CXGB4_ULD_CRYPTO
].tx_handler(skb
, dev
);
1424 #endif /* CHELSIO_TLS_DEVICE */
1426 qidx
= skb_get_queue_mapping(skb
);
1428 spin_lock(&adap
->ptp_lock
);
1429 if (!(adap
->ptp_tx_skb
)) {
1430 skb_shinfo(skb
)->tx_flags
|= SKBTX_IN_PROGRESS
;
1431 adap
->ptp_tx_skb
= skb_get(skb
);
1433 spin_unlock(&adap
->ptp_lock
);
1436 q
= &adap
->sge
.ptptxq
;
1438 q
= &adap
->sge
.ethtxq
[qidx
+ pi
->first_qset
];
1440 skb_tx_timestamp(skb
);
1442 reclaim_completed_tx(adap
, &q
->q
, -1, true);
1443 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1445 #ifdef CONFIG_CHELSIO_T4_FCOE
1446 ret
= cxgb_fcoe_offload(skb
, adap
, pi
, &cntrl
);
1447 if (unlikely(ret
== -ENOTSUPP
)) {
1449 spin_unlock(&adap
->ptp_lock
);
1452 #endif /* CONFIG_CHELSIO_T4_FCOE */
1454 chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
1455 flits
= calc_tx_flits(skb
, chip_ver
);
1456 ndesc
= flits_to_desc(flits
);
1457 credits
= txq_avail(&q
->q
) - ndesc
;
1459 if (unlikely(credits
< 0)) {
1461 dev_err(adap
->pdev_dev
,
1462 "%s: Tx ring %u full while queue awake!\n",
1465 spin_unlock(&adap
->ptp_lock
);
1466 return NETDEV_TX_BUSY
;
1469 if (is_eth_imm(skb
, chip_ver
))
1472 if (skb
->encapsulation
&& chip_ver
> CHELSIO_T5
)
1473 tnl_type
= cxgb_encap_offload_supported(skb
);
1475 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1476 if (last_desc
>= q
->q
.size
)
1477 last_desc
-= q
->q
.size
;
1478 sgl_sdesc
= &q
->q
.sdesc
[last_desc
];
1481 unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, sgl_sdesc
->addr
) < 0)) {
1482 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1485 spin_unlock(&adap
->ptp_lock
);
1489 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1490 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1491 /* After we're done injecting the Work Request for this
1492 * packet, we'll be below our "stop threshold" so stop the TX
1493 * Queue now and schedule a request for an SGE Egress Queue
1494 * Update message. The queue will get started later on when
1495 * the firmware processes this Work Request and sends us an
1496 * Egress Queue Status Update message indicating that space
1500 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1503 wr
= (void *)&q
->q
.desc
[q
->q
.pidx
];
1504 eowr
= (void *)&q
->q
.desc
[q
->q
.pidx
];
1505 wr
->equiq_to_len16
= htonl(wr_mid
);
1506 wr
->r3
= cpu_to_be64(0);
1507 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
)
1508 end
= (u64
*)eowr
+ flits
;
1510 end
= (u64
*)wr
+ flits
;
1512 len
= immediate
? skb
->len
: 0;
1513 len
+= sizeof(*cpl
);
1514 if (ssi
->gso_size
&& !(ssi
->gso_type
& SKB_GSO_UDP_L4
)) {
1515 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1516 struct cpl_tx_tnl_lso
*tnl_lso
= (void *)(wr
+ 1);
1519 len
+= sizeof(*tnl_lso
);
1521 len
+= sizeof(*lso
);
1523 wr
->op_immdlen
= htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR
) |
1524 FW_WR_IMMDLEN_V(len
));
1526 struct iphdr
*iph
= ip_hdr(skb
);
1528 t6_fill_tnl_lso(skb
, tnl_lso
, tnl_type
);
1529 cpl
= (void *)(tnl_lso
+ 1);
1530 /* Driver is expected to compute partial checksum that
1531 * does not include the IP Total Length.
1533 if (iph
->version
== 4) {
1536 iph
->check
= (u16
)(~ip_fast_csum((u8
*)iph
,
1539 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1540 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1542 cpl
= write_tso_wr(adap
, skb
, lso
);
1543 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1545 sgl
= (u64
*)(cpl
+ 1); /* sgl start here */
1547 q
->tx_cso
+= ssi
->gso_segs
;
1548 } else if (ssi
->gso_size
) {
1552 hdrlen
= eth_get_headlen(dev
, skb
->data
, skb_headlen(skb
));
1554 wr
->op_immdlen
= cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR
) |
1555 FW_ETH_TX_EO_WR_IMMDLEN_V(len
));
1556 cpl
= write_eo_udp_wr(skb
, eowr
, hdrlen
);
1557 cntrl
= hwcsum(adap
->params
.chip
, skb
);
1559 start
= (u64
*)(cpl
+ 1);
1560 sgl
= (u64
*)inline_tx_skb_header(skb
, &q
->q
, (void *)start
,
1562 if (unlikely(start
> sgl
)) {
1563 left
= (u8
*)end
- (u8
*)q
->q
.stat
;
1564 end
= (void *)q
->q
.desc
+ left
;
1568 q
->tx_cso
+= ssi
->gso_segs
;
1571 op
= FW_PTP_TX_PKT_WR
;
1573 op
= FW_ETH_TX_PKT_WR
;
1574 wr
->op_immdlen
= htonl(FW_WR_OP_V(op
) |
1575 FW_WR_IMMDLEN_V(len
));
1576 cpl
= (void *)(wr
+ 1);
1577 sgl
= (u64
*)(cpl
+ 1);
1578 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1579 cntrl
= hwcsum(adap
->params
.chip
, skb
) |
1585 if (unlikely((u8
*)sgl
>= (u8
*)q
->q
.stat
)) {
1586 /* If current position is already at the end of the
1587 * txq, reset the current to point to start of the queue
1588 * and update the end ptr as well.
1590 left
= (u8
*)end
- (u8
*)q
->q
.stat
;
1591 end
= (void *)q
->q
.desc
+ left
;
1592 sgl
= (void *)q
->q
.desc
;
1595 if (skb_vlan_tag_present(skb
)) {
1597 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1598 #ifdef CONFIG_CHELSIO_T4_FCOE
1599 if (skb
->protocol
== htons(ETH_P_FCOE
))
1600 cntrl
|= TXPKT_VLAN_V(
1601 ((skb
->priority
& 0x7) << VLAN_PRIO_SHIFT
));
1602 #endif /* CONFIG_CHELSIO_T4_FCOE */
1605 ctrl0
= TXPKT_OPCODE_V(CPL_TX_PKT_XT
) | TXPKT_INTF_V(pi
->tx_chan
) |
1606 TXPKT_PF_V(adap
->pf
);
1608 ctrl0
|= TXPKT_TSTAMP_F
;
1609 #ifdef CONFIG_CHELSIO_T4_DCB
1610 if (is_t4(adap
->params
.chip
))
1611 ctrl0
|= TXPKT_OVLAN_IDX_V(q
->dcb_prio
);
1613 ctrl0
|= TXPKT_T5_OVLAN_IDX_V(q
->dcb_prio
);
1615 cpl
->ctrl0
= htonl(ctrl0
);
1616 cpl
->pack
= htons(0);
1617 cpl
->len
= htons(skb
->len
);
1618 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1621 cxgb4_inline_tx_skb(skb
, &q
->q
, sgl
);
1622 dev_consume_skb_any(skb
);
1624 cxgb4_write_sgl(skb
, &q
->q
, (void *)sgl
, end
, sgl_off
,
1627 sgl_sdesc
->skb
= skb
;
1630 txq_advance(&q
->q
, ndesc
);
1632 cxgb4_ring_tx_db(adap
, &q
->q
, ndesc
);
1634 spin_unlock(&adap
->ptp_lock
);
1635 return NETDEV_TX_OK
;
1638 dev_kfree_skb_any(skb
);
1639 return NETDEV_TX_OK
;
1644 /* Egress Queue sizes, producer and consumer indices are all in units
1645 * of Egress Context Units bytes. Note that as far as the hardware is
1646 * concerned, the free list is an Egress Queue (the host produces free
1647 * buffers which the hardware consumes) and free list entries are
1648 * 64-bit PCI DMA addresses.
1650 EQ_UNIT
= SGE_EQ_IDXSIZE
,
1651 FL_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1652 TXD_PER_EQ_UNIT
= EQ_UNIT
/ sizeof(__be64
),
1654 T4VF_ETHTXQ_MAX_HDR
= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1655 sizeof(struct cpl_tx_pkt_lso_core
) +
1656 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
),
1660 * t4vf_is_eth_imm - can an Ethernet packet be sent as immediate data?
1663 * Returns whether an Ethernet packet is small enough to fit completely as
1666 static inline int t4vf_is_eth_imm(const struct sk_buff
*skb
)
1668 /* The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request
1669 * which does not accommodate immediate data. We could dike out all
1670 * of the support code for immediate data but that would tie our hands
1671 * too much if we ever want to enhace the firmware. It would also
1672 * create more differences between the PF and VF Drivers.
1678 * t4vf_calc_tx_flits - calculate the number of flits for a packet TX WR
1681 * Returns the number of flits needed for a TX Work Request for the
1682 * given Ethernet packet, including the needed WR and CPL headers.
1684 static inline unsigned int t4vf_calc_tx_flits(const struct sk_buff
*skb
)
1688 /* If the skb is small enough, we can pump it out as a work request
1689 * with only immediate data. In that case we just have to have the
1690 * TX Packet header plus the skb data in the Work Request.
1692 if (t4vf_is_eth_imm(skb
))
1693 return DIV_ROUND_UP(skb
->len
+ sizeof(struct cpl_tx_pkt
),
1696 /* Otherwise, we're going to have to construct a Scatter gather list
1697 * of the skb body and fragments. We also include the flits necessary
1698 * for the TX Packet Work Request and CPL. We always have a firmware
1699 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
1700 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
1701 * message or, if we're doing a Large Send Offload, an LSO CPL message
1702 * with an embedded TX Packet Write CPL message.
1704 flits
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
1705 if (skb_shinfo(skb
)->gso_size
)
1706 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1707 sizeof(struct cpl_tx_pkt_lso_core
) +
1708 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1710 flits
+= (sizeof(struct fw_eth_tx_pkt_vm_wr
) +
1711 sizeof(struct cpl_tx_pkt_core
)) / sizeof(__be64
);
1716 * cxgb4_vf_eth_xmit - add a packet to an Ethernet TX queue
1718 * @dev: the egress net device
1720 * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled.
1722 static netdev_tx_t
cxgb4_vf_eth_xmit(struct sk_buff
*skb
,
1723 struct net_device
*dev
)
1725 unsigned int last_desc
, flits
, ndesc
;
1726 const struct skb_shared_info
*ssi
;
1727 struct fw_eth_tx_pkt_vm_wr
*wr
;
1728 struct tx_sw_desc
*sgl_sdesc
;
1729 struct cpl_tx_pkt_core
*cpl
;
1730 const struct port_info
*pi
;
1731 struct sge_eth_txq
*txq
;
1732 struct adapter
*adapter
;
1733 int qidx
, credits
, ret
;
1734 size_t fw_hdr_copy_len
;
1738 /* The chip minimum packet length is 10 octets but the firmware
1739 * command that we are using requires that we copy the Ethernet header
1740 * (including the VLAN tag) into the header so we reject anything
1741 * smaller than that ...
1743 fw_hdr_copy_len
= sizeof(wr
->ethmacdst
) + sizeof(wr
->ethmacsrc
) +
1744 sizeof(wr
->ethtype
) + sizeof(wr
->vlantci
);
1745 ret
= cxgb4_validate_skb(skb
, dev
, fw_hdr_copy_len
);
1749 /* Figure out which TX Queue we're going to use. */
1750 pi
= netdev_priv(dev
);
1751 adapter
= pi
->adapter
;
1752 qidx
= skb_get_queue_mapping(skb
);
1753 WARN_ON(qidx
>= pi
->nqsets
);
1754 txq
= &adapter
->sge
.ethtxq
[pi
->first_qset
+ qidx
];
1756 /* Take this opportunity to reclaim any TX Descriptors whose DMA
1757 * transfers have completed.
1759 reclaim_completed_tx(adapter
, &txq
->q
, -1, true);
1761 /* Calculate the number of flits and TX Descriptors we're going to
1762 * need along with how many TX Descriptors will be left over after
1763 * we inject our Work Request.
1765 flits
= t4vf_calc_tx_flits(skb
);
1766 ndesc
= flits_to_desc(flits
);
1767 credits
= txq_avail(&txq
->q
) - ndesc
;
1769 if (unlikely(credits
< 0)) {
1770 /* Not enough room for this packet's Work Request. Stop the
1771 * TX Queue and return a "busy" condition. The queue will get
1772 * started later on when the firmware informs us that space
1776 dev_err(adapter
->pdev_dev
,
1777 "%s: TX ring %u full while queue awake!\n",
1779 return NETDEV_TX_BUSY
;
1782 last_desc
= txq
->q
.pidx
+ ndesc
- 1;
1783 if (last_desc
>= txq
->q
.size
)
1784 last_desc
-= txq
->q
.size
;
1785 sgl_sdesc
= &txq
->q
.sdesc
[last_desc
];
1787 if (!t4vf_is_eth_imm(skb
) &&
1788 unlikely(cxgb4_map_skb(adapter
->pdev_dev
, skb
,
1789 sgl_sdesc
->addr
) < 0)) {
1790 /* We need to map the skb into PCI DMA space (because it can't
1791 * be in-lined directly into the Work Request) and the mapping
1792 * operation failed. Record the error and drop the packet.
1794 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1799 wr_mid
= FW_WR_LEN16_V(DIV_ROUND_UP(flits
, 2));
1800 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1801 /* After we're done injecting the Work Request for this
1802 * packet, we'll be below our "stop threshold" so stop the TX
1803 * Queue now and schedule a request for an SGE Egress Queue
1804 * Update message. The queue will get started later on when
1805 * the firmware processes this Work Request and sends us an
1806 * Egress Queue Status Update message indicating that space
1810 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1813 /* Start filling in our Work Request. Note that we do _not_ handle
1814 * the WR Header wrapping around the TX Descriptor Ring. If our
1815 * maximum header size ever exceeds one TX Descriptor, we'll need to
1816 * do something else here.
1818 WARN_ON(DIV_ROUND_UP(T4VF_ETHTXQ_MAX_HDR
, TXD_PER_EQ_UNIT
) > 1);
1819 wr
= (void *)&txq
->q
.desc
[txq
->q
.pidx
];
1820 wr
->equiq_to_len16
= cpu_to_be32(wr_mid
);
1821 wr
->r3
[0] = cpu_to_be32(0);
1822 wr
->r3
[1] = cpu_to_be32(0);
1823 skb_copy_from_linear_data(skb
, (void *)wr
->ethmacdst
, fw_hdr_copy_len
);
1824 end
= (u64
*)wr
+ flits
;
1826 /* If this is a Large Send Offload packet we'll put in an LSO CPL
1827 * message with an encapsulated TX Packet CPL message. Otherwise we
1828 * just use a TX Packet CPL message.
1830 ssi
= skb_shinfo(skb
);
1831 if (ssi
->gso_size
) {
1832 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
1833 bool v6
= (ssi
->gso_type
& SKB_GSO_TCPV6
) != 0;
1834 int l3hdr_len
= skb_network_header_len(skb
);
1835 int eth_xtra_len
= skb_network_offset(skb
) - ETH_HLEN
;
1838 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1839 FW_WR_IMMDLEN_V(sizeof(*lso
) +
1841 /* Fill in the LSO CPL message. */
1843 cpu_to_be32(LSO_OPCODE_V(CPL_TX_PKT_LSO
) |
1847 LSO_ETHHDR_LEN_V(eth_xtra_len
/ 4) |
1848 LSO_IPHDR_LEN_V(l3hdr_len
/ 4) |
1849 LSO_TCPHDR_LEN_V(tcp_hdr(skb
)->doff
));
1850 lso
->ipid_ofst
= cpu_to_be16(0);
1851 lso
->mss
= cpu_to_be16(ssi
->gso_size
);
1852 lso
->seqno_offset
= cpu_to_be32(0);
1853 if (is_t4(adapter
->params
.chip
))
1854 lso
->len
= cpu_to_be32(skb
->len
);
1856 lso
->len
= cpu_to_be32(LSO_T5_XFER_SIZE_V(skb
->len
));
1858 /* Set up TX Packet CPL pointer, control word and perform
1861 cpl
= (void *)(lso
+ 1);
1863 if (CHELSIO_CHIP_VERSION(adapter
->params
.chip
) <= CHELSIO_T5
)
1864 cntrl
= TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1866 cntrl
= T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len
);
1868 cntrl
|= TXPKT_CSUM_TYPE_V(v6
?
1869 TX_CSUM_TCPIP6
: TX_CSUM_TCPIP
) |
1870 TXPKT_IPHDR_LEN_V(l3hdr_len
);
1872 txq
->tx_cso
+= ssi
->gso_segs
;
1876 len
= (t4vf_is_eth_imm(skb
)
1877 ? skb
->len
+ sizeof(*cpl
)
1880 cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_PKT_VM_WR
) |
1881 FW_WR_IMMDLEN_V(len
));
1883 /* Set up TX Packet CPL pointer, control word and perform
1886 cpl
= (void *)(wr
+ 1);
1887 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1888 cntrl
= hwcsum(adapter
->params
.chip
, skb
) |
1892 cntrl
= TXPKT_L4CSUM_DIS_F
| TXPKT_IPCSUM_DIS_F
;
1896 /* If there's a VLAN tag present, add that to the list of things to
1897 * do in this Work Request.
1899 if (skb_vlan_tag_present(skb
)) {
1901 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
1904 /* Fill in the TX Packet CPL message header. */
1905 cpl
->ctrl0
= cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT
) |
1906 TXPKT_INTF_V(pi
->port_id
) |
1908 cpl
->pack
= cpu_to_be16(0);
1909 cpl
->len
= cpu_to_be16(skb
->len
);
1910 cpl
->ctrl1
= cpu_to_be64(cntrl
);
1912 /* Fill in the body of the TX Packet CPL message with either in-lined
1913 * data or a Scatter/Gather List.
1915 if (t4vf_is_eth_imm(skb
)) {
1916 /* In-line the packet's data and free the skb since we don't
1917 * need it any longer.
1919 cxgb4_inline_tx_skb(skb
, &txq
->q
, cpl
+ 1);
1920 dev_consume_skb_any(skb
);
1922 /* Write the skb's Scatter/Gather list into the TX Packet CPL
1923 * message and retain a pointer to the skb so we can free it
1924 * later when its DMA completes. (We store the skb pointer
1925 * in the Software Descriptor corresponding to the last TX
1926 * Descriptor used by the Work Request.)
1928 * The retained skb will be freed when the corresponding TX
1929 * Descriptors are reclaimed after their DMAs complete.
1930 * However, this could take quite a while since, in general,
1931 * the hardware is set up to be lazy about sending DMA
1932 * completion notifications to us and we mostly perform TX
1933 * reclaims in the transmit routine.
1935 * This is good for performamce but means that we rely on new
1936 * TX packets arriving to run the destructors of completed
1937 * packets, which open up space in their sockets' send queues.
1938 * Sometimes we do not get such new packets causing TX to
1939 * stall. A single UDP transmitter is a good example of this
1940 * situation. We have a clean up timer that periodically
1941 * reclaims completed packets but it doesn't run often enough
1942 * (nor do we want it to) to prevent lengthy stalls. A
1943 * solution to this problem is to run the destructor early,
1944 * after the packet is queued but before it's DMAd. A con is
1945 * that we lie to socket memory accounting, but the amount of
1946 * extra memory is reasonable (limited by the number of TX
1947 * descriptors), the packets do actually get freed quickly by
1948 * new packets almost always, and for protocols like TCP that
1949 * wait for acks to really free up the data the extra memory
1950 * is even less. On the positive side we run the destructors
1951 * on the sending CPU rather than on a potentially different
1952 * completing CPU, usually a good thing.
1954 * Run the destructor before telling the DMA engine about the
1955 * packet to make sure it doesn't complete and get freed
1958 struct ulptx_sgl
*sgl
= (struct ulptx_sgl
*)(cpl
+ 1);
1959 struct sge_txq
*tq
= &txq
->q
;
1961 /* If the Work Request header was an exact multiple of our TX
1962 * Descriptor length, then it's possible that the starting SGL
1963 * pointer lines up exactly with the end of our TX Descriptor
1964 * ring. If that's the case, wrap around to the beginning
1967 if (unlikely((void *)sgl
== (void *)tq
->stat
)) {
1968 sgl
= (void *)tq
->desc
;
1969 end
= (void *)((void *)tq
->desc
+
1970 ((void *)end
- (void *)tq
->stat
));
1973 cxgb4_write_sgl(skb
, tq
, sgl
, end
, 0, sgl_sdesc
->addr
);
1975 sgl_sdesc
->skb
= skb
;
1978 /* Advance our internal TX Queue state, tell the hardware about
1979 * the new TX descriptors and return success.
1981 txq_advance(&txq
->q
, ndesc
);
1983 cxgb4_ring_tx_db(adapter
, &txq
->q
, ndesc
);
1984 return NETDEV_TX_OK
;
1987 /* An error of some sort happened. Free the TX skb and tell the
1988 * OS that we've "dealt" with the packet ...
1990 dev_kfree_skb_any(skb
);
1991 return NETDEV_TX_OK
;
1995 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1996 * @q: the SGE control Tx queue
1998 * This is a variant of cxgb4_reclaim_completed_tx() that is used
1999 * for Tx queues that send only immediate data (presently just
2000 * the control queues) and thus do not have any sk_buffs to release.
2002 static inline void reclaim_completed_tx_imm(struct sge_txq
*q
)
2004 int hw_cidx
= ntohs(READ_ONCE(q
->stat
->cidx
));
2005 int reclaim
= hw_cidx
- q
->cidx
;
2010 q
->in_use
-= reclaim
;
2014 static inline void eosw_txq_advance_index(u32
*idx
, u32 n
, u32 max
)
2024 void cxgb4_eosw_txq_free_desc(struct adapter
*adap
,
2025 struct sge_eosw_txq
*eosw_txq
, u32 ndesc
)
2027 struct tx_sw_desc
*d
;
2029 d
= &eosw_txq
->desc
[eosw_txq
->last_cidx
];
2033 unmap_skb(adap
->pdev_dev
, d
->skb
, d
->addr
);
2034 memset(d
->addr
, 0, sizeof(d
->addr
));
2036 dev_consume_skb_any(d
->skb
);
2039 eosw_txq_advance_index(&eosw_txq
->last_cidx
, 1,
2041 d
= &eosw_txq
->desc
[eosw_txq
->last_cidx
];
2045 static inline void eosw_txq_advance(struct sge_eosw_txq
*eosw_txq
, u32 n
)
2047 eosw_txq_advance_index(&eosw_txq
->pidx
, n
, eosw_txq
->ndesc
);
2048 eosw_txq
->inuse
+= n
;
2051 static inline int eosw_txq_enqueue(struct sge_eosw_txq
*eosw_txq
,
2052 struct sk_buff
*skb
)
2054 if (eosw_txq
->inuse
== eosw_txq
->ndesc
)
2057 eosw_txq
->desc
[eosw_txq
->pidx
].skb
= skb
;
2061 static inline struct sk_buff
*eosw_txq_peek(struct sge_eosw_txq
*eosw_txq
)
2063 return eosw_txq
->desc
[eosw_txq
->last_pidx
].skb
;
2066 static inline u8
ethofld_calc_tx_flits(struct adapter
*adap
,
2067 struct sk_buff
*skb
, u32 hdr_len
)
2072 wrlen
= sizeof(struct fw_eth_tx_eo_wr
) + sizeof(struct cpl_tx_pkt_core
);
2073 if (skb_shinfo(skb
)->gso_size
&&
2074 !(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
))
2075 wrlen
+= sizeof(struct cpl_tx_pkt_lso_core
);
2077 wrlen
+= roundup(hdr_len
, 16);
2079 /* Packet headers + WR + CPLs */
2080 flits
= DIV_ROUND_UP(wrlen
, 8);
2082 if (skb_shinfo(skb
)->nr_frags
> 0) {
2083 if (skb_headlen(skb
) - hdr_len
)
2084 nsgl
= sgl_len(skb_shinfo(skb
)->nr_frags
+ 1);
2086 nsgl
= sgl_len(skb_shinfo(skb
)->nr_frags
);
2087 } else if (skb
->len
- hdr_len
) {
2091 return flits
+ nsgl
;
2094 static inline void *write_eo_wr(struct adapter
*adap
,
2095 struct sge_eosw_txq
*eosw_txq
,
2096 struct sk_buff
*skb
, struct fw_eth_tx_eo_wr
*wr
,
2097 u32 hdr_len
, u32 wrlen
)
2099 const struct skb_shared_info
*ssi
= skb_shinfo(skb
);
2100 struct cpl_tx_pkt_core
*cpl
;
2101 u32 immd_len
, wrlen16
;
2105 ver
= ip_hdr(skb
)->version
;
2106 proto
= (ver
== 6) ? ipv6_hdr(skb
)->nexthdr
: ip_hdr(skb
)->protocol
;
2108 wrlen16
= DIV_ROUND_UP(wrlen
, 16);
2109 immd_len
= sizeof(struct cpl_tx_pkt_core
);
2110 if (skb_shinfo(skb
)->gso_size
&&
2111 !(skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
))
2112 immd_len
+= sizeof(struct cpl_tx_pkt_lso_core
);
2113 immd_len
+= hdr_len
;
2115 if (!eosw_txq
->ncompl
||
2116 eosw_txq
->last_compl
>= adap
->params
.ofldq_wr_cred
/ 2) {
2119 eosw_txq
->last_compl
= 0;
2122 wr
->op_immdlen
= cpu_to_be32(FW_WR_OP_V(FW_ETH_TX_EO_WR
) |
2123 FW_ETH_TX_EO_WR_IMMDLEN_V(immd_len
) |
2124 FW_WR_COMPL_V(compl));
2125 wr
->equiq_to_len16
= cpu_to_be32(FW_WR_LEN16_V(wrlen16
) |
2126 FW_WR_FLOWID_V(eosw_txq
->hwtid
));
2128 if (proto
== IPPROTO_UDP
) {
2129 cpl
= write_eo_udp_wr(skb
, wr
, hdr_len
);
2131 wr
->u
.tcpseg
.type
= FW_ETH_TX_EO_TYPE_TCPSEG
;
2132 wr
->u
.tcpseg
.ethlen
= skb_network_offset(skb
);
2133 wr
->u
.tcpseg
.iplen
= cpu_to_be16(skb_network_header_len(skb
));
2134 wr
->u
.tcpseg
.tcplen
= tcp_hdrlen(skb
);
2135 wr
->u
.tcpseg
.tsclk_tsoff
= 0;
2136 wr
->u
.tcpseg
.r4
= 0;
2137 wr
->u
.tcpseg
.r5
= 0;
2138 wr
->u
.tcpseg
.plen
= cpu_to_be32(skb
->len
- hdr_len
);
2140 if (ssi
->gso_size
) {
2141 struct cpl_tx_pkt_lso_core
*lso
= (void *)(wr
+ 1);
2143 wr
->u
.tcpseg
.mss
= cpu_to_be16(ssi
->gso_size
);
2144 cpl
= write_tso_wr(adap
, skb
, lso
);
2146 wr
->u
.tcpseg
.mss
= cpu_to_be16(0xffff);
2147 cpl
= (void *)(wr
+ 1);
2151 eosw_txq
->cred
-= wrlen16
;
2152 eosw_txq
->last_compl
+= wrlen16
;
2156 static void ethofld_hard_xmit(struct net_device
*dev
,
2157 struct sge_eosw_txq
*eosw_txq
)
2159 struct port_info
*pi
= netdev2pinfo(dev
);
2160 struct adapter
*adap
= netdev2adap(dev
);
2161 u32 wrlen
, wrlen16
, hdr_len
, data_len
;
2162 enum sge_eosw_state next_state
;
2163 u64 cntrl
, *start
, *end
, *sgl
;
2164 struct sge_eohw_txq
*eohw_txq
;
2165 struct cpl_tx_pkt_core
*cpl
;
2166 struct fw_eth_tx_eo_wr
*wr
;
2167 bool skip_eotx_wr
= false;
2168 struct tx_sw_desc
*d
;
2169 struct sk_buff
*skb
;
2173 eohw_txq
= &adap
->sge
.eohw_txq
[eosw_txq
->hwqid
];
2174 spin_lock(&eohw_txq
->lock
);
2175 reclaim_completed_tx_imm(&eohw_txq
->q
);
2177 d
= &eosw_txq
->desc
[eosw_txq
->last_pidx
];
2179 skb_tx_timestamp(skb
);
2181 wr
= (struct fw_eth_tx_eo_wr
*)&eohw_txq
->q
.desc
[eohw_txq
->q
.pidx
];
2182 if (unlikely(eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
&&
2183 eosw_txq
->last_pidx
== eosw_txq
->flowc_idx
)) {
2186 flits
= DIV_ROUND_UP(hdr_len
, 8);
2187 if (eosw_txq
->state
== CXGB4_EO_STATE_FLOWC_OPEN_SEND
)
2188 next_state
= CXGB4_EO_STATE_FLOWC_OPEN_REPLY
;
2190 next_state
= CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
;
2191 skip_eotx_wr
= true;
2193 hdr_len
= eth_get_headlen(dev
, skb
->data
, skb_headlen(skb
));
2194 data_len
= skb
->len
- hdr_len
;
2195 flits
= ethofld_calc_tx_flits(adap
, skb
, hdr_len
);
2197 ndesc
= flits_to_desc(flits
);
2199 wrlen16
= DIV_ROUND_UP(wrlen
, 16);
2201 /* If there are no CPL credits, then wait for credits
2202 * to come back and retry again
2204 if (unlikely(wrlen16
> eosw_txq
->cred
))
2207 if (unlikely(skip_eotx_wr
)) {
2209 eosw_txq
->state
= next_state
;
2210 goto write_wr_headers
;
2213 cpl
= write_eo_wr(adap
, eosw_txq
, skb
, wr
, hdr_len
, wrlen
);
2214 cntrl
= hwcsum(adap
->params
.chip
, skb
);
2215 if (skb_vlan_tag_present(skb
))
2216 cntrl
|= TXPKT_VLAN_VLD_F
| TXPKT_VLAN_V(skb_vlan_tag_get(skb
));
2218 cpl
->ctrl0
= cpu_to_be32(TXPKT_OPCODE_V(CPL_TX_PKT_XT
) |
2219 TXPKT_INTF_V(pi
->tx_chan
) |
2220 TXPKT_PF_V(adap
->pf
));
2222 cpl
->len
= cpu_to_be16(skb
->len
);
2223 cpl
->ctrl1
= cpu_to_be64(cntrl
);
2225 start
= (u64
*)(cpl
+ 1);
2228 sgl
= (u64
*)inline_tx_skb_header(skb
, &eohw_txq
->q
, (void *)start
,
2231 if (unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, d
->addr
))) {
2232 memset(d
->addr
, 0, sizeof(d
->addr
));
2233 eohw_txq
->mapping_err
++;
2237 end
= (u64
*)wr
+ flits
;
2238 if (unlikely(start
> sgl
)) {
2239 left
= (u8
*)end
- (u8
*)eohw_txq
->q
.stat
;
2240 end
= (void *)eohw_txq
->q
.desc
+ left
;
2243 if (unlikely((u8
*)sgl
>= (u8
*)eohw_txq
->q
.stat
)) {
2244 /* If current position is already at the end of the
2245 * txq, reset the current to point to start of the queue
2246 * and update the end ptr as well.
2248 left
= (u8
*)end
- (u8
*)eohw_txq
->q
.stat
;
2250 end
= (void *)eohw_txq
->q
.desc
+ left
;
2251 sgl
= (void *)eohw_txq
->q
.desc
;
2254 cxgb4_write_sgl(skb
, &eohw_txq
->q
, (void *)sgl
, end
, hdr_len
,
2258 if (skb_shinfo(skb
)->gso_size
) {
2259 if (skb_shinfo(skb
)->gso_type
& SKB_GSO_UDP_L4
)
2263 eohw_txq
->tx_cso
+= skb_shinfo(skb
)->gso_segs
;
2264 } else if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
2268 if (skb_vlan_tag_present(skb
))
2269 eohw_txq
->vlan_ins
++;
2271 txq_advance(&eohw_txq
->q
, ndesc
);
2272 cxgb4_ring_tx_db(adap
, &eohw_txq
->q
, ndesc
);
2273 eosw_txq_advance_index(&eosw_txq
->last_pidx
, 1, eosw_txq
->ndesc
);
2276 spin_unlock(&eohw_txq
->lock
);
2279 static void ethofld_xmit(struct net_device
*dev
, struct sge_eosw_txq
*eosw_txq
)
2281 struct sk_buff
*skb
;
2284 switch (eosw_txq
->state
) {
2285 case CXGB4_EO_STATE_ACTIVE
:
2286 case CXGB4_EO_STATE_FLOWC_OPEN_SEND
:
2287 case CXGB4_EO_STATE_FLOWC_CLOSE_SEND
:
2288 pktcount
= eosw_txq
->pidx
- eosw_txq
->last_pidx
;
2290 pktcount
+= eosw_txq
->ndesc
;
2292 case CXGB4_EO_STATE_FLOWC_OPEN_REPLY
:
2293 case CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
:
2294 case CXGB4_EO_STATE_CLOSED
:
2299 while (pktcount
--) {
2300 skb
= eosw_txq_peek(eosw_txq
);
2302 eosw_txq_advance_index(&eosw_txq
->last_pidx
, 1,
2307 ethofld_hard_xmit(dev
, eosw_txq
);
2311 static netdev_tx_t
cxgb4_ethofld_xmit(struct sk_buff
*skb
,
2312 struct net_device
*dev
)
2314 struct cxgb4_tc_port_mqprio
*tc_port_mqprio
;
2315 struct port_info
*pi
= netdev2pinfo(dev
);
2316 struct adapter
*adap
= netdev2adap(dev
);
2317 struct sge_eosw_txq
*eosw_txq
;
2321 ret
= cxgb4_validate_skb(skb
, dev
, ETH_HLEN
);
2325 tc_port_mqprio
= &adap
->tc_mqprio
->port_mqprio
[pi
->port_id
];
2326 qid
= skb_get_queue_mapping(skb
) - pi
->nqsets
;
2327 eosw_txq
= &tc_port_mqprio
->eosw_txq
[qid
];
2328 spin_lock_bh(&eosw_txq
->lock
);
2329 if (eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
)
2332 ret
= eosw_txq_enqueue(eosw_txq
, skb
);
2336 /* SKB is queued for processing until credits are available.
2337 * So, call the destructor now and we'll free the skb later
2338 * after it has been successfully transmitted.
2342 eosw_txq_advance(eosw_txq
, 1);
2343 ethofld_xmit(dev
, eosw_txq
);
2344 spin_unlock_bh(&eosw_txq
->lock
);
2345 return NETDEV_TX_OK
;
2348 spin_unlock_bh(&eosw_txq
->lock
);
2350 dev_kfree_skb_any(skb
);
2351 return NETDEV_TX_OK
;
2354 netdev_tx_t
t4_start_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
2356 struct port_info
*pi
= netdev_priv(dev
);
2357 u16 qid
= skb_get_queue_mapping(skb
);
2359 if (unlikely(pi
->eth_flags
& PRIV_FLAG_PORT_TX_VM
))
2360 return cxgb4_vf_eth_xmit(skb
, dev
);
2362 if (unlikely(qid
>= pi
->nqsets
))
2363 return cxgb4_ethofld_xmit(skb
, dev
);
2365 return cxgb4_eth_xmit(skb
, dev
);
2369 * cxgb4_ethofld_send_flowc - Send ETHOFLD flowc request to bind eotid to tc.
2371 * @eotid - ETHOFLD tid to bind/unbind
2372 * @tc - traffic class. If set to FW_SCHED_CLS_NONE, then unbinds the @eotid
2374 * Send a FLOWC work request to bind an ETHOFLD TID to a traffic class.
2375 * If @tc is set to FW_SCHED_CLS_NONE, then the @eotid is unbound from
2378 int cxgb4_ethofld_send_flowc(struct net_device
*dev
, u32 eotid
, u32 tc
)
2380 struct port_info
*pi
= netdev2pinfo(dev
);
2381 struct adapter
*adap
= netdev2adap(dev
);
2382 enum sge_eosw_state next_state
;
2383 struct sge_eosw_txq
*eosw_txq
;
2384 u32 len
, len16
, nparams
= 6;
2385 struct fw_flowc_wr
*flowc
;
2386 struct eotid_entry
*entry
;
2387 struct sge_ofld_rxq
*rxq
;
2388 struct sk_buff
*skb
;
2391 len
= sizeof(*flowc
) + sizeof(struct fw_flowc_mnemval
) * nparams
;
2392 len16
= DIV_ROUND_UP(len
, 16);
2394 entry
= cxgb4_lookup_eotid(&adap
->tids
, eotid
);
2398 eosw_txq
= (struct sge_eosw_txq
*)entry
->data
;
2402 skb
= alloc_skb(len
, GFP_KERNEL
);
2406 spin_lock_bh(&eosw_txq
->lock
);
2407 if (tc
!= FW_SCHED_CLS_NONE
) {
2408 if (eosw_txq
->state
!= CXGB4_EO_STATE_CLOSED
)
2411 next_state
= CXGB4_EO_STATE_FLOWC_OPEN_SEND
;
2413 if (eosw_txq
->state
!= CXGB4_EO_STATE_ACTIVE
)
2416 next_state
= CXGB4_EO_STATE_FLOWC_CLOSE_SEND
;
2419 flowc
= __skb_put(skb
, len
);
2420 memset(flowc
, 0, len
);
2422 rxq
= &adap
->sge
.eohw_rxq
[eosw_txq
->hwqid
];
2423 flowc
->flowid_len16
= cpu_to_be32(FW_WR_LEN16_V(len16
) |
2424 FW_WR_FLOWID_V(eosw_txq
->hwtid
));
2425 flowc
->op_to_nparams
= cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR
) |
2426 FW_FLOWC_WR_NPARAMS_V(nparams
) |
2428 flowc
->mnemval
[0].mnemonic
= FW_FLOWC_MNEM_PFNVFN
;
2429 flowc
->mnemval
[0].val
= cpu_to_be32(FW_PFVF_CMD_PFN_V(adap
->pf
));
2430 flowc
->mnemval
[1].mnemonic
= FW_FLOWC_MNEM_CH
;
2431 flowc
->mnemval
[1].val
= cpu_to_be32(pi
->tx_chan
);
2432 flowc
->mnemval
[2].mnemonic
= FW_FLOWC_MNEM_PORT
;
2433 flowc
->mnemval
[2].val
= cpu_to_be32(pi
->tx_chan
);
2434 flowc
->mnemval
[3].mnemonic
= FW_FLOWC_MNEM_IQID
;
2435 flowc
->mnemval
[3].val
= cpu_to_be32(rxq
->rspq
.abs_id
);
2436 flowc
->mnemval
[4].mnemonic
= FW_FLOWC_MNEM_SCHEDCLASS
;
2437 flowc
->mnemval
[4].val
= cpu_to_be32(tc
);
2438 flowc
->mnemval
[5].mnemonic
= FW_FLOWC_MNEM_EOSTATE
;
2439 flowc
->mnemval
[5].val
= cpu_to_be32(tc
== FW_SCHED_CLS_NONE
?
2440 FW_FLOWC_MNEM_EOSTATE_CLOSING
:
2441 FW_FLOWC_MNEM_EOSTATE_ESTABLISHED
);
2443 eosw_txq
->cred
-= len16
;
2445 eosw_txq
->last_compl
= 0;
2447 ret
= eosw_txq_enqueue(eosw_txq
, skb
);
2449 dev_consume_skb_any(skb
);
2453 eosw_txq
->state
= next_state
;
2454 eosw_txq
->flowc_idx
= eosw_txq
->pidx
;
2455 eosw_txq_advance(eosw_txq
, 1);
2456 ethofld_xmit(dev
, eosw_txq
);
2459 spin_unlock_bh(&eosw_txq
->lock
);
2464 * is_imm - check whether a packet can be sent as immediate data
2467 * Returns true if a packet can be sent as a WR with immediate data.
2469 static inline int is_imm(const struct sk_buff
*skb
)
2471 return skb
->len
<= MAX_CTRL_WR_LEN
;
2475 * ctrlq_check_stop - check if a control queue is full and should stop
2477 * @wr: most recent WR written to the queue
2479 * Check if a control queue has become full and should be stopped.
2480 * We clean up control queue descriptors very lazily, only when we are out.
2481 * If the queue is still full after reclaiming any completed descriptors
2482 * we suspend it and have the last WR wake it up.
2484 static void ctrlq_check_stop(struct sge_ctrl_txq
*q
, struct fw_wr_hdr
*wr
)
2486 reclaim_completed_tx_imm(&q
->q
);
2487 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2488 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2495 * ctrl_xmit - send a packet through an SGE control Tx queue
2496 * @q: the control queue
2499 * Send a packet through an SGE control Tx queue. Packets sent through
2500 * a control queue must fit entirely as immediate data.
2502 static int ctrl_xmit(struct sge_ctrl_txq
*q
, struct sk_buff
*skb
)
2505 struct fw_wr_hdr
*wr
;
2507 if (unlikely(!is_imm(skb
))) {
2510 return NET_XMIT_DROP
;
2513 ndesc
= DIV_ROUND_UP(skb
->len
, sizeof(struct tx_desc
));
2514 spin_lock(&q
->sendq
.lock
);
2516 if (unlikely(q
->full
)) {
2517 skb
->priority
= ndesc
; /* save for restart */
2518 __skb_queue_tail(&q
->sendq
, skb
);
2519 spin_unlock(&q
->sendq
.lock
);
2523 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2524 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2526 txq_advance(&q
->q
, ndesc
);
2527 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
))
2528 ctrlq_check_stop(q
, wr
);
2530 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
2531 spin_unlock(&q
->sendq
.lock
);
2534 return NET_XMIT_SUCCESS
;
2538 * restart_ctrlq - restart a suspended control queue
2539 * @data: the control queue to restart
2541 * Resumes transmission on a suspended Tx control queue.
2543 static void restart_ctrlq(unsigned long data
)
2545 struct sk_buff
*skb
;
2546 unsigned int written
= 0;
2547 struct sge_ctrl_txq
*q
= (struct sge_ctrl_txq
*)data
;
2549 spin_lock(&q
->sendq
.lock
);
2550 reclaim_completed_tx_imm(&q
->q
);
2551 BUG_ON(txq_avail(&q
->q
) < TXQ_STOP_THRES
); /* q should be empty */
2553 while ((skb
= __skb_dequeue(&q
->sendq
)) != NULL
) {
2554 struct fw_wr_hdr
*wr
;
2555 unsigned int ndesc
= skb
->priority
; /* previously saved */
2558 /* Write descriptors and free skbs outside the lock to limit
2559 * wait times. q->full is still set so new skbs will be queued.
2561 wr
= (struct fw_wr_hdr
*)&q
->q
.desc
[q
->q
.pidx
];
2562 txq_advance(&q
->q
, ndesc
);
2563 spin_unlock(&q
->sendq
.lock
);
2565 cxgb4_inline_tx_skb(skb
, &q
->q
, wr
);
2568 if (unlikely(txq_avail(&q
->q
) < TXQ_STOP_THRES
)) {
2569 unsigned long old
= q
->q
.stops
;
2571 ctrlq_check_stop(q
, wr
);
2572 if (q
->q
.stops
!= old
) { /* suspended anew */
2573 spin_lock(&q
->sendq
.lock
);
2578 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2581 spin_lock(&q
->sendq
.lock
);
2586 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2587 spin_unlock(&q
->sendq
.lock
);
2591 * t4_mgmt_tx - send a management message
2592 * @adap: the adapter
2593 * @skb: the packet containing the management message
2595 * Send a management message through control queue 0.
2597 int t4_mgmt_tx(struct adapter
*adap
, struct sk_buff
*skb
)
2602 ret
= ctrl_xmit(&adap
->sge
.ctrlq
[0], skb
);
2608 * is_ofld_imm - check whether a packet can be sent as immediate data
2611 * Returns true if a packet can be sent as an offload WR with immediate
2612 * data. We currently use the same limit as for Ethernet packets.
2614 static inline int is_ofld_imm(const struct sk_buff
*skb
)
2616 struct work_request_hdr
*req
= (struct work_request_hdr
*)skb
->data
;
2617 unsigned long opcode
= FW_WR_OP_G(ntohl(req
->wr_hi
));
2619 if (opcode
== FW_CRYPTO_LOOKASIDE_WR
)
2620 return skb
->len
<= SGE_MAX_WR_LEN
;
2622 return skb
->len
<= MAX_IMM_TX_PKT_LEN
;
2626 * calc_tx_flits_ofld - calculate # of flits for an offload packet
2629 * Returns the number of flits needed for the given offload packet.
2630 * These packets are already fully constructed and no additional headers
2633 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff
*skb
)
2635 unsigned int flits
, cnt
;
2637 if (is_ofld_imm(skb
))
2638 return DIV_ROUND_UP(skb
->len
, 8);
2640 flits
= skb_transport_offset(skb
) / 8U; /* headers */
2641 cnt
= skb_shinfo(skb
)->nr_frags
;
2642 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
2644 return flits
+ sgl_len(cnt
);
2648 * txq_stop_maperr - stop a Tx queue due to I/O MMU exhaustion
2649 * @adap: the adapter
2650 * @q: the queue to stop
2652 * Mark a Tx queue stopped due to I/O MMU exhaustion and resulting
2653 * inability to map packets. A periodic timer attempts to restart
2656 static void txq_stop_maperr(struct sge_uld_txq
*q
)
2660 set_bit(q
->q
.cntxt_id
- q
->adap
->sge
.egr_start
,
2661 q
->adap
->sge
.txq_maperr
);
2665 * ofldtxq_stop - stop an offload Tx queue that has become full
2666 * @q: the queue to stop
2667 * @wr: the Work Request causing the queue to become full
2669 * Stops an offload Tx queue that has become full and modifies the packet
2670 * being written to request a wakeup.
2672 static void ofldtxq_stop(struct sge_uld_txq
*q
, struct fw_wr_hdr
*wr
)
2674 wr
->lo
|= htonl(FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
);
2680 * service_ofldq - service/restart a suspended offload queue
2681 * @q: the offload queue
2683 * Services an offload Tx queue by moving packets from its Pending Send
2684 * Queue to the Hardware TX ring. The function starts and ends with the
2685 * Send Queue locked, but drops the lock while putting the skb at the
2686 * head of the Send Queue onto the Hardware TX Ring. Dropping the lock
2687 * allows more skbs to be added to the Send Queue by other threads.
2688 * The packet being processed at the head of the Pending Send Queue is
2689 * left on the queue in case we experience DMA Mapping errors, etc.
2690 * and need to give up and restart later.
2692 * service_ofldq() can be thought of as a task which opportunistically
2693 * uses other threads execution contexts. We use the Offload Queue
2694 * boolean "service_ofldq_running" to make sure that only one instance
2695 * is ever running at a time ...
2697 static void service_ofldq(struct sge_uld_txq
*q
)
2699 u64
*pos
, *before
, *end
;
2701 struct sk_buff
*skb
;
2702 struct sge_txq
*txq
;
2704 unsigned int written
= 0;
2705 unsigned int flits
, ndesc
;
2707 /* If another thread is currently in service_ofldq() processing the
2708 * Pending Send Queue then there's nothing to do. Otherwise, flag
2709 * that we're doing the work and continue. Examining/modifying
2710 * the Offload Queue boolean "service_ofldq_running" must be done
2711 * while holding the Pending Send Queue Lock.
2713 if (q
->service_ofldq_running
)
2715 q
->service_ofldq_running
= true;
2717 while ((skb
= skb_peek(&q
->sendq
)) != NULL
&& !q
->full
) {
2718 /* We drop the lock while we're working with the skb at the
2719 * head of the Pending Send Queue. This allows more skbs to
2720 * be added to the Pending Send Queue while we're working on
2721 * this one. We don't need to lock to guard the TX Ring
2722 * updates because only one thread of execution is ever
2723 * allowed into service_ofldq() at a time.
2725 spin_unlock(&q
->sendq
.lock
);
2727 cxgb4_reclaim_completed_tx(q
->adap
, &q
->q
, false);
2729 flits
= skb
->priority
; /* previously saved */
2730 ndesc
= flits_to_desc(flits
);
2731 credits
= txq_avail(&q
->q
) - ndesc
;
2732 BUG_ON(credits
< 0);
2733 if (unlikely(credits
< TXQ_STOP_THRES
))
2734 ofldtxq_stop(q
, (struct fw_wr_hdr
*)skb
->data
);
2736 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
2737 if (is_ofld_imm(skb
))
2738 cxgb4_inline_tx_skb(skb
, &q
->q
, pos
);
2739 else if (cxgb4_map_skb(q
->adap
->pdev_dev
, skb
,
2740 (dma_addr_t
*)skb
->head
)) {
2742 spin_lock(&q
->sendq
.lock
);
2745 int last_desc
, hdr_len
= skb_transport_offset(skb
);
2747 /* The WR headers may not fit within one descriptor.
2748 * So we need to deal with wrap-around here.
2750 before
= (u64
*)pos
;
2751 end
= (u64
*)pos
+ flits
;
2753 pos
= (void *)inline_tx_skb_header(skb
, &q
->q
,
2756 if (before
> (u64
*)pos
) {
2757 left
= (u8
*)end
- (u8
*)txq
->stat
;
2758 end
= (void *)txq
->desc
+ left
;
2761 /* If current position is already at the end of the
2762 * ofld queue, reset the current to point to
2763 * start of the queue and update the end ptr as well.
2765 if (pos
== (u64
*)txq
->stat
) {
2766 left
= (u8
*)end
- (u8
*)txq
->stat
;
2767 end
= (void *)txq
->desc
+ left
;
2768 pos
= (void *)txq
->desc
;
2771 cxgb4_write_sgl(skb
, &q
->q
, (void *)pos
,
2773 (dma_addr_t
*)skb
->head
);
2774 #ifdef CONFIG_NEED_DMA_MAP_STATE
2775 skb
->dev
= q
->adap
->port
[0];
2776 skb
->destructor
= deferred_unmap_destructor
;
2778 last_desc
= q
->q
.pidx
+ ndesc
- 1;
2779 if (last_desc
>= q
->q
.size
)
2780 last_desc
-= q
->q
.size
;
2781 q
->q
.sdesc
[last_desc
].skb
= skb
;
2784 txq_advance(&q
->q
, ndesc
);
2786 if (unlikely(written
> 32)) {
2787 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2791 /* Reacquire the Pending Send Queue Lock so we can unlink the
2792 * skb we've just successfully transferred to the TX Ring and
2793 * loop for the next skb which may be at the head of the
2794 * Pending Send Queue.
2796 spin_lock(&q
->sendq
.lock
);
2797 __skb_unlink(skb
, &q
->sendq
);
2798 if (is_ofld_imm(skb
))
2801 if (likely(written
))
2802 cxgb4_ring_tx_db(q
->adap
, &q
->q
, written
);
2804 /*Indicate that no thread is processing the Pending Send Queue
2807 q
->service_ofldq_running
= false;
2811 * ofld_xmit - send a packet through an offload queue
2812 * @q: the Tx offload queue
2815 * Send an offload packet through an SGE offload queue.
2817 static int ofld_xmit(struct sge_uld_txq
*q
, struct sk_buff
*skb
)
2819 skb
->priority
= calc_tx_flits_ofld(skb
); /* save for restart */
2820 spin_lock(&q
->sendq
.lock
);
2822 /* Queue the new skb onto the Offload Queue's Pending Send Queue. If
2823 * that results in this new skb being the only one on the queue, start
2824 * servicing it. If there are other skbs already on the list, then
2825 * either the queue is currently being processed or it's been stopped
2826 * for some reason and it'll be restarted at a later time. Restart
2827 * paths are triggered by events like experiencing a DMA Mapping Error
2828 * or filling the Hardware TX Ring.
2830 __skb_queue_tail(&q
->sendq
, skb
);
2831 if (q
->sendq
.qlen
== 1)
2834 spin_unlock(&q
->sendq
.lock
);
2835 return NET_XMIT_SUCCESS
;
2839 * restart_ofldq - restart a suspended offload queue
2840 * @data: the offload queue to restart
2842 * Resumes transmission on a suspended Tx offload queue.
2844 static void restart_ofldq(unsigned long data
)
2846 struct sge_uld_txq
*q
= (struct sge_uld_txq
*)data
;
2848 spin_lock(&q
->sendq
.lock
);
2849 q
->full
= 0; /* the queue actually is completely empty now */
2851 spin_unlock(&q
->sendq
.lock
);
2855 * skb_txq - return the Tx queue an offload packet should use
2858 * Returns the Tx queue an offload packet should use as indicated by bits
2859 * 1-15 in the packet's queue_mapping.
2861 static inline unsigned int skb_txq(const struct sk_buff
*skb
)
2863 return skb
->queue_mapping
>> 1;
2867 * is_ctrl_pkt - return whether an offload packet is a control packet
2870 * Returns whether an offload packet should use an OFLD or a CTRL
2871 * Tx queue as indicated by bit 0 in the packet's queue_mapping.
2873 static inline unsigned int is_ctrl_pkt(const struct sk_buff
*skb
)
2875 return skb
->queue_mapping
& 1;
2878 static inline int uld_send(struct adapter
*adap
, struct sk_buff
*skb
,
2879 unsigned int tx_uld_type
)
2881 struct sge_uld_txq_info
*txq_info
;
2882 struct sge_uld_txq
*txq
;
2883 unsigned int idx
= skb_txq(skb
);
2885 if (unlikely(is_ctrl_pkt(skb
))) {
2886 /* Single ctrl queue is a requirement for LE workaround path */
2887 if (adap
->tids
.nsftids
)
2889 return ctrl_xmit(&adap
->sge
.ctrlq
[idx
], skb
);
2892 txq_info
= adap
->sge
.uld_txq_info
[tx_uld_type
];
2893 if (unlikely(!txq_info
)) {
2895 return NET_XMIT_DROP
;
2898 txq
= &txq_info
->uldtxq
[idx
];
2899 return ofld_xmit(txq
, skb
);
2903 * t4_ofld_send - send an offload packet
2904 * @adap: the adapter
2907 * Sends an offload packet. We use the packet queue_mapping to select the
2908 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2909 * should be sent as regular or control, bits 1-15 select the queue.
2911 int t4_ofld_send(struct adapter
*adap
, struct sk_buff
*skb
)
2916 ret
= uld_send(adap
, skb
, CXGB4_TX_OFLD
);
2922 * cxgb4_ofld_send - send an offload packet
2923 * @dev: the net device
2926 * Sends an offload packet. This is an exported version of @t4_ofld_send,
2927 * intended for ULDs.
2929 int cxgb4_ofld_send(struct net_device
*dev
, struct sk_buff
*skb
)
2931 return t4_ofld_send(netdev2adap(dev
), skb
);
2933 EXPORT_SYMBOL(cxgb4_ofld_send
);
2935 static void *inline_tx_header(const void *src
,
2936 const struct sge_txq
*q
,
2937 void *pos
, int length
)
2939 int left
= (void *)q
->stat
- pos
;
2942 if (likely(length
<= left
)) {
2943 memcpy(pos
, src
, length
);
2946 memcpy(pos
, src
, left
);
2947 memcpy(q
->desc
, src
+ left
, length
- left
);
2948 pos
= (void *)q
->desc
+ (length
- left
);
2950 /* 0-pad to multiple of 16 */
2951 p
= PTR_ALIGN(pos
, 8);
2952 if ((uintptr_t)p
& 8) {
2960 * ofld_xmit_direct - copy a WR into offload queue
2961 * @q: the Tx offload queue
2962 * @src: location of WR
2965 * Copy an immediate WR into an uncontended SGE offload queue.
2967 static int ofld_xmit_direct(struct sge_uld_txq
*q
, const void *src
,
2974 /* Use the lower limit as the cut-off */
2975 if (len
> MAX_IMM_OFLD_TX_DATA_WR_LEN
) {
2977 return NET_XMIT_DROP
;
2980 /* Don't return NET_XMIT_CN here as the current
2981 * implementation doesn't queue the request
2982 * using an skb when the following conditions not met
2984 if (!spin_trylock(&q
->sendq
.lock
))
2985 return NET_XMIT_DROP
;
2987 if (q
->full
|| !skb_queue_empty(&q
->sendq
) ||
2988 q
->service_ofldq_running
) {
2989 spin_unlock(&q
->sendq
.lock
);
2990 return NET_XMIT_DROP
;
2992 ndesc
= flits_to_desc(DIV_ROUND_UP(len
, 8));
2993 credits
= txq_avail(&q
->q
) - ndesc
;
2994 pos
= (u64
*)&q
->q
.desc
[q
->q
.pidx
];
2996 /* ofldtxq_stop modifies WR header in-situ */
2997 inline_tx_header(src
, &q
->q
, pos
, len
);
2998 if (unlikely(credits
< TXQ_STOP_THRES
))
2999 ofldtxq_stop(q
, (struct fw_wr_hdr
*)pos
);
3000 txq_advance(&q
->q
, ndesc
);
3001 cxgb4_ring_tx_db(q
->adap
, &q
->q
, ndesc
);
3003 spin_unlock(&q
->sendq
.lock
);
3004 return NET_XMIT_SUCCESS
;
3007 int cxgb4_immdata_send(struct net_device
*dev
, unsigned int idx
,
3008 const void *src
, unsigned int len
)
3010 struct sge_uld_txq_info
*txq_info
;
3011 struct sge_uld_txq
*txq
;
3012 struct adapter
*adap
;
3015 adap
= netdev2adap(dev
);
3018 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
3019 if (unlikely(!txq_info
)) {
3022 return NET_XMIT_DROP
;
3024 txq
= &txq_info
->uldtxq
[idx
];
3026 ret
= ofld_xmit_direct(txq
, src
, len
);
3028 return net_xmit_eval(ret
);
3030 EXPORT_SYMBOL(cxgb4_immdata_send
);
3033 * t4_crypto_send - send crypto packet
3034 * @adap: the adapter
3037 * Sends crypto packet. We use the packet queue_mapping to select the
3038 * appropriate Tx queue as follows: bit 0 indicates whether the packet
3039 * should be sent as regular or control, bits 1-15 select the queue.
3041 static int t4_crypto_send(struct adapter
*adap
, struct sk_buff
*skb
)
3046 ret
= uld_send(adap
, skb
, CXGB4_TX_CRYPTO
);
3052 * cxgb4_crypto_send - send crypto packet
3053 * @dev: the net device
3056 * Sends crypto packet. This is an exported version of @t4_crypto_send,
3057 * intended for ULDs.
3059 int cxgb4_crypto_send(struct net_device
*dev
, struct sk_buff
*skb
)
3061 return t4_crypto_send(netdev2adap(dev
), skb
);
3063 EXPORT_SYMBOL(cxgb4_crypto_send
);
3065 static inline void copy_frags(struct sk_buff
*skb
,
3066 const struct pkt_gl
*gl
, unsigned int offset
)
3070 /* usually there's just one frag */
3071 __skb_fill_page_desc(skb
, 0, gl
->frags
[0].page
,
3072 gl
->frags
[0].offset
+ offset
,
3073 gl
->frags
[0].size
- offset
);
3074 skb_shinfo(skb
)->nr_frags
= gl
->nfrags
;
3075 for (i
= 1; i
< gl
->nfrags
; i
++)
3076 __skb_fill_page_desc(skb
, i
, gl
->frags
[i
].page
,
3077 gl
->frags
[i
].offset
,
3080 /* get a reference to the last page, we don't own it */
3081 get_page(gl
->frags
[gl
->nfrags
- 1].page
);
3085 * cxgb4_pktgl_to_skb - build an sk_buff from a packet gather list
3086 * @gl: the gather list
3087 * @skb_len: size of sk_buff main body if it carries fragments
3088 * @pull_len: amount of data to move to the sk_buff's main body
3090 * Builds an sk_buff from the given packet gather list. Returns the
3091 * sk_buff or %NULL if sk_buff allocation failed.
3093 struct sk_buff
*cxgb4_pktgl_to_skb(const struct pkt_gl
*gl
,
3094 unsigned int skb_len
, unsigned int pull_len
)
3096 struct sk_buff
*skb
;
3099 * Below we rely on RX_COPY_THRES being less than the smallest Rx buffer
3100 * size, which is expected since buffers are at least PAGE_SIZEd.
3101 * In this case packets up to RX_COPY_THRES have only one fragment.
3103 if (gl
->tot_len
<= RX_COPY_THRES
) {
3104 skb
= dev_alloc_skb(gl
->tot_len
);
3107 __skb_put(skb
, gl
->tot_len
);
3108 skb_copy_to_linear_data(skb
, gl
->va
, gl
->tot_len
);
3110 skb
= dev_alloc_skb(skb_len
);
3113 __skb_put(skb
, pull_len
);
3114 skb_copy_to_linear_data(skb
, gl
->va
, pull_len
);
3116 copy_frags(skb
, gl
, pull_len
);
3117 skb
->len
= gl
->tot_len
;
3118 skb
->data_len
= skb
->len
- pull_len
;
3119 skb
->truesize
+= skb
->data_len
;
3123 EXPORT_SYMBOL(cxgb4_pktgl_to_skb
);
3126 * t4_pktgl_free - free a packet gather list
3127 * @gl: the gather list
3129 * Releases the pages of a packet gather list. We do not own the last
3130 * page on the list and do not free it.
3132 static void t4_pktgl_free(const struct pkt_gl
*gl
)
3135 const struct page_frag
*p
;
3137 for (p
= gl
->frags
, n
= gl
->nfrags
- 1; n
--; p
++)
3142 * Process an MPS trace packet. Give it an unused protocol number so it won't
3143 * be delivered to anyone and send it to the stack for capture.
3145 static noinline
int handle_trace_pkt(struct adapter
*adap
,
3146 const struct pkt_gl
*gl
)
3148 struct sk_buff
*skb
;
3150 skb
= cxgb4_pktgl_to_skb(gl
, RX_PULL_LEN
, RX_PULL_LEN
);
3151 if (unlikely(!skb
)) {
3156 if (is_t4(adap
->params
.chip
))
3157 __skb_pull(skb
, sizeof(struct cpl_trace_pkt
));
3159 __skb_pull(skb
, sizeof(struct cpl_t5_trace_pkt
));
3161 skb_reset_mac_header(skb
);
3162 skb
->protocol
= htons(0xffff);
3163 skb
->dev
= adap
->port
[0];
3164 netif_receive_skb(skb
);
3169 * cxgb4_sgetim_to_hwtstamp - convert sge time stamp to hw time stamp
3170 * @adap: the adapter
3171 * @hwtstamps: time stamp structure to update
3172 * @sgetstamp: 60bit iqe timestamp
3174 * Every ingress queue entry has the 60-bit timestamp, convert that timestamp
3175 * which is in Core Clock ticks into ktime_t and assign it
3177 static void cxgb4_sgetim_to_hwtstamp(struct adapter
*adap
,
3178 struct skb_shared_hwtstamps
*hwtstamps
,
3182 u64 tmp
= (sgetstamp
* 1000 * 1000 + adap
->params
.vpd
.cclk
/ 2);
3184 ns
= div_u64(tmp
, adap
->params
.vpd
.cclk
);
3186 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
3187 hwtstamps
->hwtstamp
= ns_to_ktime(ns
);
3190 static void do_gro(struct sge_eth_rxq
*rxq
, const struct pkt_gl
*gl
,
3191 const struct cpl_rx_pkt
*pkt
, unsigned long tnl_hdr_len
)
3193 struct adapter
*adapter
= rxq
->rspq
.adap
;
3194 struct sge
*s
= &adapter
->sge
;
3195 struct port_info
*pi
;
3197 struct sk_buff
*skb
;
3199 skb
= napi_get_frags(&rxq
->rspq
.napi
);
3200 if (unlikely(!skb
)) {
3202 rxq
->stats
.rx_drops
++;
3206 copy_frags(skb
, gl
, s
->pktshift
);
3208 skb
->csum_level
= 1;
3209 skb
->len
= gl
->tot_len
- s
->pktshift
;
3210 skb
->data_len
= skb
->len
;
3211 skb
->truesize
+= skb
->data_len
;
3212 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3213 skb_record_rx_queue(skb
, rxq
->rspq
.idx
);
3214 pi
= netdev_priv(skb
->dev
);
3216 cxgb4_sgetim_to_hwtstamp(adapter
, skb_hwtstamps(skb
),
3218 if (rxq
->rspq
.netdev
->features
& NETIF_F_RXHASH
)
3219 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
3222 if (unlikely(pkt
->vlan_ex
)) {
3223 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
3224 rxq
->stats
.vlan_ex
++;
3226 ret
= napi_gro_frags(&rxq
->rspq
.napi
);
3227 if (ret
== GRO_HELD
)
3228 rxq
->stats
.lro_pkts
++;
3229 else if (ret
== GRO_MERGED
|| ret
== GRO_MERGED_FREE
)
3230 rxq
->stats
.lro_merged
++;
3232 rxq
->stats
.rx_cso
++;
3242 * t4_systim_to_hwstamp - read hardware time stamp
3243 * @adap: the adapter
3246 * Read Time Stamp from MPS packet and insert in skb which
3247 * is forwarded to PTP application
3249 static noinline
int t4_systim_to_hwstamp(struct adapter
*adapter
,
3250 struct sk_buff
*skb
)
3252 struct skb_shared_hwtstamps
*hwtstamps
;
3253 struct cpl_rx_mps_pkt
*cpl
= NULL
;
3254 unsigned char *data
;
3257 cpl
= (struct cpl_rx_mps_pkt
*)skb
->data
;
3258 if (!(CPL_RX_MPS_PKT_TYPE_G(ntohl(cpl
->op_to_r1_hi
)) &
3259 X_CPL_RX_MPS_PKT_TYPE_PTP
))
3260 return RX_PTP_PKT_ERR
;
3262 data
= skb
->data
+ sizeof(*cpl
);
3263 skb_pull(skb
, 2 * sizeof(u64
) + sizeof(struct cpl_rx_mps_pkt
));
3264 offset
= ETH_HLEN
+ IPV4_HLEN(skb
->data
) + UDP_HLEN
;
3265 if (skb
->len
< offset
+ OFF_PTP_SEQUENCE_ID
+ sizeof(short))
3266 return RX_PTP_PKT_ERR
;
3268 hwtstamps
= skb_hwtstamps(skb
);
3269 memset(hwtstamps
, 0, sizeof(*hwtstamps
));
3270 hwtstamps
->hwtstamp
= ns_to_ktime(be64_to_cpu(*((u64
*)data
)));
3272 return RX_PTP_PKT_SUC
;
3276 * t4_rx_hststamp - Recv PTP Event Message
3277 * @adap: the adapter
3278 * @rsp: the response queue descriptor holding the RX_PKT message
3281 * PTP enabled and MPS packet, read HW timestamp
3283 static int t4_rx_hststamp(struct adapter
*adapter
, const __be64
*rsp
,
3284 struct sge_eth_rxq
*rxq
, struct sk_buff
*skb
)
3288 if (unlikely((*(u8
*)rsp
== CPL_RX_MPS_PKT
) &&
3289 !is_t4(adapter
->params
.chip
))) {
3290 ret
= t4_systim_to_hwstamp(adapter
, skb
);
3291 if (ret
== RX_PTP_PKT_ERR
) {
3293 rxq
->stats
.rx_drops
++;
3297 return RX_NON_PTP_PKT
;
3301 * t4_tx_hststamp - Loopback PTP Transmit Event Message
3302 * @adap: the adapter
3304 * @dev: the ingress net device
3306 * Read hardware timestamp for the loopback PTP Tx event message
3308 static int t4_tx_hststamp(struct adapter
*adapter
, struct sk_buff
*skb
,
3309 struct net_device
*dev
)
3311 struct port_info
*pi
= netdev_priv(dev
);
3313 if (!is_t4(adapter
->params
.chip
) && adapter
->ptp_tx_skb
) {
3314 cxgb4_ptp_read_hwstamp(adapter
, pi
);
3322 * t4_tx_completion_handler - handle CPL_SGE_EGR_UPDATE messages
3323 * @rspq: Ethernet RX Response Queue associated with Ethernet TX Queue
3324 * @rsp: Response Entry pointer into Response Queue
3325 * @gl: Gather List pointer
3327 * For adapters which support the SGE Doorbell Queue Timer facility,
3328 * we configure the Ethernet TX Queues to send CIDX Updates to the
3329 * Associated Ethernet RX Response Queue with CPL_SGE_EGR_UPDATE
3330 * messages. This adds a small load to PCIe Link RX bandwidth and,
3331 * potentially, higher CPU Interrupt load, but allows us to respond
3332 * much more quickly to the CIDX Updates. This is important for
3333 * Upper Layer Software which isn't willing to have a large amount
3334 * of TX Data outstanding before receiving DMA Completions.
3336 static void t4_tx_completion_handler(struct sge_rspq
*rspq
,
3338 const struct pkt_gl
*gl
)
3340 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3341 struct port_info
*pi
= netdev_priv(rspq
->netdev
);
3342 struct adapter
*adapter
= rspq
->adap
;
3343 struct sge
*s
= &adapter
->sge
;
3344 struct sge_eth_txq
*txq
;
3346 /* skip RSS header */
3349 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
3351 if (unlikely(opcode
== CPL_FW4_MSG
&&
3352 ((const struct cpl_fw4_msg
*)rsp
)->type
==
3355 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3359 if (unlikely(opcode
!= CPL_SGE_EGR_UPDATE
)) {
3360 pr_info("%s: unexpected FW4/CPL %#x on Rx queue\n",
3365 txq
= &s
->ethtxq
[pi
->first_qset
+ rspq
->idx
];
3366 t4_sge_eth_txq_egress_update(adapter
, txq
, -1);
3370 * t4_ethrx_handler - process an ingress ethernet packet
3371 * @q: the response queue that received the packet
3372 * @rsp: the response queue descriptor holding the RX_PKT message
3373 * @si: the gather list of packet fragments
3375 * Process an ingress ethernet packet and deliver it to the stack.
3377 int t4_ethrx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
3378 const struct pkt_gl
*si
)
3381 struct sk_buff
*skb
;
3382 const struct cpl_rx_pkt
*pkt
;
3383 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
3384 struct adapter
*adapter
= q
->adap
;
3385 struct sge
*s
= &q
->adap
->sge
;
3386 int cpl_trace_pkt
= is_t4(q
->adap
->params
.chip
) ?
3387 CPL_TRACE_PKT
: CPL_TRACE_PKT_T5
;
3388 u16 err_vec
, tnl_hdr_len
= 0;
3389 struct port_info
*pi
;
3392 /* If we're looking at TX Queue CIDX Update, handle that separately
3395 if (unlikely((*(u8
*)rsp
== CPL_FW4_MSG
) ||
3396 (*(u8
*)rsp
== CPL_SGE_EGR_UPDATE
))) {
3397 t4_tx_completion_handler(q
, rsp
, si
);
3401 if (unlikely(*(u8
*)rsp
== cpl_trace_pkt
))
3402 return handle_trace_pkt(q
->adap
, si
);
3404 pkt
= (const struct cpl_rx_pkt
*)rsp
;
3405 /* Compressed error vector is enabled for T6 only */
3406 if (q
->adap
->params
.tp
.rx_pkt_encap
) {
3407 err_vec
= T6_COMPR_RXERR_VEC_G(be16_to_cpu(pkt
->err_vec
));
3408 tnl_hdr_len
= T6_RX_TNLHDR_LEN_G(ntohs(pkt
->err_vec
));
3410 err_vec
= be16_to_cpu(pkt
->err_vec
);
3413 csum_ok
= pkt
->csum_calc
&& !err_vec
&&
3414 (q
->netdev
->features
& NETIF_F_RXCSUM
);
3417 rxq
->stats
.bad_rx_pkts
++;
3419 if (((pkt
->l2info
& htonl(RXF_TCP_F
)) ||
3421 (q
->netdev
->features
& NETIF_F_GRO
) && csum_ok
&& !pkt
->ip_frag
) {
3422 do_gro(rxq
, si
, pkt
, tnl_hdr_len
);
3426 skb
= cxgb4_pktgl_to_skb(si
, RX_PKT_SKB_LEN
, RX_PULL_LEN
);
3427 if (unlikely(!skb
)) {
3429 rxq
->stats
.rx_drops
++;
3432 pi
= netdev_priv(q
->netdev
);
3434 /* Handle PTP Event Rx packet */
3435 if (unlikely(pi
->ptp_enable
)) {
3436 ret
= t4_rx_hststamp(adapter
, rsp
, rxq
, skb
);
3437 if (ret
== RX_PTP_PKT_ERR
)
3441 __skb_pull(skb
, s
->pktshift
); /* remove ethernet header pad */
3443 /* Handle the PTP Event Tx Loopback packet */
3444 if (unlikely(pi
->ptp_enable
&& !ret
&&
3445 (pkt
->l2info
& htonl(RXF_UDP_F
)) &&
3446 cxgb4_ptp_is_ptp_rx(skb
))) {
3447 if (!t4_tx_hststamp(adapter
, skb
, q
->netdev
))
3451 skb
->protocol
= eth_type_trans(skb
, q
->netdev
);
3452 skb_record_rx_queue(skb
, q
->idx
);
3453 if (skb
->dev
->features
& NETIF_F_RXHASH
)
3454 skb_set_hash(skb
, (__force u32
)pkt
->rsshdr
.hash_val
,
3460 cxgb4_sgetim_to_hwtstamp(q
->adap
, skb_hwtstamps(skb
),
3462 if (csum_ok
&& (pkt
->l2info
& htonl(RXF_UDP_F
| RXF_TCP_F
))) {
3463 if (!pkt
->ip_frag
) {
3464 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3465 rxq
->stats
.rx_cso
++;
3466 } else if (pkt
->l2info
& htonl(RXF_IP_F
)) {
3467 __sum16 c
= (__force __sum16
)pkt
->csum
;
3468 skb
->csum
= csum_unfold(c
);
3471 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3472 skb
->csum_level
= 1;
3474 skb
->ip_summed
= CHECKSUM_COMPLETE
;
3476 rxq
->stats
.rx_cso
++;
3479 skb_checksum_none_assert(skb
);
3480 #ifdef CONFIG_CHELSIO_T4_FCOE
3481 #define CPL_RX_PKT_FLAGS (RXF_PSH_F | RXF_SYN_F | RXF_UDP_F | \
3482 RXF_TCP_F | RXF_IP_F | RXF_IP6_F | RXF_LRO_F)
3484 if (!(pkt
->l2info
& cpu_to_be32(CPL_RX_PKT_FLAGS
))) {
3485 if ((pkt
->l2info
& cpu_to_be32(RXF_FCOE_F
)) &&
3486 (pi
->fcoe
.flags
& CXGB_FCOE_ENABLED
)) {
3487 if (q
->adap
->params
.tp
.rx_pkt_encap
)
3489 T6_COMPR_RXERR_SUM_F
;
3491 csum_ok
= err_vec
& RXERR_CSUM_F
;
3493 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
3497 #undef CPL_RX_PKT_FLAGS
3498 #endif /* CONFIG_CHELSIO_T4_FCOE */
3501 if (unlikely(pkt
->vlan_ex
)) {
3502 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
), ntohs(pkt
->vlan
));
3503 rxq
->stats
.vlan_ex
++;
3505 skb_mark_napi_id(skb
, &q
->napi
);
3506 netif_receive_skb(skb
);
3511 * restore_rx_bufs - put back a packet's Rx buffers
3512 * @si: the packet gather list
3513 * @q: the SGE free list
3514 * @frags: number of FL buffers to restore
3516 * Puts back on an FL the Rx buffers associated with @si. The buffers
3517 * have already been unmapped and are left unmapped, we mark them so to
3518 * prevent further unmapping attempts.
3520 * This function undoes a series of @unmap_rx_buf calls when we find out
3521 * that the current packet can't be processed right away afterall and we
3522 * need to come back to it later. This is a very rare event and there's
3523 * no effort to make this particularly efficient.
3525 static void restore_rx_bufs(const struct pkt_gl
*si
, struct sge_fl
*q
,
3528 struct rx_sw_desc
*d
;
3532 q
->cidx
= q
->size
- 1;
3535 d
= &q
->sdesc
[q
->cidx
];
3536 d
->page
= si
->frags
[frags
].page
;
3537 d
->dma_addr
|= RX_UNMAPPED_BUF
;
3543 * is_new_response - check if a response is newly written
3544 * @r: the response descriptor
3545 * @q: the response queue
3547 * Returns true if a response descriptor contains a yet unprocessed
3550 static inline bool is_new_response(const struct rsp_ctrl
*r
,
3551 const struct sge_rspq
*q
)
3553 return (r
->type_gen
>> RSPD_GEN_S
) == q
->gen
;
3557 * rspq_next - advance to the next entry in a response queue
3560 * Updates the state of a response queue to advance it to the next entry.
3562 static inline void rspq_next(struct sge_rspq
*q
)
3564 q
->cur_desc
= (void *)q
->cur_desc
+ q
->iqe_len
;
3565 if (unlikely(++q
->cidx
== q
->size
)) {
3568 q
->cur_desc
= q
->desc
;
3573 * process_responses - process responses from an SGE response queue
3574 * @q: the ingress queue to process
3575 * @budget: how many responses can be processed in this round
3577 * Process responses from an SGE response queue up to the supplied budget.
3578 * Responses include received packets as well as control messages from FW
3581 * Additionally choose the interrupt holdoff time for the next interrupt
3582 * on this queue. If the system is under memory shortage use a fairly
3583 * long delay to help recovery.
3585 static int process_responses(struct sge_rspq
*q
, int budget
)
3588 int budget_left
= budget
;
3589 const struct rsp_ctrl
*rc
;
3590 struct sge_eth_rxq
*rxq
= container_of(q
, struct sge_eth_rxq
, rspq
);
3591 struct adapter
*adapter
= q
->adap
;
3592 struct sge
*s
= &adapter
->sge
;
3594 while (likely(budget_left
)) {
3595 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3596 if (!is_new_response(rc
, q
)) {
3597 if (q
->flush_handler
)
3598 q
->flush_handler(q
);
3603 rsp_type
= RSPD_TYPE_G(rc
->type_gen
);
3604 if (likely(rsp_type
== RSPD_TYPE_FLBUF_X
)) {
3605 struct page_frag
*fp
;
3607 const struct rx_sw_desc
*rsd
;
3608 u32 len
= ntohl(rc
->pldbuflen_qid
), bufsz
, frags
;
3610 if (len
& RSPD_NEWBUF_F
) {
3611 if (likely(q
->offset
> 0)) {
3612 free_rx_bufs(q
->adap
, &rxq
->fl
, 1);
3615 len
= RSPD_LEN_G(len
);
3619 /* gather packet fragments */
3620 for (frags
= 0, fp
= si
.frags
; ; frags
++, fp
++) {
3621 rsd
= &rxq
->fl
.sdesc
[rxq
->fl
.cidx
];
3622 bufsz
= get_buf_size(adapter
, rsd
);
3623 fp
->page
= rsd
->page
;
3624 fp
->offset
= q
->offset
;
3625 fp
->size
= min(bufsz
, len
);
3629 unmap_rx_buf(q
->adap
, &rxq
->fl
);
3632 si
.sgetstamp
= SGE_TIMESTAMP_G(
3633 be64_to_cpu(rc
->last_flit
));
3635 * Last buffer remains mapped so explicitly make it
3636 * coherent for CPU access.
3638 dma_sync_single_for_cpu(q
->adap
->pdev_dev
,
3640 fp
->size
, DMA_FROM_DEVICE
);
3642 si
.va
= page_address(si
.frags
[0].page
) +
3646 si
.nfrags
= frags
+ 1;
3647 ret
= q
->handler(q
, q
->cur_desc
, &si
);
3648 if (likely(ret
== 0))
3649 q
->offset
+= ALIGN(fp
->size
, s
->fl_align
);
3651 restore_rx_bufs(&si
, &rxq
->fl
, frags
);
3652 } else if (likely(rsp_type
== RSPD_TYPE_CPL_X
)) {
3653 ret
= q
->handler(q
, q
->cur_desc
, NULL
);
3655 ret
= q
->handler(q
, (const __be64
*)rc
, CXGB4_MSG_AN
);
3658 if (unlikely(ret
)) {
3659 /* couldn't process descriptor, back off for recovery */
3660 q
->next_intr_params
= QINTR_TIMER_IDX_V(NOMEM_TMR_IDX
);
3668 if (q
->offset
>= 0 && fl_cap(&rxq
->fl
) - rxq
->fl
.avail
>= 16)
3669 __refill_fl(q
->adap
, &rxq
->fl
);
3670 return budget
- budget_left
;
3674 * napi_rx_handler - the NAPI handler for Rx processing
3675 * @napi: the napi instance
3676 * @budget: how many packets we can process in this round
3678 * Handler for new data events when using NAPI. This does not need any
3679 * locking or protection from interrupts as data interrupts are off at
3680 * this point and other adapter interrupts do not interfere (the latter
3681 * in not a concern at all with MSI-X as non-data interrupts then have
3682 * a separate handler).
3684 static int napi_rx_handler(struct napi_struct
*napi
, int budget
)
3686 unsigned int params
;
3687 struct sge_rspq
*q
= container_of(napi
, struct sge_rspq
, napi
);
3691 work_done
= process_responses(q
, budget
);
3692 if (likely(work_done
< budget
)) {
3695 napi_complete_done(napi
, work_done
);
3696 timer_index
= QINTR_TIMER_IDX_G(q
->next_intr_params
);
3698 if (q
->adaptive_rx
) {
3699 if (work_done
> max(timer_pkt_quota
[timer_index
],
3701 timer_index
= (timer_index
+ 1);
3703 timer_index
= timer_index
- 1;
3705 timer_index
= clamp(timer_index
, 0, SGE_TIMERREGS
- 1);
3706 q
->next_intr_params
=
3707 QINTR_TIMER_IDX_V(timer_index
) |
3709 params
= q
->next_intr_params
;
3711 params
= q
->next_intr_params
;
3712 q
->next_intr_params
= q
->intr_params
;
3715 params
= QINTR_TIMER_IDX_V(7);
3717 val
= CIDXINC_V(work_done
) | SEINTARM_V(params
);
3719 /* If we don't have access to the new User GTS (T5+), use the old
3720 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3722 if (unlikely(q
->bar2_addr
== NULL
)) {
3723 t4_write_reg(q
->adap
, MYPF_REG(SGE_PF_GTS_A
),
3724 val
| INGRESSQID_V((u32
)q
->cntxt_id
));
3726 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3727 q
->bar2_addr
+ SGE_UDB_GTS
);
3733 void cxgb4_ethofld_restart(unsigned long data
)
3735 struct sge_eosw_txq
*eosw_txq
= (struct sge_eosw_txq
*)data
;
3738 spin_lock(&eosw_txq
->lock
);
3739 pktcount
= eosw_txq
->cidx
- eosw_txq
->last_cidx
;
3741 pktcount
+= eosw_txq
->ndesc
;
3744 cxgb4_eosw_txq_free_desc(netdev2adap(eosw_txq
->netdev
),
3745 eosw_txq
, pktcount
);
3746 eosw_txq
->inuse
-= pktcount
;
3749 /* There may be some packets waiting for completions. So,
3750 * attempt to send these packets now.
3752 ethofld_xmit(eosw_txq
->netdev
, eosw_txq
);
3753 spin_unlock(&eosw_txq
->lock
);
3756 /* cxgb4_ethofld_rx_handler - Process ETHOFLD Tx completions
3757 * @q: the response queue that received the packet
3758 * @rsp: the response queue descriptor holding the CPL message
3759 * @si: the gather list of packet fragments
3761 * Process a ETHOFLD Tx completion. Increment the cidx here, but
3762 * free up the descriptors in a tasklet later.
3764 int cxgb4_ethofld_rx_handler(struct sge_rspq
*q
, const __be64
*rsp
,
3765 const struct pkt_gl
*si
)
3767 u8 opcode
= ((const struct rss_header
*)rsp
)->opcode
;
3769 /* skip RSS header */
3772 if (opcode
== CPL_FW4_ACK
) {
3773 const struct cpl_fw4_ack
*cpl
;
3774 struct sge_eosw_txq
*eosw_txq
;
3775 struct eotid_entry
*entry
;
3776 struct sk_buff
*skb
;
3781 cpl
= (const struct cpl_fw4_ack
*)rsp
;
3782 eotid
= CPL_FW4_ACK_FLOWID_G(ntohl(OPCODE_TID(cpl
))) -
3783 q
->adap
->tids
.eotid_base
;
3784 entry
= cxgb4_lookup_eotid(&q
->adap
->tids
, eotid
);
3788 eosw_txq
= (struct sge_eosw_txq
*)entry
->data
;
3792 spin_lock(&eosw_txq
->lock
);
3793 credits
= cpl
->credits
;
3794 while (credits
> 0) {
3795 skb
= eosw_txq
->desc
[eosw_txq
->cidx
].skb
;
3799 if (unlikely((eosw_txq
->state
==
3800 CXGB4_EO_STATE_FLOWC_OPEN_REPLY
||
3802 CXGB4_EO_STATE_FLOWC_CLOSE_REPLY
) &&
3803 eosw_txq
->cidx
== eosw_txq
->flowc_idx
)) {
3804 flits
= DIV_ROUND_UP(skb
->len
, 8);
3805 if (eosw_txq
->state
==
3806 CXGB4_EO_STATE_FLOWC_OPEN_REPLY
)
3807 eosw_txq
->state
= CXGB4_EO_STATE_ACTIVE
;
3809 eosw_txq
->state
= CXGB4_EO_STATE_CLOSED
;
3810 complete(&eosw_txq
->completion
);
3812 hdr_len
= eth_get_headlen(eosw_txq
->netdev
,
3815 flits
= ethofld_calc_tx_flits(q
->adap
, skb
,
3818 eosw_txq_advance_index(&eosw_txq
->cidx
, 1,
3820 wrlen16
= DIV_ROUND_UP(flits
* 8, 16);
3824 eosw_txq
->cred
+= cpl
->credits
;
3827 spin_unlock(&eosw_txq
->lock
);
3829 /* Schedule a tasklet to reclaim SKBs and restart ETHOFLD Tx,
3830 * if there were packets waiting for completion.
3832 tasklet_schedule(&eosw_txq
->qresume_tsk
);
3840 * The MSI-X interrupt handler for an SGE response queue.
3842 irqreturn_t
t4_sge_intr_msix(int irq
, void *cookie
)
3844 struct sge_rspq
*q
= cookie
;
3846 napi_schedule(&q
->napi
);
3851 * Process the indirect interrupt entries in the interrupt queue and kick off
3852 * NAPI for each queue that has generated an entry.
3854 static unsigned int process_intrq(struct adapter
*adap
)
3856 unsigned int credits
;
3857 const struct rsp_ctrl
*rc
;
3858 struct sge_rspq
*q
= &adap
->sge
.intrq
;
3861 spin_lock(&adap
->sge
.intrq_lock
);
3862 for (credits
= 0; ; credits
++) {
3863 rc
= (void *)q
->cur_desc
+ (q
->iqe_len
- sizeof(*rc
));
3864 if (!is_new_response(rc
, q
))
3868 if (RSPD_TYPE_G(rc
->type_gen
) == RSPD_TYPE_INTR_X
) {
3869 unsigned int qid
= ntohl(rc
->pldbuflen_qid
);
3871 qid
-= adap
->sge
.ingr_start
;
3872 napi_schedule(&adap
->sge
.ingr_map
[qid
]->napi
);
3878 val
= CIDXINC_V(credits
) | SEINTARM_V(q
->intr_params
);
3880 /* If we don't have access to the new User GTS (T5+), use the old
3881 * doorbell mechanism; otherwise use the new BAR2 mechanism.
3883 if (unlikely(q
->bar2_addr
== NULL
)) {
3884 t4_write_reg(adap
, MYPF_REG(SGE_PF_GTS_A
),
3885 val
| INGRESSQID_V(q
->cntxt_id
));
3887 writel(val
| INGRESSQID_V(q
->bar2_qid
),
3888 q
->bar2_addr
+ SGE_UDB_GTS
);
3891 spin_unlock(&adap
->sge
.intrq_lock
);
3896 * The MSI interrupt handler, which handles data events from SGE response queues
3897 * as well as error and other async events as they all use the same MSI vector.
3899 static irqreturn_t
t4_intr_msi(int irq
, void *cookie
)
3901 struct adapter
*adap
= cookie
;
3903 if (adap
->flags
& CXGB4_MASTER_PF
)
3904 t4_slow_intr_handler(adap
);
3905 process_intrq(adap
);
3910 * Interrupt handler for legacy INTx interrupts.
3911 * Handles data events from SGE response queues as well as error and other
3912 * async events as they all use the same interrupt line.
3914 static irqreturn_t
t4_intr_intx(int irq
, void *cookie
)
3916 struct adapter
*adap
= cookie
;
3918 t4_write_reg(adap
, MYPF_REG(PCIE_PF_CLI_A
), 0);
3919 if (((adap
->flags
& CXGB4_MASTER_PF
) && t4_slow_intr_handler(adap
)) |
3920 process_intrq(adap
))
3922 return IRQ_NONE
; /* probably shared interrupt */
3926 * t4_intr_handler - select the top-level interrupt handler
3927 * @adap: the adapter
3929 * Selects the top-level interrupt handler based on the type of interrupts
3930 * (MSI-X, MSI, or INTx).
3932 irq_handler_t
t4_intr_handler(struct adapter
*adap
)
3934 if (adap
->flags
& CXGB4_USING_MSIX
)
3935 return t4_sge_intr_msix
;
3936 if (adap
->flags
& CXGB4_USING_MSI
)
3938 return t4_intr_intx
;
3941 static void sge_rx_timer_cb(struct timer_list
*t
)
3945 struct adapter
*adap
= from_timer(adap
, t
, sge
.rx_timer
);
3946 struct sge
*s
= &adap
->sge
;
3948 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
3949 for (m
= s
->starving_fl
[i
]; m
; m
&= m
- 1) {
3950 struct sge_eth_rxq
*rxq
;
3951 unsigned int id
= __ffs(m
) + i
* BITS_PER_LONG
;
3952 struct sge_fl
*fl
= s
->egr_map
[id
];
3954 clear_bit(id
, s
->starving_fl
);
3955 smp_mb__after_atomic();
3957 if (fl_starving(adap
, fl
)) {
3958 rxq
= container_of(fl
, struct sge_eth_rxq
, fl
);
3959 if (napi_reschedule(&rxq
->rspq
.napi
))
3962 set_bit(id
, s
->starving_fl
);
3965 /* The remainder of the SGE RX Timer Callback routine is dedicated to
3966 * global Master PF activities like checking for chip ingress stalls,
3969 if (!(adap
->flags
& CXGB4_MASTER_PF
))
3972 t4_idma_monitor(adap
, &s
->idma_monitor
, HZ
, RX_QCHECK_PERIOD
);
3975 mod_timer(&s
->rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
3978 static void sge_tx_timer_cb(struct timer_list
*t
)
3980 struct adapter
*adap
= from_timer(adap
, t
, sge
.tx_timer
);
3981 struct sge
*s
= &adap
->sge
;
3982 unsigned long m
, period
;
3983 unsigned int i
, budget
;
3985 for (i
= 0; i
< BITS_TO_LONGS(s
->egr_sz
); i
++)
3986 for (m
= s
->txq_maperr
[i
]; m
; m
&= m
- 1) {
3987 unsigned long id
= __ffs(m
) + i
* BITS_PER_LONG
;
3988 struct sge_uld_txq
*txq
= s
->egr_map
[id
];
3990 clear_bit(id
, s
->txq_maperr
);
3991 tasklet_schedule(&txq
->qresume_tsk
);
3994 if (!is_t4(adap
->params
.chip
)) {
3995 struct sge_eth_txq
*q
= &s
->ptptxq
;
3998 spin_lock(&adap
->ptp_lock
);
3999 avail
= reclaimable(&q
->q
);
4002 free_tx_desc(adap
, &q
->q
, avail
, false);
4003 q
->q
.in_use
-= avail
;
4005 spin_unlock(&adap
->ptp_lock
);
4008 budget
= MAX_TIMER_TX_RECLAIM
;
4009 i
= s
->ethtxq_rover
;
4011 budget
-= t4_sge_eth_txq_egress_update(adap
, &s
->ethtxq
[i
],
4016 if (++i
>= s
->ethqsets
)
4018 } while (i
!= s
->ethtxq_rover
);
4019 s
->ethtxq_rover
= i
;
4022 /* If we found too many reclaimable packets schedule a timer
4023 * in the near future to continue where we left off.
4027 /* We reclaimed all reclaimable TX Descriptors, so reschedule
4028 * at the normal period.
4030 period
= TX_QCHECK_PERIOD
;
4033 mod_timer(&s
->tx_timer
, jiffies
+ period
);
4037 * bar2_address - return the BAR2 address for an SGE Queue's Registers
4038 * @adapter: the adapter
4039 * @qid: the SGE Queue ID
4040 * @qtype: the SGE Queue Type (Egress or Ingress)
4041 * @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
4043 * Returns the BAR2 address for the SGE Queue Registers associated with
4044 * @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
4045 * returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
4046 * Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
4047 * Registers are supported (e.g. the Write Combining Doorbell Buffer).
4049 static void __iomem
*bar2_address(struct adapter
*adapter
,
4051 enum t4_bar2_qtype qtype
,
4052 unsigned int *pbar2_qid
)
4057 ret
= t4_bar2_sge_qregs(adapter
, qid
, qtype
, 0,
4058 &bar2_qoffset
, pbar2_qid
);
4062 return adapter
->bar2
+ bar2_qoffset
;
4065 /* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
4066 * @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
4068 int t4_sge_alloc_rxq(struct adapter
*adap
, struct sge_rspq
*iq
, bool fwevtq
,
4069 struct net_device
*dev
, int intr_idx
,
4070 struct sge_fl
*fl
, rspq_handler_t hnd
,
4071 rspq_flush_handler_t flush_hnd
, int cong
)
4075 struct sge
*s
= &adap
->sge
;
4076 struct port_info
*pi
= netdev_priv(dev
);
4077 int relaxed
= !(adap
->flags
& CXGB4_ROOT_NO_RELAXED_ORDERING
);
4079 /* Size needs to be multiple of 16, including status entry. */
4080 iq
->size
= roundup(iq
->size
, 16);
4082 iq
->desc
= alloc_ring(adap
->pdev_dev
, iq
->size
, iq
->iqe_len
, 0,
4083 &iq
->phys_addr
, NULL
, 0,
4084 dev_to_node(adap
->pdev_dev
));
4088 memset(&c
, 0, sizeof(c
));
4089 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_IQ_CMD
) | FW_CMD_REQUEST_F
|
4090 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4091 FW_IQ_CMD_PFN_V(adap
->pf
) | FW_IQ_CMD_VFN_V(0));
4092 c
.alloc_to_len16
= htonl(FW_IQ_CMD_ALLOC_F
| FW_IQ_CMD_IQSTART_F
|
4094 c
.type_to_iqandstindex
= htonl(FW_IQ_CMD_TYPE_V(FW_IQ_TYPE_FL_INT_CAP
) |
4095 FW_IQ_CMD_IQASYNCH_V(fwevtq
) | FW_IQ_CMD_VIID_V(pi
->viid
) |
4096 FW_IQ_CMD_IQANDST_V(intr_idx
< 0) |
4097 FW_IQ_CMD_IQANUD_V(UPDATEDELIVERY_INTERRUPT_X
) |
4098 FW_IQ_CMD_IQANDSTINDEX_V(intr_idx
>= 0 ? intr_idx
:
4100 c
.iqdroprss_to_iqesize
= htons(FW_IQ_CMD_IQPCIECH_V(pi
->tx_chan
) |
4101 FW_IQ_CMD_IQGTSMODE_F
|
4102 FW_IQ_CMD_IQINTCNTTHRESH_V(iq
->pktcnt_idx
) |
4103 FW_IQ_CMD_IQESIZE_V(ilog2(iq
->iqe_len
) - 4));
4104 c
.iqsize
= htons(iq
->size
);
4105 c
.iqaddr
= cpu_to_be64(iq
->phys_addr
);
4107 c
.iqns_to_fl0congen
= htonl(FW_IQ_CMD_IQFLINTCONGEN_F
|
4108 FW_IQ_CMD_IQTYPE_V(cong
? FW_IQ_IQTYPE_NIC
4109 : FW_IQ_IQTYPE_OFLD
));
4112 unsigned int chip_ver
=
4113 CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4115 /* Allocate the ring for the hardware free list (with space
4116 * for its status page) along with the associated software
4117 * descriptor ring. The free list size needs to be a multiple
4118 * of the Egress Queue Unit and at least 2 Egress Units larger
4119 * than the SGE's Egress Congrestion Threshold
4120 * (fl_starve_thres - 1).
4122 if (fl
->size
< s
->fl_starve_thres
- 1 + 2 * 8)
4123 fl
->size
= s
->fl_starve_thres
- 1 + 2 * 8;
4124 fl
->size
= roundup(fl
->size
, 8);
4125 fl
->desc
= alloc_ring(adap
->pdev_dev
, fl
->size
, sizeof(__be64
),
4126 sizeof(struct rx_sw_desc
), &fl
->addr
,
4127 &fl
->sdesc
, s
->stat_len
,
4128 dev_to_node(adap
->pdev_dev
));
4132 flsz
= fl
->size
/ 8 + s
->stat_len
/ sizeof(struct tx_desc
);
4133 c
.iqns_to_fl0congen
|= htonl(FW_IQ_CMD_FL0PACKEN_F
|
4134 FW_IQ_CMD_FL0FETCHRO_V(relaxed
) |
4135 FW_IQ_CMD_FL0DATARO_V(relaxed
) |
4136 FW_IQ_CMD_FL0PADEN_F
);
4138 c
.iqns_to_fl0congen
|=
4139 htonl(FW_IQ_CMD_FL0CNGCHMAP_V(cong
) |
4140 FW_IQ_CMD_FL0CONGCIF_F
|
4141 FW_IQ_CMD_FL0CONGEN_F
);
4142 /* In T6, for egress queue type FL there is internal overhead
4143 * of 16B for header going into FLM module. Hence the maximum
4144 * allowed burst size is 448 bytes. For T4/T5, the hardware
4145 * doesn't coalesce fetch requests if more than 64 bytes of
4146 * Free List pointers are provided, so we use a 128-byte Fetch
4147 * Burst Minimum there (T6 implements coalescing so we can use
4148 * the smaller 64-byte value there).
4150 c
.fl0dcaen_to_fl0cidxfthresh
=
4151 htons(FW_IQ_CMD_FL0FBMIN_V(chip_ver
<= CHELSIO_T5
?
4152 FETCHBURSTMIN_128B_X
:
4153 FETCHBURSTMIN_64B_T6_X
) |
4154 FW_IQ_CMD_FL0FBMAX_V((chip_ver
<= CHELSIO_T5
) ?
4155 FETCHBURSTMAX_512B_X
:
4156 FETCHBURSTMAX_256B_X
));
4157 c
.fl0size
= htons(flsz
);
4158 c
.fl0addr
= cpu_to_be64(fl
->addr
);
4161 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4165 netif_napi_add(dev
, &iq
->napi
, napi_rx_handler
, 64);
4166 iq
->cur_desc
= iq
->desc
;
4169 iq
->next_intr_params
= iq
->intr_params
;
4170 iq
->cntxt_id
= ntohs(c
.iqid
);
4171 iq
->abs_id
= ntohs(c
.physiqid
);
4172 iq
->bar2_addr
= bar2_address(adap
,
4174 T4_BAR2_QTYPE_INGRESS
,
4176 iq
->size
--; /* subtract status entry */
4179 iq
->flush_handler
= flush_hnd
;
4181 memset(&iq
->lro_mgr
, 0, sizeof(struct t4_lro_mgr
));
4182 skb_queue_head_init(&iq
->lro_mgr
.lroq
);
4184 /* set offset to -1 to distinguish ingress queues without FL */
4185 iq
->offset
= fl
? 0 : -1;
4187 adap
->sge
.ingr_map
[iq
->cntxt_id
- adap
->sge
.ingr_start
] = iq
;
4190 fl
->cntxt_id
= ntohs(c
.fl0id
);
4191 fl
->avail
= fl
->pend_cred
= 0;
4192 fl
->pidx
= fl
->cidx
= 0;
4193 fl
->alloc_failed
= fl
->large_alloc_failed
= fl
->starving
= 0;
4194 adap
->sge
.egr_map
[fl
->cntxt_id
- adap
->sge
.egr_start
] = fl
;
4196 /* Note, we must initialize the BAR2 Free List User Doorbell
4197 * information before refilling the Free List!
4199 fl
->bar2_addr
= bar2_address(adap
,
4201 T4_BAR2_QTYPE_EGRESS
,
4203 refill_fl(adap
, fl
, fl_cap(fl
), GFP_KERNEL
);
4206 /* For T5 and later we attempt to set up the Congestion Manager values
4207 * of the new RX Ethernet Queue. This should really be handled by
4208 * firmware because it's more complex than any host driver wants to
4209 * get involved with and it's different per chip and this is almost
4210 * certainly wrong. Firmware would be wrong as well, but it would be
4211 * a lot easier to fix in one place ... For now we do something very
4212 * simple (and hopefully less wrong).
4214 if (!is_t4(adap
->params
.chip
) && cong
>= 0) {
4215 u32 param
, val
, ch_map
= 0;
4217 u16 cng_ch_bits_log
= adap
->params
.arch
.cng_ch_bits_log
;
4219 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
4220 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_CONM_CTXT
) |
4221 FW_PARAMS_PARAM_YZ_V(iq
->cntxt_id
));
4223 val
= CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_QUEUE_X
);
4226 CONMCTXT_CNGTPMODE_V(CONMCTXT_CNGTPMODE_CHANNEL_X
);
4227 for (i
= 0; i
< 4; i
++) {
4228 if (cong
& (1 << i
))
4229 ch_map
|= 1 << (i
<< cng_ch_bits_log
);
4231 val
|= CONMCTXT_CNGCHMAP_V(ch_map
);
4233 ret
= t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1,
4236 dev_warn(adap
->pdev_dev
, "Failed to set Congestion"
4237 " Manager Context for Ingress Queue %d: %d\n",
4238 iq
->cntxt_id
, -ret
);
4247 dma_free_coherent(adap
->pdev_dev
, iq
->size
* iq
->iqe_len
,
4248 iq
->desc
, iq
->phys_addr
);
4251 if (fl
&& fl
->desc
) {
4254 dma_free_coherent(adap
->pdev_dev
, flsz
* sizeof(struct tx_desc
),
4255 fl
->desc
, fl
->addr
);
4261 static void init_txq(struct adapter
*adap
, struct sge_txq
*q
, unsigned int id
)
4264 q
->bar2_addr
= bar2_address(adap
,
4266 T4_BAR2_QTYPE_EGRESS
,
4269 q
->cidx
= q
->pidx
= 0;
4270 q
->stops
= q
->restarts
= 0;
4271 q
->stat
= (void *)&q
->desc
[q
->size
];
4272 spin_lock_init(&q
->db_lock
);
4273 adap
->sge
.egr_map
[id
- adap
->sge
.egr_start
] = q
;
4277 * t4_sge_alloc_eth_txq - allocate an Ethernet TX Queue
4278 * @adap: the adapter
4279 * @txq: the SGE Ethernet TX Queue to initialize
4280 * @dev: the Linux Network Device
4281 * @netdevq: the corresponding Linux TX Queue
4282 * @iqid: the Ingress Queue to which to deliver CIDX Update messages
4283 * @dbqt: whether this TX Queue will use the SGE Doorbell Queue Timers
4285 int t4_sge_alloc_eth_txq(struct adapter
*adap
, struct sge_eth_txq
*txq
,
4286 struct net_device
*dev
, struct netdev_queue
*netdevq
,
4287 unsigned int iqid
, u8 dbqt
)
4289 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4290 struct port_info
*pi
= netdev_priv(dev
);
4291 struct sge
*s
= &adap
->sge
;
4292 struct fw_eq_eth_cmd c
;
4295 /* Add status entries */
4296 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4298 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, txq
->q
.size
,
4299 sizeof(struct tx_desc
), sizeof(struct tx_sw_desc
),
4300 &txq
->q
.phys_addr
, &txq
->q
.sdesc
, s
->stat_len
,
4301 netdev_queue_numa_node_read(netdevq
));
4305 memset(&c
, 0, sizeof(c
));
4306 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_ETH_CMD
) | FW_CMD_REQUEST_F
|
4307 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4308 FW_EQ_ETH_CMD_PFN_V(adap
->pf
) |
4309 FW_EQ_ETH_CMD_VFN_V(0));
4310 c
.alloc_to_len16
= htonl(FW_EQ_ETH_CMD_ALLOC_F
|
4311 FW_EQ_ETH_CMD_EQSTART_F
| FW_LEN16(c
));
4313 /* For TX Ethernet Queues using the SGE Doorbell Queue Timer
4314 * mechanism, we use Ingress Queue messages for Hardware Consumer
4315 * Index Updates on the TX Queue. Otherwise we have the Hardware
4316 * write the CIDX Updates into the Status Page at the end of the
4319 c
.autoequiqe_to_viid
= htonl(FW_EQ_ETH_CMD_AUTOEQUEQE_F
|
4320 FW_EQ_ETH_CMD_VIID_V(pi
->viid
));
4322 c
.fetchszm_to_iqid
=
4323 htonl(FW_EQ_ETH_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4324 FW_EQ_ETH_CMD_PCIECHN_V(pi
->tx_chan
) |
4325 FW_EQ_ETH_CMD_FETCHRO_F
| FW_EQ_ETH_CMD_IQID_V(iqid
));
4327 /* Note that the CIDX Flush Threshold should match MAX_TX_RECLAIM. */
4329 htonl(FW_EQ_ETH_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
4330 ? FETCHBURSTMIN_64B_X
4331 : FETCHBURSTMIN_64B_T6_X
) |
4332 FW_EQ_ETH_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4333 FW_EQ_ETH_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4334 FW_EQ_ETH_CMD_EQSIZE_V(nentries
));
4336 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
4338 /* If we're using the SGE Doorbell Queue Timer mechanism, pass in the
4339 * currently configured Timer Index. THis can be changed later via an
4340 * ethtool -C tx-usecs {Timer Val} command. Note that the SGE
4341 * Doorbell Queue mode is currently automatically enabled in the
4342 * Firmware by setting either AUTOEQUEQE or AUTOEQUIQE ...
4346 cpu_to_be32(FW_EQ_ETH_CMD_TIMEREN_F
|
4347 FW_EQ_ETH_CMD_TIMERIX_V(txq
->dbqtimerix
));
4349 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4351 kfree(txq
->q
.sdesc
);
4352 txq
->q
.sdesc
= NULL
;
4353 dma_free_coherent(adap
->pdev_dev
,
4354 nentries
* sizeof(struct tx_desc
),
4355 txq
->q
.desc
, txq
->q
.phys_addr
);
4360 txq
->q
.q_type
= CXGB4_TXQ_ETH
;
4361 init_txq(adap
, &txq
->q
, FW_EQ_ETH_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
4367 txq
->mapping_err
= 0;
4373 int t4_sge_alloc_ctrl_txq(struct adapter
*adap
, struct sge_ctrl_txq
*txq
,
4374 struct net_device
*dev
, unsigned int iqid
,
4375 unsigned int cmplqid
)
4377 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4378 struct port_info
*pi
= netdev_priv(dev
);
4379 struct sge
*s
= &adap
->sge
;
4380 struct fw_eq_ctrl_cmd c
;
4383 /* Add status entries */
4384 nentries
= txq
->q
.size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4386 txq
->q
.desc
= alloc_ring(adap
->pdev_dev
, nentries
,
4387 sizeof(struct tx_desc
), 0, &txq
->q
.phys_addr
,
4388 NULL
, 0, dev_to_node(adap
->pdev_dev
));
4392 c
.op_to_vfn
= htonl(FW_CMD_OP_V(FW_EQ_CTRL_CMD
) | FW_CMD_REQUEST_F
|
4393 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4394 FW_EQ_CTRL_CMD_PFN_V(adap
->pf
) |
4395 FW_EQ_CTRL_CMD_VFN_V(0));
4396 c
.alloc_to_len16
= htonl(FW_EQ_CTRL_CMD_ALLOC_F
|
4397 FW_EQ_CTRL_CMD_EQSTART_F
| FW_LEN16(c
));
4398 c
.cmpliqid_eqid
= htonl(FW_EQ_CTRL_CMD_CMPLIQID_V(cmplqid
));
4399 c
.physeqid_pkd
= htonl(0);
4400 c
.fetchszm_to_iqid
=
4401 htonl(FW_EQ_CTRL_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4402 FW_EQ_CTRL_CMD_PCIECHN_V(pi
->tx_chan
) |
4403 FW_EQ_CTRL_CMD_FETCHRO_F
| FW_EQ_CTRL_CMD_IQID_V(iqid
));
4405 htonl(FW_EQ_CTRL_CMD_FBMIN_V(chip_ver
<= CHELSIO_T5
4406 ? FETCHBURSTMIN_64B_X
4407 : FETCHBURSTMIN_64B_T6_X
) |
4408 FW_EQ_CTRL_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4409 FW_EQ_CTRL_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4410 FW_EQ_CTRL_CMD_EQSIZE_V(nentries
));
4411 c
.eqaddr
= cpu_to_be64(txq
->q
.phys_addr
);
4413 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4415 dma_free_coherent(adap
->pdev_dev
,
4416 nentries
* sizeof(struct tx_desc
),
4417 txq
->q
.desc
, txq
->q
.phys_addr
);
4422 txq
->q
.q_type
= CXGB4_TXQ_CTRL
;
4423 init_txq(adap
, &txq
->q
, FW_EQ_CTRL_CMD_EQID_G(ntohl(c
.cmpliqid_eqid
)));
4425 skb_queue_head_init(&txq
->sendq
);
4426 tasklet_init(&txq
->qresume_tsk
, restart_ctrlq
, (unsigned long)txq
);
4431 int t4_sge_mod_ctrl_txq(struct adapter
*adap
, unsigned int eqid
,
4432 unsigned int cmplqid
)
4436 param
= (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ
) |
4437 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DMAQ_EQ_CMPLIQID_CTRL
) |
4438 FW_PARAMS_PARAM_YZ_V(eqid
));
4440 return t4_set_params(adap
, adap
->mbox
, adap
->pf
, 0, 1, ¶m
, &val
);
4443 static int t4_sge_alloc_ofld_txq(struct adapter
*adap
, struct sge_txq
*q
,
4444 struct net_device
*dev
, u32 cmd
, u32 iqid
)
4446 unsigned int chip_ver
= CHELSIO_CHIP_VERSION(adap
->params
.chip
);
4447 struct port_info
*pi
= netdev_priv(dev
);
4448 struct sge
*s
= &adap
->sge
;
4449 struct fw_eq_ofld_cmd c
;
4450 u32 fb_min
, nentries
;
4453 /* Add status entries */
4454 nentries
= q
->size
+ s
->stat_len
/ sizeof(struct tx_desc
);
4455 q
->desc
= alloc_ring(adap
->pdev_dev
, q
->size
, sizeof(struct tx_desc
),
4456 sizeof(struct tx_sw_desc
), &q
->phys_addr
,
4457 &q
->sdesc
, s
->stat_len
, NUMA_NO_NODE
);
4461 if (chip_ver
<= CHELSIO_T5
)
4462 fb_min
= FETCHBURSTMIN_64B_X
;
4464 fb_min
= FETCHBURSTMIN_64B_T6_X
;
4466 memset(&c
, 0, sizeof(c
));
4467 c
.op_to_vfn
= htonl(FW_CMD_OP_V(cmd
) | FW_CMD_REQUEST_F
|
4468 FW_CMD_WRITE_F
| FW_CMD_EXEC_F
|
4469 FW_EQ_OFLD_CMD_PFN_V(adap
->pf
) |
4470 FW_EQ_OFLD_CMD_VFN_V(0));
4471 c
.alloc_to_len16
= htonl(FW_EQ_OFLD_CMD_ALLOC_F
|
4472 FW_EQ_OFLD_CMD_EQSTART_F
| FW_LEN16(c
));
4473 c
.fetchszm_to_iqid
=
4474 htonl(FW_EQ_OFLD_CMD_HOSTFCMODE_V(HOSTFCMODE_STATUS_PAGE_X
) |
4475 FW_EQ_OFLD_CMD_PCIECHN_V(pi
->tx_chan
) |
4476 FW_EQ_OFLD_CMD_FETCHRO_F
| FW_EQ_OFLD_CMD_IQID_V(iqid
));
4478 htonl(FW_EQ_OFLD_CMD_FBMIN_V(fb_min
) |
4479 FW_EQ_OFLD_CMD_FBMAX_V(FETCHBURSTMAX_512B_X
) |
4480 FW_EQ_OFLD_CMD_CIDXFTHRESH_V(CIDXFLUSHTHRESH_32_X
) |
4481 FW_EQ_OFLD_CMD_EQSIZE_V(nentries
));
4482 c
.eqaddr
= cpu_to_be64(q
->phys_addr
);
4484 ret
= t4_wr_mbox(adap
, adap
->mbox
, &c
, sizeof(c
), &c
);
4488 dma_free_coherent(adap
->pdev_dev
,
4489 nentries
* sizeof(struct tx_desc
),
4490 q
->desc
, q
->phys_addr
);
4495 init_txq(adap
, q
, FW_EQ_OFLD_CMD_EQID_G(ntohl(c
.eqid_pkd
)));
4499 int t4_sge_alloc_uld_txq(struct adapter
*adap
, struct sge_uld_txq
*txq
,
4500 struct net_device
*dev
, unsigned int iqid
,
4501 unsigned int uld_type
)
4503 u32 cmd
= FW_EQ_OFLD_CMD
;
4506 if (unlikely(uld_type
== CXGB4_TX_CRYPTO
))
4507 cmd
= FW_EQ_CTRL_CMD
;
4509 ret
= t4_sge_alloc_ofld_txq(adap
, &txq
->q
, dev
, cmd
, iqid
);
4513 txq
->q
.q_type
= CXGB4_TXQ_ULD
;
4515 skb_queue_head_init(&txq
->sendq
);
4516 tasklet_init(&txq
->qresume_tsk
, restart_ofldq
, (unsigned long)txq
);
4518 txq
->mapping_err
= 0;
4522 int t4_sge_alloc_ethofld_txq(struct adapter
*adap
, struct sge_eohw_txq
*txq
,
4523 struct net_device
*dev
, u32 iqid
)
4527 ret
= t4_sge_alloc_ofld_txq(adap
, &txq
->q
, dev
, FW_EQ_OFLD_CMD
, iqid
);
4531 txq
->q
.q_type
= CXGB4_TXQ_ULD
;
4532 spin_lock_init(&txq
->lock
);
4538 txq
->mapping_err
= 0;
4542 void free_txq(struct adapter
*adap
, struct sge_txq
*q
)
4544 struct sge
*s
= &adap
->sge
;
4546 dma_free_coherent(adap
->pdev_dev
,
4547 q
->size
* sizeof(struct tx_desc
) + s
->stat_len
,
4548 q
->desc
, q
->phys_addr
);
4554 void free_rspq_fl(struct adapter
*adap
, struct sge_rspq
*rq
,
4557 struct sge
*s
= &adap
->sge
;
4558 unsigned int fl_id
= fl
? fl
->cntxt_id
: 0xffff;
4560 adap
->sge
.ingr_map
[rq
->cntxt_id
- adap
->sge
.ingr_start
] = NULL
;
4561 t4_iq_free(adap
, adap
->mbox
, adap
->pf
, 0, FW_IQ_TYPE_FL_INT_CAP
,
4562 rq
->cntxt_id
, fl_id
, 0xffff);
4563 dma_free_coherent(adap
->pdev_dev
, (rq
->size
+ 1) * rq
->iqe_len
,
4564 rq
->desc
, rq
->phys_addr
);
4565 netif_napi_del(&rq
->napi
);
4567 rq
->cntxt_id
= rq
->abs_id
= 0;
4571 free_rx_bufs(adap
, fl
, fl
->avail
);
4572 dma_free_coherent(adap
->pdev_dev
, fl
->size
* 8 + s
->stat_len
,
4573 fl
->desc
, fl
->addr
);
4582 * t4_free_ofld_rxqs - free a block of consecutive Rx queues
4583 * @adap: the adapter
4584 * @n: number of queues
4585 * @q: pointer to first queue
4587 * Release the resources of a consecutive block of offload Rx queues.
4589 void t4_free_ofld_rxqs(struct adapter
*adap
, int n
, struct sge_ofld_rxq
*q
)
4591 for ( ; n
; n
--, q
++)
4593 free_rspq_fl(adap
, &q
->rspq
,
4594 q
->fl
.size
? &q
->fl
: NULL
);
4597 void t4_sge_free_ethofld_txq(struct adapter
*adap
, struct sge_eohw_txq
*txq
)
4600 t4_ofld_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4602 free_tx_desc(adap
, &txq
->q
, txq
->q
.in_use
, false);
4603 kfree(txq
->q
.sdesc
);
4604 free_txq(adap
, &txq
->q
);
4609 * t4_free_sge_resources - free SGE resources
4610 * @adap: the adapter
4612 * Frees resources used by the SGE queue sets.
4614 void t4_free_sge_resources(struct adapter
*adap
)
4617 struct sge_eth_rxq
*eq
;
4618 struct sge_eth_txq
*etq
;
4620 /* stop all Rx queues in order to start them draining */
4621 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4622 eq
= &adap
->sge
.ethrxq
[i
];
4624 t4_iq_stop(adap
, adap
->mbox
, adap
->pf
, 0,
4625 FW_IQ_TYPE_FL_INT_CAP
,
4627 eq
->fl
.size
? eq
->fl
.cntxt_id
: 0xffff,
4631 /* clean up Ethernet Tx/Rx queues */
4632 for (i
= 0; i
< adap
->sge
.ethqsets
; i
++) {
4633 eq
= &adap
->sge
.ethrxq
[i
];
4635 free_rspq_fl(adap
, &eq
->rspq
,
4636 eq
->fl
.size
? &eq
->fl
: NULL
);
4638 cxgb4_free_msix_idx_in_bmap(adap
, eq
->msix
->idx
);
4642 etq
= &adap
->sge
.ethtxq
[i
];
4644 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4646 __netif_tx_lock_bh(etq
->txq
);
4647 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4648 __netif_tx_unlock_bh(etq
->txq
);
4649 kfree(etq
->q
.sdesc
);
4650 free_txq(adap
, &etq
->q
);
4654 /* clean up control Tx queues */
4655 for (i
= 0; i
< ARRAY_SIZE(adap
->sge
.ctrlq
); i
++) {
4656 struct sge_ctrl_txq
*cq
= &adap
->sge
.ctrlq
[i
];
4659 tasklet_kill(&cq
->qresume_tsk
);
4660 t4_ctrl_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4662 __skb_queue_purge(&cq
->sendq
);
4663 free_txq(adap
, &cq
->q
);
4667 if (adap
->sge
.fw_evtq
.desc
) {
4668 free_rspq_fl(adap
, &adap
->sge
.fw_evtq
, NULL
);
4669 if (adap
->sge
.fwevtq_msix_idx
>= 0)
4670 cxgb4_free_msix_idx_in_bmap(adap
,
4671 adap
->sge
.fwevtq_msix_idx
);
4674 if (adap
->sge
.nd_msix_idx
>= 0)
4675 cxgb4_free_msix_idx_in_bmap(adap
, adap
->sge
.nd_msix_idx
);
4677 if (adap
->sge
.intrq
.desc
)
4678 free_rspq_fl(adap
, &adap
->sge
.intrq
, NULL
);
4680 if (!is_t4(adap
->params
.chip
)) {
4681 etq
= &adap
->sge
.ptptxq
;
4683 t4_eth_eq_free(adap
, adap
->mbox
, adap
->pf
, 0,
4685 spin_lock_bh(&adap
->ptp_lock
);
4686 free_tx_desc(adap
, &etq
->q
, etq
->q
.in_use
, true);
4687 spin_unlock_bh(&adap
->ptp_lock
);
4688 kfree(etq
->q
.sdesc
);
4689 free_txq(adap
, &etq
->q
);
4693 /* clear the reverse egress queue map */
4694 memset(adap
->sge
.egr_map
, 0,
4695 adap
->sge
.egr_sz
* sizeof(*adap
->sge
.egr_map
));
4698 void t4_sge_start(struct adapter
*adap
)
4700 adap
->sge
.ethtxq_rover
= 0;
4701 mod_timer(&adap
->sge
.rx_timer
, jiffies
+ RX_QCHECK_PERIOD
);
4702 mod_timer(&adap
->sge
.tx_timer
, jiffies
+ TX_QCHECK_PERIOD
);
4706 * t4_sge_stop - disable SGE operation
4707 * @adap: the adapter
4709 * Stop tasklets and timers associated with the DMA engine. Note that
4710 * this is effective only if measures have been taken to disable any HW
4711 * events that may restart them.
4713 void t4_sge_stop(struct adapter
*adap
)
4716 struct sge
*s
= &adap
->sge
;
4718 if (in_interrupt()) /* actions below require waiting */
4721 if (s
->rx_timer
.function
)
4722 del_timer_sync(&s
->rx_timer
);
4723 if (s
->tx_timer
.function
)
4724 del_timer_sync(&s
->tx_timer
);
4726 if (is_offload(adap
)) {
4727 struct sge_uld_txq_info
*txq_info
;
4729 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_OFLD
];
4731 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4733 for_each_ofldtxq(&adap
->sge
, i
) {
4735 tasklet_kill(&txq
->qresume_tsk
);
4740 if (is_pci_uld(adap
)) {
4741 struct sge_uld_txq_info
*txq_info
;
4743 txq_info
= adap
->sge
.uld_txq_info
[CXGB4_TX_CRYPTO
];
4745 struct sge_uld_txq
*txq
= txq_info
->uldtxq
;
4747 for_each_ofldtxq(&adap
->sge
, i
) {
4749 tasklet_kill(&txq
->qresume_tsk
);
4754 for (i
= 0; i
< ARRAY_SIZE(s
->ctrlq
); i
++) {
4755 struct sge_ctrl_txq
*cq
= &s
->ctrlq
[i
];
4758 tasklet_kill(&cq
->qresume_tsk
);
4763 * t4_sge_init_soft - grab core SGE values needed by SGE code
4764 * @adap: the adapter
4766 * We need to grab the SGE operating parameters that we need to have
4767 * in order to do our job and make sure we can live with them.
4770 static int t4_sge_init_soft(struct adapter
*adap
)
4772 struct sge
*s
= &adap
->sge
;
4773 u32 fl_small_pg
, fl_large_pg
, fl_small_mtu
, fl_large_mtu
;
4774 u32 timer_value_0_and_1
, timer_value_2_and_3
, timer_value_4_and_5
;
4775 u32 ingress_rx_threshold
;
4778 * Verify that CPL messages are going to the Ingress Queue for
4779 * process_responses() and that only packet data is going to the
4782 if ((t4_read_reg(adap
, SGE_CONTROL_A
) & RXPKTCPLMODE_F
) !=
4783 RXPKTCPLMODE_V(RXPKTCPLMODE_SPLIT_X
)) {
4784 dev_err(adap
->pdev_dev
, "bad SGE CPL MODE\n");
4789 * Validate the Host Buffer Register Array indices that we want to
4792 * XXX Note that we should really read through the Host Buffer Size
4793 * XXX register array and find the indices of the Buffer Sizes which
4794 * XXX meet our needs!
4796 #define READ_FL_BUF(x) \
4797 t4_read_reg(adap, SGE_FL_BUFFER_SIZE0_A+(x)*sizeof(u32))
4799 fl_small_pg
= READ_FL_BUF(RX_SMALL_PG_BUF
);
4800 fl_large_pg
= READ_FL_BUF(RX_LARGE_PG_BUF
);
4801 fl_small_mtu
= READ_FL_BUF(RX_SMALL_MTU_BUF
);
4802 fl_large_mtu
= READ_FL_BUF(RX_LARGE_MTU_BUF
);
4804 /* We only bother using the Large Page logic if the Large Page Buffer
4805 * is larger than our Page Size Buffer.
4807 if (fl_large_pg
<= fl_small_pg
)
4812 /* The Page Size Buffer must be exactly equal to our Page Size and the
4813 * Large Page Size Buffer should be 0 (per above) or a power of 2.
4815 if (fl_small_pg
!= PAGE_SIZE
||
4816 (fl_large_pg
& (fl_large_pg
-1)) != 0) {
4817 dev_err(adap
->pdev_dev
, "bad SGE FL page buffer sizes [%d, %d]\n",
4818 fl_small_pg
, fl_large_pg
);
4822 s
->fl_pg_order
= ilog2(fl_large_pg
) - PAGE_SHIFT
;
4824 if (fl_small_mtu
< FL_MTU_SMALL_BUFSIZE(adap
) ||
4825 fl_large_mtu
< FL_MTU_LARGE_BUFSIZE(adap
)) {
4826 dev_err(adap
->pdev_dev
, "bad SGE FL MTU sizes [%d, %d]\n",
4827 fl_small_mtu
, fl_large_mtu
);
4832 * Retrieve our RX interrupt holdoff timer values and counter
4833 * threshold values from the SGE parameters.
4835 timer_value_0_and_1
= t4_read_reg(adap
, SGE_TIMER_VALUE_0_AND_1_A
);
4836 timer_value_2_and_3
= t4_read_reg(adap
, SGE_TIMER_VALUE_2_AND_3_A
);
4837 timer_value_4_and_5
= t4_read_reg(adap
, SGE_TIMER_VALUE_4_AND_5_A
);
4838 s
->timer_val
[0] = core_ticks_to_us(adap
,
4839 TIMERVALUE0_G(timer_value_0_and_1
));
4840 s
->timer_val
[1] = core_ticks_to_us(adap
,
4841 TIMERVALUE1_G(timer_value_0_and_1
));
4842 s
->timer_val
[2] = core_ticks_to_us(adap
,
4843 TIMERVALUE2_G(timer_value_2_and_3
));
4844 s
->timer_val
[3] = core_ticks_to_us(adap
,
4845 TIMERVALUE3_G(timer_value_2_and_3
));
4846 s
->timer_val
[4] = core_ticks_to_us(adap
,
4847 TIMERVALUE4_G(timer_value_4_and_5
));
4848 s
->timer_val
[5] = core_ticks_to_us(adap
,
4849 TIMERVALUE5_G(timer_value_4_and_5
));
4851 ingress_rx_threshold
= t4_read_reg(adap
, SGE_INGRESS_RX_THRESHOLD_A
);
4852 s
->counter_val
[0] = THRESHOLD_0_G(ingress_rx_threshold
);
4853 s
->counter_val
[1] = THRESHOLD_1_G(ingress_rx_threshold
);
4854 s
->counter_val
[2] = THRESHOLD_2_G(ingress_rx_threshold
);
4855 s
->counter_val
[3] = THRESHOLD_3_G(ingress_rx_threshold
);
4861 * t4_sge_init - initialize SGE
4862 * @adap: the adapter
4864 * Perform low-level SGE code initialization needed every time after a
4867 int t4_sge_init(struct adapter
*adap
)
4869 struct sge
*s
= &adap
->sge
;
4870 u32 sge_control
, sge_conm_ctrl
;
4871 int ret
, egress_threshold
;
4874 * Ingress Padding Boundary and Egress Status Page Size are set up by
4875 * t4_fixup_host_params().
4877 sge_control
= t4_read_reg(adap
, SGE_CONTROL_A
);
4878 s
->pktshift
= PKTSHIFT_G(sge_control
);
4879 s
->stat_len
= (sge_control
& EGRSTATUSPAGESIZE_F
) ? 128 : 64;
4881 s
->fl_align
= t4_fl_pkt_align(adap
);
4882 ret
= t4_sge_init_soft(adap
);
4887 * A FL with <= fl_starve_thres buffers is starving and a periodic
4888 * timer will attempt to refill it. This needs to be larger than the
4889 * SGE's Egress Congestion Threshold. If it isn't, then we can get
4890 * stuck waiting for new packets while the SGE is waiting for us to
4891 * give it more Free List entries. (Note that the SGE's Egress
4892 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
4893 * there was only a single field to control this. For T5 there's the
4894 * original field which now only applies to Unpacked Mode Free List
4895 * buffers and a new field which only applies to Packed Mode Free List
4898 sge_conm_ctrl
= t4_read_reg(adap
, SGE_CONM_CTRL_A
);
4899 switch (CHELSIO_CHIP_VERSION(adap
->params
.chip
)) {
4901 egress_threshold
= EGRTHRESHOLD_G(sge_conm_ctrl
);
4904 egress_threshold
= EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4907 egress_threshold
= T6_EGRTHRESHOLDPACKING_G(sge_conm_ctrl
);
4910 dev_err(adap
->pdev_dev
, "Unsupported Chip version %d\n",
4911 CHELSIO_CHIP_VERSION(adap
->params
.chip
));
4914 s
->fl_starve_thres
= 2*egress_threshold
+ 1;
4916 t4_idma_monitor_init(adap
, &s
->idma_monitor
);
4918 /* Set up timers used for recuring callbacks to process RX and TX
4919 * administrative tasks.
4921 timer_setup(&s
->rx_timer
, sge_rx_timer_cb
, 0);
4922 timer_setup(&s
->tx_timer
, sge_tx_timer_cb
, 0);
4924 spin_lock_init(&s
->intrq_lock
);