2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/mlx4/cq.h>
36 #include <linux/slab.h>
37 #include <linux/mlx4/qp.h>
38 #include <linux/skbuff.h>
39 #include <linux/if_vlan.h>
40 #include <linux/prefetch.h>
41 #include <linux/vmalloc.h>
42 #include <linux/tcp.h>
44 #include <linux/ipv6.h>
45 #include <linux/moduleparam.h>
46 #include <linux/indirect_call_wrapper.h>
50 int mlx4_en_create_tx_ring(struct mlx4_en_priv
*priv
,
51 struct mlx4_en_tx_ring
**pring
, u32 size
,
52 u16 stride
, int node
, int queue_index
)
54 struct mlx4_en_dev
*mdev
= priv
->mdev
;
55 struct mlx4_en_tx_ring
*ring
;
59 ring
= kzalloc_node(sizeof(*ring
), GFP_KERNEL
, node
);
61 en_err(priv
, "Failed allocating TX ring\n");
66 ring
->size_mask
= size
- 1;
67 ring
->sp_stride
= stride
;
68 ring
->full_size
= ring
->size
- HEADROOM
- MAX_DESC_TXBBS
;
70 tmp
= size
* sizeof(struct mlx4_en_tx_info
);
71 ring
->tx_info
= kvmalloc_node(tmp
, GFP_KERNEL
, node
);
77 en_dbg(DRV
, priv
, "Allocated tx_info ring at addr:%p size:%d\n",
80 ring
->bounce_buf
= kmalloc_node(MAX_DESC_SIZE
, GFP_KERNEL
, node
);
81 if (!ring
->bounce_buf
) {
82 ring
->bounce_buf
= kmalloc(MAX_DESC_SIZE
, GFP_KERNEL
);
83 if (!ring
->bounce_buf
) {
88 ring
->buf_size
= ALIGN(size
* ring
->sp_stride
, MLX4_EN_PAGE_SIZE
);
90 /* Allocate HW buffers on provided NUMA node */
91 set_dev_node(&mdev
->dev
->persist
->pdev
->dev
, node
);
92 err
= mlx4_alloc_hwq_res(mdev
->dev
, &ring
->sp_wqres
, ring
->buf_size
);
93 set_dev_node(&mdev
->dev
->persist
->pdev
->dev
, mdev
->dev
->numa_node
);
95 en_err(priv
, "Failed allocating hwq resources\n");
99 ring
->buf
= ring
->sp_wqres
.buf
.direct
.buf
;
101 en_dbg(DRV
, priv
, "Allocated TX ring (addr:%p) - buf:%p size:%d buf_size:%d dma:%llx\n",
102 ring
, ring
->buf
, ring
->size
, ring
->buf_size
,
103 (unsigned long long) ring
->sp_wqres
.buf
.direct
.map
);
105 err
= mlx4_qp_reserve_range(mdev
->dev
, 1, 1, &ring
->qpn
,
106 MLX4_RESERVE_ETH_BF_QP
,
107 MLX4_RES_USAGE_DRIVER
);
109 en_err(priv
, "failed reserving qp for TX ring\n");
113 err
= mlx4_qp_alloc(mdev
->dev
, ring
->qpn
, &ring
->sp_qp
);
115 en_err(priv
, "Failed allocating qp %d\n", ring
->qpn
);
118 ring
->sp_qp
.event
= mlx4_en_sqp_event
;
120 err
= mlx4_bf_alloc(mdev
->dev
, &ring
->bf
, node
);
122 en_dbg(DRV
, priv
, "working without blueflame (%d)\n", err
);
123 ring
->bf
.uar
= &mdev
->priv_uar
;
124 ring
->bf
.uar
->map
= mdev
->uar_map
;
125 ring
->bf_enabled
= false;
126 ring
->bf_alloced
= false;
127 priv
->pflags
&= ~MLX4_EN_PRIV_FLAGS_BLUEFLAME
;
129 ring
->bf_alloced
= true;
130 ring
->bf_enabled
= !!(priv
->pflags
&
131 MLX4_EN_PRIV_FLAGS_BLUEFLAME
);
134 ring
->hwtstamp_tx_type
= priv
->hwtstamp_config
.tx_type
;
135 ring
->queue_index
= queue_index
;
137 if (queue_index
< priv
->num_tx_rings_p_up
)
138 cpumask_set_cpu(cpumask_local_spread(queue_index
,
139 priv
->mdev
->dev
->numa_node
),
140 &ring
->sp_affinity_mask
);
146 mlx4_qp_release_range(mdev
->dev
, ring
->qpn
, 1);
148 mlx4_free_hwq_res(mdev
->dev
, &ring
->sp_wqres
, ring
->buf_size
);
150 kfree(ring
->bounce_buf
);
151 ring
->bounce_buf
= NULL
;
153 kvfree(ring
->tx_info
);
154 ring
->tx_info
= NULL
;
161 void mlx4_en_destroy_tx_ring(struct mlx4_en_priv
*priv
,
162 struct mlx4_en_tx_ring
**pring
)
164 struct mlx4_en_dev
*mdev
= priv
->mdev
;
165 struct mlx4_en_tx_ring
*ring
= *pring
;
166 en_dbg(DRV
, priv
, "Destroying tx ring, qpn: %d\n", ring
->qpn
);
168 if (ring
->bf_alloced
)
169 mlx4_bf_free(mdev
->dev
, &ring
->bf
);
170 mlx4_qp_remove(mdev
->dev
, &ring
->sp_qp
);
171 mlx4_qp_free(mdev
->dev
, &ring
->sp_qp
);
172 mlx4_qp_release_range(priv
->mdev
->dev
, ring
->qpn
, 1);
173 mlx4_free_hwq_res(mdev
->dev
, &ring
->sp_wqres
, ring
->buf_size
);
174 kfree(ring
->bounce_buf
);
175 ring
->bounce_buf
= NULL
;
176 kvfree(ring
->tx_info
);
177 ring
->tx_info
= NULL
;
182 int mlx4_en_activate_tx_ring(struct mlx4_en_priv
*priv
,
183 struct mlx4_en_tx_ring
*ring
,
184 int cq
, int user_prio
)
186 struct mlx4_en_dev
*mdev
= priv
->mdev
;
191 ring
->cons
= 0xffffffff;
192 ring
->last_nr_txbb
= 1;
193 memset(ring
->tx_info
, 0, ring
->size
* sizeof(struct mlx4_en_tx_info
));
194 memset(ring
->buf
, 0, ring
->buf_size
);
195 ring
->free_tx_desc
= mlx4_en_free_tx_desc
;
197 ring
->sp_qp_state
= MLX4_QP_STATE_RST
;
198 ring
->doorbell_qpn
= cpu_to_be32(ring
->sp_qp
.qpn
<< 8);
199 ring
->mr_key
= cpu_to_be32(mdev
->mr
.key
);
201 mlx4_en_fill_qp_context(priv
, ring
->size
, ring
->sp_stride
, 1, 0, ring
->qpn
,
202 ring
->sp_cqn
, user_prio
, &ring
->sp_context
);
203 if (ring
->bf_alloced
)
204 ring
->sp_context
.usr_page
=
205 cpu_to_be32(mlx4_to_hw_uar_index(mdev
->dev
,
206 ring
->bf
.uar
->index
));
208 err
= mlx4_qp_to_ready(mdev
->dev
, &ring
->sp_wqres
.mtt
, &ring
->sp_context
,
209 &ring
->sp_qp
, &ring
->sp_qp_state
);
210 if (!cpumask_empty(&ring
->sp_affinity_mask
))
211 netif_set_xps_queue(priv
->dev
, &ring
->sp_affinity_mask
,
217 void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv
*priv
,
218 struct mlx4_en_tx_ring
*ring
)
220 struct mlx4_en_dev
*mdev
= priv
->mdev
;
222 mlx4_qp_modify(mdev
->dev
, NULL
, ring
->sp_qp_state
,
223 MLX4_QP_STATE_RST
, NULL
, 0, 0, &ring
->sp_qp
);
226 static inline bool mlx4_en_is_tx_ring_full(struct mlx4_en_tx_ring
*ring
)
228 return ring
->prod
- ring
->cons
> ring
->full_size
;
231 static void mlx4_en_stamp_wqe(struct mlx4_en_priv
*priv
,
232 struct mlx4_en_tx_ring
*ring
, int index
,
235 __be32 stamp
= cpu_to_be32(STAMP_VAL
| (!!owner
<< STAMP_SHIFT
));
236 struct mlx4_en_tx_desc
*tx_desc
= ring
->buf
+ (index
<< LOG_TXBB_SIZE
);
237 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[index
];
238 void *end
= ring
->buf
+ ring
->buf_size
;
239 __be32
*ptr
= (__be32
*)tx_desc
;
242 /* Optimize the common case when there are no wraparounds */
243 if (likely((void *)tx_desc
+
244 (tx_info
->nr_txbb
<< LOG_TXBB_SIZE
) <= end
)) {
245 /* Stamp the freed descriptor */
246 for (i
= 0; i
< tx_info
->nr_txbb
<< LOG_TXBB_SIZE
;
252 /* Stamp the freed descriptor */
253 for (i
= 0; i
< tx_info
->nr_txbb
<< LOG_TXBB_SIZE
;
257 if ((void *)ptr
>= end
) {
259 stamp
^= cpu_to_be32(0x80000000);
265 INDIRECT_CALLABLE_DECLARE(u32
mlx4_en_free_tx_desc(struct mlx4_en_priv
*priv
,
266 struct mlx4_en_tx_ring
*ring
,
267 int index
, u64 timestamp
,
270 u32
mlx4_en_free_tx_desc(struct mlx4_en_priv
*priv
,
271 struct mlx4_en_tx_ring
*ring
,
272 int index
, u64 timestamp
,
275 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[index
];
276 struct mlx4_en_tx_desc
*tx_desc
= ring
->buf
+ (index
<< LOG_TXBB_SIZE
);
277 struct mlx4_wqe_data_seg
*data
= (void *) tx_desc
+ tx_info
->data_offset
;
278 void *end
= ring
->buf
+ ring
->buf_size
;
279 struct sk_buff
*skb
= tx_info
->skb
;
280 int nr_maps
= tx_info
->nr_maps
;
283 /* We do not touch skb here, so prefetch skb->users location
284 * to speedup consume_skb()
286 prefetchw(&skb
->users
);
288 if (unlikely(timestamp
)) {
289 struct skb_shared_hwtstamps hwts
;
291 mlx4_en_fill_hwtstamps(priv
->mdev
, &hwts
, timestamp
);
292 skb_tstamp_tx(skb
, &hwts
);
297 dma_unmap_single(priv
->ddev
,
299 tx_info
->map0_byte_count
,
302 dma_unmap_page(priv
->ddev
,
304 tx_info
->map0_byte_count
,
306 /* Optimize the common case when there are no wraparounds */
307 if (likely((void *)tx_desc
+
308 (tx_info
->nr_txbb
<< LOG_TXBB_SIZE
) <= end
)) {
309 for (i
= 1; i
< nr_maps
; i
++) {
311 dma_unmap_page(priv
->ddev
,
312 (dma_addr_t
)be64_to_cpu(data
->addr
),
313 be32_to_cpu(data
->byte_count
),
317 if ((void *)data
>= end
)
318 data
= ring
->buf
+ ((void *)data
- end
);
320 for (i
= 1; i
< nr_maps
; i
++) {
322 /* Check for wraparound before unmapping */
323 if ((void *) data
>= end
)
325 dma_unmap_page(priv
->ddev
,
326 (dma_addr_t
)be64_to_cpu(data
->addr
),
327 be32_to_cpu(data
->byte_count
),
332 napi_consume_skb(skb
, napi_mode
);
334 return tx_info
->nr_txbb
;
337 INDIRECT_CALLABLE_DECLARE(u32
mlx4_en_recycle_tx_desc(struct mlx4_en_priv
*priv
,
338 struct mlx4_en_tx_ring
*ring
,
339 int index
, u64 timestamp
,
342 u32
mlx4_en_recycle_tx_desc(struct mlx4_en_priv
*priv
,
343 struct mlx4_en_tx_ring
*ring
,
344 int index
, u64 timestamp
,
347 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[index
];
348 struct mlx4_en_rx_alloc frame
= {
349 .page
= tx_info
->page
,
350 .dma
= tx_info
->map0_dma
,
353 if (!mlx4_en_rx_recycle(ring
->recycle_ring
, &frame
)) {
354 dma_unmap_page(priv
->ddev
, tx_info
->map0_dma
,
355 PAGE_SIZE
, priv
->dma_dir
);
356 put_page(tx_info
->page
);
359 return tx_info
->nr_txbb
;
362 int mlx4_en_free_tx_buf(struct net_device
*dev
, struct mlx4_en_tx_ring
*ring
)
364 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
367 /* Skip last polled descriptor */
368 ring
->cons
+= ring
->last_nr_txbb
;
369 en_dbg(DRV
, priv
, "Freeing Tx buf - cons:0x%x prod:0x%x\n",
370 ring
->cons
, ring
->prod
);
372 if ((u32
) (ring
->prod
- ring
->cons
) > ring
->size
) {
373 if (netif_msg_tx_err(priv
))
374 en_warn(priv
, "Tx consumer passed producer!\n");
378 while (ring
->cons
!= ring
->prod
) {
379 ring
->last_nr_txbb
= ring
->free_tx_desc(priv
, ring
,
380 ring
->cons
& ring
->size_mask
,
381 0, 0 /* Non-NAPI caller */);
382 ring
->cons
+= ring
->last_nr_txbb
;
387 netdev_tx_reset_queue(ring
->tx_queue
);
390 en_dbg(DRV
, priv
, "Freed %d uncompleted tx descriptors\n", cnt
);
395 bool mlx4_en_process_tx_cq(struct net_device
*dev
,
396 struct mlx4_en_cq
*cq
, int napi_budget
)
398 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
399 struct mlx4_cq
*mcq
= &cq
->mcq
;
400 struct mlx4_en_tx_ring
*ring
= priv
->tx_ring
[cq
->type
][cq
->ring
];
401 struct mlx4_cqe
*cqe
;
402 u16 index
, ring_index
, stamp_index
;
403 u32 txbbs_skipped
= 0;
405 u32 cons_index
= mcq
->cons_index
;
407 u32 size_mask
= ring
->size_mask
;
408 struct mlx4_cqe
*buf
= cq
->buf
;
411 int factor
= priv
->cqe_factor
;
413 int budget
= priv
->tx_work_limit
;
417 if (unlikely(!priv
->port_up
))
420 netdev_txq_bql_complete_prefetchw(ring
->tx_queue
);
422 index
= cons_index
& size_mask
;
423 cqe
= mlx4_en_get_cqe(buf
, index
, priv
->cqe_size
) + factor
;
424 last_nr_txbb
= READ_ONCE(ring
->last_nr_txbb
);
425 ring_cons
= READ_ONCE(ring
->cons
);
426 ring_index
= ring_cons
& size_mask
;
427 stamp_index
= ring_index
;
429 /* Process all completed CQEs */
430 while (XNOR(cqe
->owner_sr_opcode
& MLX4_CQE_OWNER_MASK
,
431 cons_index
& size
) && (done
< budget
)) {
435 * make sure we read the CQE after we read the
440 if (unlikely((cqe
->owner_sr_opcode
& MLX4_CQE_OPCODE_MASK
) ==
441 MLX4_CQE_OPCODE_ERROR
)) {
442 struct mlx4_err_cqe
*cqe_err
= (struct mlx4_err_cqe
*)cqe
;
444 en_err(priv
, "CQE error - vendor syndrome: 0x%x syndrome: 0x%x\n",
445 cqe_err
->vendor_err_syndrome
,
449 /* Skip over last polled CQE */
450 new_index
= be16_to_cpu(cqe
->wqe_index
) & size_mask
;
455 txbbs_skipped
+= last_nr_txbb
;
456 ring_index
= (ring_index
+ last_nr_txbb
) & size_mask
;
458 if (unlikely(ring
->tx_info
[ring_index
].ts_requested
))
459 timestamp
= mlx4_en_get_cqe_ts(cqe
);
461 /* free next descriptor */
462 last_nr_txbb
= INDIRECT_CALL_2(ring
->free_tx_desc
,
463 mlx4_en_free_tx_desc
,
464 mlx4_en_recycle_tx_desc
,
465 priv
, ring
, ring_index
,
466 timestamp
, napi_budget
);
468 mlx4_en_stamp_wqe(priv
, ring
, stamp_index
,
469 !!((ring_cons
+ txbbs_stamp
) &
471 stamp_index
= ring_index
;
472 txbbs_stamp
= txbbs_skipped
;
474 bytes
+= ring
->tx_info
[ring_index
].nr_bytes
;
475 } while ((++done
< budget
) && (ring_index
!= new_index
));
478 index
= cons_index
& size_mask
;
479 cqe
= mlx4_en_get_cqe(buf
, index
, priv
->cqe_size
) + factor
;
483 * To prevent CQ overflow we first update CQ consumer and only then
486 mcq
->cons_index
= cons_index
;
490 /* we want to dirty this cache line once */
491 WRITE_ONCE(ring
->last_nr_txbb
, last_nr_txbb
);
492 WRITE_ONCE(ring
->cons
, ring_cons
+ txbbs_skipped
);
494 if (cq
->type
== TX_XDP
)
495 return done
< budget
;
497 netdev_tx_completed_queue(ring
->tx_queue
, packets
, bytes
);
499 /* Wakeup Tx queue if this stopped, and ring is not full.
501 if (netif_tx_queue_stopped(ring
->tx_queue
) &&
502 !mlx4_en_is_tx_ring_full(ring
)) {
503 netif_tx_wake_queue(ring
->tx_queue
);
507 return done
< budget
;
510 void mlx4_en_tx_irq(struct mlx4_cq
*mcq
)
512 struct mlx4_en_cq
*cq
= container_of(mcq
, struct mlx4_en_cq
, mcq
);
513 struct mlx4_en_priv
*priv
= netdev_priv(cq
->dev
);
515 if (likely(priv
->port_up
))
516 napi_schedule_irqoff(&cq
->napi
);
518 mlx4_en_arm_cq(priv
, cq
);
521 /* TX CQ polling - called by NAPI */
522 int mlx4_en_poll_tx_cq(struct napi_struct
*napi
, int budget
)
524 struct mlx4_en_cq
*cq
= container_of(napi
, struct mlx4_en_cq
, napi
);
525 struct net_device
*dev
= cq
->dev
;
526 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
529 clean_complete
= mlx4_en_process_tx_cq(dev
, cq
, budget
);
534 mlx4_en_arm_cq(priv
, cq
);
539 static struct mlx4_en_tx_desc
*mlx4_en_bounce_to_desc(struct mlx4_en_priv
*priv
,
540 struct mlx4_en_tx_ring
*ring
,
542 unsigned int desc_size
)
544 u32 copy
= (ring
->size
- index
) << LOG_TXBB_SIZE
;
547 for (i
= desc_size
- copy
- 4; i
>= 0; i
-= 4) {
548 if ((i
& (TXBB_SIZE
- 1)) == 0)
551 *((u32
*) (ring
->buf
+ i
)) =
552 *((u32
*) (ring
->bounce_buf
+ copy
+ i
));
555 for (i
= copy
- 4; i
>= 4 ; i
-= 4) {
556 if ((i
& (TXBB_SIZE
- 1)) == 0)
559 *((u32
*)(ring
->buf
+ (index
<< LOG_TXBB_SIZE
) + i
)) =
560 *((u32
*) (ring
->bounce_buf
+ i
));
563 /* Return real descriptor location */
564 return ring
->buf
+ (index
<< LOG_TXBB_SIZE
);
567 /* Decide if skb can be inlined in tx descriptor to avoid dma mapping
569 * It seems strange we do not simply use skb_copy_bits().
570 * This would allow to inline all skbs iff skb->len <= inline_thold
572 * Note that caller already checked skb was not a gso packet
574 static bool is_inline(int inline_thold
, const struct sk_buff
*skb
,
575 const struct skb_shared_info
*shinfo
,
580 if (skb
->len
> inline_thold
|| !inline_thold
)
583 if (shinfo
->nr_frags
== 1) {
584 ptr
= skb_frag_address_safe(&shinfo
->frags
[0]);
590 if (shinfo
->nr_frags
)
595 static int inline_size(const struct sk_buff
*skb
)
597 if (skb
->len
+ CTRL_SIZE
+ sizeof(struct mlx4_wqe_inline_seg
)
598 <= MLX4_INLINE_ALIGN
)
599 return ALIGN(skb
->len
+ CTRL_SIZE
+
600 sizeof(struct mlx4_wqe_inline_seg
), 16);
602 return ALIGN(skb
->len
+ CTRL_SIZE
+ 2 *
603 sizeof(struct mlx4_wqe_inline_seg
), 16);
606 static int get_real_size(const struct sk_buff
*skb
,
607 const struct skb_shared_info
*shinfo
,
608 struct net_device
*dev
,
609 int *lso_header_size
,
613 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
616 if (shinfo
->gso_size
) {
618 if (skb
->encapsulation
)
619 *lso_header_size
= (skb_inner_transport_header(skb
) - skb
->data
) + inner_tcp_hdrlen(skb
);
621 *lso_header_size
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
622 real_size
= CTRL_SIZE
+ shinfo
->nr_frags
* DS_SIZE
+
623 ALIGN(*lso_header_size
+ 4, DS_SIZE
);
624 if (unlikely(*lso_header_size
!= skb_headlen(skb
))) {
625 /* We add a segment for the skb linear buffer only if
626 * it contains data */
627 if (*lso_header_size
< skb_headlen(skb
))
628 real_size
+= DS_SIZE
;
630 if (netif_msg_tx_err(priv
))
631 en_warn(priv
, "Non-linear headers\n");
636 *lso_header_size
= 0;
637 *inline_ok
= is_inline(priv
->prof
->inline_thold
, skb
,
641 real_size
= inline_size(skb
);
643 real_size
= CTRL_SIZE
+
644 (shinfo
->nr_frags
+ 1) * DS_SIZE
;
650 static void build_inline_wqe(struct mlx4_en_tx_desc
*tx_desc
,
651 const struct sk_buff
*skb
,
652 const struct skb_shared_info
*shinfo
,
655 struct mlx4_wqe_inline_seg
*inl
= &tx_desc
->inl
;
656 int spc
= MLX4_INLINE_ALIGN
- CTRL_SIZE
- sizeof(*inl
);
657 unsigned int hlen
= skb_headlen(skb
);
659 if (skb
->len
<= spc
) {
660 if (likely(skb
->len
>= MIN_PKT_LEN
)) {
661 inl
->byte_count
= cpu_to_be32(1 << 31 | skb
->len
);
663 inl
->byte_count
= cpu_to_be32(1 << 31 | MIN_PKT_LEN
);
664 memset(((void *)(inl
+ 1)) + skb
->len
, 0,
665 MIN_PKT_LEN
- skb
->len
);
667 skb_copy_from_linear_data(skb
, inl
+ 1, hlen
);
668 if (shinfo
->nr_frags
)
669 memcpy(((void *)(inl
+ 1)) + hlen
, fragptr
,
670 skb_frag_size(&shinfo
->frags
[0]));
673 inl
->byte_count
= cpu_to_be32(1 << 31 | spc
);
675 skb_copy_from_linear_data(skb
, inl
+ 1, hlen
);
677 memcpy(((void *)(inl
+ 1)) + hlen
,
678 fragptr
, spc
- hlen
);
679 fragptr
+= spc
- hlen
;
681 inl
= (void *) (inl
+ 1) + spc
;
682 memcpy(((void *)(inl
+ 1)), fragptr
, skb
->len
- spc
);
684 skb_copy_from_linear_data(skb
, inl
+ 1, spc
);
685 inl
= (void *) (inl
+ 1) + spc
;
686 skb_copy_from_linear_data_offset(skb
, spc
, inl
+ 1,
688 if (shinfo
->nr_frags
)
689 memcpy(((void *)(inl
+ 1)) + hlen
- spc
,
691 skb_frag_size(&shinfo
->frags
[0]));
695 inl
->byte_count
= cpu_to_be32(1 << 31 | (skb
->len
- spc
));
699 u16
mlx4_en_select_queue(struct net_device
*dev
, struct sk_buff
*skb
,
700 struct net_device
*sb_dev
)
702 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
703 u16 rings_p_up
= priv
->num_tx_rings_p_up
;
705 if (netdev_get_num_tc(dev
))
706 return netdev_pick_tx(dev
, skb
, NULL
);
708 return netdev_pick_tx(dev
, skb
, NULL
) % rings_p_up
;
711 static void mlx4_bf_copy(void __iomem
*dst
, const void *src
,
712 unsigned int bytecnt
)
714 __iowrite64_copy(dst
, src
, bytecnt
/ 8);
717 void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring
*ring
)
720 /* Since there is no iowrite*_native() that writes the
721 * value as is, without byteswapping - using the one
722 * the doesn't do byteswapping in the relevant arch
725 #if defined(__LITTLE_ENDIAN)
730 (__force u32
)ring
->doorbell_qpn
,
731 ring
->bf
.uar
->map
+ MLX4_SEND_DOORBELL
);
734 static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring
*ring
,
735 struct mlx4_en_tx_desc
*tx_desc
,
736 union mlx4_wqe_qpn_vlan qpn_vlan
,
737 int desc_size
, int bf_index
,
738 __be32 op_own
, bool bf_ok
,
741 tx_desc
->ctrl
.qpn_vlan
= qpn_vlan
;
744 op_own
|= htonl((bf_index
& 0xffff) << 8);
745 /* Ensure new descriptor hits memory
746 * before setting ownership of this descriptor to HW
749 tx_desc
->ctrl
.owner_opcode
= op_own
;
753 mlx4_bf_copy(ring
->bf
.reg
+ ring
->bf
.offset
, &tx_desc
->ctrl
,
758 ring
->bf
.offset
^= ring
->bf
.buf_size
;
760 /* Ensure new descriptor hits memory
761 * before setting ownership of this descriptor to HW
764 tx_desc
->ctrl
.owner_opcode
= op_own
;
766 mlx4_en_xmit_doorbell(ring
);
772 static bool mlx4_en_build_dma_wqe(struct mlx4_en_priv
*priv
,
773 struct skb_shared_info
*shinfo
,
774 struct mlx4_wqe_data_seg
*data
,
778 struct mlx4_en_tx_info
*tx_info
)
780 struct device
*ddev
= priv
->ddev
;
785 /* Map fragments if any */
786 for (i_frag
= shinfo
->nr_frags
- 1; i_frag
>= 0; i_frag
--) {
787 const skb_frag_t
*frag
= &shinfo
->frags
[i_frag
];
788 byte_count
= skb_frag_size(frag
);
789 dma
= skb_frag_dma_map(ddev
, frag
,
792 if (dma_mapping_error(ddev
, dma
))
795 data
->addr
= cpu_to_be64(dma
);
798 data
->byte_count
= cpu_to_be32(byte_count
);
802 /* Map linear part if needed */
803 if (tx_info
->linear
) {
804 byte_count
= skb_headlen(skb
) - lso_header_size
;
806 dma
= dma_map_single(ddev
, skb
->data
+
807 lso_header_size
, byte_count
,
809 if (dma_mapping_error(ddev
, dma
))
812 data
->addr
= cpu_to_be64(dma
);
815 data
->byte_count
= cpu_to_be32(byte_count
);
817 /* tx completion can avoid cache line miss for common cases */
818 tx_info
->map0_dma
= dma
;
819 tx_info
->map0_byte_count
= byte_count
;
824 en_err(priv
, "DMA mapping error\n");
826 while (++i_frag
< shinfo
->nr_frags
) {
828 dma_unmap_page(ddev
, (dma_addr_t
)be64_to_cpu(data
->addr
),
829 be32_to_cpu(data
->byte_count
),
836 netdev_tx_t
mlx4_en_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
838 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
839 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
840 union mlx4_wqe_qpn_vlan qpn_vlan
= {};
841 struct mlx4_en_tx_ring
*ring
;
842 struct mlx4_en_tx_desc
*tx_desc
;
843 struct mlx4_wqe_data_seg
*data
;
844 struct mlx4_en_tx_info
*tx_info
;
852 void *fragptr
= NULL
;
861 tx_ind
= skb_get_queue_mapping(skb
);
862 ring
= priv
->tx_ring
[TX
][tx_ind
];
864 if (unlikely(!priv
->port_up
))
867 /* fetch ring->cons far ahead before needing it to avoid stall */
868 ring_cons
= READ_ONCE(ring
->cons
);
870 real_size
= get_real_size(skb
, shinfo
, dev
, &lso_header_size
,
871 &inline_ok
, &fragptr
);
872 if (unlikely(!real_size
))
875 /* Align descriptor to TXBB size */
876 desc_size
= ALIGN(real_size
, TXBB_SIZE
);
877 nr_txbb
= desc_size
>> LOG_TXBB_SIZE
;
878 if (unlikely(nr_txbb
> MAX_DESC_TXBBS
)) {
879 if (netif_msg_tx_err(priv
))
880 en_warn(priv
, "Oversized header or SG list\n");
884 bf_ok
= ring
->bf_enabled
;
885 if (skb_vlan_tag_present(skb
)) {
888 qpn_vlan
.vlan_tag
= cpu_to_be16(skb_vlan_tag_get(skb
));
889 vlan_proto
= be16_to_cpu(skb
->vlan_proto
);
890 if (vlan_proto
== ETH_P_8021AD
)
891 qpn_vlan
.ins_vlan
= MLX4_WQE_CTRL_INS_SVLAN
;
892 else if (vlan_proto
== ETH_P_8021Q
)
893 qpn_vlan
.ins_vlan
= MLX4_WQE_CTRL_INS_CVLAN
;
895 qpn_vlan
.ins_vlan
= 0;
899 netdev_txq_bql_enqueue_prefetchw(ring
->tx_queue
);
901 /* Track current inflight packets for performance analysis */
902 AVG_PERF_COUNTER(priv
->pstats
.inflight_avg
,
903 (u32
)(ring
->prod
- ring_cons
- 1));
905 /* Packet is good - grab an index and transmit it */
906 index
= ring
->prod
& ring
->size_mask
;
907 bf_index
= ring
->prod
;
909 /* See if we have enough space for whole descriptor TXBB for setting
910 * SW ownership on next descriptor; if not, use a bounce buffer. */
911 if (likely(index
+ nr_txbb
<= ring
->size
))
912 tx_desc
= ring
->buf
+ (index
<< LOG_TXBB_SIZE
);
914 tx_desc
= (struct mlx4_en_tx_desc
*) ring
->bounce_buf
;
919 /* Save skb in tx_info ring */
920 tx_info
= &ring
->tx_info
[index
];
922 tx_info
->nr_txbb
= nr_txbb
;
924 if (!lso_header_size
) {
925 data
= &tx_desc
->data
;
926 data_offset
= offsetof(struct mlx4_en_tx_desc
, data
);
928 int lso_align
= ALIGN(lso_header_size
+ 4, DS_SIZE
);
930 data
= (void *)&tx_desc
->lso
+ lso_align
;
931 data_offset
= offsetof(struct mlx4_en_tx_desc
, lso
) + lso_align
;
934 /* valid only for none inline segments */
935 tx_info
->data_offset
= data_offset
;
937 tx_info
->inl
= inline_ok
;
939 tx_info
->linear
= lso_header_size
< skb_headlen(skb
) && !inline_ok
;
941 tx_info
->nr_maps
= shinfo
->nr_frags
+ tx_info
->linear
;
942 data
+= tx_info
->nr_maps
- 1;
945 if (!mlx4_en_build_dma_wqe(priv
, shinfo
, data
, skb
,
946 lso_header_size
, ring
->mr_key
,
951 * For timestamping add flag to skb_shinfo and
952 * set flag for further reference
954 tx_info
->ts_requested
= 0;
955 if (unlikely(ring
->hwtstamp_tx_type
== HWTSTAMP_TX_ON
&&
956 shinfo
->tx_flags
& SKBTX_HW_TSTAMP
)) {
957 shinfo
->tx_flags
|= SKBTX_IN_PROGRESS
;
958 tx_info
->ts_requested
= 1;
961 /* Prepare ctrl segement apart opcode+ownership, which depends on
962 * whether LSO is used */
963 tx_desc
->ctrl
.srcrb_flags
= priv
->ctrl_flags
;
964 if (likely(skb
->ip_summed
== CHECKSUM_PARTIAL
)) {
965 if (!skb
->encapsulation
)
966 tx_desc
->ctrl
.srcrb_flags
|= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM
|
967 MLX4_WQE_CTRL_TCP_UDP_CSUM
);
969 tx_desc
->ctrl
.srcrb_flags
|= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM
);
973 if (priv
->flags
& MLX4_EN_FLAG_ENABLE_HW_LOOPBACK
) {
976 /* Copy dst mac address to wqe. This allows loopback in eSwitch,
977 * so that VFs and PF can communicate with each other
979 ethh
= (struct ethhdr
*)skb
->data
;
980 tx_desc
->ctrl
.srcrb_flags16
[0] = get_unaligned((__be16
*)ethh
->h_dest
);
981 tx_desc
->ctrl
.imm
= get_unaligned((__be32
*)(ethh
->h_dest
+ 2));
984 /* Handle LSO (TSO) packets */
985 if (lso_header_size
) {
988 /* Mark opcode as LSO */
989 op_own
= cpu_to_be32(MLX4_OPCODE_LSO
| (1 << 6)) |
990 ((ring
->prod
& ring
->size
) ?
991 cpu_to_be32(MLX4_EN_BIT_DESC_OWN
) : 0);
993 /* Fill in the LSO prefix */
994 tx_desc
->lso
.mss_hdr_size
= cpu_to_be32(
995 shinfo
->gso_size
<< 16 | lso_header_size
);
998 * note that we already verified that it is linear */
999 memcpy(tx_desc
->lso
.header
, skb
->data
, lso_header_size
);
1001 ring
->tso_packets
++;
1003 i
= shinfo
->gso_segs
;
1004 tx_info
->nr_bytes
= skb
->len
+ (i
- 1) * lso_header_size
;
1007 /* Normal (Non LSO) packet */
1008 op_own
= cpu_to_be32(MLX4_OPCODE_SEND
) |
1009 ((ring
->prod
& ring
->size
) ?
1010 cpu_to_be32(MLX4_EN_BIT_DESC_OWN
) : 0);
1011 tx_info
->nr_bytes
= max_t(unsigned int, skb
->len
, ETH_ZLEN
);
1014 ring
->bytes
+= tx_info
->nr_bytes
;
1015 AVG_PERF_COUNTER(priv
->pstats
.tx_pktsz_avg
, skb
->len
);
1018 build_inline_wqe(tx_desc
, skb
, shinfo
, fragptr
);
1020 if (skb
->encapsulation
) {
1028 ip
.hdr
= skb_inner_network_header(skb
);
1029 proto
= (ip
.v4
->version
== 4) ? ip
.v4
->protocol
:
1032 if (proto
== IPPROTO_TCP
|| proto
== IPPROTO_UDP
)
1033 op_own
|= cpu_to_be32(MLX4_WQE_CTRL_IIP
| MLX4_WQE_CTRL_ILP
);
1035 op_own
|= cpu_to_be32(MLX4_WQE_CTRL_IIP
);
1038 ring
->prod
+= nr_txbb
;
1040 /* If we used a bounce buffer then copy descriptor back into place */
1041 if (unlikely(bounce
))
1042 tx_desc
= mlx4_en_bounce_to_desc(priv
, ring
, index
, desc_size
);
1044 skb_tx_timestamp(skb
);
1046 /* Check available TXBBs And 2K spare for prefetch */
1047 stop_queue
= mlx4_en_is_tx_ring_full(ring
);
1048 if (unlikely(stop_queue
)) {
1049 netif_tx_stop_queue(ring
->tx_queue
);
1050 ring
->queue_stopped
++;
1053 send_doorbell
= __netdev_tx_sent_queue(ring
->tx_queue
,
1055 netdev_xmit_more());
1057 real_size
= (real_size
/ 16) & 0x3f;
1059 bf_ok
&= desc_size
<= MAX_BF
&& send_doorbell
;
1062 qpn_vlan
.bf_qpn
= ring
->doorbell_qpn
| cpu_to_be32(real_size
);
1064 qpn_vlan
.fence_size
= real_size
;
1066 mlx4_en_tx_write_desc(ring
, tx_desc
, qpn_vlan
, desc_size
, bf_index
,
1067 op_own
, bf_ok
, send_doorbell
);
1069 if (unlikely(stop_queue
)) {
1070 /* If queue was emptied after the if (stop_queue) , and before
1071 * the netif_tx_stop_queue() - need to wake the queue,
1072 * or else it will remain stopped forever.
1073 * Need a memory barrier to make sure ring->cons was not
1074 * updated before queue was stopped.
1078 ring_cons
= READ_ONCE(ring
->cons
);
1079 if (unlikely(!mlx4_en_is_tx_ring_full(ring
))) {
1080 netif_tx_wake_queue(ring
->tx_queue
);
1084 return NETDEV_TX_OK
;
1089 dev_kfree_skb_any(skb
);
1090 return NETDEV_TX_OK
;
1093 #define MLX4_EN_XDP_TX_NRTXBB 1
1094 #define MLX4_EN_XDP_TX_REAL_SZ (((CTRL_SIZE + MLX4_EN_XDP_TX_NRTXBB * DS_SIZE) \
1097 void mlx4_en_init_tx_xdp_ring_descs(struct mlx4_en_priv
*priv
,
1098 struct mlx4_en_tx_ring
*ring
)
1102 for (i
= 0; i
< ring
->size
; i
++) {
1103 struct mlx4_en_tx_info
*tx_info
= &ring
->tx_info
[i
];
1104 struct mlx4_en_tx_desc
*tx_desc
= ring
->buf
+
1105 (i
<< LOG_TXBB_SIZE
);
1107 tx_info
->map0_byte_count
= PAGE_SIZE
;
1108 tx_info
->nr_txbb
= MLX4_EN_XDP_TX_NRTXBB
;
1109 tx_info
->data_offset
= offsetof(struct mlx4_en_tx_desc
, data
);
1110 tx_info
->ts_requested
= 0;
1111 tx_info
->nr_maps
= 1;
1112 tx_info
->linear
= 1;
1115 tx_desc
->data
.lkey
= ring
->mr_key
;
1116 tx_desc
->ctrl
.qpn_vlan
.fence_size
= MLX4_EN_XDP_TX_REAL_SZ
;
1117 tx_desc
->ctrl
.srcrb_flags
= priv
->ctrl_flags
;
1121 netdev_tx_t
mlx4_en_xmit_frame(struct mlx4_en_rx_ring
*rx_ring
,
1122 struct mlx4_en_rx_alloc
*frame
,
1123 struct mlx4_en_priv
*priv
, unsigned int length
,
1124 int tx_ind
, bool *doorbell_pending
)
1126 struct mlx4_en_tx_desc
*tx_desc
;
1127 struct mlx4_en_tx_info
*tx_info
;
1128 struct mlx4_wqe_data_seg
*data
;
1129 struct mlx4_en_tx_ring
*ring
;
1134 if (unlikely(!priv
->port_up
))
1137 ring
= priv
->tx_ring
[TX_XDP
][tx_ind
];
1139 if (unlikely(mlx4_en_is_tx_ring_full(ring
)))
1142 index
= ring
->prod
& ring
->size_mask
;
1143 tx_info
= &ring
->tx_info
[index
];
1145 /* Track current inflight packets for performance analysis */
1146 AVG_PERF_COUNTER(priv
->pstats
.inflight_avg
,
1147 (u32
)(ring
->prod
- READ_ONCE(ring
->cons
) - 1));
1149 tx_desc
= ring
->buf
+ (index
<< LOG_TXBB_SIZE
);
1150 data
= &tx_desc
->data
;
1154 tx_info
->page
= frame
->page
;
1156 tx_info
->map0_dma
= dma
;
1157 tx_info
->nr_bytes
= max_t(unsigned int, length
, ETH_ZLEN
);
1159 dma_sync_single_range_for_device(priv
->ddev
, dma
, frame
->page_offset
,
1160 length
, PCI_DMA_TODEVICE
);
1162 data
->addr
= cpu_to_be64(dma
+ frame
->page_offset
);
1164 data
->byte_count
= cpu_to_be32(length
);
1166 /* tx completion can avoid cache line miss for common cases */
1168 op_own
= cpu_to_be32(MLX4_OPCODE_SEND
) |
1169 ((ring
->prod
& ring
->size
) ?
1170 cpu_to_be32(MLX4_EN_BIT_DESC_OWN
) : 0);
1173 AVG_PERF_COUNTER(priv
->pstats
.tx_pktsz_avg
, length
);
1175 ring
->prod
+= MLX4_EN_XDP_TX_NRTXBB
;
1177 /* Ensure new descriptor hits memory
1178 * before setting ownership of this descriptor to HW
1181 tx_desc
->ctrl
.owner_opcode
= op_own
;
1184 *doorbell_pending
= true;
1186 return NETDEV_TX_OK
;
1189 rx_ring
->xdp_tx_full
++;
1190 *doorbell_pending
= true;
1192 return NETDEV_TX_BUSY
;