2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/prefetch.h>
35 #include <linux/ipv6.h>
36 #include <linux/tcp.h>
37 #include <net/ip6_checksum.h>
38 #include <net/page_pool.h>
39 #include <net/inet_ecn.h>
44 #include "ipoib/ipoib.h"
45 #include "en_accel/ipsec_rxtx.h"
46 #include "en_accel/tls_rxtx.h"
47 #include "lib/clock.h"
50 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config
*config
)
52 return config
->rx_filter
== HWTSTAMP_FILTER_ALL
;
55 static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq
*wq
,
58 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
60 memcpy(data
, mlx5_cqwq_get_wqe(wq
, ci
), sizeof(struct mlx5_cqe64
));
63 static inline void mlx5e_read_title_slot(struct mlx5e_rq
*rq
,
67 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
68 struct mlx5_cqe64
*title
= &cqd
->title
;
70 mlx5e_read_cqe_slot(wq
, cqcc
, title
);
71 cqd
->left
= be32_to_cpu(title
->byte_cnt
);
72 cqd
->wqe_counter
= be16_to_cpu(title
->wqe_counter
);
73 rq
->stats
->cqe_compress_blks
++;
76 static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq
*wq
,
77 struct mlx5e_cq_decomp
*cqd
,
80 mlx5e_read_cqe_slot(wq
, cqcc
, cqd
->mini_arr
);
81 cqd
->mini_arr_idx
= 0;
84 static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq
*wq
, int n
)
87 u8 op_own
= mlx5_cqwq_get_ctr_wrap_cnt(wq
, cqcc
) & 1;
88 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
89 u32 wq_sz
= mlx5_cqwq_get_size(wq
);
90 u32 ci_top
= min_t(u32
, wq_sz
, ci
+ n
);
92 for (; ci
< ci_top
; ci
++, n
--) {
93 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
98 if (unlikely(ci
== wq_sz
)) {
100 for (ci
= 0; ci
< n
; ci
++) {
101 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
103 cqe
->op_own
= op_own
;
108 static inline void mlx5e_decompress_cqe(struct mlx5e_rq
*rq
,
109 struct mlx5_cqwq
*wq
,
112 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
113 struct mlx5_mini_cqe8
*mini_cqe
= &cqd
->mini_arr
[cqd
->mini_arr_idx
];
114 struct mlx5_cqe64
*title
= &cqd
->title
;
116 title
->byte_cnt
= mini_cqe
->byte_cnt
;
117 title
->check_sum
= mini_cqe
->checksum
;
118 title
->op_own
&= 0xf0;
119 title
->op_own
|= 0x01 & (cqcc
>> wq
->fbc
.log_sz
);
120 title
->wqe_counter
= cpu_to_be16(cqd
->wqe_counter
);
122 if (rq
->wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
)
123 cqd
->wqe_counter
+= mpwrq_get_cqe_consumed_strides(title
);
126 mlx5_wq_cyc_ctr2ix(&rq
->wqe
.wq
, cqd
->wqe_counter
+ 1);
129 static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq
*rq
,
130 struct mlx5_cqwq
*wq
,
133 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
135 mlx5e_decompress_cqe(rq
, wq
, cqcc
);
136 cqd
->title
.rss_hash_type
= 0;
137 cqd
->title
.rss_hash_result
= 0;
140 static inline u32
mlx5e_decompress_cqes_cont(struct mlx5e_rq
*rq
,
141 struct mlx5_cqwq
*wq
,
142 int update_owner_only
,
145 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
146 u32 cqcc
= wq
->cc
+ update_owner_only
;
150 cqe_count
= min_t(u32
, cqd
->left
, budget_rem
);
152 for (i
= update_owner_only
; i
< cqe_count
;
153 i
++, cqd
->mini_arr_idx
++, cqcc
++) {
154 if (cqd
->mini_arr_idx
== MLX5_MINI_CQE_ARRAY_SIZE
)
155 mlx5e_read_mini_arr_slot(wq
, cqd
, cqcc
);
157 mlx5e_decompress_cqe_no_hash(rq
, wq
, cqcc
);
158 rq
->handle_rx_cqe(rq
, &cqd
->title
);
160 mlx5e_cqes_update_owner(wq
, cqcc
- wq
->cc
);
162 cqd
->left
-= cqe_count
;
163 rq
->stats
->cqe_compress_pkts
+= cqe_count
;
168 static inline u32
mlx5e_decompress_cqes_start(struct mlx5e_rq
*rq
,
169 struct mlx5_cqwq
*wq
,
172 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
175 mlx5e_read_title_slot(rq
, wq
, cc
);
176 mlx5e_read_mini_arr_slot(wq
, cqd
, cc
+ 1);
177 mlx5e_decompress_cqe(rq
, wq
, cc
);
178 rq
->handle_rx_cqe(rq
, &cqd
->title
);
181 return mlx5e_decompress_cqes_cont(rq
, wq
, 1, budget_rem
) - 1;
184 static inline bool mlx5e_page_is_reserved(struct page
*page
)
186 return page_is_pfmemalloc(page
) || page_to_nid(page
) != numa_mem_id();
189 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq
*rq
,
190 struct mlx5e_dma_info
*dma_info
)
192 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
193 u32 tail_next
= (cache
->tail
+ 1) & (MLX5E_CACHE_SIZE
- 1);
194 struct mlx5e_rq_stats
*stats
= rq
->stats
;
196 if (tail_next
== cache
->head
) {
201 if (unlikely(mlx5e_page_is_reserved(dma_info
->page
))) {
202 stats
->cache_waive
++;
206 cache
->page_cache
[cache
->tail
] = *dma_info
;
207 cache
->tail
= tail_next
;
211 static inline bool mlx5e_rx_cache_get(struct mlx5e_rq
*rq
,
212 struct mlx5e_dma_info
*dma_info
)
214 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
215 struct mlx5e_rq_stats
*stats
= rq
->stats
;
217 if (unlikely(cache
->head
== cache
->tail
)) {
218 stats
->cache_empty
++;
222 if (page_ref_count(cache
->page_cache
[cache
->head
].page
) != 1) {
227 *dma_info
= cache
->page_cache
[cache
->head
];
228 cache
->head
= (cache
->head
+ 1) & (MLX5E_CACHE_SIZE
- 1);
229 stats
->cache_reuse
++;
231 dma_sync_single_for_device(rq
->pdev
, dma_info
->addr
,
237 static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq
*rq
,
238 struct mlx5e_dma_info
*dma_info
)
240 if (mlx5e_rx_cache_get(rq
, dma_info
))
243 dma_info
->page
= page_pool_dev_alloc_pages(rq
->page_pool
);
244 if (unlikely(!dma_info
->page
))
247 dma_info
->addr
= dma_map_page(rq
->pdev
, dma_info
->page
, 0,
248 PAGE_SIZE
, rq
->buff
.map_dir
);
249 if (unlikely(dma_mapping_error(rq
->pdev
, dma_info
->addr
))) {
250 put_page(dma_info
->page
);
251 dma_info
->page
= NULL
;
258 void mlx5e_page_dma_unmap(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
)
260 dma_unmap_page(rq
->pdev
, dma_info
->addr
, PAGE_SIZE
, rq
->buff
.map_dir
);
263 void mlx5e_page_release(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
,
266 if (likely(recycle
)) {
267 if (mlx5e_rx_cache_put(rq
, dma_info
))
270 mlx5e_page_dma_unmap(rq
, dma_info
);
271 page_pool_recycle_direct(rq
->page_pool
, dma_info
->page
);
273 mlx5e_page_dma_unmap(rq
, dma_info
);
274 put_page(dma_info
->page
);
278 static inline int mlx5e_get_rx_frag(struct mlx5e_rq
*rq
,
279 struct mlx5e_wqe_frag_info
*frag
)
284 /* On first frag (offset == 0), replenish page (dma_info actually).
285 * Other frags that point to the same dma_info (with a different
286 * offset) should just use the new one without replenishing again
289 err
= mlx5e_page_alloc_mapped(rq
, frag
->di
);
294 static inline void mlx5e_put_rx_frag(struct mlx5e_rq
*rq
,
295 struct mlx5e_wqe_frag_info
*frag
,
298 if (frag
->last_in_page
)
299 mlx5e_page_release(rq
, frag
->di
, recycle
);
302 static inline struct mlx5e_wqe_frag_info
*get_frag(struct mlx5e_rq
*rq
, u16 ix
)
304 return &rq
->wqe
.frags
[ix
<< rq
->wqe
.info
.log_num_frags
];
307 static int mlx5e_alloc_rx_wqe(struct mlx5e_rq
*rq
, struct mlx5e_rx_wqe_cyc
*wqe
,
310 struct mlx5e_wqe_frag_info
*frag
= get_frag(rq
, ix
);
314 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, frag
++) {
315 err
= mlx5e_get_rx_frag(rq
, frag
);
319 wqe
->data
[i
].addr
= cpu_to_be64(frag
->di
->addr
+
320 frag
->offset
+ rq
->buff
.headroom
);
327 mlx5e_put_rx_frag(rq
, --frag
, true);
332 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq
*rq
,
333 struct mlx5e_wqe_frag_info
*wi
,
338 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, wi
++)
339 mlx5e_put_rx_frag(rq
, wi
, recycle
);
342 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq
*rq
, u16 ix
)
344 struct mlx5e_wqe_frag_info
*wi
= get_frag(rq
, ix
);
346 mlx5e_free_rx_wqe(rq
, wi
, false);
349 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq
*rq
, u16 ix
, u8 wqe_bulk
)
351 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
355 for (i
= 0; i
< wqe_bulk
; i
++) {
356 struct mlx5e_rx_wqe_cyc
*wqe
= mlx5_wq_cyc_get_wqe(wq
, ix
+ i
);
358 err
= mlx5e_alloc_rx_wqe(rq
, wqe
, ix
+ i
);
367 mlx5e_dealloc_rx_wqe(rq
, ix
+ i
);
373 mlx5e_add_skb_frag(struct mlx5e_rq
*rq
, struct sk_buff
*skb
,
374 struct mlx5e_dma_info
*di
, u32 frag_offset
, u32 len
,
375 unsigned int truesize
)
377 dma_sync_single_for_cpu(rq
->pdev
,
378 di
->addr
+ frag_offset
,
379 len
, DMA_FROM_DEVICE
);
380 page_ref_inc(di
->page
);
381 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
,
382 di
->page
, frag_offset
, len
, truesize
);
386 mlx5e_copy_skb_header(struct device
*pdev
, struct sk_buff
*skb
,
387 struct mlx5e_dma_info
*dma_info
,
388 int offset_from
, u32 headlen
)
390 const void *from
= page_address(dma_info
->page
) + offset_from
;
391 /* Aligning len to sizeof(long) optimizes memcpy performance */
392 unsigned int len
= ALIGN(headlen
, sizeof(long));
394 dma_sync_single_for_cpu(pdev
, dma_info
->addr
+ offset_from
, len
,
396 skb_copy_to_linear_data(skb
, from
, len
);
400 mlx5e_free_rx_mpwqe(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
, bool recycle
)
402 const bool no_xdp_xmit
=
403 bitmap_empty(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
404 struct mlx5e_dma_info
*dma_info
= wi
->umr
.dma_info
;
407 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++)
408 if (no_xdp_xmit
|| !test_bit(i
, wi
->xdp_xmit_bitmap
))
409 mlx5e_page_release(rq
, &dma_info
[i
], recycle
);
412 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq
*rq
)
414 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
415 struct mlx5e_rx_wqe_ll
*wqe
= mlx5_wq_ll_get_wqe(wq
, wq
->head
);
417 rq
->mpwqe
.umr_in_progress
= false;
419 mlx5_wq_ll_push(wq
, be16_to_cpu(wqe
->next
.next_wqe_index
));
421 /* ensure wqes are visible to device before updating doorbell record */
424 mlx5_wq_ll_update_db_record(wq
);
427 static inline u16
mlx5e_icosq_wrap_cnt(struct mlx5e_icosq
*sq
)
429 return sq
->pc
>> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE
;
432 static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq
*sq
,
433 struct mlx5_wq_cyc
*wq
,
436 struct mlx5e_sq_wqe_info
*edge_wi
, *wi
= &sq
->db
.ico_wqe
[pi
];
438 edge_wi
= wi
+ nnops
;
440 /* fill sq frag edge with nops to avoid wqe wrapping two pages */
441 for (; wi
< edge_wi
; wi
++) {
442 wi
->opcode
= MLX5_OPCODE_NOP
;
443 mlx5e_post_nop(wq
, sq
->sqn
, &sq
->pc
);
447 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
449 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
450 struct mlx5e_dma_info
*dma_info
= &wi
->umr
.dma_info
[0];
451 struct mlx5e_icosq
*sq
= &rq
->channel
->icosq
;
452 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
453 struct mlx5e_umr_wqe
*umr_wqe
;
454 u16 xlt_offset
= ix
<< (MLX5E_LOG_ALIGNED_MPWQE_PPW
- 1);
455 u16 pi
, contig_wqebbs_room
;
459 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
460 contig_wqebbs_room
= mlx5_wq_cyc_get_contig_wqebbs(wq
, pi
);
461 if (unlikely(contig_wqebbs_room
< MLX5E_UMR_WQEBBS
)) {
462 mlx5e_fill_icosq_frag_edge(sq
, wq
, pi
, contig_wqebbs_room
);
463 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
466 umr_wqe
= mlx5_wq_cyc_get_wqe(wq
, pi
);
467 if (unlikely(mlx5e_icosq_wrap_cnt(sq
) < 2))
468 memcpy(umr_wqe
, &rq
->mpwqe
.umr_wqe
,
469 offsetof(struct mlx5e_umr_wqe
, inline_mtts
));
471 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++, dma_info
++) {
472 err
= mlx5e_page_alloc_mapped(rq
, dma_info
);
475 umr_wqe
->inline_mtts
[i
].ptag
= cpu_to_be64(dma_info
->addr
| MLX5_EN_WR
);
478 bitmap_zero(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
479 wi
->consumed_strides
= 0;
481 rq
->mpwqe
.umr_in_progress
= true;
483 umr_wqe
->ctrl
.opmod_idx_opcode
=
484 cpu_to_be32((sq
->pc
<< MLX5_WQE_CTRL_WQE_INDEX_SHIFT
) |
486 umr_wqe
->uctrl
.xlt_offset
= cpu_to_be16(xlt_offset
);
488 sq
->db
.ico_wqe
[pi
].opcode
= MLX5_OPCODE_UMR
;
489 sq
->pc
+= MLX5E_UMR_WQEBBS
;
490 mlx5e_notify_hw(wq
, sq
->pc
, sq
->uar_map
, &umr_wqe
->ctrl
);
497 mlx5e_page_release(rq
, dma_info
, true);
499 rq
->stats
->buff_alloc_err
++;
504 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
506 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
507 /* Don't recycle, this function is called on rq/netdev close */
508 mlx5e_free_rx_mpwqe(rq
, wi
, false);
511 bool mlx5e_post_rx_wqes(struct mlx5e_rq
*rq
)
513 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
517 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
520 wqe_bulk
= rq
->wqe
.info
.wqe_bulk
;
522 if (mlx5_wq_cyc_missing(wq
) < wqe_bulk
)
526 u16 head
= mlx5_wq_cyc_get_head(wq
);
528 err
= mlx5e_alloc_rx_wqes(rq
, head
, wqe_bulk
);
530 rq
->stats
->buff_alloc_err
++;
534 mlx5_wq_cyc_push_n(wq
, wqe_bulk
);
535 } while (mlx5_wq_cyc_missing(wq
) >= wqe_bulk
);
537 /* ensure wqes are visible to device before updating doorbell record */
540 mlx5_wq_cyc_update_db_record(wq
);
545 static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq
*cq
,
546 struct mlx5e_icosq
*sq
,
548 struct mlx5_cqe64
*cqe
)
550 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
551 u16 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
552 struct mlx5e_sq_wqe_info
*icowi
= &sq
->db
.ico_wqe
[ci
];
554 mlx5_cqwq_pop(&cq
->wq
);
556 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_REQ
)) {
557 netdev_WARN_ONCE(cq
->channel
->netdev
,
558 "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe
));
562 if (likely(icowi
->opcode
== MLX5_OPCODE_UMR
)) {
563 mlx5e_post_rx_mpwqe(rq
);
567 if (unlikely(icowi
->opcode
!= MLX5_OPCODE_NOP
))
568 netdev_WARN_ONCE(cq
->channel
->netdev
,
569 "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi
->opcode
);
572 static void mlx5e_poll_ico_cq(struct mlx5e_cq
*cq
, struct mlx5e_rq
*rq
)
574 struct mlx5e_icosq
*sq
= container_of(cq
, struct mlx5e_icosq
, cq
);
575 struct mlx5_cqe64
*cqe
;
577 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
)))
580 cqe
= mlx5_cqwq_get_cqe(&cq
->wq
);
584 /* by design, there's only a single cqe */
585 mlx5e_poll_ico_single_cqe(cq
, sq
, rq
, cqe
);
587 mlx5_cqwq_update_db_record(&cq
->wq
);
590 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq
*rq
)
592 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
594 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
597 mlx5e_poll_ico_cq(&rq
->channel
->icosq
.cq
, rq
);
599 if (mlx5_wq_ll_is_full(wq
))
602 if (!rq
->mpwqe
.umr_in_progress
)
603 mlx5e_alloc_rx_mpwqe(rq
, wq
->head
);
605 rq
->stats
->congst_umr
+= mlx5_wq_ll_missing(wq
) > 2;
610 static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64
*cqe
, struct tcphdr
*tcp
)
612 u8 l4_hdr_type
= get_cqe_l4_hdr_type(cqe
);
613 u8 tcp_ack
= (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA
) ||
614 (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA
);
617 tcp
->psh
= get_cqe_lro_tcppsh(cqe
);
621 tcp
->ack_seq
= cqe
->lro_ack_seq_num
;
622 tcp
->window
= cqe
->lro_tcp_win
;
626 static void mlx5e_lro_update_hdr(struct sk_buff
*skb
, struct mlx5_cqe64
*cqe
,
629 struct ethhdr
*eth
= (struct ethhdr
*)(skb
->data
);
631 int network_depth
= 0;
637 proto
= __vlan_get_protocol(skb
, eth
->h_proto
, &network_depth
);
639 tot_len
= cqe_bcnt
- network_depth
;
640 ip_p
= skb
->data
+ network_depth
;
642 if (proto
== htons(ETH_P_IP
)) {
643 struct iphdr
*ipv4
= ip_p
;
645 tcp
= ip_p
+ sizeof(struct iphdr
);
646 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
648 ipv4
->ttl
= cqe
->lro_min_ttl
;
649 ipv4
->tot_len
= cpu_to_be16(tot_len
);
651 ipv4
->check
= ip_fast_csum((unsigned char *)ipv4
,
654 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
655 check
= csum_partial(tcp
, tcp
->doff
* 4,
656 csum_unfold((__force __sum16
)cqe
->check_sum
));
657 /* Almost done, don't forget the pseudo header */
658 tcp
->check
= csum_tcpudp_magic(ipv4
->saddr
, ipv4
->daddr
,
659 tot_len
- sizeof(struct iphdr
),
662 u16 payload_len
= tot_len
- sizeof(struct ipv6hdr
);
663 struct ipv6hdr
*ipv6
= ip_p
;
665 tcp
= ip_p
+ sizeof(struct ipv6hdr
);
666 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
668 ipv6
->hop_limit
= cqe
->lro_min_ttl
;
669 ipv6
->payload_len
= cpu_to_be16(payload_len
);
671 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
672 check
= csum_partial(tcp
, tcp
->doff
* 4,
673 csum_unfold((__force __sum16
)cqe
->check_sum
));
674 /* Almost done, don't forget the pseudo header */
675 tcp
->check
= csum_ipv6_magic(&ipv6
->saddr
, &ipv6
->daddr
, payload_len
,
680 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64
*cqe
,
683 u8 cht
= cqe
->rss_hash_type
;
684 int ht
= (cht
& CQE_RSS_HTYPE_L4
) ? PKT_HASH_TYPE_L4
:
685 (cht
& CQE_RSS_HTYPE_IP
) ? PKT_HASH_TYPE_L3
:
687 skb_set_hash(skb
, be32_to_cpu(cqe
->rss_hash_result
), ht
);
690 static inline bool is_last_ethertype_ip(struct sk_buff
*skb
, int *network_depth
,
693 *proto
= ((struct ethhdr
*)skb
->data
)->h_proto
;
694 *proto
= __vlan_get_protocol(skb
, *proto
, network_depth
);
695 return (*proto
== htons(ETH_P_IP
) || *proto
== htons(ETH_P_IPV6
));
698 static inline void mlx5e_enable_ecn(struct mlx5e_rq
*rq
, struct sk_buff
*skb
)
700 int network_depth
= 0;
705 if (unlikely(!is_last_ethertype_ip(skb
, &network_depth
, &proto
)))
708 ip
= skb
->data
+ network_depth
;
709 rc
= ((proto
== htons(ETH_P_IP
)) ? IP_ECN_set_ce((struct iphdr
*)ip
) :
710 IP6_ECN_set_ce(skb
, (struct ipv6hdr
*)ip
));
712 rq
->stats
->ecn_mark
+= !!rc
;
715 static u32
mlx5e_get_fcs(const struct sk_buff
*skb
)
717 const void *fcs_bytes
;
720 fcs_bytes
= skb_header_pointer(skb
, skb
->len
- ETH_FCS_LEN
,
721 ETH_FCS_LEN
, &_fcs_bytes
);
723 return __get_unaligned_cpu32(fcs_bytes
);
726 static u8
get_ip_proto(struct sk_buff
*skb
, int network_depth
, __be16 proto
)
728 void *ip_p
= skb
->data
+ network_depth
;
730 return (proto
== htons(ETH_P_IP
)) ? ((struct iphdr
*)ip_p
)->protocol
:
731 ((struct ipv6hdr
*)ip_p
)->nexthdr
;
734 #define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
736 static inline void mlx5e_handle_csum(struct net_device
*netdev
,
737 struct mlx5_cqe64
*cqe
,
742 struct mlx5e_rq_stats
*stats
= rq
->stats
;
743 int network_depth
= 0;
746 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
750 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
751 stats
->csum_unnecessary
++;
755 if (unlikely(test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE
, &rq
->state
)))
756 goto csum_unnecessary
;
758 /* CQE csum doesn't cover padding octets in short ethernet
759 * frames. And the pad field is appended prior to calculating
760 * and appending the FCS field.
762 * Detecting these padded frames requires to verify and parse
763 * IP headers, so we simply force all those small frames to be
764 * CHECKSUM_UNNECESSARY even if they are not padded.
766 if (short_frame(skb
->len
))
767 goto csum_unnecessary
;
769 if (likely(is_last_ethertype_ip(skb
, &network_depth
, &proto
))) {
770 if (unlikely(get_ip_proto(skb
, network_depth
, proto
) == IPPROTO_SCTP
))
771 goto csum_unnecessary
;
773 skb
->ip_summed
= CHECKSUM_COMPLETE
;
774 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
775 if (network_depth
> ETH_HLEN
)
776 /* CQE csum is calculated from the IP header and does
777 * not cover VLAN headers (if present). This will add
778 * the checksum manually.
780 skb
->csum
= csum_partial(skb
->data
+ ETH_HLEN
,
781 network_depth
- ETH_HLEN
,
783 if (unlikely(netdev
->features
& NETIF_F_RXFCS
))
784 skb
->csum
= csum_block_add(skb
->csum
,
785 (__force __wsum
)mlx5e_get_fcs(skb
),
786 skb
->len
- ETH_FCS_LEN
);
787 stats
->csum_complete
++;
792 if (likely((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
793 ((cqe
->hds_ip_ext
& CQE_L4_OK
) ||
794 (get_cqe_l4_hdr_type(cqe
) == CQE_L4_HDR_TYPE_NONE
)))) {
795 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
796 if (cqe_is_tunneled(cqe
)) {
798 skb
->encapsulation
= 1;
799 stats
->csum_unnecessary_inner
++;
802 stats
->csum_unnecessary
++;
806 skb
->ip_summed
= CHECKSUM_NONE
;
810 #define MLX5E_CE_BIT_MASK 0x80
812 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64
*cqe
,
817 u8 lro_num_seg
= be32_to_cpu(cqe
->srqn
) >> 24;
818 struct mlx5e_rq_stats
*stats
= rq
->stats
;
819 struct net_device
*netdev
= rq
->netdev
;
821 skb
->mac_len
= ETH_HLEN
;
823 #ifdef CONFIG_MLX5_EN_TLS
824 mlx5e_tls_handle_rx_skb(netdev
, skb
, &cqe_bcnt
);
827 if (lro_num_seg
> 1) {
828 mlx5e_lro_update_hdr(skb
, cqe
, cqe_bcnt
);
829 skb_shinfo(skb
)->gso_size
= DIV_ROUND_UP(cqe_bcnt
, lro_num_seg
);
830 /* Subtract one since we already counted this as one
831 * "regular" packet in mlx5e_complete_rx_cqe()
833 stats
->packets
+= lro_num_seg
- 1;
834 stats
->lro_packets
++;
835 stats
->lro_bytes
+= cqe_bcnt
;
838 if (unlikely(mlx5e_rx_hw_stamp(rq
->tstamp
)))
839 skb_hwtstamps(skb
)->hwtstamp
=
840 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
842 skb_record_rx_queue(skb
, rq
->ix
);
844 if (likely(netdev
->features
& NETIF_F_RXHASH
))
845 mlx5e_skb_set_hash(cqe
, skb
);
847 if (cqe_has_vlan(cqe
)) {
848 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
849 be16_to_cpu(cqe
->vlan_info
));
850 stats
->removed_vlan_packets
++;
853 skb
->mark
= be32_to_cpu(cqe
->sop_drop_qpn
) & MLX5E_TC_FLOW_ID_MASK
;
855 mlx5e_handle_csum(netdev
, cqe
, rq
, skb
, !!lro_num_seg
);
856 /* checking CE bit in cqe - MSB in ml_path field */
857 if (unlikely(cqe
->ml_path
& MLX5E_CE_BIT_MASK
))
858 mlx5e_enable_ecn(rq
, skb
);
860 skb
->protocol
= eth_type_trans(skb
, netdev
);
863 static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq
*rq
,
864 struct mlx5_cqe64
*cqe
,
868 struct mlx5e_rq_stats
*stats
= rq
->stats
;
871 stats
->bytes
+= cqe_bcnt
;
872 mlx5e_build_rx_skb(cqe
, cqe_bcnt
, rq
, skb
);
876 struct sk_buff
*mlx5e_build_linear_skb(struct mlx5e_rq
*rq
, void *va
,
877 u32 frag_size
, u16 headroom
,
880 struct sk_buff
*skb
= build_skb(va
, frag_size
);
882 if (unlikely(!skb
)) {
883 rq
->stats
->buff_alloc_err
++;
887 skb_reserve(skb
, headroom
);
888 skb_put(skb
, cqe_bcnt
);
894 mlx5e_skb_from_cqe_linear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
895 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
897 struct mlx5e_dma_info
*di
= wi
->di
;
898 u16 rx_headroom
= rq
->buff
.headroom
;
904 va
= page_address(di
->page
) + wi
->offset
;
905 data
= va
+ rx_headroom
;
906 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt
);
908 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, wi
->offset
,
909 frag_size
, DMA_FROM_DEVICE
);
910 prefetchw(va
); /* xdp_frame data area */
913 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
914 rq
->stats
->wqe_err
++;
919 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt
);
922 return NULL
; /* page/packet was consumed by XDP */
924 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt
);
928 /* queue up for recycling/reuse */
929 page_ref_inc(di
->page
);
935 mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
936 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
938 struct mlx5e_rq_frag_info
*frag_info
= &rq
->wqe
.info
.arr
[0];
939 struct mlx5e_wqe_frag_info
*head_wi
= wi
;
940 u16 headlen
= min_t(u32
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
941 u16 frag_headlen
= headlen
;
942 u16 byte_cnt
= cqe_bcnt
- headlen
;
945 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
946 rq
->stats
->wqe_err
++;
950 /* XDP is not supported in this configuration, as incoming packets
951 * might spread among multiple pages.
953 skb
= napi_alloc_skb(rq
->cq
.napi
,
954 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
955 if (unlikely(!skb
)) {
956 rq
->stats
->buff_alloc_err
++;
960 prefetchw(skb
->data
);
963 u16 frag_consumed_bytes
=
964 min_t(u16
, frag_info
->frag_size
- frag_headlen
, byte_cnt
);
966 mlx5e_add_skb_frag(rq
, skb
, wi
->di
, wi
->offset
+ frag_headlen
,
967 frag_consumed_bytes
, frag_info
->frag_stride
);
968 byte_cnt
-= frag_consumed_bytes
;
975 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_wi
->di
, head_wi
->offset
, headlen
);
976 /* skb linear part was allocated with headlen and aligned to long */
977 skb
->tail
+= headlen
;
983 void mlx5e_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
985 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
986 struct mlx5e_wqe_frag_info
*wi
;
991 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
992 wi
= get_frag(rq
, ci
);
993 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
995 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
997 /* probably for XDP */
998 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
999 /* do not return page to cache,
1000 * it will be returned on XDP_TX completion.
1007 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1008 napi_gro_receive(rq
->cq
.napi
, skb
);
1011 mlx5e_free_rx_wqe(rq
, wi
, true);
1013 mlx5_wq_cyc_pop(wq
);
1016 #ifdef CONFIG_MLX5_ESWITCH
1017 void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1019 struct net_device
*netdev
= rq
->netdev
;
1020 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
1021 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1022 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
1023 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1024 struct mlx5e_wqe_frag_info
*wi
;
1025 struct sk_buff
*skb
;
1029 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1030 wi
= get_frag(rq
, ci
);
1031 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1033 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1035 /* probably for XDP */
1036 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
1037 /* do not return page to cache,
1038 * it will be returned on XDP_TX completion.
1045 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1047 if (rep
->vlan
&& skb_vlan_tag_present(skb
))
1050 napi_gro_receive(rq
->cq
.napi
, skb
);
1053 mlx5e_free_rx_wqe(rq
, wi
, true);
1055 mlx5_wq_cyc_pop(wq
);
1060 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1061 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1063 u16 headlen
= min_t(u16
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
1064 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1065 u32 frag_offset
= head_offset
+ headlen
;
1066 u32 byte_cnt
= cqe_bcnt
- headlen
;
1067 struct mlx5e_dma_info
*head_di
= di
;
1068 struct sk_buff
*skb
;
1070 skb
= napi_alloc_skb(rq
->cq
.napi
,
1071 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
1072 if (unlikely(!skb
)) {
1073 rq
->stats
->buff_alloc_err
++;
1077 prefetchw(skb
->data
);
1079 if (unlikely(frag_offset
>= PAGE_SIZE
)) {
1081 frag_offset
-= PAGE_SIZE
;
1085 u32 pg_consumed_bytes
=
1086 min_t(u32
, PAGE_SIZE
- frag_offset
, byte_cnt
);
1087 unsigned int truesize
=
1088 ALIGN(pg_consumed_bytes
, BIT(rq
->mpwqe
.log_stride_sz
));
1090 mlx5e_add_skb_frag(rq
, skb
, di
, frag_offset
,
1091 pg_consumed_bytes
, truesize
);
1092 byte_cnt
-= pg_consumed_bytes
;
1097 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_di
, head_offset
, headlen
);
1098 /* skb linear part was allocated with headlen and aligned to long */
1099 skb
->tail
+= headlen
;
1100 skb
->len
+= headlen
;
1106 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1107 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1109 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1110 u16 rx_headroom
= rq
->buff
.headroom
;
1111 u32 cqe_bcnt32
= cqe_bcnt
;
1112 struct sk_buff
*skb
;
1117 /* Check packet size. Note LRO doesn't use linear SKB */
1118 if (unlikely(cqe_bcnt
> rq
->hw_mtu
)) {
1119 rq
->stats
->oversize_pkts_sw_drop
++;
1123 va
= page_address(di
->page
) + head_offset
;
1124 data
= va
+ rx_headroom
;
1125 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt32
);
1127 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, head_offset
,
1128 frag_size
, DMA_FROM_DEVICE
);
1129 prefetchw(va
); /* xdp_frame data area */
1133 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt32
);
1136 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
))
1137 __set_bit(page_idx
, wi
->xdp_xmit_bitmap
); /* non-atomic */
1138 return NULL
; /* page/packet was consumed by XDP */
1141 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt32
);
1145 /* queue up for recycling/reuse */
1146 page_ref_inc(di
->page
);
1151 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1153 u16 cstrides
= mpwrq_get_cqe_consumed_strides(cqe
);
1154 u16 wqe_id
= be16_to_cpu(cqe
->wqe_id
);
1155 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[wqe_id
];
1156 u16 stride_ix
= mpwrq_get_cqe_stride_index(cqe
);
1157 u32 wqe_offset
= stride_ix
<< rq
->mpwqe
.log_stride_sz
;
1158 u32 head_offset
= wqe_offset
& (PAGE_SIZE
- 1);
1159 u32 page_idx
= wqe_offset
>> PAGE_SHIFT
;
1160 struct mlx5e_rx_wqe_ll
*wqe
;
1161 struct mlx5_wq_ll
*wq
;
1162 struct sk_buff
*skb
;
1165 wi
->consumed_strides
+= cstrides
;
1167 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
1168 rq
->stats
->wqe_err
++;
1172 if (unlikely(mpwrq_is_filler_cqe(cqe
))) {
1173 struct mlx5e_rq_stats
*stats
= rq
->stats
;
1175 stats
->mpwqe_filler_cqes
++;
1176 stats
->mpwqe_filler_strides
+= cstrides
;
1180 cqe_bcnt
= mpwrq_get_cqe_byte_cnt(cqe
);
1182 skb
= rq
->mpwqe
.skb_from_cqe_mpwrq(rq
, wi
, cqe_bcnt
, head_offset
,
1187 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1188 napi_gro_receive(rq
->cq
.napi
, skb
);
1191 if (likely(wi
->consumed_strides
< rq
->mpwqe
.num_strides
))
1195 wqe
= mlx5_wq_ll_get_wqe(wq
, wqe_id
);
1196 mlx5e_free_rx_mpwqe(rq
, wi
, true);
1197 mlx5_wq_ll_pop(wq
, cqe
->wqe_id
, &wqe
->next
.next_wqe_index
);
1200 int mlx5e_poll_rx_cq(struct mlx5e_cq
*cq
, int budget
)
1202 struct mlx5e_rq
*rq
= container_of(cq
, struct mlx5e_rq
, cq
);
1203 struct mlx5_cqwq
*cqwq
= &cq
->wq
;
1204 struct mlx5_cqe64
*cqe
;
1207 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
1211 work_done
+= mlx5e_decompress_cqes_cont(rq
, cqwq
, 0, budget
);
1213 cqe
= mlx5_cqwq_get_cqe(cqwq
);
1215 if (unlikely(work_done
))
1221 if (mlx5_get_cqe_format(cqe
) == MLX5_COMPRESSED
) {
1223 mlx5e_decompress_cqes_start(rq
, cqwq
,
1224 budget
- work_done
);
1228 mlx5_cqwq_pop(cqwq
);
1230 rq
->handle_rx_cqe(rq
, cqe
);
1231 } while ((++work_done
< budget
) && (cqe
= mlx5_cqwq_get_cqe(cqwq
)));
1235 mlx5e_xdp_rx_poll_complete(rq
);
1237 mlx5_cqwq_update_db_record(cqwq
);
1239 /* ensure cq space is freed before enabling more cqes */
1245 #ifdef CONFIG_MLX5_CORE_IPOIB
1247 #define MLX5_IB_GRH_DGID_OFFSET 24
1248 #define MLX5_GID_SIZE 16
1250 static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq
*rq
,
1251 struct mlx5_cqe64
*cqe
,
1253 struct sk_buff
*skb
)
1255 struct hwtstamp_config
*tstamp
;
1256 struct mlx5e_rq_stats
*stats
;
1257 struct net_device
*netdev
;
1258 struct mlx5e_priv
*priv
;
1259 char *pseudo_header
;
1264 qpn
= be32_to_cpu(cqe
->sop_drop_qpn
) & 0xffffff;
1265 netdev
= mlx5i_pkey_get_netdev(rq
->netdev
, qpn
);
1267 /* No mapping present, cannot process SKB. This might happen if a child
1268 * interface is going down while having unprocessed CQEs on parent RQ
1270 if (unlikely(!netdev
)) {
1271 /* TODO: add drop counters support */
1273 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn
);
1277 priv
= mlx5i_epriv(netdev
);
1278 tstamp
= &priv
->tstamp
;
1279 stats
= &priv
->channel_stats
[rq
->ix
].rq
;
1281 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
1282 dgid
= skb
->data
+ MLX5_IB_GRH_DGID_OFFSET
;
1283 if ((!g
) || dgid
[0] != 0xff)
1284 skb
->pkt_type
= PACKET_HOST
;
1285 else if (memcmp(dgid
, netdev
->broadcast
+ 4, MLX5_GID_SIZE
) == 0)
1286 skb
->pkt_type
= PACKET_BROADCAST
;
1288 skb
->pkt_type
= PACKET_MULTICAST
;
1290 /* TODO: IB/ipoib: Allow mcast packets from other VFs
1291 * 68996a6e760e5c74654723eeb57bf65628ae87f4
1294 skb_pull(skb
, MLX5_IB_GRH_BYTES
);
1296 skb
->protocol
= *((__be16
*)(skb
->data
));
1298 skb
->ip_summed
= CHECKSUM_COMPLETE
;
1299 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
1301 if (unlikely(mlx5e_rx_hw_stamp(tstamp
)))
1302 skb_hwtstamps(skb
)->hwtstamp
=
1303 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
1305 skb_record_rx_queue(skb
, rq
->ix
);
1307 if (likely(netdev
->features
& NETIF_F_RXHASH
))
1308 mlx5e_skb_set_hash(cqe
, skb
);
1310 /* 20 bytes of ipoib header and 4 for encap existing */
1311 pseudo_header
= skb_push(skb
, MLX5_IPOIB_PSEUDO_LEN
);
1312 memset(pseudo_header
, 0, MLX5_IPOIB_PSEUDO_LEN
);
1313 skb_reset_mac_header(skb
);
1314 skb_pull(skb
, MLX5_IPOIB_HARD_LEN
);
1318 stats
->csum_complete
++;
1320 stats
->bytes
+= cqe_bcnt
;
1323 void mlx5i_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1325 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1326 struct mlx5e_wqe_frag_info
*wi
;
1327 struct sk_buff
*skb
;
1331 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1332 wi
= get_frag(rq
, ci
);
1333 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1335 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1339 mlx5i_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1340 if (unlikely(!skb
->dev
)) {
1341 dev_kfree_skb_any(skb
);
1344 napi_gro_receive(rq
->cq
.napi
, skb
);
1347 mlx5e_free_rx_wqe(rq
, wi
, true);
1348 mlx5_wq_cyc_pop(wq
);
1351 #endif /* CONFIG_MLX5_CORE_IPOIB */
1353 #ifdef CONFIG_MLX5_EN_IPSEC
1355 void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1357 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1358 struct mlx5e_wqe_frag_info
*wi
;
1359 struct sk_buff
*skb
;
1363 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1364 wi
= get_frag(rq
, ci
);
1365 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1367 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1368 if (unlikely(!skb
)) {
1369 /* a DROP, save the page-reuse checks */
1370 mlx5e_free_rx_wqe(rq
, wi
, true);
1373 skb
= mlx5e_ipsec_handle_rx_skb(rq
->netdev
, skb
, &cqe_bcnt
);
1374 if (unlikely(!skb
)) {
1375 mlx5e_free_rx_wqe(rq
, wi
, true);
1379 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1380 napi_gro_receive(rq
->cq
.napi
, skb
);
1382 mlx5e_free_rx_wqe(rq
, wi
, true);
1384 mlx5_wq_cyc_pop(wq
);
1387 #endif /* CONFIG_MLX5_EN_IPSEC */