2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/prefetch.h>
35 #include <linux/ipv6.h>
36 #include <linux/tcp.h>
37 #include <net/ip6_checksum.h>
38 #include <net/page_pool.h>
39 #include <net/inet_ecn.h>
44 #include "ipoib/ipoib.h"
45 #include "en_accel/ipsec_rxtx.h"
46 #include "en_accel/tls_rxtx.h"
47 #include "lib/clock.h"
50 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config
*config
)
52 return config
->rx_filter
== HWTSTAMP_FILTER_ALL
;
55 static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq
*wq
,
58 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
60 memcpy(data
, mlx5_cqwq_get_wqe(wq
, ci
), sizeof(struct mlx5_cqe64
));
63 static inline void mlx5e_read_title_slot(struct mlx5e_rq
*rq
,
67 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
68 struct mlx5_cqe64
*title
= &cqd
->title
;
70 mlx5e_read_cqe_slot(wq
, cqcc
, title
);
71 cqd
->left
= be32_to_cpu(title
->byte_cnt
);
72 cqd
->wqe_counter
= be16_to_cpu(title
->wqe_counter
);
73 rq
->stats
->cqe_compress_blks
++;
76 static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq
*wq
,
77 struct mlx5e_cq_decomp
*cqd
,
80 mlx5e_read_cqe_slot(wq
, cqcc
, cqd
->mini_arr
);
81 cqd
->mini_arr_idx
= 0;
84 static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq
*wq
, int n
)
87 u8 op_own
= mlx5_cqwq_get_ctr_wrap_cnt(wq
, cqcc
) & 1;
88 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
89 u32 wq_sz
= mlx5_cqwq_get_size(wq
);
90 u32 ci_top
= min_t(u32
, wq_sz
, ci
+ n
);
92 for (; ci
< ci_top
; ci
++, n
--) {
93 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
98 if (unlikely(ci
== wq_sz
)) {
100 for (ci
= 0; ci
< n
; ci
++) {
101 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
103 cqe
->op_own
= op_own
;
108 static inline void mlx5e_decompress_cqe(struct mlx5e_rq
*rq
,
109 struct mlx5_cqwq
*wq
,
112 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
113 struct mlx5_mini_cqe8
*mini_cqe
= &cqd
->mini_arr
[cqd
->mini_arr_idx
];
114 struct mlx5_cqe64
*title
= &cqd
->title
;
116 title
->byte_cnt
= mini_cqe
->byte_cnt
;
117 title
->check_sum
= mini_cqe
->checksum
;
118 title
->op_own
&= 0xf0;
119 title
->op_own
|= 0x01 & (cqcc
>> wq
->fbc
.log_sz
);
120 title
->wqe_counter
= cpu_to_be16(cqd
->wqe_counter
);
122 if (rq
->wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
)
123 cqd
->wqe_counter
+= mpwrq_get_cqe_consumed_strides(title
);
126 mlx5_wq_cyc_ctr2ix(&rq
->wqe
.wq
, cqd
->wqe_counter
+ 1);
129 static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq
*rq
,
130 struct mlx5_cqwq
*wq
,
133 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
135 mlx5e_decompress_cqe(rq
, wq
, cqcc
);
136 cqd
->title
.rss_hash_type
= 0;
137 cqd
->title
.rss_hash_result
= 0;
140 static inline u32
mlx5e_decompress_cqes_cont(struct mlx5e_rq
*rq
,
141 struct mlx5_cqwq
*wq
,
142 int update_owner_only
,
145 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
146 u32 cqcc
= wq
->cc
+ update_owner_only
;
150 cqe_count
= min_t(u32
, cqd
->left
, budget_rem
);
152 for (i
= update_owner_only
; i
< cqe_count
;
153 i
++, cqd
->mini_arr_idx
++, cqcc
++) {
154 if (cqd
->mini_arr_idx
== MLX5_MINI_CQE_ARRAY_SIZE
)
155 mlx5e_read_mini_arr_slot(wq
, cqd
, cqcc
);
157 mlx5e_decompress_cqe_no_hash(rq
, wq
, cqcc
);
158 rq
->handle_rx_cqe(rq
, &cqd
->title
);
160 mlx5e_cqes_update_owner(wq
, cqcc
- wq
->cc
);
162 cqd
->left
-= cqe_count
;
163 rq
->stats
->cqe_compress_pkts
+= cqe_count
;
168 static inline u32
mlx5e_decompress_cqes_start(struct mlx5e_rq
*rq
,
169 struct mlx5_cqwq
*wq
,
172 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
175 mlx5e_read_title_slot(rq
, wq
, cc
);
176 mlx5e_read_mini_arr_slot(wq
, cqd
, cc
+ 1);
177 mlx5e_decompress_cqe(rq
, wq
, cc
);
178 rq
->handle_rx_cqe(rq
, &cqd
->title
);
181 return mlx5e_decompress_cqes_cont(rq
, wq
, 1, budget_rem
) - 1;
184 static inline bool mlx5e_page_is_reserved(struct page
*page
)
186 return page_is_pfmemalloc(page
) || page_to_nid(page
) != numa_mem_id();
189 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq
*rq
,
190 struct mlx5e_dma_info
*dma_info
)
192 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
193 u32 tail_next
= (cache
->tail
+ 1) & (MLX5E_CACHE_SIZE
- 1);
194 struct mlx5e_rq_stats
*stats
= rq
->stats
;
196 if (tail_next
== cache
->head
) {
201 if (unlikely(mlx5e_page_is_reserved(dma_info
->page
))) {
202 stats
->cache_waive
++;
206 cache
->page_cache
[cache
->tail
] = *dma_info
;
207 cache
->tail
= tail_next
;
211 static inline bool mlx5e_rx_cache_get(struct mlx5e_rq
*rq
,
212 struct mlx5e_dma_info
*dma_info
)
214 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
215 struct mlx5e_rq_stats
*stats
= rq
->stats
;
217 if (unlikely(cache
->head
== cache
->tail
)) {
218 stats
->cache_empty
++;
222 if (page_ref_count(cache
->page_cache
[cache
->head
].page
) != 1) {
227 *dma_info
= cache
->page_cache
[cache
->head
];
228 cache
->head
= (cache
->head
+ 1) & (MLX5E_CACHE_SIZE
- 1);
229 stats
->cache_reuse
++;
231 dma_sync_single_for_device(rq
->pdev
, dma_info
->addr
,
237 static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq
*rq
,
238 struct mlx5e_dma_info
*dma_info
)
240 if (mlx5e_rx_cache_get(rq
, dma_info
))
243 dma_info
->page
= page_pool_dev_alloc_pages(rq
->page_pool
);
244 if (unlikely(!dma_info
->page
))
247 dma_info
->addr
= dma_map_page(rq
->pdev
, dma_info
->page
, 0,
248 PAGE_SIZE
, rq
->buff
.map_dir
);
249 if (unlikely(dma_mapping_error(rq
->pdev
, dma_info
->addr
))) {
250 put_page(dma_info
->page
);
251 dma_info
->page
= NULL
;
258 void mlx5e_page_dma_unmap(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
)
260 dma_unmap_page(rq
->pdev
, dma_info
->addr
, PAGE_SIZE
, rq
->buff
.map_dir
);
263 void mlx5e_page_release(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
,
266 if (likely(recycle
)) {
267 if (mlx5e_rx_cache_put(rq
, dma_info
))
270 mlx5e_page_dma_unmap(rq
, dma_info
);
271 page_pool_recycle_direct(rq
->page_pool
, dma_info
->page
);
273 mlx5e_page_dma_unmap(rq
, dma_info
);
274 put_page(dma_info
->page
);
278 static inline int mlx5e_get_rx_frag(struct mlx5e_rq
*rq
,
279 struct mlx5e_wqe_frag_info
*frag
)
284 /* On first frag (offset == 0), replenish page (dma_info actually).
285 * Other frags that point to the same dma_info (with a different
286 * offset) should just use the new one without replenishing again
289 err
= mlx5e_page_alloc_mapped(rq
, frag
->di
);
294 static inline void mlx5e_put_rx_frag(struct mlx5e_rq
*rq
,
295 struct mlx5e_wqe_frag_info
*frag
,
298 if (frag
->last_in_page
)
299 mlx5e_page_release(rq
, frag
->di
, recycle
);
302 static inline struct mlx5e_wqe_frag_info
*get_frag(struct mlx5e_rq
*rq
, u16 ix
)
304 return &rq
->wqe
.frags
[ix
<< rq
->wqe
.info
.log_num_frags
];
307 static int mlx5e_alloc_rx_wqe(struct mlx5e_rq
*rq
, struct mlx5e_rx_wqe_cyc
*wqe
,
310 struct mlx5e_wqe_frag_info
*frag
= get_frag(rq
, ix
);
314 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, frag
++) {
315 err
= mlx5e_get_rx_frag(rq
, frag
);
319 wqe
->data
[i
].addr
= cpu_to_be64(frag
->di
->addr
+
320 frag
->offset
+ rq
->buff
.headroom
);
327 mlx5e_put_rx_frag(rq
, --frag
, true);
332 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq
*rq
,
333 struct mlx5e_wqe_frag_info
*wi
,
338 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, wi
++)
339 mlx5e_put_rx_frag(rq
, wi
, recycle
);
342 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq
*rq
, u16 ix
)
344 struct mlx5e_wqe_frag_info
*wi
= get_frag(rq
, ix
);
346 mlx5e_free_rx_wqe(rq
, wi
, false);
349 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq
*rq
, u16 ix
, u8 wqe_bulk
)
351 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
355 for (i
= 0; i
< wqe_bulk
; i
++) {
356 struct mlx5e_rx_wqe_cyc
*wqe
= mlx5_wq_cyc_get_wqe(wq
, ix
+ i
);
358 err
= mlx5e_alloc_rx_wqe(rq
, wqe
, ix
+ i
);
367 mlx5e_dealloc_rx_wqe(rq
, ix
+ i
);
373 mlx5e_add_skb_frag(struct mlx5e_rq
*rq
, struct sk_buff
*skb
,
374 struct mlx5e_dma_info
*di
, u32 frag_offset
, u32 len
,
375 unsigned int truesize
)
377 dma_sync_single_for_cpu(rq
->pdev
,
378 di
->addr
+ frag_offset
,
379 len
, DMA_FROM_DEVICE
);
380 page_ref_inc(di
->page
);
381 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
,
382 di
->page
, frag_offset
, len
, truesize
);
386 mlx5e_copy_skb_header(struct device
*pdev
, struct sk_buff
*skb
,
387 struct mlx5e_dma_info
*dma_info
,
388 int offset_from
, u32 headlen
)
390 const void *from
= page_address(dma_info
->page
) + offset_from
;
391 /* Aligning len to sizeof(long) optimizes memcpy performance */
392 unsigned int len
= ALIGN(headlen
, sizeof(long));
394 dma_sync_single_for_cpu(pdev
, dma_info
->addr
+ offset_from
, len
,
396 skb_copy_to_linear_data(skb
, from
, len
);
400 mlx5e_free_rx_mpwqe(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
, bool recycle
)
402 const bool no_xdp_xmit
=
403 bitmap_empty(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
404 struct mlx5e_dma_info
*dma_info
= wi
->umr
.dma_info
;
407 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++)
408 if (no_xdp_xmit
|| !test_bit(i
, wi
->xdp_xmit_bitmap
))
409 mlx5e_page_release(rq
, &dma_info
[i
], recycle
);
412 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq
*rq
)
414 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
415 struct mlx5e_rx_wqe_ll
*wqe
= mlx5_wq_ll_get_wqe(wq
, wq
->head
);
417 rq
->mpwqe
.umr_in_progress
= false;
419 mlx5_wq_ll_push(wq
, be16_to_cpu(wqe
->next
.next_wqe_index
));
421 /* ensure wqes are visible to device before updating doorbell record */
424 mlx5_wq_ll_update_db_record(wq
);
427 static inline u16
mlx5e_icosq_wrap_cnt(struct mlx5e_icosq
*sq
)
429 return sq
->pc
>> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE
;
432 static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq
*sq
,
433 struct mlx5_wq_cyc
*wq
,
436 struct mlx5e_sq_wqe_info
*edge_wi
, *wi
= &sq
->db
.ico_wqe
[pi
];
438 edge_wi
= wi
+ nnops
;
440 /* fill sq frag edge with nops to avoid wqe wrapping two pages */
441 for (; wi
< edge_wi
; wi
++) {
442 wi
->opcode
= MLX5_OPCODE_NOP
;
443 mlx5e_post_nop(wq
, sq
->sqn
, &sq
->pc
);
447 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
449 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
450 struct mlx5e_dma_info
*dma_info
= &wi
->umr
.dma_info
[0];
451 struct mlx5e_icosq
*sq
= &rq
->channel
->icosq
;
452 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
453 struct mlx5e_umr_wqe
*umr_wqe
;
454 u16 xlt_offset
= ix
<< (MLX5E_LOG_ALIGNED_MPWQE_PPW
- 1);
455 u16 pi
, contig_wqebbs_room
;
459 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
460 contig_wqebbs_room
= mlx5_wq_cyc_get_contig_wqebbs(wq
, pi
);
461 if (unlikely(contig_wqebbs_room
< MLX5E_UMR_WQEBBS
)) {
462 mlx5e_fill_icosq_frag_edge(sq
, wq
, pi
, contig_wqebbs_room
);
463 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
466 umr_wqe
= mlx5_wq_cyc_get_wqe(wq
, pi
);
467 if (unlikely(mlx5e_icosq_wrap_cnt(sq
) < 2))
468 memcpy(umr_wqe
, &rq
->mpwqe
.umr_wqe
,
469 offsetof(struct mlx5e_umr_wqe
, inline_mtts
));
471 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++, dma_info
++) {
472 err
= mlx5e_page_alloc_mapped(rq
, dma_info
);
475 umr_wqe
->inline_mtts
[i
].ptag
= cpu_to_be64(dma_info
->addr
| MLX5_EN_WR
);
478 bitmap_zero(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
479 wi
->consumed_strides
= 0;
481 rq
->mpwqe
.umr_in_progress
= true;
483 umr_wqe
->ctrl
.opmod_idx_opcode
=
484 cpu_to_be32((sq
->pc
<< MLX5_WQE_CTRL_WQE_INDEX_SHIFT
) |
486 umr_wqe
->uctrl
.xlt_offset
= cpu_to_be16(xlt_offset
);
488 sq
->db
.ico_wqe
[pi
].opcode
= MLX5_OPCODE_UMR
;
489 sq
->pc
+= MLX5E_UMR_WQEBBS
;
490 mlx5e_notify_hw(wq
, sq
->pc
, sq
->uar_map
, &umr_wqe
->ctrl
);
497 mlx5e_page_release(rq
, dma_info
, true);
499 rq
->stats
->buff_alloc_err
++;
504 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
506 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
507 /* Don't recycle, this function is called on rq/netdev close */
508 mlx5e_free_rx_mpwqe(rq
, wi
, false);
511 bool mlx5e_post_rx_wqes(struct mlx5e_rq
*rq
)
513 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
517 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
520 wqe_bulk
= rq
->wqe
.info
.wqe_bulk
;
522 if (mlx5_wq_cyc_missing(wq
) < wqe_bulk
)
526 u16 head
= mlx5_wq_cyc_get_head(wq
);
528 err
= mlx5e_alloc_rx_wqes(rq
, head
, wqe_bulk
);
530 rq
->stats
->buff_alloc_err
++;
534 mlx5_wq_cyc_push_n(wq
, wqe_bulk
);
535 } while (mlx5_wq_cyc_missing(wq
) >= wqe_bulk
);
537 /* ensure wqes are visible to device before updating doorbell record */
540 mlx5_wq_cyc_update_db_record(wq
);
545 static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq
*cq
,
546 struct mlx5e_icosq
*sq
,
548 struct mlx5_cqe64
*cqe
)
550 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
551 u16 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
552 struct mlx5e_sq_wqe_info
*icowi
= &sq
->db
.ico_wqe
[ci
];
554 mlx5_cqwq_pop(&cq
->wq
);
556 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_REQ
)) {
557 netdev_WARN_ONCE(cq
->channel
->netdev
,
558 "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe
));
562 if (likely(icowi
->opcode
== MLX5_OPCODE_UMR
)) {
563 mlx5e_post_rx_mpwqe(rq
);
567 if (unlikely(icowi
->opcode
!= MLX5_OPCODE_NOP
))
568 netdev_WARN_ONCE(cq
->channel
->netdev
,
569 "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi
->opcode
);
572 static void mlx5e_poll_ico_cq(struct mlx5e_cq
*cq
, struct mlx5e_rq
*rq
)
574 struct mlx5e_icosq
*sq
= container_of(cq
, struct mlx5e_icosq
, cq
);
575 struct mlx5_cqe64
*cqe
;
577 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
)))
580 cqe
= mlx5_cqwq_get_cqe(&cq
->wq
);
584 /* by design, there's only a single cqe */
585 mlx5e_poll_ico_single_cqe(cq
, sq
, rq
, cqe
);
587 mlx5_cqwq_update_db_record(&cq
->wq
);
590 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq
*rq
)
592 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
594 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
597 mlx5e_poll_ico_cq(&rq
->channel
->icosq
.cq
, rq
);
599 if (mlx5_wq_ll_is_full(wq
))
602 if (!rq
->mpwqe
.umr_in_progress
)
603 mlx5e_alloc_rx_mpwqe(rq
, wq
->head
);
605 rq
->stats
->congst_umr
+= mlx5_wq_ll_missing(wq
) > 2;
610 static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64
*cqe
, struct tcphdr
*tcp
)
612 u8 l4_hdr_type
= get_cqe_l4_hdr_type(cqe
);
613 u8 tcp_ack
= (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA
) ||
614 (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA
);
617 tcp
->psh
= get_cqe_lro_tcppsh(cqe
);
621 tcp
->ack_seq
= cqe
->lro_ack_seq_num
;
622 tcp
->window
= cqe
->lro_tcp_win
;
626 static void mlx5e_lro_update_hdr(struct sk_buff
*skb
, struct mlx5_cqe64
*cqe
,
629 struct ethhdr
*eth
= (struct ethhdr
*)(skb
->data
);
631 int network_depth
= 0;
637 proto
= __vlan_get_protocol(skb
, eth
->h_proto
, &network_depth
);
639 tot_len
= cqe_bcnt
- network_depth
;
640 ip_p
= skb
->data
+ network_depth
;
642 if (proto
== htons(ETH_P_IP
)) {
643 struct iphdr
*ipv4
= ip_p
;
645 tcp
= ip_p
+ sizeof(struct iphdr
);
646 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
648 ipv4
->ttl
= cqe
->lro_min_ttl
;
649 ipv4
->tot_len
= cpu_to_be16(tot_len
);
651 ipv4
->check
= ip_fast_csum((unsigned char *)ipv4
,
654 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
655 check
= csum_partial(tcp
, tcp
->doff
* 4,
656 csum_unfold((__force __sum16
)cqe
->check_sum
));
657 /* Almost done, don't forget the pseudo header */
658 tcp
->check
= csum_tcpudp_magic(ipv4
->saddr
, ipv4
->daddr
,
659 tot_len
- sizeof(struct iphdr
),
662 u16 payload_len
= tot_len
- sizeof(struct ipv6hdr
);
663 struct ipv6hdr
*ipv6
= ip_p
;
665 tcp
= ip_p
+ sizeof(struct ipv6hdr
);
666 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
668 ipv6
->hop_limit
= cqe
->lro_min_ttl
;
669 ipv6
->payload_len
= cpu_to_be16(payload_len
);
671 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
672 check
= csum_partial(tcp
, tcp
->doff
* 4,
673 csum_unfold((__force __sum16
)cqe
->check_sum
));
674 /* Almost done, don't forget the pseudo header */
675 tcp
->check
= csum_ipv6_magic(&ipv6
->saddr
, &ipv6
->daddr
, payload_len
,
680 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64
*cqe
,
683 u8 cht
= cqe
->rss_hash_type
;
684 int ht
= (cht
& CQE_RSS_HTYPE_L4
) ? PKT_HASH_TYPE_L4
:
685 (cht
& CQE_RSS_HTYPE_IP
) ? PKT_HASH_TYPE_L3
:
687 skb_set_hash(skb
, be32_to_cpu(cqe
->rss_hash_result
), ht
);
690 static inline bool is_last_ethertype_ip(struct sk_buff
*skb
, int *network_depth
,
693 *proto
= ((struct ethhdr
*)skb
->data
)->h_proto
;
694 *proto
= __vlan_get_protocol(skb
, *proto
, network_depth
);
695 return (*proto
== htons(ETH_P_IP
) || *proto
== htons(ETH_P_IPV6
));
698 static inline void mlx5e_enable_ecn(struct mlx5e_rq
*rq
, struct sk_buff
*skb
)
700 int network_depth
= 0;
705 if (unlikely(!is_last_ethertype_ip(skb
, &network_depth
, &proto
)))
708 ip
= skb
->data
+ network_depth
;
709 rc
= ((proto
== htons(ETH_P_IP
)) ? IP_ECN_set_ce((struct iphdr
*)ip
) :
710 IP6_ECN_set_ce(skb
, (struct ipv6hdr
*)ip
));
712 rq
->stats
->ecn_mark
+= !!rc
;
715 static u8
get_ip_proto(struct sk_buff
*skb
, int network_depth
, __be16 proto
)
717 void *ip_p
= skb
->data
+ network_depth
;
719 return (proto
== htons(ETH_P_IP
)) ? ((struct iphdr
*)ip_p
)->protocol
:
720 ((struct ipv6hdr
*)ip_p
)->nexthdr
;
723 #define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
725 #define MAX_PADDING 8
728 tail_padding_csum_slow(struct sk_buff
*skb
, int offset
, int len
,
729 struct mlx5e_rq_stats
*stats
)
731 stats
->csum_complete_tail_slow
++;
732 skb
->csum
= csum_block_add(skb
->csum
,
733 skb_checksum(skb
, offset
, len
, 0),
738 tail_padding_csum(struct sk_buff
*skb
, int offset
,
739 struct mlx5e_rq_stats
*stats
)
741 u8 tail_padding
[MAX_PADDING
];
742 int len
= skb
->len
- offset
;
745 if (unlikely(len
> MAX_PADDING
)) {
746 tail_padding_csum_slow(skb
, offset
, len
, stats
);
750 tail
= skb_header_pointer(skb
, offset
, len
, tail_padding
);
751 if (unlikely(!tail
)) {
752 tail_padding_csum_slow(skb
, offset
, len
, stats
);
756 stats
->csum_complete_tail
++;
757 skb
->csum
= csum_block_add(skb
->csum
, csum_partial(tail
, len
, 0), offset
);
761 mlx5e_skb_padding_csum(struct sk_buff
*skb
, int network_depth
, __be16 proto
,
762 struct mlx5e_rq_stats
*stats
)
769 case htons(ETH_P_IP
):
770 ip4
= (struct iphdr
*)(skb
->data
+ network_depth
);
771 pkt_len
= network_depth
+ ntohs(ip4
->tot_len
);
773 case htons(ETH_P_IPV6
):
774 ip6
= (struct ipv6hdr
*)(skb
->data
+ network_depth
);
775 pkt_len
= network_depth
+ sizeof(*ip6
) + ntohs(ip6
->payload_len
);
781 if (likely(pkt_len
>= skb
->len
))
784 tail_padding_csum(skb
, pkt_len
, stats
);
787 static inline void mlx5e_handle_csum(struct net_device
*netdev
,
788 struct mlx5_cqe64
*cqe
,
793 struct mlx5e_rq_stats
*stats
= rq
->stats
;
794 int network_depth
= 0;
797 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
801 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
802 stats
->csum_unnecessary
++;
806 /* True when explicitly set via priv flag, or XDP prog is loaded */
807 if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE
, &rq
->state
))
808 goto csum_unnecessary
;
810 /* CQE csum doesn't cover padding octets in short ethernet
811 * frames. And the pad field is appended prior to calculating
812 * and appending the FCS field.
814 * Detecting these padded frames requires to verify and parse
815 * IP headers, so we simply force all those small frames to be
816 * CHECKSUM_UNNECESSARY even if they are not padded.
818 if (short_frame(skb
->len
))
819 goto csum_unnecessary
;
821 if (likely(is_last_ethertype_ip(skb
, &network_depth
, &proto
))) {
822 if (unlikely(get_ip_proto(skb
, network_depth
, proto
) == IPPROTO_SCTP
))
823 goto csum_unnecessary
;
825 skb
->ip_summed
= CHECKSUM_COMPLETE
;
826 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
827 if (network_depth
> ETH_HLEN
)
828 /* CQE csum is calculated from the IP header and does
829 * not cover VLAN headers (if present). This will add
830 * the checksum manually.
832 skb
->csum
= csum_partial(skb
->data
+ ETH_HLEN
,
833 network_depth
- ETH_HLEN
,
836 mlx5e_skb_padding_csum(skb
, network_depth
, proto
, stats
);
837 stats
->csum_complete
++;
842 if (likely((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
843 ((cqe
->hds_ip_ext
& CQE_L4_OK
) ||
844 (get_cqe_l4_hdr_type(cqe
) == CQE_L4_HDR_TYPE_NONE
)))) {
845 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
846 if (cqe_is_tunneled(cqe
)) {
848 skb
->encapsulation
= 1;
849 stats
->csum_unnecessary_inner
++;
852 stats
->csum_unnecessary
++;
856 skb
->ip_summed
= CHECKSUM_NONE
;
860 #define MLX5E_CE_BIT_MASK 0x80
862 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64
*cqe
,
867 u8 lro_num_seg
= be32_to_cpu(cqe
->srqn
) >> 24;
868 struct mlx5e_rq_stats
*stats
= rq
->stats
;
869 struct net_device
*netdev
= rq
->netdev
;
871 skb
->mac_len
= ETH_HLEN
;
873 #ifdef CONFIG_MLX5_EN_TLS
874 mlx5e_tls_handle_rx_skb(netdev
, skb
, &cqe_bcnt
);
877 if (lro_num_seg
> 1) {
878 mlx5e_lro_update_hdr(skb
, cqe
, cqe_bcnt
);
879 skb_shinfo(skb
)->gso_size
= DIV_ROUND_UP(cqe_bcnt
, lro_num_seg
);
880 /* Subtract one since we already counted this as one
881 * "regular" packet in mlx5e_complete_rx_cqe()
883 stats
->packets
+= lro_num_seg
- 1;
884 stats
->lro_packets
++;
885 stats
->lro_bytes
+= cqe_bcnt
;
888 if (unlikely(mlx5e_rx_hw_stamp(rq
->tstamp
)))
889 skb_hwtstamps(skb
)->hwtstamp
=
890 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
892 skb_record_rx_queue(skb
, rq
->ix
);
894 if (likely(netdev
->features
& NETIF_F_RXHASH
))
895 mlx5e_skb_set_hash(cqe
, skb
);
897 if (cqe_has_vlan(cqe
)) {
898 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
899 be16_to_cpu(cqe
->vlan_info
));
900 stats
->removed_vlan_packets
++;
903 skb
->mark
= be32_to_cpu(cqe
->sop_drop_qpn
) & MLX5E_TC_FLOW_ID_MASK
;
905 mlx5e_handle_csum(netdev
, cqe
, rq
, skb
, !!lro_num_seg
);
906 /* checking CE bit in cqe - MSB in ml_path field */
907 if (unlikely(cqe
->ml_path
& MLX5E_CE_BIT_MASK
))
908 mlx5e_enable_ecn(rq
, skb
);
910 skb
->protocol
= eth_type_trans(skb
, netdev
);
913 static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq
*rq
,
914 struct mlx5_cqe64
*cqe
,
918 struct mlx5e_rq_stats
*stats
= rq
->stats
;
921 stats
->bytes
+= cqe_bcnt
;
922 mlx5e_build_rx_skb(cqe
, cqe_bcnt
, rq
, skb
);
926 struct sk_buff
*mlx5e_build_linear_skb(struct mlx5e_rq
*rq
, void *va
,
927 u32 frag_size
, u16 headroom
,
930 struct sk_buff
*skb
= build_skb(va
, frag_size
);
932 if (unlikely(!skb
)) {
933 rq
->stats
->buff_alloc_err
++;
937 skb_reserve(skb
, headroom
);
938 skb_put(skb
, cqe_bcnt
);
944 mlx5e_skb_from_cqe_linear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
945 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
947 struct mlx5e_dma_info
*di
= wi
->di
;
948 u16 rx_headroom
= rq
->buff
.headroom
;
954 va
= page_address(di
->page
) + wi
->offset
;
955 data
= va
+ rx_headroom
;
956 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt
);
958 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, wi
->offset
,
959 frag_size
, DMA_FROM_DEVICE
);
960 prefetchw(va
); /* xdp_frame data area */
963 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
964 rq
->stats
->wqe_err
++;
969 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt
);
972 return NULL
; /* page/packet was consumed by XDP */
974 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt
);
978 /* queue up for recycling/reuse */
979 page_ref_inc(di
->page
);
985 mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
986 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
988 struct mlx5e_rq_frag_info
*frag_info
= &rq
->wqe
.info
.arr
[0];
989 struct mlx5e_wqe_frag_info
*head_wi
= wi
;
990 u16 headlen
= min_t(u32
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
991 u16 frag_headlen
= headlen
;
992 u16 byte_cnt
= cqe_bcnt
- headlen
;
995 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
996 rq
->stats
->wqe_err
++;
1000 /* XDP is not supported in this configuration, as incoming packets
1001 * might spread among multiple pages.
1003 skb
= napi_alloc_skb(rq
->cq
.napi
,
1004 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
1005 if (unlikely(!skb
)) {
1006 rq
->stats
->buff_alloc_err
++;
1010 prefetchw(skb
->data
);
1013 u16 frag_consumed_bytes
=
1014 min_t(u16
, frag_info
->frag_size
- frag_headlen
, byte_cnt
);
1016 mlx5e_add_skb_frag(rq
, skb
, wi
->di
, wi
->offset
+ frag_headlen
,
1017 frag_consumed_bytes
, frag_info
->frag_stride
);
1018 byte_cnt
-= frag_consumed_bytes
;
1025 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_wi
->di
, head_wi
->offset
, headlen
);
1026 /* skb linear part was allocated with headlen and aligned to long */
1027 skb
->tail
+= headlen
;
1028 skb
->len
+= headlen
;
1033 void mlx5e_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1035 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1036 struct mlx5e_wqe_frag_info
*wi
;
1037 struct sk_buff
*skb
;
1041 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1042 wi
= get_frag(rq
, ci
);
1043 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1045 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1047 /* probably for XDP */
1048 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
1049 /* do not return page to cache,
1050 * it will be returned on XDP_TX completion.
1057 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1058 napi_gro_receive(rq
->cq
.napi
, skb
);
1061 mlx5e_free_rx_wqe(rq
, wi
, true);
1063 mlx5_wq_cyc_pop(wq
);
1066 #ifdef CONFIG_MLX5_ESWITCH
1067 void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1069 struct net_device
*netdev
= rq
->netdev
;
1070 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
1071 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1072 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
1073 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1074 struct mlx5e_wqe_frag_info
*wi
;
1075 struct sk_buff
*skb
;
1079 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1080 wi
= get_frag(rq
, ci
);
1081 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1083 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1085 /* probably for XDP */
1086 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
1087 /* do not return page to cache,
1088 * it will be returned on XDP_TX completion.
1095 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1097 if (rep
->vlan
&& skb_vlan_tag_present(skb
))
1100 napi_gro_receive(rq
->cq
.napi
, skb
);
1103 mlx5e_free_rx_wqe(rq
, wi
, true);
1105 mlx5_wq_cyc_pop(wq
);
1110 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1111 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1113 u16 headlen
= min_t(u16
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
1114 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1115 u32 frag_offset
= head_offset
+ headlen
;
1116 u32 byte_cnt
= cqe_bcnt
- headlen
;
1117 struct mlx5e_dma_info
*head_di
= di
;
1118 struct sk_buff
*skb
;
1120 skb
= napi_alloc_skb(rq
->cq
.napi
,
1121 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
1122 if (unlikely(!skb
)) {
1123 rq
->stats
->buff_alloc_err
++;
1127 prefetchw(skb
->data
);
1129 if (unlikely(frag_offset
>= PAGE_SIZE
)) {
1131 frag_offset
-= PAGE_SIZE
;
1135 u32 pg_consumed_bytes
=
1136 min_t(u32
, PAGE_SIZE
- frag_offset
, byte_cnt
);
1137 unsigned int truesize
=
1138 ALIGN(pg_consumed_bytes
, BIT(rq
->mpwqe
.log_stride_sz
));
1140 mlx5e_add_skb_frag(rq
, skb
, di
, frag_offset
,
1141 pg_consumed_bytes
, truesize
);
1142 byte_cnt
-= pg_consumed_bytes
;
1147 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_di
, head_offset
, headlen
);
1148 /* skb linear part was allocated with headlen and aligned to long */
1149 skb
->tail
+= headlen
;
1150 skb
->len
+= headlen
;
1156 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1157 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1159 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1160 u16 rx_headroom
= rq
->buff
.headroom
;
1161 u32 cqe_bcnt32
= cqe_bcnt
;
1162 struct sk_buff
*skb
;
1167 /* Check packet size. Note LRO doesn't use linear SKB */
1168 if (unlikely(cqe_bcnt
> rq
->hw_mtu
)) {
1169 rq
->stats
->oversize_pkts_sw_drop
++;
1173 va
= page_address(di
->page
) + head_offset
;
1174 data
= va
+ rx_headroom
;
1175 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt32
);
1177 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, head_offset
,
1178 frag_size
, DMA_FROM_DEVICE
);
1179 prefetchw(va
); /* xdp_frame data area */
1183 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt32
);
1186 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
))
1187 __set_bit(page_idx
, wi
->xdp_xmit_bitmap
); /* non-atomic */
1188 return NULL
; /* page/packet was consumed by XDP */
1191 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt32
);
1195 /* queue up for recycling/reuse */
1196 page_ref_inc(di
->page
);
1201 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1203 u16 cstrides
= mpwrq_get_cqe_consumed_strides(cqe
);
1204 u16 wqe_id
= be16_to_cpu(cqe
->wqe_id
);
1205 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[wqe_id
];
1206 u16 stride_ix
= mpwrq_get_cqe_stride_index(cqe
);
1207 u32 wqe_offset
= stride_ix
<< rq
->mpwqe
.log_stride_sz
;
1208 u32 head_offset
= wqe_offset
& (PAGE_SIZE
- 1);
1209 u32 page_idx
= wqe_offset
>> PAGE_SHIFT
;
1210 struct mlx5e_rx_wqe_ll
*wqe
;
1211 struct mlx5_wq_ll
*wq
;
1212 struct sk_buff
*skb
;
1215 wi
->consumed_strides
+= cstrides
;
1217 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
1218 rq
->stats
->wqe_err
++;
1222 if (unlikely(mpwrq_is_filler_cqe(cqe
))) {
1223 struct mlx5e_rq_stats
*stats
= rq
->stats
;
1225 stats
->mpwqe_filler_cqes
++;
1226 stats
->mpwqe_filler_strides
+= cstrides
;
1230 cqe_bcnt
= mpwrq_get_cqe_byte_cnt(cqe
);
1232 skb
= rq
->mpwqe
.skb_from_cqe_mpwrq(rq
, wi
, cqe_bcnt
, head_offset
,
1237 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1238 napi_gro_receive(rq
->cq
.napi
, skb
);
1241 if (likely(wi
->consumed_strides
< rq
->mpwqe
.num_strides
))
1245 wqe
= mlx5_wq_ll_get_wqe(wq
, wqe_id
);
1246 mlx5e_free_rx_mpwqe(rq
, wi
, true);
1247 mlx5_wq_ll_pop(wq
, cqe
->wqe_id
, &wqe
->next
.next_wqe_index
);
1250 int mlx5e_poll_rx_cq(struct mlx5e_cq
*cq
, int budget
)
1252 struct mlx5e_rq
*rq
= container_of(cq
, struct mlx5e_rq
, cq
);
1253 struct mlx5_cqwq
*cqwq
= &cq
->wq
;
1254 struct mlx5_cqe64
*cqe
;
1257 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
1261 work_done
+= mlx5e_decompress_cqes_cont(rq
, cqwq
, 0, budget
);
1263 cqe
= mlx5_cqwq_get_cqe(cqwq
);
1265 if (unlikely(work_done
))
1271 if (mlx5_get_cqe_format(cqe
) == MLX5_COMPRESSED
) {
1273 mlx5e_decompress_cqes_start(rq
, cqwq
,
1274 budget
- work_done
);
1278 mlx5_cqwq_pop(cqwq
);
1280 rq
->handle_rx_cqe(rq
, cqe
);
1281 } while ((++work_done
< budget
) && (cqe
= mlx5_cqwq_get_cqe(cqwq
)));
1285 mlx5e_xdp_rx_poll_complete(rq
);
1287 mlx5_cqwq_update_db_record(cqwq
);
1289 /* ensure cq space is freed before enabling more cqes */
1295 #ifdef CONFIG_MLX5_CORE_IPOIB
1297 #define MLX5_IB_GRH_DGID_OFFSET 24
1298 #define MLX5_GID_SIZE 16
1300 static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq
*rq
,
1301 struct mlx5_cqe64
*cqe
,
1303 struct sk_buff
*skb
)
1305 struct hwtstamp_config
*tstamp
;
1306 struct mlx5e_rq_stats
*stats
;
1307 struct net_device
*netdev
;
1308 struct mlx5e_priv
*priv
;
1309 char *pseudo_header
;
1314 qpn
= be32_to_cpu(cqe
->sop_drop_qpn
) & 0xffffff;
1315 netdev
= mlx5i_pkey_get_netdev(rq
->netdev
, qpn
);
1317 /* No mapping present, cannot process SKB. This might happen if a child
1318 * interface is going down while having unprocessed CQEs on parent RQ
1320 if (unlikely(!netdev
)) {
1321 /* TODO: add drop counters support */
1323 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn
);
1327 priv
= mlx5i_epriv(netdev
);
1328 tstamp
= &priv
->tstamp
;
1329 stats
= &priv
->channel_stats
[rq
->ix
].rq
;
1331 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
1332 dgid
= skb
->data
+ MLX5_IB_GRH_DGID_OFFSET
;
1333 if ((!g
) || dgid
[0] != 0xff)
1334 skb
->pkt_type
= PACKET_HOST
;
1335 else if (memcmp(dgid
, netdev
->broadcast
+ 4, MLX5_GID_SIZE
) == 0)
1336 skb
->pkt_type
= PACKET_BROADCAST
;
1338 skb
->pkt_type
= PACKET_MULTICAST
;
1340 /* TODO: IB/ipoib: Allow mcast packets from other VFs
1341 * 68996a6e760e5c74654723eeb57bf65628ae87f4
1344 skb_pull(skb
, MLX5_IB_GRH_BYTES
);
1346 skb
->protocol
= *((__be16
*)(skb
->data
));
1348 if (netdev
->features
& NETIF_F_RXCSUM
) {
1349 skb
->ip_summed
= CHECKSUM_COMPLETE
;
1350 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
1351 stats
->csum_complete
++;
1353 skb
->ip_summed
= CHECKSUM_NONE
;
1357 if (unlikely(mlx5e_rx_hw_stamp(tstamp
)))
1358 skb_hwtstamps(skb
)->hwtstamp
=
1359 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
1361 skb_record_rx_queue(skb
, rq
->ix
);
1363 if (likely(netdev
->features
& NETIF_F_RXHASH
))
1364 mlx5e_skb_set_hash(cqe
, skb
);
1366 /* 20 bytes of ipoib header and 4 for encap existing */
1367 pseudo_header
= skb_push(skb
, MLX5_IPOIB_PSEUDO_LEN
);
1368 memset(pseudo_header
, 0, MLX5_IPOIB_PSEUDO_LEN
);
1369 skb_reset_mac_header(skb
);
1370 skb_pull(skb
, MLX5_IPOIB_HARD_LEN
);
1375 stats
->bytes
+= cqe_bcnt
;
1378 void mlx5i_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1380 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1381 struct mlx5e_wqe_frag_info
*wi
;
1382 struct sk_buff
*skb
;
1386 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1387 wi
= get_frag(rq
, ci
);
1388 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1390 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1394 mlx5i_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1395 if (unlikely(!skb
->dev
)) {
1396 dev_kfree_skb_any(skb
);
1399 napi_gro_receive(rq
->cq
.napi
, skb
);
1402 mlx5e_free_rx_wqe(rq
, wi
, true);
1403 mlx5_wq_cyc_pop(wq
);
1406 #endif /* CONFIG_MLX5_CORE_IPOIB */
1408 #ifdef CONFIG_MLX5_EN_IPSEC
1410 void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1412 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1413 struct mlx5e_wqe_frag_info
*wi
;
1414 struct sk_buff
*skb
;
1418 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1419 wi
= get_frag(rq
, ci
);
1420 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1422 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1423 if (unlikely(!skb
)) {
1424 /* a DROP, save the page-reuse checks */
1425 mlx5e_free_rx_wqe(rq
, wi
, true);
1428 skb
= mlx5e_ipsec_handle_rx_skb(rq
->netdev
, skb
, &cqe_bcnt
);
1429 if (unlikely(!skb
)) {
1430 mlx5e_free_rx_wqe(rq
, wi
, true);
1434 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1435 napi_gro_receive(rq
->cq
.napi
, skb
);
1437 mlx5e_free_rx_wqe(rq
, wi
, true);
1439 mlx5_wq_cyc_pop(wq
);
1442 #endif /* CONFIG_MLX5_EN_IPSEC */