2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/prefetch.h>
35 #include <linux/ipv6.h>
36 #include <linux/tcp.h>
37 #include <net/ip6_checksum.h>
38 #include <net/page_pool.h>
39 #include <net/inet_ecn.h>
44 #include "ipoib/ipoib.h"
45 #include "en_accel/ipsec_rxtx.h"
46 #include "en_accel/tls_rxtx.h"
47 #include "lib/clock.h"
50 static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config
*config
)
52 return config
->rx_filter
== HWTSTAMP_FILTER_ALL
;
55 static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq
*wq
,
58 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
60 memcpy(data
, mlx5_cqwq_get_wqe(wq
, ci
), sizeof(struct mlx5_cqe64
));
63 static inline void mlx5e_read_title_slot(struct mlx5e_rq
*rq
,
67 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
68 struct mlx5_cqe64
*title
= &cqd
->title
;
70 mlx5e_read_cqe_slot(wq
, cqcc
, title
);
71 cqd
->left
= be32_to_cpu(title
->byte_cnt
);
72 cqd
->wqe_counter
= be16_to_cpu(title
->wqe_counter
);
73 rq
->stats
->cqe_compress_blks
++;
76 static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq
*wq
,
77 struct mlx5e_cq_decomp
*cqd
,
80 mlx5e_read_cqe_slot(wq
, cqcc
, cqd
->mini_arr
);
81 cqd
->mini_arr_idx
= 0;
84 static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq
*wq
, int n
)
87 u8 op_own
= mlx5_cqwq_get_ctr_wrap_cnt(wq
, cqcc
) & 1;
88 u32 ci
= mlx5_cqwq_ctr2ix(wq
, cqcc
);
89 u32 wq_sz
= mlx5_cqwq_get_size(wq
);
90 u32 ci_top
= min_t(u32
, wq_sz
, ci
+ n
);
92 for (; ci
< ci_top
; ci
++, n
--) {
93 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
98 if (unlikely(ci
== wq_sz
)) {
100 for (ci
= 0; ci
< n
; ci
++) {
101 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(wq
, ci
);
103 cqe
->op_own
= op_own
;
108 static inline void mlx5e_decompress_cqe(struct mlx5e_rq
*rq
,
109 struct mlx5_cqwq
*wq
,
112 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
113 struct mlx5_mini_cqe8
*mini_cqe
= &cqd
->mini_arr
[cqd
->mini_arr_idx
];
114 struct mlx5_cqe64
*title
= &cqd
->title
;
116 title
->byte_cnt
= mini_cqe
->byte_cnt
;
117 title
->check_sum
= mini_cqe
->checksum
;
118 title
->op_own
&= 0xf0;
119 title
->op_own
|= 0x01 & (cqcc
>> wq
->fbc
.log_sz
);
120 title
->wqe_counter
= cpu_to_be16(cqd
->wqe_counter
);
122 if (rq
->wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
)
123 cqd
->wqe_counter
+= mpwrq_get_cqe_consumed_strides(title
);
126 mlx5_wq_cyc_ctr2ix(&rq
->wqe
.wq
, cqd
->wqe_counter
+ 1);
129 static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq
*rq
,
130 struct mlx5_cqwq
*wq
,
133 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
135 mlx5e_decompress_cqe(rq
, wq
, cqcc
);
136 cqd
->title
.rss_hash_type
= 0;
137 cqd
->title
.rss_hash_result
= 0;
140 static inline u32
mlx5e_decompress_cqes_cont(struct mlx5e_rq
*rq
,
141 struct mlx5_cqwq
*wq
,
142 int update_owner_only
,
145 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
146 u32 cqcc
= wq
->cc
+ update_owner_only
;
150 cqe_count
= min_t(u32
, cqd
->left
, budget_rem
);
152 for (i
= update_owner_only
; i
< cqe_count
;
153 i
++, cqd
->mini_arr_idx
++, cqcc
++) {
154 if (cqd
->mini_arr_idx
== MLX5_MINI_CQE_ARRAY_SIZE
)
155 mlx5e_read_mini_arr_slot(wq
, cqd
, cqcc
);
157 mlx5e_decompress_cqe_no_hash(rq
, wq
, cqcc
);
158 rq
->handle_rx_cqe(rq
, &cqd
->title
);
160 mlx5e_cqes_update_owner(wq
, cqcc
- wq
->cc
);
162 cqd
->left
-= cqe_count
;
163 rq
->stats
->cqe_compress_pkts
+= cqe_count
;
168 static inline u32
mlx5e_decompress_cqes_start(struct mlx5e_rq
*rq
,
169 struct mlx5_cqwq
*wq
,
172 struct mlx5e_cq_decomp
*cqd
= &rq
->cqd
;
175 mlx5e_read_title_slot(rq
, wq
, cc
);
176 mlx5e_read_mini_arr_slot(wq
, cqd
, cc
+ 1);
177 mlx5e_decompress_cqe(rq
, wq
, cc
);
178 rq
->handle_rx_cqe(rq
, &cqd
->title
);
181 return mlx5e_decompress_cqes_cont(rq
, wq
, 1, budget_rem
) - 1;
184 static inline bool mlx5e_page_is_reserved(struct page
*page
)
186 return page_is_pfmemalloc(page
) || page_to_nid(page
) != numa_mem_id();
189 static inline bool mlx5e_rx_cache_put(struct mlx5e_rq
*rq
,
190 struct mlx5e_dma_info
*dma_info
)
192 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
193 u32 tail_next
= (cache
->tail
+ 1) & (MLX5E_CACHE_SIZE
- 1);
194 struct mlx5e_rq_stats
*stats
= rq
->stats
;
196 if (tail_next
== cache
->head
) {
201 if (unlikely(mlx5e_page_is_reserved(dma_info
->page
))) {
202 stats
->cache_waive
++;
206 cache
->page_cache
[cache
->tail
] = *dma_info
;
207 cache
->tail
= tail_next
;
211 static inline bool mlx5e_rx_cache_get(struct mlx5e_rq
*rq
,
212 struct mlx5e_dma_info
*dma_info
)
214 struct mlx5e_page_cache
*cache
= &rq
->page_cache
;
215 struct mlx5e_rq_stats
*stats
= rq
->stats
;
217 if (unlikely(cache
->head
== cache
->tail
)) {
218 stats
->cache_empty
++;
222 if (page_ref_count(cache
->page_cache
[cache
->head
].page
) != 1) {
227 *dma_info
= cache
->page_cache
[cache
->head
];
228 cache
->head
= (cache
->head
+ 1) & (MLX5E_CACHE_SIZE
- 1);
229 stats
->cache_reuse
++;
231 dma_sync_single_for_device(rq
->pdev
, dma_info
->addr
,
237 static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq
*rq
,
238 struct mlx5e_dma_info
*dma_info
)
240 if (mlx5e_rx_cache_get(rq
, dma_info
))
243 dma_info
->page
= page_pool_dev_alloc_pages(rq
->page_pool
);
244 if (unlikely(!dma_info
->page
))
247 dma_info
->addr
= dma_map_page(rq
->pdev
, dma_info
->page
, 0,
248 PAGE_SIZE
, rq
->buff
.map_dir
);
249 if (unlikely(dma_mapping_error(rq
->pdev
, dma_info
->addr
))) {
250 put_page(dma_info
->page
);
251 dma_info
->page
= NULL
;
258 void mlx5e_page_dma_unmap(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
)
260 dma_unmap_page(rq
->pdev
, dma_info
->addr
, PAGE_SIZE
, rq
->buff
.map_dir
);
263 void mlx5e_page_release(struct mlx5e_rq
*rq
, struct mlx5e_dma_info
*dma_info
,
266 if (likely(recycle
)) {
267 if (mlx5e_rx_cache_put(rq
, dma_info
))
270 mlx5e_page_dma_unmap(rq
, dma_info
);
271 page_pool_recycle_direct(rq
->page_pool
, dma_info
->page
);
273 mlx5e_page_dma_unmap(rq
, dma_info
);
274 put_page(dma_info
->page
);
278 static inline int mlx5e_get_rx_frag(struct mlx5e_rq
*rq
,
279 struct mlx5e_wqe_frag_info
*frag
)
284 /* On first frag (offset == 0), replenish page (dma_info actually).
285 * Other frags that point to the same dma_info (with a different
286 * offset) should just use the new one without replenishing again
289 err
= mlx5e_page_alloc_mapped(rq
, frag
->di
);
294 static inline void mlx5e_put_rx_frag(struct mlx5e_rq
*rq
,
295 struct mlx5e_wqe_frag_info
*frag
,
298 if (frag
->last_in_page
)
299 mlx5e_page_release(rq
, frag
->di
, recycle
);
302 static inline struct mlx5e_wqe_frag_info
*get_frag(struct mlx5e_rq
*rq
, u16 ix
)
304 return &rq
->wqe
.frags
[ix
<< rq
->wqe
.info
.log_num_frags
];
307 static int mlx5e_alloc_rx_wqe(struct mlx5e_rq
*rq
, struct mlx5e_rx_wqe_cyc
*wqe
,
310 struct mlx5e_wqe_frag_info
*frag
= get_frag(rq
, ix
);
314 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, frag
++) {
315 err
= mlx5e_get_rx_frag(rq
, frag
);
319 wqe
->data
[i
].addr
= cpu_to_be64(frag
->di
->addr
+
320 frag
->offset
+ rq
->buff
.headroom
);
327 mlx5e_put_rx_frag(rq
, --frag
, true);
332 static inline void mlx5e_free_rx_wqe(struct mlx5e_rq
*rq
,
333 struct mlx5e_wqe_frag_info
*wi
,
338 for (i
= 0; i
< rq
->wqe
.info
.num_frags
; i
++, wi
++)
339 mlx5e_put_rx_frag(rq
, wi
, recycle
);
342 void mlx5e_dealloc_rx_wqe(struct mlx5e_rq
*rq
, u16 ix
)
344 struct mlx5e_wqe_frag_info
*wi
= get_frag(rq
, ix
);
346 mlx5e_free_rx_wqe(rq
, wi
, false);
349 static int mlx5e_alloc_rx_wqes(struct mlx5e_rq
*rq
, u16 ix
, u8 wqe_bulk
)
351 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
355 for (i
= 0; i
< wqe_bulk
; i
++) {
356 struct mlx5e_rx_wqe_cyc
*wqe
= mlx5_wq_cyc_get_wqe(wq
, ix
+ i
);
358 err
= mlx5e_alloc_rx_wqe(rq
, wqe
, ix
+ i
);
367 mlx5e_dealloc_rx_wqe(rq
, ix
+ i
);
373 mlx5e_add_skb_frag(struct mlx5e_rq
*rq
, struct sk_buff
*skb
,
374 struct mlx5e_dma_info
*di
, u32 frag_offset
, u32 len
,
375 unsigned int truesize
)
377 dma_sync_single_for_cpu(rq
->pdev
,
378 di
->addr
+ frag_offset
,
379 len
, DMA_FROM_DEVICE
);
380 page_ref_inc(di
->page
);
381 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
,
382 di
->page
, frag_offset
, len
, truesize
);
386 mlx5e_copy_skb_header(struct device
*pdev
, struct sk_buff
*skb
,
387 struct mlx5e_dma_info
*dma_info
,
388 int offset_from
, u32 headlen
)
390 const void *from
= page_address(dma_info
->page
) + offset_from
;
391 /* Aligning len to sizeof(long) optimizes memcpy performance */
392 unsigned int len
= ALIGN(headlen
, sizeof(long));
394 dma_sync_single_for_cpu(pdev
, dma_info
->addr
+ offset_from
, len
,
396 skb_copy_to_linear_data(skb
, from
, len
);
400 mlx5e_free_rx_mpwqe(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
, bool recycle
)
402 const bool no_xdp_xmit
=
403 bitmap_empty(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
404 struct mlx5e_dma_info
*dma_info
= wi
->umr
.dma_info
;
407 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++)
408 if (no_xdp_xmit
|| !test_bit(i
, wi
->xdp_xmit_bitmap
))
409 mlx5e_page_release(rq
, &dma_info
[i
], recycle
);
412 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq
*rq
)
414 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
415 struct mlx5e_rx_wqe_ll
*wqe
= mlx5_wq_ll_get_wqe(wq
, wq
->head
);
417 rq
->mpwqe
.umr_in_progress
= false;
419 mlx5_wq_ll_push(wq
, be16_to_cpu(wqe
->next
.next_wqe_index
));
421 /* ensure wqes are visible to device before updating doorbell record */
424 mlx5_wq_ll_update_db_record(wq
);
427 static inline u16
mlx5e_icosq_wrap_cnt(struct mlx5e_icosq
*sq
)
429 return sq
->pc
>> MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE
;
432 static inline void mlx5e_fill_icosq_frag_edge(struct mlx5e_icosq
*sq
,
433 struct mlx5_wq_cyc
*wq
,
436 struct mlx5e_sq_wqe_info
*edge_wi
, *wi
= &sq
->db
.ico_wqe
[pi
];
438 edge_wi
= wi
+ nnops
;
440 /* fill sq frag edge with nops to avoid wqe wrapping two pages */
441 for (; wi
< edge_wi
; wi
++) {
442 wi
->opcode
= MLX5_OPCODE_NOP
;
443 mlx5e_post_nop(wq
, sq
->sqn
, &sq
->pc
);
447 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
449 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
450 struct mlx5e_dma_info
*dma_info
= &wi
->umr
.dma_info
[0];
451 struct mlx5e_icosq
*sq
= &rq
->channel
->icosq
;
452 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
453 struct mlx5e_umr_wqe
*umr_wqe
;
454 u16 xlt_offset
= ix
<< (MLX5E_LOG_ALIGNED_MPWQE_PPW
- 1);
455 u16 pi
, contig_wqebbs_room
;
459 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
460 contig_wqebbs_room
= mlx5_wq_cyc_get_contig_wqebbs(wq
, pi
);
461 if (unlikely(contig_wqebbs_room
< MLX5E_UMR_WQEBBS
)) {
462 mlx5e_fill_icosq_frag_edge(sq
, wq
, pi
, contig_wqebbs_room
);
463 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
466 umr_wqe
= mlx5_wq_cyc_get_wqe(wq
, pi
);
467 if (unlikely(mlx5e_icosq_wrap_cnt(sq
) < 2))
468 memcpy(umr_wqe
, &rq
->mpwqe
.umr_wqe
,
469 offsetof(struct mlx5e_umr_wqe
, inline_mtts
));
471 for (i
= 0; i
< MLX5_MPWRQ_PAGES_PER_WQE
; i
++, dma_info
++) {
472 err
= mlx5e_page_alloc_mapped(rq
, dma_info
);
475 umr_wqe
->inline_mtts
[i
].ptag
= cpu_to_be64(dma_info
->addr
| MLX5_EN_WR
);
478 bitmap_zero(wi
->xdp_xmit_bitmap
, MLX5_MPWRQ_PAGES_PER_WQE
);
479 wi
->consumed_strides
= 0;
481 rq
->mpwqe
.umr_in_progress
= true;
483 umr_wqe
->ctrl
.opmod_idx_opcode
=
484 cpu_to_be32((sq
->pc
<< MLX5_WQE_CTRL_WQE_INDEX_SHIFT
) |
486 umr_wqe
->uctrl
.xlt_offset
= cpu_to_be16(xlt_offset
);
488 sq
->db
.ico_wqe
[pi
].opcode
= MLX5_OPCODE_UMR
;
489 sq
->pc
+= MLX5E_UMR_WQEBBS
;
490 mlx5e_notify_hw(wq
, sq
->pc
, sq
->uar_map
, &umr_wqe
->ctrl
);
497 mlx5e_page_release(rq
, dma_info
, true);
499 rq
->stats
->buff_alloc_err
++;
504 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq
*rq
, u16 ix
)
506 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[ix
];
507 /* Don't recycle, this function is called on rq/netdev close */
508 mlx5e_free_rx_mpwqe(rq
, wi
, false);
511 bool mlx5e_post_rx_wqes(struct mlx5e_rq
*rq
)
513 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
517 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
520 wqe_bulk
= rq
->wqe
.info
.wqe_bulk
;
522 if (mlx5_wq_cyc_missing(wq
) < wqe_bulk
)
526 u16 head
= mlx5_wq_cyc_get_head(wq
);
528 err
= mlx5e_alloc_rx_wqes(rq
, head
, wqe_bulk
);
530 rq
->stats
->buff_alloc_err
++;
534 mlx5_wq_cyc_push_n(wq
, wqe_bulk
);
535 } while (mlx5_wq_cyc_missing(wq
) >= wqe_bulk
);
537 /* ensure wqes are visible to device before updating doorbell record */
540 mlx5_wq_cyc_update_db_record(wq
);
545 static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq
*cq
,
546 struct mlx5e_icosq
*sq
,
548 struct mlx5_cqe64
*cqe
)
550 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
551 u16 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
552 struct mlx5e_sq_wqe_info
*icowi
= &sq
->db
.ico_wqe
[ci
];
554 mlx5_cqwq_pop(&cq
->wq
);
556 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_REQ
)) {
557 netdev_WARN_ONCE(cq
->channel
->netdev
,
558 "Bad OP in ICOSQ CQE: 0x%x\n", get_cqe_opcode(cqe
));
562 if (likely(icowi
->opcode
== MLX5_OPCODE_UMR
)) {
563 mlx5e_post_rx_mpwqe(rq
);
567 if (unlikely(icowi
->opcode
!= MLX5_OPCODE_NOP
))
568 netdev_WARN_ONCE(cq
->channel
->netdev
,
569 "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi
->opcode
);
572 static void mlx5e_poll_ico_cq(struct mlx5e_cq
*cq
, struct mlx5e_rq
*rq
)
574 struct mlx5e_icosq
*sq
= container_of(cq
, struct mlx5e_icosq
, cq
);
575 struct mlx5_cqe64
*cqe
;
577 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
)))
580 cqe
= mlx5_cqwq_get_cqe(&cq
->wq
);
584 /* by design, there's only a single cqe */
585 mlx5e_poll_ico_single_cqe(cq
, sq
, rq
, cqe
);
587 mlx5_cqwq_update_db_record(&cq
->wq
);
590 bool mlx5e_post_rx_mpwqes(struct mlx5e_rq
*rq
)
592 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
594 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
597 mlx5e_poll_ico_cq(&rq
->channel
->icosq
.cq
, rq
);
599 if (mlx5_wq_ll_is_full(wq
))
602 if (!rq
->mpwqe
.umr_in_progress
)
603 mlx5e_alloc_rx_mpwqe(rq
, wq
->head
);
605 rq
->stats
->congst_umr
+= mlx5_wq_ll_missing(wq
) > 2;
610 static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64
*cqe
, struct tcphdr
*tcp
)
612 u8 l4_hdr_type
= get_cqe_l4_hdr_type(cqe
);
613 u8 tcp_ack
= (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA
) ||
614 (l4_hdr_type
== CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA
);
617 tcp
->psh
= get_cqe_lro_tcppsh(cqe
);
621 tcp
->ack_seq
= cqe
->lro_ack_seq_num
;
622 tcp
->window
= cqe
->lro_tcp_win
;
626 static void mlx5e_lro_update_hdr(struct sk_buff
*skb
, struct mlx5_cqe64
*cqe
,
629 struct ethhdr
*eth
= (struct ethhdr
*)(skb
->data
);
631 int network_depth
= 0;
637 proto
= __vlan_get_protocol(skb
, eth
->h_proto
, &network_depth
);
639 tot_len
= cqe_bcnt
- network_depth
;
640 ip_p
= skb
->data
+ network_depth
;
642 if (proto
== htons(ETH_P_IP
)) {
643 struct iphdr
*ipv4
= ip_p
;
645 tcp
= ip_p
+ sizeof(struct iphdr
);
646 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV4
;
648 ipv4
->ttl
= cqe
->lro_min_ttl
;
649 ipv4
->tot_len
= cpu_to_be16(tot_len
);
651 ipv4
->check
= ip_fast_csum((unsigned char *)ipv4
,
654 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
655 check
= csum_partial(tcp
, tcp
->doff
* 4,
656 csum_unfold((__force __sum16
)cqe
->check_sum
));
657 /* Almost done, don't forget the pseudo header */
658 tcp
->check
= csum_tcpudp_magic(ipv4
->saddr
, ipv4
->daddr
,
659 tot_len
- sizeof(struct iphdr
),
662 u16 payload_len
= tot_len
- sizeof(struct ipv6hdr
);
663 struct ipv6hdr
*ipv6
= ip_p
;
665 tcp
= ip_p
+ sizeof(struct ipv6hdr
);
666 skb_shinfo(skb
)->gso_type
= SKB_GSO_TCPV6
;
668 ipv6
->hop_limit
= cqe
->lro_min_ttl
;
669 ipv6
->payload_len
= cpu_to_be16(payload_len
);
671 mlx5e_lro_update_tcp_hdr(cqe
, tcp
);
672 check
= csum_partial(tcp
, tcp
->doff
* 4,
673 csum_unfold((__force __sum16
)cqe
->check_sum
));
674 /* Almost done, don't forget the pseudo header */
675 tcp
->check
= csum_ipv6_magic(&ipv6
->saddr
, &ipv6
->daddr
, payload_len
,
680 static inline void mlx5e_skb_set_hash(struct mlx5_cqe64
*cqe
,
683 u8 cht
= cqe
->rss_hash_type
;
684 int ht
= (cht
& CQE_RSS_HTYPE_L4
) ? PKT_HASH_TYPE_L4
:
685 (cht
& CQE_RSS_HTYPE_IP
) ? PKT_HASH_TYPE_L3
:
687 skb_set_hash(skb
, be32_to_cpu(cqe
->rss_hash_result
), ht
);
690 static inline bool is_last_ethertype_ip(struct sk_buff
*skb
, int *network_depth
,
693 *proto
= ((struct ethhdr
*)skb
->data
)->h_proto
;
694 *proto
= __vlan_get_protocol(skb
, *proto
, network_depth
);
696 if (*proto
== htons(ETH_P_IP
))
697 return pskb_may_pull(skb
, *network_depth
+ sizeof(struct iphdr
));
699 if (*proto
== htons(ETH_P_IPV6
))
700 return pskb_may_pull(skb
, *network_depth
+ sizeof(struct ipv6hdr
));
705 static inline void mlx5e_enable_ecn(struct mlx5e_rq
*rq
, struct sk_buff
*skb
)
707 int network_depth
= 0;
712 if (unlikely(!is_last_ethertype_ip(skb
, &network_depth
, &proto
)))
715 ip
= skb
->data
+ network_depth
;
716 rc
= ((proto
== htons(ETH_P_IP
)) ? IP_ECN_set_ce((struct iphdr
*)ip
) :
717 IP6_ECN_set_ce(skb
, (struct ipv6hdr
*)ip
));
719 rq
->stats
->ecn_mark
+= !!rc
;
722 static u8
get_ip_proto(struct sk_buff
*skb
, int network_depth
, __be16 proto
)
724 void *ip_p
= skb
->data
+ network_depth
;
726 return (proto
== htons(ETH_P_IP
)) ? ((struct iphdr
*)ip_p
)->protocol
:
727 ((struct ipv6hdr
*)ip_p
)->nexthdr
;
730 #define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
732 #define MAX_PADDING 8
735 tail_padding_csum_slow(struct sk_buff
*skb
, int offset
, int len
,
736 struct mlx5e_rq_stats
*stats
)
738 stats
->csum_complete_tail_slow
++;
739 skb
->csum
= csum_block_add(skb
->csum
,
740 skb_checksum(skb
, offset
, len
, 0),
745 tail_padding_csum(struct sk_buff
*skb
, int offset
,
746 struct mlx5e_rq_stats
*stats
)
748 u8 tail_padding
[MAX_PADDING
];
749 int len
= skb
->len
- offset
;
752 if (unlikely(len
> MAX_PADDING
)) {
753 tail_padding_csum_slow(skb
, offset
, len
, stats
);
757 tail
= skb_header_pointer(skb
, offset
, len
, tail_padding
);
758 if (unlikely(!tail
)) {
759 tail_padding_csum_slow(skb
, offset
, len
, stats
);
763 stats
->csum_complete_tail
++;
764 skb
->csum
= csum_block_add(skb
->csum
, csum_partial(tail
, len
, 0), offset
);
768 mlx5e_skb_padding_csum(struct sk_buff
*skb
, int network_depth
, __be16 proto
,
769 struct mlx5e_rq_stats
*stats
)
776 case htons(ETH_P_IP
):
777 ip4
= (struct iphdr
*)(skb
->data
+ network_depth
);
778 pkt_len
= network_depth
+ ntohs(ip4
->tot_len
);
780 case htons(ETH_P_IPV6
):
781 ip6
= (struct ipv6hdr
*)(skb
->data
+ network_depth
);
782 pkt_len
= network_depth
+ sizeof(*ip6
) + ntohs(ip6
->payload_len
);
788 if (likely(pkt_len
>= skb
->len
))
791 tail_padding_csum(skb
, pkt_len
, stats
);
794 static inline void mlx5e_handle_csum(struct net_device
*netdev
,
795 struct mlx5_cqe64
*cqe
,
800 struct mlx5e_rq_stats
*stats
= rq
->stats
;
801 int network_depth
= 0;
804 if (unlikely(!(netdev
->features
& NETIF_F_RXCSUM
)))
808 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
809 stats
->csum_unnecessary
++;
813 /* True when explicitly set via priv flag, or XDP prog is loaded */
814 if (test_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE
, &rq
->state
))
815 goto csum_unnecessary
;
817 /* CQE csum doesn't cover padding octets in short ethernet
818 * frames. And the pad field is appended prior to calculating
819 * and appending the FCS field.
821 * Detecting these padded frames requires to verify and parse
822 * IP headers, so we simply force all those small frames to be
823 * CHECKSUM_UNNECESSARY even if they are not padded.
825 if (short_frame(skb
->len
))
826 goto csum_unnecessary
;
828 if (likely(is_last_ethertype_ip(skb
, &network_depth
, &proto
))) {
829 if (unlikely(get_ip_proto(skb
, network_depth
, proto
) == IPPROTO_SCTP
))
830 goto csum_unnecessary
;
832 skb
->ip_summed
= CHECKSUM_COMPLETE
;
833 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
834 if (network_depth
> ETH_HLEN
)
835 /* CQE csum is calculated from the IP header and does
836 * not cover VLAN headers (if present). This will add
837 * the checksum manually.
839 skb
->csum
= csum_partial(skb
->data
+ ETH_HLEN
,
840 network_depth
- ETH_HLEN
,
843 mlx5e_skb_padding_csum(skb
, network_depth
, proto
, stats
);
844 stats
->csum_complete
++;
849 if (likely((cqe
->hds_ip_ext
& CQE_L3_OK
) &&
850 (cqe
->hds_ip_ext
& CQE_L4_OK
))) {
851 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
852 if (cqe_is_tunneled(cqe
)) {
854 skb
->encapsulation
= 1;
855 stats
->csum_unnecessary_inner
++;
858 stats
->csum_unnecessary
++;
862 skb
->ip_summed
= CHECKSUM_NONE
;
866 #define MLX5E_CE_BIT_MASK 0x80
868 static inline void mlx5e_build_rx_skb(struct mlx5_cqe64
*cqe
,
873 u8 lro_num_seg
= be32_to_cpu(cqe
->srqn
) >> 24;
874 struct mlx5e_rq_stats
*stats
= rq
->stats
;
875 struct net_device
*netdev
= rq
->netdev
;
877 skb
->mac_len
= ETH_HLEN
;
879 #ifdef CONFIG_MLX5_EN_TLS
880 mlx5e_tls_handle_rx_skb(netdev
, skb
, &cqe_bcnt
);
883 if (lro_num_seg
> 1) {
884 mlx5e_lro_update_hdr(skb
, cqe
, cqe_bcnt
);
885 skb_shinfo(skb
)->gso_size
= DIV_ROUND_UP(cqe_bcnt
, lro_num_seg
);
886 /* Subtract one since we already counted this as one
887 * "regular" packet in mlx5e_complete_rx_cqe()
889 stats
->packets
+= lro_num_seg
- 1;
890 stats
->lro_packets
++;
891 stats
->lro_bytes
+= cqe_bcnt
;
894 if (unlikely(mlx5e_rx_hw_stamp(rq
->tstamp
)))
895 skb_hwtstamps(skb
)->hwtstamp
=
896 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
898 skb_record_rx_queue(skb
, rq
->ix
);
900 if (likely(netdev
->features
& NETIF_F_RXHASH
))
901 mlx5e_skb_set_hash(cqe
, skb
);
903 if (cqe_has_vlan(cqe
)) {
904 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
905 be16_to_cpu(cqe
->vlan_info
));
906 stats
->removed_vlan_packets
++;
909 skb
->mark
= be32_to_cpu(cqe
->sop_drop_qpn
) & MLX5E_TC_FLOW_ID_MASK
;
911 mlx5e_handle_csum(netdev
, cqe
, rq
, skb
, !!lro_num_seg
);
912 /* checking CE bit in cqe - MSB in ml_path field */
913 if (unlikely(cqe
->ml_path
& MLX5E_CE_BIT_MASK
))
914 mlx5e_enable_ecn(rq
, skb
);
916 skb
->protocol
= eth_type_trans(skb
, netdev
);
919 static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq
*rq
,
920 struct mlx5_cqe64
*cqe
,
924 struct mlx5e_rq_stats
*stats
= rq
->stats
;
927 stats
->bytes
+= cqe_bcnt
;
928 mlx5e_build_rx_skb(cqe
, cqe_bcnt
, rq
, skb
);
932 struct sk_buff
*mlx5e_build_linear_skb(struct mlx5e_rq
*rq
, void *va
,
933 u32 frag_size
, u16 headroom
,
936 struct sk_buff
*skb
= build_skb(va
, frag_size
);
938 if (unlikely(!skb
)) {
939 rq
->stats
->buff_alloc_err
++;
943 skb_reserve(skb
, headroom
);
944 skb_put(skb
, cqe_bcnt
);
950 mlx5e_skb_from_cqe_linear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
951 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
953 struct mlx5e_dma_info
*di
= wi
->di
;
954 u16 rx_headroom
= rq
->buff
.headroom
;
960 va
= page_address(di
->page
) + wi
->offset
;
961 data
= va
+ rx_headroom
;
962 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt
);
964 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, wi
->offset
,
965 frag_size
, DMA_FROM_DEVICE
);
966 prefetchw(va
); /* xdp_frame data area */
969 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
970 rq
->stats
->wqe_err
++;
975 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt
);
978 return NULL
; /* page/packet was consumed by XDP */
980 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt
);
984 /* queue up for recycling/reuse */
985 page_ref_inc(di
->page
);
991 mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
,
992 struct mlx5e_wqe_frag_info
*wi
, u32 cqe_bcnt
)
994 struct mlx5e_rq_frag_info
*frag_info
= &rq
->wqe
.info
.arr
[0];
995 struct mlx5e_wqe_frag_info
*head_wi
= wi
;
996 u16 headlen
= min_t(u32
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
997 u16 frag_headlen
= headlen
;
998 u16 byte_cnt
= cqe_bcnt
- headlen
;
1001 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
1002 rq
->stats
->wqe_err
++;
1006 /* XDP is not supported in this configuration, as incoming packets
1007 * might spread among multiple pages.
1009 skb
= napi_alloc_skb(rq
->cq
.napi
,
1010 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
1011 if (unlikely(!skb
)) {
1012 rq
->stats
->buff_alloc_err
++;
1016 prefetchw(skb
->data
);
1019 u16 frag_consumed_bytes
=
1020 min_t(u16
, frag_info
->frag_size
- frag_headlen
, byte_cnt
);
1022 mlx5e_add_skb_frag(rq
, skb
, wi
->di
, wi
->offset
+ frag_headlen
,
1023 frag_consumed_bytes
, frag_info
->frag_stride
);
1024 byte_cnt
-= frag_consumed_bytes
;
1031 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_wi
->di
, head_wi
->offset
, headlen
);
1032 /* skb linear part was allocated with headlen and aligned to long */
1033 skb
->tail
+= headlen
;
1034 skb
->len
+= headlen
;
1039 void mlx5e_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1041 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1042 struct mlx5e_wqe_frag_info
*wi
;
1043 struct sk_buff
*skb
;
1047 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1048 wi
= get_frag(rq
, ci
);
1049 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1051 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1053 /* probably for XDP */
1054 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
1055 /* do not return page to cache,
1056 * it will be returned on XDP_TX completion.
1063 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1064 napi_gro_receive(rq
->cq
.napi
, skb
);
1067 mlx5e_free_rx_wqe(rq
, wi
, true);
1069 mlx5_wq_cyc_pop(wq
);
1072 #ifdef CONFIG_MLX5_ESWITCH
1073 void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1075 struct net_device
*netdev
= rq
->netdev
;
1076 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
1077 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1078 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
1079 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1080 struct mlx5e_wqe_frag_info
*wi
;
1081 struct sk_buff
*skb
;
1085 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1086 wi
= get_frag(rq
, ci
);
1087 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1089 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1091 /* probably for XDP */
1092 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
)) {
1093 /* do not return page to cache,
1094 * it will be returned on XDP_TX completion.
1101 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1103 if (rep
->vlan
&& skb_vlan_tag_present(skb
))
1106 napi_gro_receive(rq
->cq
.napi
, skb
);
1109 mlx5e_free_rx_wqe(rq
, wi
, true);
1111 mlx5_wq_cyc_pop(wq
);
1116 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1117 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1119 u16 headlen
= min_t(u16
, MLX5E_RX_MAX_HEAD
, cqe_bcnt
);
1120 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1121 u32 frag_offset
= head_offset
+ headlen
;
1122 u32 byte_cnt
= cqe_bcnt
- headlen
;
1123 struct mlx5e_dma_info
*head_di
= di
;
1124 struct sk_buff
*skb
;
1126 skb
= napi_alloc_skb(rq
->cq
.napi
,
1127 ALIGN(MLX5E_RX_MAX_HEAD
, sizeof(long)));
1128 if (unlikely(!skb
)) {
1129 rq
->stats
->buff_alloc_err
++;
1133 prefetchw(skb
->data
);
1135 if (unlikely(frag_offset
>= PAGE_SIZE
)) {
1137 frag_offset
-= PAGE_SIZE
;
1141 u32 pg_consumed_bytes
=
1142 min_t(u32
, PAGE_SIZE
- frag_offset
, byte_cnt
);
1143 unsigned int truesize
=
1144 ALIGN(pg_consumed_bytes
, BIT(rq
->mpwqe
.log_stride_sz
));
1146 mlx5e_add_skb_frag(rq
, skb
, di
, frag_offset
,
1147 pg_consumed_bytes
, truesize
);
1148 byte_cnt
-= pg_consumed_bytes
;
1153 mlx5e_copy_skb_header(rq
->pdev
, skb
, head_di
, head_offset
, headlen
);
1154 /* skb linear part was allocated with headlen and aligned to long */
1155 skb
->tail
+= headlen
;
1156 skb
->len
+= headlen
;
1162 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq
*rq
, struct mlx5e_mpw_info
*wi
,
1163 u16 cqe_bcnt
, u32 head_offset
, u32 page_idx
)
1165 struct mlx5e_dma_info
*di
= &wi
->umr
.dma_info
[page_idx
];
1166 u16 rx_headroom
= rq
->buff
.headroom
;
1167 u32 cqe_bcnt32
= cqe_bcnt
;
1168 struct sk_buff
*skb
;
1173 /* Check packet size. Note LRO doesn't use linear SKB */
1174 if (unlikely(cqe_bcnt
> rq
->hw_mtu
)) {
1175 rq
->stats
->oversize_pkts_sw_drop
++;
1179 va
= page_address(di
->page
) + head_offset
;
1180 data
= va
+ rx_headroom
;
1181 frag_size
= MLX5_SKB_FRAG_SZ(rx_headroom
+ cqe_bcnt32
);
1183 dma_sync_single_range_for_cpu(rq
->pdev
, di
->addr
, head_offset
,
1184 frag_size
, DMA_FROM_DEVICE
);
1185 prefetchw(va
); /* xdp_frame data area */
1189 consumed
= mlx5e_xdp_handle(rq
, di
, va
, &rx_headroom
, &cqe_bcnt32
);
1192 if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT
, rq
->flags
))
1193 __set_bit(page_idx
, wi
->xdp_xmit_bitmap
); /* non-atomic */
1194 return NULL
; /* page/packet was consumed by XDP */
1197 skb
= mlx5e_build_linear_skb(rq
, va
, frag_size
, rx_headroom
, cqe_bcnt32
);
1201 /* queue up for recycling/reuse */
1202 page_ref_inc(di
->page
);
1207 void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1209 u16 cstrides
= mpwrq_get_cqe_consumed_strides(cqe
);
1210 u16 wqe_id
= be16_to_cpu(cqe
->wqe_id
);
1211 struct mlx5e_mpw_info
*wi
= &rq
->mpwqe
.info
[wqe_id
];
1212 u16 stride_ix
= mpwrq_get_cqe_stride_index(cqe
);
1213 u32 wqe_offset
= stride_ix
<< rq
->mpwqe
.log_stride_sz
;
1214 u32 head_offset
= wqe_offset
& (PAGE_SIZE
- 1);
1215 u32 page_idx
= wqe_offset
>> PAGE_SHIFT
;
1216 struct mlx5e_rx_wqe_ll
*wqe
;
1217 struct mlx5_wq_ll
*wq
;
1218 struct sk_buff
*skb
;
1221 wi
->consumed_strides
+= cstrides
;
1223 if (unlikely(get_cqe_opcode(cqe
) != MLX5_CQE_RESP_SEND
)) {
1224 rq
->stats
->wqe_err
++;
1228 if (unlikely(mpwrq_is_filler_cqe(cqe
))) {
1229 struct mlx5e_rq_stats
*stats
= rq
->stats
;
1231 stats
->mpwqe_filler_cqes
++;
1232 stats
->mpwqe_filler_strides
+= cstrides
;
1236 cqe_bcnt
= mpwrq_get_cqe_byte_cnt(cqe
);
1238 skb
= rq
->mpwqe
.skb_from_cqe_mpwrq(rq
, wi
, cqe_bcnt
, head_offset
,
1243 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1244 napi_gro_receive(rq
->cq
.napi
, skb
);
1247 if (likely(wi
->consumed_strides
< rq
->mpwqe
.num_strides
))
1251 wqe
= mlx5_wq_ll_get_wqe(wq
, wqe_id
);
1252 mlx5e_free_rx_mpwqe(rq
, wi
, true);
1253 mlx5_wq_ll_pop(wq
, cqe
->wqe_id
, &wqe
->next
.next_wqe_index
);
1256 int mlx5e_poll_rx_cq(struct mlx5e_cq
*cq
, int budget
)
1258 struct mlx5e_rq
*rq
= container_of(cq
, struct mlx5e_rq
, cq
);
1259 struct mlx5_cqwq
*cqwq
= &cq
->wq
;
1260 struct mlx5_cqe64
*cqe
;
1263 if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
)))
1267 work_done
+= mlx5e_decompress_cqes_cont(rq
, cqwq
, 0, budget
);
1269 cqe
= mlx5_cqwq_get_cqe(cqwq
);
1271 if (unlikely(work_done
))
1277 if (mlx5_get_cqe_format(cqe
) == MLX5_COMPRESSED
) {
1279 mlx5e_decompress_cqes_start(rq
, cqwq
,
1280 budget
- work_done
);
1284 mlx5_cqwq_pop(cqwq
);
1286 rq
->handle_rx_cqe(rq
, cqe
);
1287 } while ((++work_done
< budget
) && (cqe
= mlx5_cqwq_get_cqe(cqwq
)));
1291 mlx5e_xdp_rx_poll_complete(rq
);
1293 mlx5_cqwq_update_db_record(cqwq
);
1295 /* ensure cq space is freed before enabling more cqes */
1301 #ifdef CONFIG_MLX5_CORE_IPOIB
1303 #define MLX5_IB_GRH_DGID_OFFSET 24
1304 #define MLX5_GID_SIZE 16
1306 static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq
*rq
,
1307 struct mlx5_cqe64
*cqe
,
1309 struct sk_buff
*skb
)
1311 struct hwtstamp_config
*tstamp
;
1312 struct mlx5e_rq_stats
*stats
;
1313 struct net_device
*netdev
;
1314 struct mlx5e_priv
*priv
;
1315 char *pseudo_header
;
1320 qpn
= be32_to_cpu(cqe
->sop_drop_qpn
) & 0xffffff;
1321 netdev
= mlx5i_pkey_get_netdev(rq
->netdev
, qpn
);
1323 /* No mapping present, cannot process SKB. This might happen if a child
1324 * interface is going down while having unprocessed CQEs on parent RQ
1326 if (unlikely(!netdev
)) {
1327 /* TODO: add drop counters support */
1329 pr_warn_once("Unable to map QPN %u to dev - dropping skb\n", qpn
);
1333 priv
= mlx5i_epriv(netdev
);
1334 tstamp
= &priv
->tstamp
;
1335 stats
= &priv
->channel_stats
[rq
->ix
].rq
;
1337 g
= (be32_to_cpu(cqe
->flags_rqpn
) >> 28) & 3;
1338 dgid
= skb
->data
+ MLX5_IB_GRH_DGID_OFFSET
;
1339 if ((!g
) || dgid
[0] != 0xff)
1340 skb
->pkt_type
= PACKET_HOST
;
1341 else if (memcmp(dgid
, netdev
->broadcast
+ 4, MLX5_GID_SIZE
) == 0)
1342 skb
->pkt_type
= PACKET_BROADCAST
;
1344 skb
->pkt_type
= PACKET_MULTICAST
;
1346 /* TODO: IB/ipoib: Allow mcast packets from other VFs
1347 * 68996a6e760e5c74654723eeb57bf65628ae87f4
1350 skb_pull(skb
, MLX5_IB_GRH_BYTES
);
1352 skb
->protocol
= *((__be16
*)(skb
->data
));
1354 if (netdev
->features
& NETIF_F_RXCSUM
) {
1355 skb
->ip_summed
= CHECKSUM_COMPLETE
;
1356 skb
->csum
= csum_unfold((__force __sum16
)cqe
->check_sum
);
1357 stats
->csum_complete
++;
1359 skb
->ip_summed
= CHECKSUM_NONE
;
1363 if (unlikely(mlx5e_rx_hw_stamp(tstamp
)))
1364 skb_hwtstamps(skb
)->hwtstamp
=
1365 mlx5_timecounter_cyc2time(rq
->clock
, get_cqe_ts(cqe
));
1367 skb_record_rx_queue(skb
, rq
->ix
);
1369 if (likely(netdev
->features
& NETIF_F_RXHASH
))
1370 mlx5e_skb_set_hash(cqe
, skb
);
1372 /* 20 bytes of ipoib header and 4 for encap existing */
1373 pseudo_header
= skb_push(skb
, MLX5_IPOIB_PSEUDO_LEN
);
1374 memset(pseudo_header
, 0, MLX5_IPOIB_PSEUDO_LEN
);
1375 skb_reset_mac_header(skb
);
1376 skb_pull(skb
, MLX5_IPOIB_HARD_LEN
);
1381 stats
->bytes
+= cqe_bcnt
;
1384 void mlx5i_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1386 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1387 struct mlx5e_wqe_frag_info
*wi
;
1388 struct sk_buff
*skb
;
1392 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1393 wi
= get_frag(rq
, ci
);
1394 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1396 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1400 mlx5i_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1401 if (unlikely(!skb
->dev
)) {
1402 dev_kfree_skb_any(skb
);
1405 napi_gro_receive(rq
->cq
.napi
, skb
);
1408 mlx5e_free_rx_wqe(rq
, wi
, true);
1409 mlx5_wq_cyc_pop(wq
);
1412 #endif /* CONFIG_MLX5_CORE_IPOIB */
1414 #ifdef CONFIG_MLX5_EN_IPSEC
1416 void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq
*rq
, struct mlx5_cqe64
*cqe
)
1418 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
1419 struct mlx5e_wqe_frag_info
*wi
;
1420 struct sk_buff
*skb
;
1424 ci
= mlx5_wq_cyc_ctr2ix(wq
, be16_to_cpu(cqe
->wqe_counter
));
1425 wi
= get_frag(rq
, ci
);
1426 cqe_bcnt
= be32_to_cpu(cqe
->byte_cnt
);
1428 skb
= rq
->wqe
.skb_from_cqe(rq
, cqe
, wi
, cqe_bcnt
);
1429 if (unlikely(!skb
)) {
1430 /* a DROP, save the page-reuse checks */
1431 mlx5e_free_rx_wqe(rq
, wi
, true);
1434 skb
= mlx5e_ipsec_handle_rx_skb(rq
->netdev
, skb
, &cqe_bcnt
);
1435 if (unlikely(!skb
)) {
1436 mlx5e_free_rx_wqe(rq
, wi
, true);
1440 mlx5e_complete_rx_cqe(rq
, cqe
, cqe_bcnt
, skb
);
1441 napi_gro_receive(rq
->cq
.napi
, skb
);
1443 mlx5e_free_rx_wqe(rq
, wi
, true);
1445 mlx5_wq_cyc_pop(wq
);
1448 #endif /* CONFIG_MLX5_EN_IPSEC */