2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <net/tc_act/tc_gact.h>
34 #include <net/pkt_cls.h>
35 #include <linux/mlx5/fs.h>
36 #include <net/vxlan.h>
37 #include <net/geneve.h>
38 #include <linux/bpf.h>
39 #include <linux/if_bridge.h>
40 #include <net/page_pool.h>
41 #include <net/xdp_sock_drv.h>
47 #include "en_accel/ipsec.h"
48 #include "en_accel/en_accel.h"
49 #include "en_accel/tls.h"
50 #include "accel/ipsec.h"
51 #include "accel/tls.h"
52 #include "lib/vxlan.h"
53 #include "lib/clock.h"
57 #include "en/monitor_stats.h"
58 #include "en/health.h"
59 #include "en/params.h"
60 #include "en/xsk/pool.h"
61 #include "en/xsk/setup.h"
62 #include "en/xsk/rx.h"
63 #include "en/xsk/tx.h"
64 #include "en/hv_vhca_stats.h"
65 #include "en/devlink.h"
70 #include "fpga/ipsec.h"
72 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev
*mdev
)
74 bool striding_rq_umr
= MLX5_CAP_GEN(mdev
, striding_rq
) &&
75 MLX5_CAP_GEN(mdev
, umr_ptr_rlky
) &&
76 MLX5_CAP_ETH(mdev
, reg_umr_sq
);
77 u16 max_wqe_sz_cap
= MLX5_CAP_GEN(mdev
, max_wqe_sz_sq
);
78 bool inline_umr
= MLX5E_UMR_WQE_INLINE_SZ
<= max_wqe_sz_cap
;
83 mlx5_core_warn(mdev
, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
84 (int)MLX5E_UMR_WQE_INLINE_SZ
, max_wqe_sz_cap
);
90 void mlx5e_update_carrier(struct mlx5e_priv
*priv
)
92 struct mlx5_core_dev
*mdev
= priv
->mdev
;
96 port_state
= mlx5_query_vport_state(mdev
,
97 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT
,
100 up
= port_state
== VPORT_STATE_UP
;
101 if (up
== netif_carrier_ok(priv
->netdev
))
102 netif_carrier_event(priv
->netdev
);
104 netdev_info(priv
->netdev
, "Link up\n");
105 netif_carrier_on(priv
->netdev
);
107 netdev_info(priv
->netdev
, "Link down\n");
108 netif_carrier_off(priv
->netdev
);
112 static void mlx5e_update_carrier_work(struct work_struct
*work
)
114 struct mlx5e_priv
*priv
= container_of(work
, struct mlx5e_priv
,
115 update_carrier_work
);
117 mutex_lock(&priv
->state_lock
);
118 if (test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
119 if (priv
->profile
->update_carrier
)
120 priv
->profile
->update_carrier(priv
);
121 mutex_unlock(&priv
->state_lock
);
124 static void mlx5e_update_stats_work(struct work_struct
*work
)
126 struct mlx5e_priv
*priv
= container_of(work
, struct mlx5e_priv
,
129 mutex_lock(&priv
->state_lock
);
130 priv
->profile
->update_stats(priv
);
131 mutex_unlock(&priv
->state_lock
);
134 void mlx5e_queue_update_stats(struct mlx5e_priv
*priv
)
136 if (!priv
->profile
->update_stats
)
139 if (unlikely(test_bit(MLX5E_STATE_DESTROYING
, &priv
->state
)))
142 queue_work(priv
->wq
, &priv
->update_stats_work
);
145 static int async_event(struct notifier_block
*nb
, unsigned long event
, void *data
)
147 struct mlx5e_priv
*priv
= container_of(nb
, struct mlx5e_priv
, events_nb
);
148 struct mlx5_eqe
*eqe
= data
;
150 if (event
!= MLX5_EVENT_TYPE_PORT_CHANGE
)
153 switch (eqe
->sub_type
) {
154 case MLX5_PORT_CHANGE_SUBTYPE_DOWN
:
155 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE
:
156 queue_work(priv
->wq
, &priv
->update_carrier_work
);
165 static void mlx5e_enable_async_events(struct mlx5e_priv
*priv
)
167 priv
->events_nb
.notifier_call
= async_event
;
168 mlx5_notifier_register(priv
->mdev
, &priv
->events_nb
);
171 static void mlx5e_disable_async_events(struct mlx5e_priv
*priv
)
173 mlx5_notifier_unregister(priv
->mdev
, &priv
->events_nb
);
176 static int blocking_event(struct notifier_block
*nb
, unsigned long event
, void *data
)
178 struct mlx5e_priv
*priv
= container_of(nb
, struct mlx5e_priv
, blocking_events_nb
);
182 case MLX5_DRIVER_EVENT_TYPE_TRAP
:
183 err
= mlx5e_handle_trap_event(priv
, data
);
186 netdev_warn(priv
->netdev
, "Sync event: Unknown event %ld\n", event
);
192 static void mlx5e_enable_blocking_events(struct mlx5e_priv
*priv
)
194 priv
->blocking_events_nb
.notifier_call
= blocking_event
;
195 mlx5_blocking_notifier_register(priv
->mdev
, &priv
->blocking_events_nb
);
198 static void mlx5e_disable_blocking_events(struct mlx5e_priv
*priv
)
200 mlx5_blocking_notifier_unregister(priv
->mdev
, &priv
->blocking_events_nb
);
203 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq
*rq
,
204 struct mlx5e_icosq
*sq
,
205 struct mlx5e_umr_wqe
*wqe
)
207 struct mlx5_wqe_ctrl_seg
*cseg
= &wqe
->ctrl
;
208 struct mlx5_wqe_umr_ctrl_seg
*ucseg
= &wqe
->uctrl
;
209 u8 ds_cnt
= DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ
, MLX5_SEND_WQE_DS
);
211 cseg
->qpn_ds
= cpu_to_be32((sq
->sqn
<< MLX5_WQE_CTRL_QPN_SHIFT
) |
213 cseg
->umr_mkey
= rq
->mkey_be
;
215 ucseg
->flags
= MLX5_UMR_TRANSLATION_OFFSET_EN
| MLX5_UMR_INLINE
;
216 ucseg
->xlt_octowords
=
217 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE
));
218 ucseg
->mkey_mask
= cpu_to_be64(MLX5_MKEY_MASK_FREE
);
221 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq
*rq
, int node
)
223 int wq_sz
= mlx5_wq_ll_get_size(&rq
->mpwqe
.wq
);
225 rq
->mpwqe
.info
= kvzalloc_node(array_size(wq_sz
,
226 sizeof(*rq
->mpwqe
.info
)),
231 mlx5e_build_umr_wqe(rq
, rq
->icosq
, &rq
->mpwqe
.umr_wqe
);
236 static int mlx5e_create_umr_mkey(struct mlx5_core_dev
*mdev
,
237 u64 npages
, u8 page_shift
,
238 struct mlx5_core_mkey
*umr_mkey
,
239 dma_addr_t filler_addr
)
241 struct mlx5_mtt
*mtt
;
248 inlen
= MLX5_ST_SZ_BYTES(create_mkey_in
) + sizeof(*mtt
) * npages
;
250 in
= kvzalloc(inlen
, GFP_KERNEL
);
254 mkc
= MLX5_ADDR_OF(create_mkey_in
, in
, memory_key_mkey_entry
);
256 MLX5_SET(mkc
, mkc
, free
, 1);
257 MLX5_SET(mkc
, mkc
, umr_en
, 1);
258 MLX5_SET(mkc
, mkc
, lw
, 1);
259 MLX5_SET(mkc
, mkc
, lr
, 1);
260 MLX5_SET(mkc
, mkc
, access_mode_1_0
, MLX5_MKC_ACCESS_MODE_MTT
);
261 mlx5e_mkey_set_relaxed_ordering(mdev
, mkc
);
262 MLX5_SET(mkc
, mkc
, qpn
, 0xffffff);
263 MLX5_SET(mkc
, mkc
, pd
, mdev
->mlx5e_res
.hw_objs
.pdn
);
264 MLX5_SET64(mkc
, mkc
, len
, npages
<< page_shift
);
265 MLX5_SET(mkc
, mkc
, translations_octword_size
,
266 MLX5_MTT_OCTW(npages
));
267 MLX5_SET(mkc
, mkc
, log_page_size
, page_shift
);
268 MLX5_SET(create_mkey_in
, in
, translations_octword_actual_size
,
269 MLX5_MTT_OCTW(npages
));
271 /* Initialize the mkey with all MTTs pointing to a default
272 * page (filler_addr). When the channels are activated, UMR
273 * WQEs will redirect the RX WQEs to the actual memory from
274 * the RQ's pool, while the gaps (wqe_overflow) remain mapped
275 * to the default page.
277 mtt
= MLX5_ADDR_OF(create_mkey_in
, in
, klm_pas_mtt
);
278 for (i
= 0 ; i
< npages
; i
++)
279 mtt
[i
].ptag
= cpu_to_be64(filler_addr
);
281 err
= mlx5_core_create_mkey(mdev
, umr_mkey
, in
, inlen
);
287 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev
*mdev
, struct mlx5e_rq
*rq
)
289 u64 num_mtts
= MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq
->mpwqe
.wq
));
291 return mlx5e_create_umr_mkey(mdev
, num_mtts
, PAGE_SHIFT
, &rq
->umr_mkey
,
292 rq
->wqe_overflow
.addr
);
295 static u64
mlx5e_get_mpwqe_offset(u16 wqe_ix
)
297 return MLX5E_REQUIRED_MTTS(wqe_ix
) << PAGE_SHIFT
;
300 static void mlx5e_init_frags_partition(struct mlx5e_rq
*rq
)
302 struct mlx5e_wqe_frag_info next_frag
= {};
303 struct mlx5e_wqe_frag_info
*prev
= NULL
;
306 next_frag
.di
= &rq
->wqe
.di
[0];
308 for (i
= 0; i
< mlx5_wq_cyc_get_size(&rq
->wqe
.wq
); i
++) {
309 struct mlx5e_rq_frag_info
*frag_info
= &rq
->wqe
.info
.arr
[0];
310 struct mlx5e_wqe_frag_info
*frag
=
311 &rq
->wqe
.frags
[i
<< rq
->wqe
.info
.log_num_frags
];
314 for (f
= 0; f
< rq
->wqe
.info
.num_frags
; f
++, frag
++) {
315 if (next_frag
.offset
+ frag_info
[f
].frag_stride
> PAGE_SIZE
) {
317 next_frag
.offset
= 0;
319 prev
->last_in_page
= true;
324 next_frag
.offset
+= frag_info
[f
].frag_stride
;
330 prev
->last_in_page
= true;
333 int mlx5e_init_di_list(struct mlx5e_rq
*rq
, int wq_sz
, int node
)
335 int len
= wq_sz
<< rq
->wqe
.info
.log_num_frags
;
337 rq
->wqe
.di
= kvzalloc_node(array_size(len
, sizeof(*rq
->wqe
.di
)), GFP_KERNEL
, node
);
341 mlx5e_init_frags_partition(rq
);
346 void mlx5e_free_di_list(struct mlx5e_rq
*rq
)
351 static void mlx5e_rq_err_cqe_work(struct work_struct
*recover_work
)
353 struct mlx5e_rq
*rq
= container_of(recover_work
, struct mlx5e_rq
, recover_work
);
355 mlx5e_reporter_rq_cqe_err(rq
);
358 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq
*rq
)
360 rq
->wqe_overflow
.page
= alloc_page(GFP_KERNEL
);
361 if (!rq
->wqe_overflow
.page
)
364 rq
->wqe_overflow
.addr
= dma_map_page(rq
->pdev
, rq
->wqe_overflow
.page
, 0,
365 PAGE_SIZE
, rq
->buff
.map_dir
);
366 if (dma_mapping_error(rq
->pdev
, rq
->wqe_overflow
.addr
)) {
367 __free_page(rq
->wqe_overflow
.page
);
373 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq
*rq
)
375 dma_unmap_page(rq
->pdev
, rq
->wqe_overflow
.addr
, PAGE_SIZE
,
377 __free_page(rq
->wqe_overflow
.page
);
380 static int mlx5e_init_rxq_rq(struct mlx5e_channel
*c
, struct mlx5e_params
*params
,
383 struct mlx5_core_dev
*mdev
= c
->mdev
;
386 rq
->wq_type
= params
->rq_wq_type
;
388 rq
->netdev
= c
->netdev
;
390 rq
->tstamp
= c
->tstamp
;
391 rq
->clock
= &mdev
->clock
;
392 rq
->icosq
= &c
->icosq
;
395 rq
->hw_mtu
= MLX5E_SW2HW_MTU(params
, params
->sw_mtu
);
396 rq
->xdpsq
= &c
->rq_xdpsq
;
397 rq
->stats
= &c
->priv
->channel_stats
[c
->ix
].rq
;
398 rq
->ptp_cyc2time
= mlx5_rq_ts_translator(mdev
);
399 err
= mlx5e_rq_set_handlers(rq
, params
, NULL
);
403 return xdp_rxq_info_reg(&rq
->xdp_rxq
, rq
->netdev
, rq
->ix
, 0);
406 static int mlx5e_alloc_rq(struct mlx5e_params
*params
,
407 struct mlx5e_xsk_param
*xsk
,
408 struct mlx5e_rq_param
*rqp
,
409 int node
, struct mlx5e_rq
*rq
)
411 struct page_pool_params pp_params
= { 0 };
412 struct mlx5_core_dev
*mdev
= rq
->mdev
;
413 void *rqc
= rqp
->rqc
;
414 void *rqc_wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
420 rqp
->wq
.db_numa_node
= node
;
421 INIT_WORK(&rq
->recover_work
, mlx5e_rq_err_cqe_work
);
423 if (params
->xdp_prog
)
424 bpf_prog_inc(params
->xdp_prog
);
425 RCU_INIT_POINTER(rq
->xdp_prog
, params
->xdp_prog
);
427 rq
->buff
.map_dir
= params
->xdp_prog
? DMA_BIDIRECTIONAL
: DMA_FROM_DEVICE
;
428 rq
->buff
.headroom
= mlx5e_get_rq_headroom(mdev
, params
, xsk
);
429 pool_size
= 1 << params
->log_rq_mtu_frames
;
431 switch (rq
->wq_type
) {
432 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
433 err
= mlx5_wq_ll_create(mdev
, &rqp
->wq
, rqc_wq
, &rq
->mpwqe
.wq
,
436 goto err_rq_xdp_prog
;
438 err
= mlx5e_alloc_mpwqe_rq_drop_page(rq
);
440 goto err_rq_wq_destroy
;
442 rq
->mpwqe
.wq
.db
= &rq
->mpwqe
.wq
.db
[MLX5_RCV_DBR
];
444 wq_sz
= mlx5_wq_ll_get_size(&rq
->mpwqe
.wq
);
446 pool_size
= MLX5_MPWRQ_PAGES_PER_WQE
<<
447 mlx5e_mpwqe_get_log_rq_size(params
, xsk
);
449 rq
->mpwqe
.log_stride_sz
= mlx5e_mpwqe_get_log_stride_size(mdev
, params
, xsk
);
450 rq
->mpwqe
.num_strides
=
451 BIT(mlx5e_mpwqe_get_log_num_strides(mdev
, params
, xsk
));
453 rq
->buff
.frame0_sz
= (1 << rq
->mpwqe
.log_stride_sz
);
455 err
= mlx5e_create_rq_umr_mkey(mdev
, rq
);
457 goto err_rq_drop_page
;
458 rq
->mkey_be
= cpu_to_be32(rq
->umr_mkey
.key
);
460 err
= mlx5e_rq_alloc_mpwqe_info(rq
, node
);
464 default: /* MLX5_WQ_TYPE_CYCLIC */
465 err
= mlx5_wq_cyc_create(mdev
, &rqp
->wq
, rqc_wq
, &rq
->wqe
.wq
,
468 goto err_rq_xdp_prog
;
470 rq
->wqe
.wq
.db
= &rq
->wqe
.wq
.db
[MLX5_RCV_DBR
];
472 wq_sz
= mlx5_wq_cyc_get_size(&rq
->wqe
.wq
);
474 rq
->wqe
.info
= rqp
->frags_info
;
475 rq
->buff
.frame0_sz
= rq
->wqe
.info
.arr
[0].frag_stride
;
478 kvzalloc_node(array_size(sizeof(*rq
->wqe
.frags
),
479 (wq_sz
<< rq
->wqe
.info
.log_num_frags
)),
481 if (!rq
->wqe
.frags
) {
483 goto err_rq_wq_destroy
;
486 err
= mlx5e_init_di_list(rq
, wq_sz
, node
);
490 rq
->mkey_be
= cpu_to_be32(mdev
->mlx5e_res
.hw_objs
.mkey
.key
);
494 err
= xdp_rxq_info_reg_mem_model(&rq
->xdp_rxq
,
495 MEM_TYPE_XSK_BUFF_POOL
, NULL
);
496 xsk_pool_set_rxq_info(rq
->xsk_pool
, &rq
->xdp_rxq
);
498 /* Create a page_pool and register it with rxq */
500 pp_params
.flags
= 0; /* No-internal DMA mapping in page_pool */
501 pp_params
.pool_size
= pool_size
;
502 pp_params
.nid
= node
;
503 pp_params
.dev
= rq
->pdev
;
504 pp_params
.dma_dir
= rq
->buff
.map_dir
;
506 /* page_pool can be used even when there is no rq->xdp_prog,
507 * given page_pool does not handle DMA mapping there is no
508 * required state to clear. And page_pool gracefully handle
511 rq
->page_pool
= page_pool_create(&pp_params
);
512 if (IS_ERR(rq
->page_pool
)) {
513 err
= PTR_ERR(rq
->page_pool
);
514 rq
->page_pool
= NULL
;
515 goto err_free_by_rq_type
;
517 if (xdp_rxq_info_is_reg(&rq
->xdp_rxq
))
518 err
= xdp_rxq_info_reg_mem_model(&rq
->xdp_rxq
,
519 MEM_TYPE_PAGE_POOL
, rq
->page_pool
);
522 goto err_free_by_rq_type
;
524 for (i
= 0; i
< wq_sz
; i
++) {
525 if (rq
->wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
) {
526 struct mlx5e_rx_wqe_ll
*wqe
=
527 mlx5_wq_ll_get_wqe(&rq
->mpwqe
.wq
, i
);
529 rq
->mpwqe
.num_strides
<< rq
->mpwqe
.log_stride_sz
;
530 u64 dma_offset
= mlx5e_get_mpwqe_offset(i
);
532 wqe
->data
[0].addr
= cpu_to_be64(dma_offset
+ rq
->buff
.headroom
);
533 wqe
->data
[0].byte_count
= cpu_to_be32(byte_count
);
534 wqe
->data
[0].lkey
= rq
->mkey_be
;
536 struct mlx5e_rx_wqe_cyc
*wqe
=
537 mlx5_wq_cyc_get_wqe(&rq
->wqe
.wq
, i
);
540 for (f
= 0; f
< rq
->wqe
.info
.num_frags
; f
++) {
541 u32 frag_size
= rq
->wqe
.info
.arr
[f
].frag_size
|
542 MLX5_HW_START_PADDING
;
544 wqe
->data
[f
].byte_count
= cpu_to_be32(frag_size
);
545 wqe
->data
[f
].lkey
= rq
->mkey_be
;
547 /* check if num_frags is not a pow of two */
548 if (rq
->wqe
.info
.num_frags
< (1 << rq
->wqe
.info
.log_num_frags
)) {
549 wqe
->data
[f
].byte_count
= 0;
550 wqe
->data
[f
].lkey
= cpu_to_be32(MLX5_INVALID_LKEY
);
551 wqe
->data
[f
].addr
= 0;
556 INIT_WORK(&rq
->dim
.work
, mlx5e_rx_dim_work
);
558 switch (params
->rx_cq_moderation
.cq_period_mode
) {
559 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE
:
560 rq
->dim
.mode
= DIM_CQ_PERIOD_MODE_START_FROM_CQE
;
562 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE
:
564 rq
->dim
.mode
= DIM_CQ_PERIOD_MODE_START_FROM_EQE
;
567 rq
->page_cache
.head
= 0;
568 rq
->page_cache
.tail
= 0;
573 switch (rq
->wq_type
) {
574 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
575 kvfree(rq
->mpwqe
.info
);
577 mlx5_core_destroy_mkey(mdev
, &rq
->umr_mkey
);
579 mlx5e_free_mpwqe_rq_drop_page(rq
);
581 default: /* MLX5_WQ_TYPE_CYCLIC */
582 mlx5e_free_di_list(rq
);
584 kvfree(rq
->wqe
.frags
);
587 mlx5_wq_destroy(&rq
->wq_ctrl
);
589 if (params
->xdp_prog
)
590 bpf_prog_put(params
->xdp_prog
);
595 static void mlx5e_free_rq(struct mlx5e_rq
*rq
)
597 struct bpf_prog
*old_prog
;
600 if (xdp_rxq_info_is_reg(&rq
->xdp_rxq
)) {
601 old_prog
= rcu_dereference_protected(rq
->xdp_prog
,
602 lockdep_is_held(&rq
->priv
->state_lock
));
604 bpf_prog_put(old_prog
);
607 switch (rq
->wq_type
) {
608 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
:
609 kvfree(rq
->mpwqe
.info
);
610 mlx5_core_destroy_mkey(rq
->mdev
, &rq
->umr_mkey
);
611 mlx5e_free_mpwqe_rq_drop_page(rq
);
613 default: /* MLX5_WQ_TYPE_CYCLIC */
614 kvfree(rq
->wqe
.frags
);
615 mlx5e_free_di_list(rq
);
618 for (i
= rq
->page_cache
.head
; i
!= rq
->page_cache
.tail
;
619 i
= (i
+ 1) & (MLX5E_CACHE_SIZE
- 1)) {
620 struct mlx5e_dma_info
*dma_info
= &rq
->page_cache
.page_cache
[i
];
622 /* With AF_XDP, page_cache is not used, so this loop is not
623 * entered, and it's safe to call mlx5e_page_release_dynamic
626 mlx5e_page_release_dynamic(rq
, dma_info
, false);
629 xdp_rxq_info_unreg(&rq
->xdp_rxq
);
630 page_pool_destroy(rq
->page_pool
);
631 mlx5_wq_destroy(&rq
->wq_ctrl
);
634 int mlx5e_create_rq(struct mlx5e_rq
*rq
, struct mlx5e_rq_param
*param
)
636 struct mlx5_core_dev
*mdev
= rq
->mdev
;
644 inlen
= MLX5_ST_SZ_BYTES(create_rq_in
) +
645 sizeof(u64
) * rq
->wq_ctrl
.buf
.npages
;
646 in
= kvzalloc(inlen
, GFP_KERNEL
);
650 ts_format
= mlx5_is_real_time_rq(mdev
) ?
651 MLX5_TIMESTAMP_FORMAT_REAL_TIME
:
652 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING
;
653 rqc
= MLX5_ADDR_OF(create_rq_in
, in
, ctx
);
654 wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
656 memcpy(rqc
, param
->rqc
, sizeof(param
->rqc
));
658 MLX5_SET(rqc
, rqc
, cqn
, rq
->cq
.mcq
.cqn
);
659 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RST
);
660 MLX5_SET(rqc
, rqc
, ts_format
, ts_format
);
661 MLX5_SET(wq
, wq
, log_wq_pg_sz
, rq
->wq_ctrl
.buf
.page_shift
-
662 MLX5_ADAPTER_PAGE_SHIFT
);
663 MLX5_SET64(wq
, wq
, dbr_addr
, rq
->wq_ctrl
.db
.dma
);
665 mlx5_fill_page_frag_array(&rq
->wq_ctrl
.buf
,
666 (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
));
668 err
= mlx5_core_create_rq(mdev
, in
, inlen
, &rq
->rqn
);
675 int mlx5e_modify_rq_state(struct mlx5e_rq
*rq
, int curr_state
, int next_state
)
677 struct mlx5_core_dev
*mdev
= rq
->mdev
;
684 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
685 in
= kvzalloc(inlen
, GFP_KERNEL
);
689 if (curr_state
== MLX5_RQC_STATE_RST
&& next_state
== MLX5_RQC_STATE_RDY
)
690 mlx5e_rqwq_reset(rq
);
692 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
694 MLX5_SET(modify_rq_in
, in
, rq_state
, curr_state
);
695 MLX5_SET(rqc
, rqc
, state
, next_state
);
697 err
= mlx5_core_modify_rq(mdev
, rq
->rqn
, in
);
704 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq
*rq
, bool enable
)
706 struct mlx5_core_dev
*mdev
= rq
->mdev
;
713 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
714 in
= kvzalloc(inlen
, GFP_KERNEL
);
718 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
720 MLX5_SET(modify_rq_in
, in
, rq_state
, MLX5_RQC_STATE_RDY
);
721 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
722 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS
);
723 MLX5_SET(rqc
, rqc
, scatter_fcs
, enable
);
724 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RDY
);
726 err
= mlx5_core_modify_rq(mdev
, rq
->rqn
, in
);
733 static int mlx5e_modify_rq_vsd(struct mlx5e_rq
*rq
, bool vsd
)
735 struct mlx5_core_dev
*mdev
= rq
->mdev
;
741 inlen
= MLX5_ST_SZ_BYTES(modify_rq_in
);
742 in
= kvzalloc(inlen
, GFP_KERNEL
);
746 rqc
= MLX5_ADDR_OF(modify_rq_in
, in
, ctx
);
748 MLX5_SET(modify_rq_in
, in
, rq_state
, MLX5_RQC_STATE_RDY
);
749 MLX5_SET64(modify_rq_in
, in
, modify_bitmask
,
750 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD
);
751 MLX5_SET(rqc
, rqc
, vsd
, vsd
);
752 MLX5_SET(rqc
, rqc
, state
, MLX5_RQC_STATE_RDY
);
754 err
= mlx5_core_modify_rq(mdev
, rq
->rqn
, in
);
761 void mlx5e_destroy_rq(struct mlx5e_rq
*rq
)
763 mlx5_core_destroy_rq(rq
->mdev
, rq
->rqn
);
766 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq
*rq
, int wait_time
)
768 unsigned long exp_time
= jiffies
+ msecs_to_jiffies(wait_time
);
770 u16 min_wqes
= mlx5_min_rx_wqes(rq
->wq_type
, mlx5e_rqwq_get_size(rq
));
773 if (mlx5e_rqwq_get_cur_sz(rq
) >= min_wqes
)
777 } while (time_before(jiffies
, exp_time
));
779 netdev_warn(rq
->netdev
, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
780 rq
->ix
, rq
->rqn
, mlx5e_rqwq_get_cur_sz(rq
), min_wqes
);
782 mlx5e_reporter_rx_timeout(rq
);
786 void mlx5e_free_rx_in_progress_descs(struct mlx5e_rq
*rq
)
788 struct mlx5_wq_ll
*wq
;
792 if (rq
->wq_type
!= MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
)
798 /* Outstanding UMR WQEs (in progress) start at wq->head */
799 for (i
= 0; i
< rq
->mpwqe
.umr_in_progress
; i
++) {
800 rq
->dealloc_wqe(rq
, head
);
801 head
= mlx5_wq_ll_get_wqe_next_ix(wq
, head
);
804 rq
->mpwqe
.actual_wq_head
= wq
->head
;
805 rq
->mpwqe
.umr_in_progress
= 0;
806 rq
->mpwqe
.umr_completed
= 0;
809 void mlx5e_free_rx_descs(struct mlx5e_rq
*rq
)
814 if (rq
->wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
) {
815 struct mlx5_wq_ll
*wq
= &rq
->mpwqe
.wq
;
817 mlx5e_free_rx_in_progress_descs(rq
);
819 while (!mlx5_wq_ll_is_empty(wq
)) {
820 struct mlx5e_rx_wqe_ll
*wqe
;
822 wqe_ix_be
= *wq
->tail_next
;
823 wqe_ix
= be16_to_cpu(wqe_ix_be
);
824 wqe
= mlx5_wq_ll_get_wqe(wq
, wqe_ix
);
825 rq
->dealloc_wqe(rq
, wqe_ix
);
826 mlx5_wq_ll_pop(wq
, wqe_ix_be
,
827 &wqe
->next
.next_wqe_index
);
830 struct mlx5_wq_cyc
*wq
= &rq
->wqe
.wq
;
832 while (!mlx5_wq_cyc_is_empty(wq
)) {
833 wqe_ix
= mlx5_wq_cyc_get_tail(wq
);
834 rq
->dealloc_wqe(rq
, wqe_ix
);
841 int mlx5e_open_rq(struct mlx5e_params
*params
, struct mlx5e_rq_param
*param
,
842 struct mlx5e_xsk_param
*xsk
, int node
,
845 struct mlx5_core_dev
*mdev
= rq
->mdev
;
848 err
= mlx5e_alloc_rq(params
, xsk
, param
, node
, rq
);
852 err
= mlx5e_create_rq(rq
, param
);
856 err
= mlx5e_modify_rq_state(rq
, MLX5_RQC_STATE_RST
, MLX5_RQC_STATE_RDY
);
860 if (mlx5e_is_tls_on(rq
->priv
) && !mlx5e_accel_is_ktls_device(mdev
))
861 __set_bit(MLX5E_RQ_STATE_FPGA_TLS
, &rq
->state
); /* must be FPGA */
863 if (MLX5_CAP_ETH(mdev
, cqe_checksum_full
))
864 __set_bit(MLX5E_RQ_STATE_CSUM_FULL
, &rq
->state
);
866 if (params
->rx_dim_enabled
)
867 __set_bit(MLX5E_RQ_STATE_AM
, &rq
->state
);
869 /* We disable csum_complete when XDP is enabled since
870 * XDP programs might manipulate packets which will render
871 * skb->checksum incorrect.
873 if (MLX5E_GET_PFLAG(params
, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE
) || params
->xdp_prog
)
874 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE
, &rq
->state
);
876 /* For CQE compression on striding RQ, use stride index provided by
877 * HW if capability is supported.
879 if (MLX5E_GET_PFLAG(params
, MLX5E_PFLAG_RX_STRIDING_RQ
) &&
880 MLX5_CAP_GEN(mdev
, mini_cqe_resp_stride_index
))
881 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX
, &rq
->state
);
886 mlx5e_destroy_rq(rq
);
893 void mlx5e_activate_rq(struct mlx5e_rq
*rq
)
895 set_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
);
897 mlx5e_trigger_irq(rq
->icosq
);
900 napi_schedule(rq
->cq
.napi
);
905 void mlx5e_deactivate_rq(struct mlx5e_rq
*rq
)
907 clear_bit(MLX5E_RQ_STATE_ENABLED
, &rq
->state
);
908 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
911 void mlx5e_close_rq(struct mlx5e_rq
*rq
)
913 cancel_work_sync(&rq
->dim
.work
);
915 cancel_work_sync(&rq
->icosq
->recover_work
);
916 cancel_work_sync(&rq
->recover_work
);
917 mlx5e_destroy_rq(rq
);
918 mlx5e_free_rx_descs(rq
);
922 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq
*sq
)
924 kvfree(sq
->db
.xdpi_fifo
.xi
);
925 kvfree(sq
->db
.wqe_info
);
928 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq
*sq
, int numa
)
930 struct mlx5e_xdp_info_fifo
*xdpi_fifo
= &sq
->db
.xdpi_fifo
;
931 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
932 int dsegs_per_wq
= wq_sz
* MLX5_SEND_WQEBB_NUM_DS
;
934 xdpi_fifo
->xi
= kvzalloc_node(sizeof(*xdpi_fifo
->xi
) * dsegs_per_wq
,
939 xdpi_fifo
->pc
= &sq
->xdpi_fifo_pc
;
940 xdpi_fifo
->cc
= &sq
->xdpi_fifo_cc
;
941 xdpi_fifo
->mask
= dsegs_per_wq
- 1;
946 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq
*sq
, int numa
)
948 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
951 sq
->db
.wqe_info
= kvzalloc_node(sizeof(*sq
->db
.wqe_info
) * wq_sz
,
953 if (!sq
->db
.wqe_info
)
956 err
= mlx5e_alloc_xdpsq_fifo(sq
, numa
);
958 mlx5e_free_xdpsq_db(sq
);
965 static int mlx5e_alloc_xdpsq(struct mlx5e_channel
*c
,
966 struct mlx5e_params
*params
,
967 struct xsk_buff_pool
*xsk_pool
,
968 struct mlx5e_sq_param
*param
,
969 struct mlx5e_xdpsq
*sq
,
972 void *sqc_wq
= MLX5_ADDR_OF(sqc
, param
->sqc
, wq
);
973 struct mlx5_core_dev
*mdev
= c
->mdev
;
974 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
978 sq
->mkey_be
= c
->mkey_be
;
980 sq
->uar_map
= mdev
->mlx5e_res
.hw_objs
.bfreg
.map
;
981 sq
->min_inline_mode
= params
->tx_min_inline_mode
;
982 sq
->hw_mtu
= MLX5E_SW2HW_MTU(params
, params
->sw_mtu
);
983 sq
->xsk_pool
= xsk_pool
;
985 sq
->stats
= sq
->xsk_pool
?
986 &c
->priv
->channel_stats
[c
->ix
].xsksq
:
988 &c
->priv
->channel_stats
[c
->ix
].xdpsq
:
989 &c
->priv
->channel_stats
[c
->ix
].rq_xdpsq
;
991 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
992 err
= mlx5_wq_cyc_create(mdev
, ¶m
->wq
, sqc_wq
, wq
, &sq
->wq_ctrl
);
995 wq
->db
= &wq
->db
[MLX5_SND_DBR
];
997 err
= mlx5e_alloc_xdpsq_db(sq
, cpu_to_node(c
->cpu
));
999 goto err_sq_wq_destroy
;
1004 mlx5_wq_destroy(&sq
->wq_ctrl
);
1009 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq
*sq
)
1011 mlx5e_free_xdpsq_db(sq
);
1012 mlx5_wq_destroy(&sq
->wq_ctrl
);
1015 static void mlx5e_free_icosq_db(struct mlx5e_icosq
*sq
)
1017 kvfree(sq
->db
.wqe_info
);
1020 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq
*sq
, int numa
)
1022 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
1025 size
= array_size(wq_sz
, sizeof(*sq
->db
.wqe_info
));
1026 sq
->db
.wqe_info
= kvzalloc_node(size
, GFP_KERNEL
, numa
);
1027 if (!sq
->db
.wqe_info
)
1033 static void mlx5e_icosq_err_cqe_work(struct work_struct
*recover_work
)
1035 struct mlx5e_icosq
*sq
= container_of(recover_work
, struct mlx5e_icosq
,
1038 mlx5e_reporter_icosq_cqe_err(sq
);
1041 static int mlx5e_alloc_icosq(struct mlx5e_channel
*c
,
1042 struct mlx5e_sq_param
*param
,
1043 struct mlx5e_icosq
*sq
)
1045 void *sqc_wq
= MLX5_ADDR_OF(sqc
, param
->sqc
, wq
);
1046 struct mlx5_core_dev
*mdev
= c
->mdev
;
1047 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
1051 sq
->uar_map
= mdev
->mlx5e_res
.hw_objs
.bfreg
.map
;
1052 sq
->reserved_room
= param
->stop_room
;
1054 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
1055 err
= mlx5_wq_cyc_create(mdev
, ¶m
->wq
, sqc_wq
, wq
, &sq
->wq_ctrl
);
1058 wq
->db
= &wq
->db
[MLX5_SND_DBR
];
1060 err
= mlx5e_alloc_icosq_db(sq
, cpu_to_node(c
->cpu
));
1062 goto err_sq_wq_destroy
;
1064 INIT_WORK(&sq
->recover_work
, mlx5e_icosq_err_cqe_work
);
1069 mlx5_wq_destroy(&sq
->wq_ctrl
);
1074 static void mlx5e_free_icosq(struct mlx5e_icosq
*sq
)
1076 mlx5e_free_icosq_db(sq
);
1077 mlx5_wq_destroy(&sq
->wq_ctrl
);
1080 void mlx5e_free_txqsq_db(struct mlx5e_txqsq
*sq
)
1082 kvfree(sq
->db
.wqe_info
);
1083 kvfree(sq
->db
.skb_fifo
.fifo
);
1084 kvfree(sq
->db
.dma_fifo
);
1087 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq
*sq
, int numa
)
1089 int wq_sz
= mlx5_wq_cyc_get_size(&sq
->wq
);
1090 int df_sz
= wq_sz
* MLX5_SEND_WQEBB_NUM_DS
;
1092 sq
->db
.dma_fifo
= kvzalloc_node(array_size(df_sz
,
1093 sizeof(*sq
->db
.dma_fifo
)),
1095 sq
->db
.skb_fifo
.fifo
= kvzalloc_node(array_size(df_sz
,
1096 sizeof(*sq
->db
.skb_fifo
.fifo
)),
1098 sq
->db
.wqe_info
= kvzalloc_node(array_size(wq_sz
,
1099 sizeof(*sq
->db
.wqe_info
)),
1101 if (!sq
->db
.dma_fifo
|| !sq
->db
.skb_fifo
.fifo
|| !sq
->db
.wqe_info
) {
1102 mlx5e_free_txqsq_db(sq
);
1106 sq
->dma_fifo_mask
= df_sz
- 1;
1108 sq
->db
.skb_fifo
.pc
= &sq
->skb_fifo_pc
;
1109 sq
->db
.skb_fifo
.cc
= &sq
->skb_fifo_cc
;
1110 sq
->db
.skb_fifo
.mask
= df_sz
- 1;
1115 static int mlx5e_alloc_txqsq(struct mlx5e_channel
*c
,
1117 struct mlx5e_params
*params
,
1118 struct mlx5e_sq_param
*param
,
1119 struct mlx5e_txqsq
*sq
,
1122 void *sqc_wq
= MLX5_ADDR_OF(sqc
, param
->sqc
, wq
);
1123 struct mlx5_core_dev
*mdev
= c
->mdev
;
1124 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
1128 sq
->tstamp
= c
->tstamp
;
1129 sq
->clock
= &mdev
->clock
;
1130 sq
->mkey_be
= c
->mkey_be
;
1131 sq
->netdev
= c
->netdev
;
1135 sq
->txq_ix
= txq_ix
;
1136 sq
->uar_map
= mdev
->mlx5e_res
.hw_objs
.bfreg
.map
;
1137 sq
->min_inline_mode
= params
->tx_min_inline_mode
;
1138 sq
->hw_mtu
= MLX5E_SW2HW_MTU(params
, params
->sw_mtu
);
1139 INIT_WORK(&sq
->recover_work
, mlx5e_tx_err_cqe_work
);
1140 if (!MLX5_CAP_ETH(mdev
, wqe_vlan_insert
))
1141 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE
, &sq
->state
);
1142 if (MLX5_IPSEC_DEV(c
->priv
->mdev
))
1143 set_bit(MLX5E_SQ_STATE_IPSEC
, &sq
->state
);
1145 set_bit(MLX5E_SQ_STATE_MPWQE
, &sq
->state
);
1146 sq
->stop_room
= param
->stop_room
;
1147 sq
->ptp_cyc2time
= mlx5_sq_ts_translator(mdev
);
1149 param
->wq
.db_numa_node
= cpu_to_node(c
->cpu
);
1150 err
= mlx5_wq_cyc_create(mdev
, ¶m
->wq
, sqc_wq
, wq
, &sq
->wq_ctrl
);
1153 wq
->db
= &wq
->db
[MLX5_SND_DBR
];
1155 err
= mlx5e_alloc_txqsq_db(sq
, cpu_to_node(c
->cpu
));
1157 goto err_sq_wq_destroy
;
1159 INIT_WORK(&sq
->dim
.work
, mlx5e_tx_dim_work
);
1160 sq
->dim
.mode
= params
->tx_cq_moderation
.cq_period_mode
;
1165 mlx5_wq_destroy(&sq
->wq_ctrl
);
1170 void mlx5e_free_txqsq(struct mlx5e_txqsq
*sq
)
1172 mlx5e_free_txqsq_db(sq
);
1173 mlx5_wq_destroy(&sq
->wq_ctrl
);
1176 static int mlx5e_create_sq(struct mlx5_core_dev
*mdev
,
1177 struct mlx5e_sq_param
*param
,
1178 struct mlx5e_create_sq_param
*csp
,
1188 inlen
= MLX5_ST_SZ_BYTES(create_sq_in
) +
1189 sizeof(u64
) * csp
->wq_ctrl
->buf
.npages
;
1190 in
= kvzalloc(inlen
, GFP_KERNEL
);
1194 ts_format
= mlx5_is_real_time_sq(mdev
) ?
1195 MLX5_TIMESTAMP_FORMAT_REAL_TIME
:
1196 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING
;
1197 sqc
= MLX5_ADDR_OF(create_sq_in
, in
, ctx
);
1198 wq
= MLX5_ADDR_OF(sqc
, sqc
, wq
);
1200 memcpy(sqc
, param
->sqc
, sizeof(param
->sqc
));
1201 MLX5_SET(sqc
, sqc
, tis_lst_sz
, csp
->tis_lst_sz
);
1202 MLX5_SET(sqc
, sqc
, tis_num_0
, csp
->tisn
);
1203 MLX5_SET(sqc
, sqc
, cqn
, csp
->cqn
);
1204 MLX5_SET(sqc
, sqc
, ts_cqe_to_dest_cqn
, csp
->ts_cqe_to_dest_cqn
);
1205 MLX5_SET(sqc
, sqc
, ts_format
, ts_format
);
1208 if (MLX5_CAP_ETH(mdev
, wqe_inline_mode
) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
)
1209 MLX5_SET(sqc
, sqc
, min_wqe_inline_mode
, csp
->min_inline_mode
);
1211 MLX5_SET(sqc
, sqc
, state
, MLX5_SQC_STATE_RST
);
1212 MLX5_SET(sqc
, sqc
, flush_in_error_en
, 1);
1214 MLX5_SET(wq
, wq
, wq_type
, MLX5_WQ_TYPE_CYCLIC
);
1215 MLX5_SET(wq
, wq
, uar_page
, mdev
->mlx5e_res
.hw_objs
.bfreg
.index
);
1216 MLX5_SET(wq
, wq
, log_wq_pg_sz
, csp
->wq_ctrl
->buf
.page_shift
-
1217 MLX5_ADAPTER_PAGE_SHIFT
);
1218 MLX5_SET64(wq
, wq
, dbr_addr
, csp
->wq_ctrl
->db
.dma
);
1220 mlx5_fill_page_frag_array(&csp
->wq_ctrl
->buf
,
1221 (__be64
*)MLX5_ADDR_OF(wq
, wq
, pas
));
1223 err
= mlx5_core_create_sq(mdev
, in
, inlen
, sqn
);
1230 int mlx5e_modify_sq(struct mlx5_core_dev
*mdev
, u32 sqn
,
1231 struct mlx5e_modify_sq_param
*p
)
1239 inlen
= MLX5_ST_SZ_BYTES(modify_sq_in
);
1240 in
= kvzalloc(inlen
, GFP_KERNEL
);
1244 sqc
= MLX5_ADDR_OF(modify_sq_in
, in
, ctx
);
1246 MLX5_SET(modify_sq_in
, in
, sq_state
, p
->curr_state
);
1247 MLX5_SET(sqc
, sqc
, state
, p
->next_state
);
1248 if (p
->rl_update
&& p
->next_state
== MLX5_SQC_STATE_RDY
) {
1250 MLX5_SET(sqc
, sqc
, packet_pacing_rate_limit_index
, p
->rl_index
);
1252 if (p
->qos_update
&& p
->next_state
== MLX5_SQC_STATE_RDY
) {
1254 MLX5_SET(sqc
, sqc
, qos_queue_group_id
, p
->qos_queue_group_id
);
1256 MLX5_SET64(modify_sq_in
, in
, modify_bitmask
, bitmask
);
1258 err
= mlx5_core_modify_sq(mdev
, sqn
, in
);
1265 static void mlx5e_destroy_sq(struct mlx5_core_dev
*mdev
, u32 sqn
)
1267 mlx5_core_destroy_sq(mdev
, sqn
);
1270 int mlx5e_create_sq_rdy(struct mlx5_core_dev
*mdev
,
1271 struct mlx5e_sq_param
*param
,
1272 struct mlx5e_create_sq_param
*csp
,
1273 u16 qos_queue_group_id
,
1276 struct mlx5e_modify_sq_param msp
= {0};
1279 err
= mlx5e_create_sq(mdev
, param
, csp
, sqn
);
1283 msp
.curr_state
= MLX5_SQC_STATE_RST
;
1284 msp
.next_state
= MLX5_SQC_STATE_RDY
;
1285 if (qos_queue_group_id
) {
1286 msp
.qos_update
= true;
1287 msp
.qos_queue_group_id
= qos_queue_group_id
;
1289 err
= mlx5e_modify_sq(mdev
, *sqn
, &msp
);
1291 mlx5e_destroy_sq(mdev
, *sqn
);
1296 static int mlx5e_set_sq_maxrate(struct net_device
*dev
,
1297 struct mlx5e_txqsq
*sq
, u32 rate
);
1299 int mlx5e_open_txqsq(struct mlx5e_channel
*c
, u32 tisn
, int txq_ix
,
1300 struct mlx5e_params
*params
, struct mlx5e_sq_param
*param
,
1301 struct mlx5e_txqsq
*sq
, int tc
, u16 qos_queue_group_id
, u16 qos_qid
)
1303 struct mlx5e_create_sq_param csp
= {};
1307 err
= mlx5e_alloc_txqsq(c
, txq_ix
, params
, param
, sq
, tc
);
1311 if (qos_queue_group_id
)
1312 sq
->stats
= c
->priv
->htb
.qos_sq_stats
[qos_qid
];
1314 sq
->stats
= &c
->priv
->channel_stats
[c
->ix
].sq
[tc
];
1318 csp
.cqn
= sq
->cq
.mcq
.cqn
;
1319 csp
.wq_ctrl
= &sq
->wq_ctrl
;
1320 csp
.min_inline_mode
= sq
->min_inline_mode
;
1321 err
= mlx5e_create_sq_rdy(c
->mdev
, param
, &csp
, qos_queue_group_id
, &sq
->sqn
);
1323 goto err_free_txqsq
;
1325 tx_rate
= c
->priv
->tx_rates
[sq
->txq_ix
];
1327 mlx5e_set_sq_maxrate(c
->netdev
, sq
, tx_rate
);
1329 if (params
->tx_dim_enabled
)
1330 sq
->state
|= BIT(MLX5E_SQ_STATE_AM
);
1335 mlx5e_free_txqsq(sq
);
1340 void mlx5e_activate_txqsq(struct mlx5e_txqsq
*sq
)
1342 sq
->txq
= netdev_get_tx_queue(sq
->netdev
, sq
->txq_ix
);
1343 set_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
);
1344 netdev_tx_reset_queue(sq
->txq
);
1345 netif_tx_start_queue(sq
->txq
);
1348 void mlx5e_tx_disable_queue(struct netdev_queue
*txq
)
1350 __netif_tx_lock_bh(txq
);
1351 netif_tx_stop_queue(txq
);
1352 __netif_tx_unlock_bh(txq
);
1355 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq
*sq
)
1357 struct mlx5_wq_cyc
*wq
= &sq
->wq
;
1359 clear_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
);
1360 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
1362 mlx5e_tx_disable_queue(sq
->txq
);
1364 /* last doorbell out, godspeed .. */
1365 if (mlx5e_wqc_has_room_for(wq
, sq
->cc
, sq
->pc
, 1)) {
1366 u16 pi
= mlx5_wq_cyc_ctr2ix(wq
, sq
->pc
);
1367 struct mlx5e_tx_wqe
*nop
;
1369 sq
->db
.wqe_info
[pi
] = (struct mlx5e_tx_wqe_info
) {
1373 nop
= mlx5e_post_nop(wq
, sq
->sqn
, &sq
->pc
);
1374 mlx5e_notify_hw(wq
, sq
->pc
, sq
->uar_map
, &nop
->ctrl
);
1378 void mlx5e_close_txqsq(struct mlx5e_txqsq
*sq
)
1380 struct mlx5_core_dev
*mdev
= sq
->mdev
;
1381 struct mlx5_rate_limit rl
= {0};
1383 cancel_work_sync(&sq
->dim
.work
);
1384 cancel_work_sync(&sq
->recover_work
);
1385 mlx5e_destroy_sq(mdev
, sq
->sqn
);
1386 if (sq
->rate_limit
) {
1387 rl
.rate
= sq
->rate_limit
;
1388 mlx5_rl_remove_rate(mdev
, &rl
);
1390 mlx5e_free_txqsq_descs(sq
);
1391 mlx5e_free_txqsq(sq
);
1394 void mlx5e_tx_err_cqe_work(struct work_struct
*recover_work
)
1396 struct mlx5e_txqsq
*sq
= container_of(recover_work
, struct mlx5e_txqsq
,
1399 mlx5e_reporter_tx_err_cqe(sq
);
1402 int mlx5e_open_icosq(struct mlx5e_channel
*c
, struct mlx5e_params
*params
,
1403 struct mlx5e_sq_param
*param
, struct mlx5e_icosq
*sq
)
1405 struct mlx5e_create_sq_param csp
= {};
1408 err
= mlx5e_alloc_icosq(c
, param
, sq
);
1412 csp
.cqn
= sq
->cq
.mcq
.cqn
;
1413 csp
.wq_ctrl
= &sq
->wq_ctrl
;
1414 csp
.min_inline_mode
= params
->tx_min_inline_mode
;
1415 err
= mlx5e_create_sq_rdy(c
->mdev
, param
, &csp
, 0, &sq
->sqn
);
1417 goto err_free_icosq
;
1419 if (param
->is_tls
) {
1420 sq
->ktls_resync
= mlx5e_ktls_rx_resync_create_resp_list();
1421 if (IS_ERR(sq
->ktls_resync
)) {
1422 err
= PTR_ERR(sq
->ktls_resync
);
1423 goto err_destroy_icosq
;
1429 mlx5e_destroy_sq(c
->mdev
, sq
->sqn
);
1431 mlx5e_free_icosq(sq
);
1436 void mlx5e_activate_icosq(struct mlx5e_icosq
*icosq
)
1438 set_bit(MLX5E_SQ_STATE_ENABLED
, &icosq
->state
);
1441 void mlx5e_deactivate_icosq(struct mlx5e_icosq
*icosq
)
1443 clear_bit(MLX5E_SQ_STATE_ENABLED
, &icosq
->state
);
1444 synchronize_net(); /* Sync with NAPI. */
1447 void mlx5e_close_icosq(struct mlx5e_icosq
*sq
)
1449 struct mlx5e_channel
*c
= sq
->channel
;
1451 if (sq
->ktls_resync
)
1452 mlx5e_ktls_rx_resync_destroy_resp_list(sq
->ktls_resync
);
1453 mlx5e_destroy_sq(c
->mdev
, sq
->sqn
);
1454 mlx5e_free_icosq_descs(sq
);
1455 mlx5e_free_icosq(sq
);
1458 int mlx5e_open_xdpsq(struct mlx5e_channel
*c
, struct mlx5e_params
*params
,
1459 struct mlx5e_sq_param
*param
, struct xsk_buff_pool
*xsk_pool
,
1460 struct mlx5e_xdpsq
*sq
, bool is_redirect
)
1462 struct mlx5e_create_sq_param csp
= {};
1465 err
= mlx5e_alloc_xdpsq(c
, params
, xsk_pool
, param
, sq
, is_redirect
);
1470 csp
.tisn
= c
->priv
->tisn
[c
->lag_port
][0]; /* tc = 0 */
1471 csp
.cqn
= sq
->cq
.mcq
.cqn
;
1472 csp
.wq_ctrl
= &sq
->wq_ctrl
;
1473 csp
.min_inline_mode
= sq
->min_inline_mode
;
1474 set_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
);
1475 err
= mlx5e_create_sq_rdy(c
->mdev
, param
, &csp
, 0, &sq
->sqn
);
1477 goto err_free_xdpsq
;
1479 mlx5e_set_xmit_fp(sq
, param
->is_mpw
);
1481 if (!param
->is_mpw
) {
1482 unsigned int ds_cnt
= MLX5E_XDP_TX_DS_COUNT
;
1483 unsigned int inline_hdr_sz
= 0;
1486 if (sq
->min_inline_mode
!= MLX5_INLINE_MODE_NONE
) {
1487 inline_hdr_sz
= MLX5E_XDP_MIN_INLINE
;
1491 /* Pre initialize fixed WQE fields */
1492 for (i
= 0; i
< mlx5_wq_cyc_get_size(&sq
->wq
); i
++) {
1493 struct mlx5e_tx_wqe
*wqe
= mlx5_wq_cyc_get_wqe(&sq
->wq
, i
);
1494 struct mlx5_wqe_ctrl_seg
*cseg
= &wqe
->ctrl
;
1495 struct mlx5_wqe_eth_seg
*eseg
= &wqe
->eth
;
1496 struct mlx5_wqe_data_seg
*dseg
;
1498 sq
->db
.wqe_info
[i
] = (struct mlx5e_xdp_wqe_info
) {
1503 cseg
->qpn_ds
= cpu_to_be32((sq
->sqn
<< 8) | ds_cnt
);
1504 eseg
->inline_hdr
.sz
= cpu_to_be16(inline_hdr_sz
);
1506 dseg
= (struct mlx5_wqe_data_seg
*)cseg
+ (ds_cnt
- 1);
1507 dseg
->lkey
= sq
->mkey_be
;
1514 clear_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
);
1515 mlx5e_free_xdpsq(sq
);
1520 void mlx5e_close_xdpsq(struct mlx5e_xdpsq
*sq
)
1522 struct mlx5e_channel
*c
= sq
->channel
;
1524 clear_bit(MLX5E_SQ_STATE_ENABLED
, &sq
->state
);
1525 synchronize_net(); /* Sync with NAPI. */
1527 mlx5e_destroy_sq(c
->mdev
, sq
->sqn
);
1528 mlx5e_free_xdpsq_descs(sq
);
1529 mlx5e_free_xdpsq(sq
);
1532 static int mlx5e_alloc_cq_common(struct mlx5e_priv
*priv
,
1533 struct mlx5e_cq_param
*param
,
1534 struct mlx5e_cq
*cq
)
1536 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1537 struct mlx5_core_cq
*mcq
= &cq
->mcq
;
1541 err
= mlx5_cqwq_create(mdev
, ¶m
->wq
, param
->cqc
, &cq
->wq
,
1547 mcq
->set_ci_db
= cq
->wq_ctrl
.db
.db
;
1548 mcq
->arm_db
= cq
->wq_ctrl
.db
.db
+ 1;
1549 *mcq
->set_ci_db
= 0;
1551 mcq
->vector
= param
->eq_ix
;
1552 mcq
->comp
= mlx5e_completion_event
;
1553 mcq
->event
= mlx5e_cq_error_event
;
1555 for (i
= 0; i
< mlx5_cqwq_get_size(&cq
->wq
); i
++) {
1556 struct mlx5_cqe64
*cqe
= mlx5_cqwq_get_wqe(&cq
->wq
, i
);
1562 cq
->netdev
= priv
->netdev
;
1568 static int mlx5e_alloc_cq(struct mlx5e_priv
*priv
,
1569 struct mlx5e_cq_param
*param
,
1570 struct mlx5e_create_cq_param
*ccp
,
1571 struct mlx5e_cq
*cq
)
1575 param
->wq
.buf_numa_node
= ccp
->node
;
1576 param
->wq
.db_numa_node
= ccp
->node
;
1577 param
->eq_ix
= ccp
->ix
;
1579 err
= mlx5e_alloc_cq_common(priv
, param
, cq
);
1581 cq
->napi
= ccp
->napi
;
1582 cq
->ch_stats
= ccp
->ch_stats
;
1587 static void mlx5e_free_cq(struct mlx5e_cq
*cq
)
1589 mlx5_wq_destroy(&cq
->wq_ctrl
);
1592 static int mlx5e_create_cq(struct mlx5e_cq
*cq
, struct mlx5e_cq_param
*param
)
1594 u32 out
[MLX5_ST_SZ_DW(create_cq_out
)];
1595 struct mlx5_core_dev
*mdev
= cq
->mdev
;
1596 struct mlx5_core_cq
*mcq
= &cq
->mcq
;
1604 err
= mlx5_vector2eqn(mdev
, param
->eq_ix
, &eqn
);
1608 inlen
= MLX5_ST_SZ_BYTES(create_cq_in
) +
1609 sizeof(u64
) * cq
->wq_ctrl
.buf
.npages
;
1610 in
= kvzalloc(inlen
, GFP_KERNEL
);
1614 cqc
= MLX5_ADDR_OF(create_cq_in
, in
, cq_context
);
1616 memcpy(cqc
, param
->cqc
, sizeof(param
->cqc
));
1618 mlx5_fill_page_frag_array(&cq
->wq_ctrl
.buf
,
1619 (__be64
*)MLX5_ADDR_OF(create_cq_in
, in
, pas
));
1621 MLX5_SET(cqc
, cqc
, cq_period_mode
, param
->cq_period_mode
);
1622 MLX5_SET(cqc
, cqc
, c_eqn_or_apu_element
, eqn
);
1623 MLX5_SET(cqc
, cqc
, uar_page
, mdev
->priv
.uar
->index
);
1624 MLX5_SET(cqc
, cqc
, log_page_size
, cq
->wq_ctrl
.buf
.page_shift
-
1625 MLX5_ADAPTER_PAGE_SHIFT
);
1626 MLX5_SET64(cqc
, cqc
, dbr_addr
, cq
->wq_ctrl
.db
.dma
);
1628 err
= mlx5_core_create_cq(mdev
, mcq
, in
, inlen
, out
, sizeof(out
));
1640 static void mlx5e_destroy_cq(struct mlx5e_cq
*cq
)
1642 mlx5_core_destroy_cq(cq
->mdev
, &cq
->mcq
);
1645 int mlx5e_open_cq(struct mlx5e_priv
*priv
, struct dim_cq_moder moder
,
1646 struct mlx5e_cq_param
*param
, struct mlx5e_create_cq_param
*ccp
,
1647 struct mlx5e_cq
*cq
)
1649 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1652 err
= mlx5e_alloc_cq(priv
, param
, ccp
, cq
);
1656 err
= mlx5e_create_cq(cq
, param
);
1660 if (MLX5_CAP_GEN(mdev
, cq_moderation
))
1661 mlx5_core_modify_cq_moderation(mdev
, &cq
->mcq
, moder
.usec
, moder
.pkts
);
1670 void mlx5e_close_cq(struct mlx5e_cq
*cq
)
1672 mlx5e_destroy_cq(cq
);
1676 static int mlx5e_open_tx_cqs(struct mlx5e_channel
*c
,
1677 struct mlx5e_params
*params
,
1678 struct mlx5e_create_cq_param
*ccp
,
1679 struct mlx5e_channel_param
*cparam
)
1684 for (tc
= 0; tc
< c
->num_tc
; tc
++) {
1685 err
= mlx5e_open_cq(c
->priv
, params
->tx_cq_moderation
, &cparam
->txq_sq
.cqp
,
1686 ccp
, &c
->sq
[tc
].cq
);
1688 goto err_close_tx_cqs
;
1694 for (tc
--; tc
>= 0; tc
--)
1695 mlx5e_close_cq(&c
->sq
[tc
].cq
);
1700 static void mlx5e_close_tx_cqs(struct mlx5e_channel
*c
)
1704 for (tc
= 0; tc
< c
->num_tc
; tc
++)
1705 mlx5e_close_cq(&c
->sq
[tc
].cq
);
1708 static int mlx5e_open_sqs(struct mlx5e_channel
*c
,
1709 struct mlx5e_params
*params
,
1710 struct mlx5e_channel_param
*cparam
)
1714 for (tc
= 0; tc
< params
->num_tc
; tc
++) {
1715 int txq_ix
= c
->ix
+ tc
* params
->num_channels
;
1717 err
= mlx5e_open_txqsq(c
, c
->priv
->tisn
[c
->lag_port
][tc
], txq_ix
,
1718 params
, &cparam
->txq_sq
, &c
->sq
[tc
], tc
, 0, 0);
1726 for (tc
--; tc
>= 0; tc
--)
1727 mlx5e_close_txqsq(&c
->sq
[tc
]);
1732 static void mlx5e_close_sqs(struct mlx5e_channel
*c
)
1736 for (tc
= 0; tc
< c
->num_tc
; tc
++)
1737 mlx5e_close_txqsq(&c
->sq
[tc
]);
1740 static int mlx5e_set_sq_maxrate(struct net_device
*dev
,
1741 struct mlx5e_txqsq
*sq
, u32 rate
)
1743 struct mlx5e_priv
*priv
= netdev_priv(dev
);
1744 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1745 struct mlx5e_modify_sq_param msp
= {0};
1746 struct mlx5_rate_limit rl
= {0};
1750 if (rate
== sq
->rate_limit
)
1754 if (sq
->rate_limit
) {
1755 rl
.rate
= sq
->rate_limit
;
1756 /* remove current rl index to free space to next ones */
1757 mlx5_rl_remove_rate(mdev
, &rl
);
1764 err
= mlx5_rl_add_rate(mdev
, &rl_index
, &rl
);
1766 netdev_err(dev
, "Failed configuring rate %u: %d\n",
1772 msp
.curr_state
= MLX5_SQC_STATE_RDY
;
1773 msp
.next_state
= MLX5_SQC_STATE_RDY
;
1774 msp
.rl_index
= rl_index
;
1775 msp
.rl_update
= true;
1776 err
= mlx5e_modify_sq(mdev
, sq
->sqn
, &msp
);
1778 netdev_err(dev
, "Failed configuring rate %u: %d\n",
1780 /* remove the rate from the table */
1782 mlx5_rl_remove_rate(mdev
, &rl
);
1786 sq
->rate_limit
= rate
;
1790 static int mlx5e_set_tx_maxrate(struct net_device
*dev
, int index
, u32 rate
)
1792 struct mlx5e_priv
*priv
= netdev_priv(dev
);
1793 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1794 struct mlx5e_txqsq
*sq
= priv
->txq2sq
[index
];
1797 if (!mlx5_rl_is_supported(mdev
)) {
1798 netdev_err(dev
, "Rate limiting is not supported on this device\n");
1802 /* rate is given in Mb/sec, HW config is in Kb/sec */
1805 /* Check whether rate in valid range, 0 is always valid */
1806 if (rate
&& !mlx5_rl_is_in_range(mdev
, rate
)) {
1807 netdev_err(dev
, "TX rate %u, is not in range\n", rate
);
1811 mutex_lock(&priv
->state_lock
);
1812 if (test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
1813 err
= mlx5e_set_sq_maxrate(dev
, sq
, rate
);
1815 priv
->tx_rates
[index
] = rate
;
1816 mutex_unlock(&priv
->state_lock
);
1821 static int mlx5e_open_rxq_rq(struct mlx5e_channel
*c
, struct mlx5e_params
*params
,
1822 struct mlx5e_rq_param
*rq_params
)
1826 err
= mlx5e_init_rxq_rq(c
, params
, &c
->rq
);
1830 return mlx5e_open_rq(params
, rq_params
, NULL
, cpu_to_node(c
->cpu
), &c
->rq
);
1833 static int mlx5e_open_queues(struct mlx5e_channel
*c
,
1834 struct mlx5e_params
*params
,
1835 struct mlx5e_channel_param
*cparam
)
1837 struct dim_cq_moder icocq_moder
= {0, 0};
1838 struct mlx5e_create_cq_param ccp
;
1841 mlx5e_build_create_cq_param(&ccp
, c
);
1843 err
= mlx5e_open_cq(c
->priv
, icocq_moder
, &cparam
->async_icosq
.cqp
, &ccp
,
1844 &c
->async_icosq
.cq
);
1848 err
= mlx5e_open_cq(c
->priv
, icocq_moder
, &cparam
->icosq
.cqp
, &ccp
,
1851 goto err_close_async_icosq_cq
;
1853 err
= mlx5e_open_tx_cqs(c
, params
, &ccp
, cparam
);
1855 goto err_close_icosq_cq
;
1857 err
= mlx5e_open_cq(c
->priv
, params
->tx_cq_moderation
, &cparam
->xdp_sq
.cqp
, &ccp
,
1860 goto err_close_tx_cqs
;
1862 err
= mlx5e_open_cq(c
->priv
, params
->rx_cq_moderation
, &cparam
->rq
.cqp
, &ccp
,
1865 goto err_close_xdp_tx_cqs
;
1867 err
= c
->xdp
? mlx5e_open_cq(c
->priv
, params
->tx_cq_moderation
, &cparam
->xdp_sq
.cqp
,
1868 &ccp
, &c
->rq_xdpsq
.cq
) : 0;
1870 goto err_close_rx_cq
;
1872 spin_lock_init(&c
->async_icosq_lock
);
1874 err
= mlx5e_open_icosq(c
, params
, &cparam
->async_icosq
, &c
->async_icosq
);
1876 goto err_close_xdpsq_cq
;
1878 err
= mlx5e_open_icosq(c
, params
, &cparam
->icosq
, &c
->icosq
);
1880 goto err_close_async_icosq
;
1882 err
= mlx5e_open_sqs(c
, params
, cparam
);
1884 goto err_close_icosq
;
1886 err
= mlx5e_open_rxq_rq(c
, params
, &cparam
->rq
);
1891 err
= mlx5e_open_xdpsq(c
, params
, &cparam
->xdp_sq
, NULL
,
1892 &c
->rq_xdpsq
, false);
1897 err
= mlx5e_open_xdpsq(c
, params
, &cparam
->xdp_sq
, NULL
, &c
->xdpsq
, true);
1899 goto err_close_xdp_sq
;
1905 mlx5e_close_xdpsq(&c
->rq_xdpsq
);
1908 mlx5e_close_rq(&c
->rq
);
1914 mlx5e_close_icosq(&c
->icosq
);
1916 err_close_async_icosq
:
1917 mlx5e_close_icosq(&c
->async_icosq
);
1921 mlx5e_close_cq(&c
->rq_xdpsq
.cq
);
1924 mlx5e_close_cq(&c
->rq
.cq
);
1926 err_close_xdp_tx_cqs
:
1927 mlx5e_close_cq(&c
->xdpsq
.cq
);
1930 mlx5e_close_tx_cqs(c
);
1933 mlx5e_close_cq(&c
->icosq
.cq
);
1935 err_close_async_icosq_cq
:
1936 mlx5e_close_cq(&c
->async_icosq
.cq
);
1941 static void mlx5e_close_queues(struct mlx5e_channel
*c
)
1943 mlx5e_close_xdpsq(&c
->xdpsq
);
1945 mlx5e_close_xdpsq(&c
->rq_xdpsq
);
1946 mlx5e_close_rq(&c
->rq
);
1948 mlx5e_close_icosq(&c
->icosq
);
1949 mlx5e_close_icosq(&c
->async_icosq
);
1951 mlx5e_close_cq(&c
->rq_xdpsq
.cq
);
1952 mlx5e_close_cq(&c
->rq
.cq
);
1953 mlx5e_close_cq(&c
->xdpsq
.cq
);
1954 mlx5e_close_tx_cqs(c
);
1955 mlx5e_close_cq(&c
->icosq
.cq
);
1956 mlx5e_close_cq(&c
->async_icosq
.cq
);
1959 static u8
mlx5e_enumerate_lag_port(struct mlx5_core_dev
*mdev
, int ix
)
1961 u16 port_aff_bias
= mlx5_core_is_pf(mdev
) ? 0 : MLX5_CAP_GEN(mdev
, vhca_id
);
1963 return (ix
+ port_aff_bias
) % mlx5e_get_num_lag_ports(mdev
);
1966 static int mlx5e_open_channel(struct mlx5e_priv
*priv
, int ix
,
1967 struct mlx5e_params
*params
,
1968 struct mlx5e_channel_param
*cparam
,
1969 struct xsk_buff_pool
*xsk_pool
,
1970 struct mlx5e_channel
**cp
)
1972 int cpu
= cpumask_first(mlx5_comp_irq_get_affinity_mask(priv
->mdev
, ix
));
1973 struct net_device
*netdev
= priv
->netdev
;
1974 struct mlx5e_xsk_param xsk
;
1975 struct mlx5e_channel
*c
;
1979 err
= mlx5_vector2irqn(priv
->mdev
, ix
, &irq
);
1983 c
= kvzalloc_node(sizeof(*c
), GFP_KERNEL
, cpu_to_node(cpu
));
1988 c
->mdev
= priv
->mdev
;
1989 c
->tstamp
= &priv
->tstamp
;
1992 c
->pdev
= mlx5_core_dma_dev(priv
->mdev
);
1993 c
->netdev
= priv
->netdev
;
1994 c
->mkey_be
= cpu_to_be32(priv
->mdev
->mlx5e_res
.hw_objs
.mkey
.key
);
1995 c
->num_tc
= params
->num_tc
;
1996 c
->xdp
= !!params
->xdp_prog
;
1997 c
->stats
= &priv
->channel_stats
[ix
].ch
;
1998 c
->aff_mask
= irq_get_effective_affinity_mask(irq
);
1999 c
->lag_port
= mlx5e_enumerate_lag_port(priv
->mdev
, ix
);
2001 netif_napi_add(netdev
, &c
->napi
, mlx5e_napi_poll
, 64);
2003 err
= mlx5e_open_queues(c
, params
, cparam
);
2008 mlx5e_build_xsk_param(xsk_pool
, &xsk
);
2009 err
= mlx5e_open_xsk(priv
, params
, &xsk
, xsk_pool
, c
);
2011 goto err_close_queues
;
2019 mlx5e_close_queues(c
);
2022 netif_napi_del(&c
->napi
);
2029 static void mlx5e_activate_channel(struct mlx5e_channel
*c
)
2033 napi_enable(&c
->napi
);
2035 for (tc
= 0; tc
< c
->num_tc
; tc
++)
2036 mlx5e_activate_txqsq(&c
->sq
[tc
]);
2037 mlx5e_activate_icosq(&c
->icosq
);
2038 mlx5e_activate_icosq(&c
->async_icosq
);
2039 mlx5e_activate_rq(&c
->rq
);
2041 if (test_bit(MLX5E_CHANNEL_STATE_XSK
, c
->state
))
2042 mlx5e_activate_xsk(c
);
2045 static void mlx5e_deactivate_channel(struct mlx5e_channel
*c
)
2049 if (test_bit(MLX5E_CHANNEL_STATE_XSK
, c
->state
))
2050 mlx5e_deactivate_xsk(c
);
2052 mlx5e_deactivate_rq(&c
->rq
);
2053 mlx5e_deactivate_icosq(&c
->async_icosq
);
2054 mlx5e_deactivate_icosq(&c
->icosq
);
2055 for (tc
= 0; tc
< c
->num_tc
; tc
++)
2056 mlx5e_deactivate_txqsq(&c
->sq
[tc
]);
2057 mlx5e_qos_deactivate_queues(c
);
2059 napi_disable(&c
->napi
);
2062 static void mlx5e_close_channel(struct mlx5e_channel
*c
)
2064 if (test_bit(MLX5E_CHANNEL_STATE_XSK
, c
->state
))
2066 mlx5e_close_queues(c
);
2067 mlx5e_qos_close_queues(c
);
2068 netif_napi_del(&c
->napi
);
2073 int mlx5e_open_channels(struct mlx5e_priv
*priv
,
2074 struct mlx5e_channels
*chs
)
2076 struct mlx5e_channel_param
*cparam
;
2080 chs
->num
= chs
->params
.num_channels
;
2082 chs
->c
= kcalloc(chs
->num
, sizeof(struct mlx5e_channel
*), GFP_KERNEL
);
2083 cparam
= kvzalloc(sizeof(struct mlx5e_channel_param
), GFP_KERNEL
);
2084 if (!chs
->c
|| !cparam
)
2087 err
= mlx5e_build_channel_param(priv
->mdev
, &chs
->params
, priv
->q_counter
, cparam
);
2091 for (i
= 0; i
< chs
->num
; i
++) {
2092 struct xsk_buff_pool
*xsk_pool
= NULL
;
2094 if (chs
->params
.xdp_prog
)
2095 xsk_pool
= mlx5e_xsk_get_pool(&chs
->params
, chs
->params
.xsk
, i
);
2097 err
= mlx5e_open_channel(priv
, i
, &chs
->params
, cparam
, xsk_pool
, &chs
->c
[i
]);
2099 goto err_close_channels
;
2102 if (MLX5E_GET_PFLAG(&chs
->params
, MLX5E_PFLAG_TX_PORT_TS
) || chs
->params
.ptp_rx
) {
2103 err
= mlx5e_ptp_open(priv
, &chs
->params
, chs
->c
[0]->lag_port
, &chs
->ptp
);
2105 goto err_close_channels
;
2108 err
= mlx5e_qos_open_queues(priv
, chs
);
2112 mlx5e_health_channels_update(priv
);
2118 mlx5e_ptp_close(chs
->ptp
);
2121 for (i
--; i
>= 0; i
--)
2122 mlx5e_close_channel(chs
->c
[i
]);
2131 static void mlx5e_activate_channels(struct mlx5e_channels
*chs
)
2135 for (i
= 0; i
< chs
->num
; i
++)
2136 mlx5e_activate_channel(chs
->c
[i
]);
2139 mlx5e_ptp_activate_channel(chs
->ptp
);
2142 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
2144 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels
*chs
)
2149 for (i
= 0; i
< chs
->num
; i
++) {
2150 int timeout
= err
? 0 : MLX5E_RQ_WQES_TIMEOUT
;
2152 err
|= mlx5e_wait_for_min_rx_wqes(&chs
->c
[i
]->rq
, timeout
);
2154 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2155 * doesn't provide any Fill Ring entries at the setup stage.
2159 return err
? -ETIMEDOUT
: 0;
2162 static void mlx5e_deactivate_channels(struct mlx5e_channels
*chs
)
2167 mlx5e_ptp_deactivate_channel(chs
->ptp
);
2169 for (i
= 0; i
< chs
->num
; i
++)
2170 mlx5e_deactivate_channel(chs
->c
[i
]);
2173 void mlx5e_close_channels(struct mlx5e_channels
*chs
)
2178 mlx5e_ptp_close(chs
->ptp
);
2181 for (i
= 0; i
< chs
->num
; i
++)
2182 mlx5e_close_channel(chs
->c
[i
]);
2189 mlx5e_create_rqt(struct mlx5e_priv
*priv
, int sz
, struct mlx5e_rqt
*rqt
)
2191 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2198 inlen
= MLX5_ST_SZ_BYTES(create_rqt_in
) + sizeof(u32
) * sz
;
2199 in
= kvzalloc(inlen
, GFP_KERNEL
);
2203 rqtc
= MLX5_ADDR_OF(create_rqt_in
, in
, rqt_context
);
2205 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
2206 MLX5_SET(rqtc
, rqtc
, rqt_max_size
, sz
);
2208 for (i
= 0; i
< sz
; i
++)
2209 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], priv
->drop_rq
.rqn
);
2211 err
= mlx5_core_create_rqt(mdev
, in
, inlen
, &rqt
->rqtn
);
2213 rqt
->enabled
= true;
2219 void mlx5e_destroy_rqt(struct mlx5e_priv
*priv
, struct mlx5e_rqt
*rqt
)
2221 rqt
->enabled
= false;
2222 mlx5_core_destroy_rqt(priv
->mdev
, rqt
->rqtn
);
2225 int mlx5e_create_indirect_rqt(struct mlx5e_priv
*priv
)
2227 struct mlx5e_rqt
*rqt
= &priv
->indir_rqt
;
2230 err
= mlx5e_create_rqt(priv
, MLX5E_INDIR_RQT_SIZE
, rqt
);
2232 mlx5_core_warn(priv
->mdev
, "create indirect rqts failed, %d\n", err
);
2236 int mlx5e_create_direct_rqts(struct mlx5e_priv
*priv
, struct mlx5e_tir
*tirs
, int n
)
2241 for (ix
= 0; ix
< n
; ix
++) {
2242 err
= mlx5e_create_rqt(priv
, 1 /*size */, &tirs
[ix
].rqt
);
2244 goto err_destroy_rqts
;
2250 mlx5_core_warn(priv
->mdev
, "create rqts failed, %d\n", err
);
2251 for (ix
--; ix
>= 0; ix
--)
2252 mlx5e_destroy_rqt(priv
, &tirs
[ix
].rqt
);
2257 void mlx5e_destroy_direct_rqts(struct mlx5e_priv
*priv
, struct mlx5e_tir
*tirs
, int n
)
2261 for (i
= 0; i
< n
; i
++)
2262 mlx5e_destroy_rqt(priv
, &tirs
[i
].rqt
);
2265 static int mlx5e_rx_hash_fn(int hfunc
)
2267 return (hfunc
== ETH_RSS_HASH_TOP
) ?
2268 MLX5_RX_HASH_FN_TOEPLITZ
:
2269 MLX5_RX_HASH_FN_INVERTED_XOR8
;
2272 int mlx5e_bits_invert(unsigned long a
, int size
)
2277 for (i
= 0; i
< size
; i
++)
2278 inv
|= (test_bit(size
- i
- 1, &a
) ? 1 : 0) << i
;
2283 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv
*priv
, int sz
,
2284 struct mlx5e_redirect_rqt_param rrp
, void *rqtc
)
2288 for (i
= 0; i
< sz
; i
++) {
2294 if (rrp
.rss
.hfunc
== ETH_RSS_HASH_XOR
)
2295 ix
= mlx5e_bits_invert(i
, ilog2(sz
));
2297 ix
= priv
->rss_params
.indirection_rqt
[ix
];
2298 rqn
= rrp
.rss
.channels
->c
[ix
]->rq
.rqn
;
2302 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], rqn
);
2306 int mlx5e_redirect_rqt(struct mlx5e_priv
*priv
, u32 rqtn
, int sz
,
2307 struct mlx5e_redirect_rqt_param rrp
)
2309 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2315 inlen
= MLX5_ST_SZ_BYTES(modify_rqt_in
) + sizeof(u32
) * sz
;
2316 in
= kvzalloc(inlen
, GFP_KERNEL
);
2320 rqtc
= MLX5_ADDR_OF(modify_rqt_in
, in
, ctx
);
2322 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
2323 MLX5_SET(modify_rqt_in
, in
, bitmask
.rqn_list
, 1);
2324 mlx5e_fill_rqt_rqns(priv
, sz
, rrp
, rqtc
);
2325 err
= mlx5_core_modify_rqt(mdev
, rqtn
, in
, inlen
);
2331 static u32
mlx5e_get_direct_rqn(struct mlx5e_priv
*priv
, int ix
,
2332 struct mlx5e_redirect_rqt_param rrp
)
2337 if (ix
>= rrp
.rss
.channels
->num
)
2338 return priv
->drop_rq
.rqn
;
2340 return rrp
.rss
.channels
->c
[ix
]->rq
.rqn
;
2343 static void mlx5e_redirect_rqts(struct mlx5e_priv
*priv
,
2344 struct mlx5e_redirect_rqt_param rrp
,
2345 struct mlx5e_redirect_rqt_param
*ptp_rrp
)
2350 if (priv
->indir_rqt
.enabled
) {
2352 rqtn
= priv
->indir_rqt
.rqtn
;
2353 mlx5e_redirect_rqt(priv
, rqtn
, MLX5E_INDIR_RQT_SIZE
, rrp
);
2356 for (ix
= 0; ix
< priv
->max_nch
; ix
++) {
2357 struct mlx5e_redirect_rqt_param direct_rrp
= {
2360 .rqn
= mlx5e_get_direct_rqn(priv
, ix
, rrp
)
2364 /* Direct RQ Tables */
2365 if (!priv
->direct_tir
[ix
].rqt
.enabled
)
2368 rqtn
= priv
->direct_tir
[ix
].rqt
.rqtn
;
2369 mlx5e_redirect_rqt(priv
, rqtn
, 1, direct_rrp
);
2372 rqtn
= priv
->ptp_tir
.rqt
.rqtn
;
2373 mlx5e_redirect_rqt(priv
, rqtn
, 1, *ptp_rrp
);
2377 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv
*priv
,
2378 struct mlx5e_channels
*chs
)
2380 bool rx_ptp_support
= priv
->profile
->rx_ptp_support
;
2381 struct mlx5e_redirect_rqt_param
*ptp_rrp_p
= NULL
;
2382 struct mlx5e_redirect_rqt_param rrp
= {
2387 .hfunc
= priv
->rss_params
.hfunc
,
2391 struct mlx5e_redirect_rqt_param ptp_rrp
;
2393 if (rx_ptp_support
) {
2396 ptp_rrp
.is_rss
= false;
2397 ptp_rrp
.rqn
= mlx5e_ptp_get_rqn(priv
->channels
.ptp
, &ptp_rqn
) ?
2398 priv
->drop_rq
.rqn
: ptp_rqn
;
2399 ptp_rrp_p
= &ptp_rrp
;
2401 mlx5e_redirect_rqts(priv
, rrp
, ptp_rrp_p
);
2404 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv
*priv
)
2406 bool rx_ptp_support
= priv
->profile
->rx_ptp_support
;
2407 struct mlx5e_redirect_rqt_param drop_rrp
= {
2410 .rqn
= priv
->drop_rq
.rqn
,
2414 mlx5e_redirect_rqts(priv
, drop_rrp
, rx_ptp_support
? &drop_rrp
: NULL
);
2417 static const struct mlx5e_tirc_config tirc_default_config
[MLX5E_NUM_INDIR_TIRS
] = {
2418 [MLX5E_TT_IPV4_TCP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV4
,
2419 .l4_prot_type
= MLX5_L4_PROT_TYPE_TCP
,
2420 .rx_hash_fields
= MLX5_HASH_IP_L4PORTS
,
2422 [MLX5E_TT_IPV6_TCP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV6
,
2423 .l4_prot_type
= MLX5_L4_PROT_TYPE_TCP
,
2424 .rx_hash_fields
= MLX5_HASH_IP_L4PORTS
,
2426 [MLX5E_TT_IPV4_UDP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV4
,
2427 .l4_prot_type
= MLX5_L4_PROT_TYPE_UDP
,
2428 .rx_hash_fields
= MLX5_HASH_IP_L4PORTS
,
2430 [MLX5E_TT_IPV6_UDP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV6
,
2431 .l4_prot_type
= MLX5_L4_PROT_TYPE_UDP
,
2432 .rx_hash_fields
= MLX5_HASH_IP_L4PORTS
,
2434 [MLX5E_TT_IPV4_IPSEC_AH
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV4
,
2436 .rx_hash_fields
= MLX5_HASH_IP_IPSEC_SPI
,
2438 [MLX5E_TT_IPV6_IPSEC_AH
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV6
,
2440 .rx_hash_fields
= MLX5_HASH_IP_IPSEC_SPI
,
2442 [MLX5E_TT_IPV4_IPSEC_ESP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV4
,
2444 .rx_hash_fields
= MLX5_HASH_IP_IPSEC_SPI
,
2446 [MLX5E_TT_IPV6_IPSEC_ESP
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV6
,
2448 .rx_hash_fields
= MLX5_HASH_IP_IPSEC_SPI
,
2450 [MLX5E_TT_IPV4
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV4
,
2452 .rx_hash_fields
= MLX5_HASH_IP
,
2454 [MLX5E_TT_IPV6
] = { .l3_prot_type
= MLX5_L3_PROT_TYPE_IPV6
,
2456 .rx_hash_fields
= MLX5_HASH_IP
,
2460 struct mlx5e_tirc_config
mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt
)
2462 return tirc_default_config
[tt
];
2465 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params
*params
, void *tirc
)
2467 if (!params
->lro_en
)
2470 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2472 MLX5_SET(tirc
, tirc
, lro_enable_mask
,
2473 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO
|
2474 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO
);
2475 MLX5_SET(tirc
, tirc
, lro_max_ip_payload_size
,
2476 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ
- ROUGH_MAX_L2_L3_HDR_SZ
) >> 8);
2477 MLX5_SET(tirc
, tirc
, lro_timeout_period_usecs
, params
->lro_timeout
);
2480 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params
*rss_params
,
2481 const struct mlx5e_tirc_config
*ttconfig
,
2482 void *tirc
, bool inner
)
2484 void *hfso
= inner
? MLX5_ADDR_OF(tirc
, tirc
, rx_hash_field_selector_inner
) :
2485 MLX5_ADDR_OF(tirc
, tirc
, rx_hash_field_selector_outer
);
2487 MLX5_SET(tirc
, tirc
, rx_hash_fn
, mlx5e_rx_hash_fn(rss_params
->hfunc
));
2488 if (rss_params
->hfunc
== ETH_RSS_HASH_TOP
) {
2489 void *rss_key
= MLX5_ADDR_OF(tirc
, tirc
,
2490 rx_hash_toeplitz_key
);
2491 size_t len
= MLX5_FLD_SZ_BYTES(tirc
,
2492 rx_hash_toeplitz_key
);
2494 MLX5_SET(tirc
, tirc
, rx_hash_symmetric
, 1);
2495 memcpy(rss_key
, rss_params
->toeplitz_hash_key
, len
);
2497 MLX5_SET(rx_hash_field_select
, hfso
, l3_prot_type
,
2498 ttconfig
->l3_prot_type
);
2499 MLX5_SET(rx_hash_field_select
, hfso
, l4_prot_type
,
2500 ttconfig
->l4_prot_type
);
2501 MLX5_SET(rx_hash_field_select
, hfso
, selected_fields
,
2502 ttconfig
->rx_hash_fields
);
2505 static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config
*ttconfig
,
2506 enum mlx5e_traffic_types tt
,
2509 *ttconfig
= tirc_default_config
[tt
];
2510 ttconfig
->rx_hash_fields
= rx_hash_fields
;
2513 void mlx5e_modify_tirs_hash(struct mlx5e_priv
*priv
, void *in
)
2515 void *tirc
= MLX5_ADDR_OF(modify_tir_in
, in
, ctx
);
2516 struct mlx5e_rss_params
*rss
= &priv
->rss_params
;
2517 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2518 int ctxlen
= MLX5_ST_SZ_BYTES(tirc
);
2519 struct mlx5e_tirc_config ttconfig
;
2522 MLX5_SET(modify_tir_in
, in
, bitmask
.hash
, 1);
2524 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
2525 memset(tirc
, 0, ctxlen
);
2526 mlx5e_update_rx_hash_fields(&ttconfig
, tt
,
2527 rss
->rx_hash_fields
[tt
]);
2528 mlx5e_build_indir_tir_ctx_hash(rss
, &ttconfig
, tirc
, false);
2529 mlx5_core_modify_tir(mdev
, priv
->indir_tir
[tt
].tirn
, in
);
2532 /* Verify inner tirs resources allocated */
2533 if (!priv
->inner_indir_tir
[0].tirn
)
2536 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
2537 memset(tirc
, 0, ctxlen
);
2538 mlx5e_update_rx_hash_fields(&ttconfig
, tt
,
2539 rss
->rx_hash_fields
[tt
]);
2540 mlx5e_build_indir_tir_ctx_hash(rss
, &ttconfig
, tirc
, true);
2541 mlx5_core_modify_tir(mdev
, priv
->inner_indir_tir
[tt
].tirn
, in
);
2545 static int mlx5e_modify_tirs_lro(struct mlx5e_priv
*priv
)
2547 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2556 inlen
= MLX5_ST_SZ_BYTES(modify_tir_in
);
2557 in
= kvzalloc(inlen
, GFP_KERNEL
);
2561 MLX5_SET(modify_tir_in
, in
, bitmask
.lro
, 1);
2562 tirc
= MLX5_ADDR_OF(modify_tir_in
, in
, ctx
);
2564 mlx5e_build_tir_ctx_lro(&priv
->channels
.params
, tirc
);
2566 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
2567 err
= mlx5_core_modify_tir(mdev
, priv
->indir_tir
[tt
].tirn
, in
);
2572 for (ix
= 0; ix
< priv
->max_nch
; ix
++) {
2573 err
= mlx5_core_modify_tir(mdev
, priv
->direct_tir
[ix
].tirn
, in
);
2584 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_lro
);
2586 static int mlx5e_set_mtu(struct mlx5_core_dev
*mdev
,
2587 struct mlx5e_params
*params
, u16 mtu
)
2589 u16 hw_mtu
= MLX5E_SW2HW_MTU(params
, mtu
);
2592 err
= mlx5_set_port_mtu(mdev
, hw_mtu
, 1);
2596 /* Update vport context MTU */
2597 mlx5_modify_nic_vport_mtu(mdev
, hw_mtu
);
2601 static void mlx5e_query_mtu(struct mlx5_core_dev
*mdev
,
2602 struct mlx5e_params
*params
, u16
*mtu
)
2607 err
= mlx5_query_nic_vport_mtu(mdev
, &hw_mtu
);
2608 if (err
|| !hw_mtu
) /* fallback to port oper mtu */
2609 mlx5_query_port_oper_mtu(mdev
, &hw_mtu
, 1);
2611 *mtu
= MLX5E_HW2SW_MTU(params
, hw_mtu
);
2614 int mlx5e_set_dev_port_mtu(struct mlx5e_priv
*priv
)
2616 struct mlx5e_params
*params
= &priv
->channels
.params
;
2617 struct net_device
*netdev
= priv
->netdev
;
2618 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2622 err
= mlx5e_set_mtu(mdev
, params
, params
->sw_mtu
);
2626 mlx5e_query_mtu(mdev
, params
, &mtu
);
2627 if (mtu
!= params
->sw_mtu
)
2628 netdev_warn(netdev
, "%s: VPort MTU %d is different than netdev mtu %d\n",
2629 __func__
, mtu
, params
->sw_mtu
);
2631 params
->sw_mtu
= mtu
;
2635 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu
);
2637 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv
*priv
)
2639 struct mlx5e_params
*params
= &priv
->channels
.params
;
2640 struct net_device
*netdev
= priv
->netdev
;
2641 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2644 /* MTU range: 68 - hw-specific max */
2645 netdev
->min_mtu
= ETH_MIN_MTU
;
2647 mlx5_query_port_max_mtu(mdev
, &max_mtu
, 1);
2648 netdev
->max_mtu
= min_t(unsigned int, MLX5E_HW2SW_MTU(params
, max_mtu
),
2652 static void mlx5e_netdev_set_tcs(struct net_device
*netdev
, u16 nch
, u8 ntc
)
2656 netdev_reset_tc(netdev
);
2661 netdev_set_num_tc(netdev
, ntc
);
2663 /* Map netdev TCs to offset 0
2664 * We have our own UP to TXQ mapping for QoS
2666 for (tc
= 0; tc
< ntc
; tc
++)
2667 netdev_set_tc_queue(netdev
, tc
, nch
, 0);
2670 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv
*priv
)
2672 int qos_queues
, nch
, ntc
, num_txqs
, err
;
2674 qos_queues
= mlx5e_qos_cur_leaf_nodes(priv
);
2676 nch
= priv
->channels
.params
.num_channels
;
2677 ntc
= priv
->channels
.params
.num_tc
;
2678 num_txqs
= nch
* ntc
+ qos_queues
;
2679 if (MLX5E_GET_PFLAG(&priv
->channels
.params
, MLX5E_PFLAG_TX_PORT_TS
))
2682 mlx5e_dbg(DRV
, priv
, "Setting num_txqs %d\n", num_txqs
);
2683 err
= netif_set_real_num_tx_queues(priv
->netdev
, num_txqs
);
2685 netdev_warn(priv
->netdev
, "netif_set_real_num_tx_queues failed, %d\n", err
);
2690 static int mlx5e_update_netdev_queues(struct mlx5e_priv
*priv
)
2692 struct net_device
*netdev
= priv
->netdev
;
2693 int old_num_txqs
, old_ntc
;
2694 int num_rxqs
, nch
, ntc
;
2697 old_num_txqs
= netdev
->real_num_tx_queues
;
2698 old_ntc
= netdev
->num_tc
? : 1;
2700 nch
= priv
->channels
.params
.num_channels
;
2701 ntc
= priv
->channels
.params
.num_tc
;
2702 num_rxqs
= nch
* priv
->profile
->rq_groups
;
2704 mlx5e_netdev_set_tcs(netdev
, nch
, ntc
);
2706 err
= mlx5e_update_tx_netdev_queues(priv
);
2709 err
= netif_set_real_num_rx_queues(netdev
, num_rxqs
);
2711 netdev_warn(netdev
, "netif_set_real_num_rx_queues failed, %d\n", err
);
2718 /* netif_set_real_num_rx_queues could fail only when nch increased. Only
2719 * one of nch and ntc is changed in this function. That means, the call
2720 * to netif_set_real_num_tx_queues below should not fail, because it
2721 * decreases the number of TX queues.
2723 WARN_ON_ONCE(netif_set_real_num_tx_queues(netdev
, old_num_txqs
));
2726 mlx5e_netdev_set_tcs(netdev
, old_num_txqs
/ old_ntc
, old_ntc
);
2730 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv
*priv
,
2731 struct mlx5e_params
*params
)
2733 struct mlx5_core_dev
*mdev
= priv
->mdev
;
2734 int num_comp_vectors
, ix
, irq
;
2736 num_comp_vectors
= mlx5_comp_vectors_count(mdev
);
2738 for (ix
= 0; ix
< params
->num_channels
; ix
++) {
2739 cpumask_clear(priv
->scratchpad
.cpumask
);
2741 for (irq
= ix
; irq
< num_comp_vectors
; irq
+= params
->num_channels
) {
2742 int cpu
= cpumask_first(mlx5_comp_irq_get_affinity_mask(mdev
, irq
));
2744 cpumask_set_cpu(cpu
, priv
->scratchpad
.cpumask
);
2747 netif_set_xps_queue(priv
->netdev
, priv
->scratchpad
.cpumask
, ix
);
2751 int mlx5e_num_channels_changed(struct mlx5e_priv
*priv
)
2753 u16 count
= priv
->channels
.params
.num_channels
;
2756 err
= mlx5e_update_netdev_queues(priv
);
2760 mlx5e_set_default_xps_cpumasks(priv
, &priv
->channels
.params
);
2762 if (!netif_is_rxfh_configured(priv
->netdev
))
2763 mlx5e_build_default_indir_rqt(priv
->rss_params
.indirection_rqt
,
2764 MLX5E_INDIR_RQT_SIZE
, count
);
2769 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed
);
2771 static void mlx5e_build_txq_maps(struct mlx5e_priv
*priv
)
2773 int i
, ch
, tc
, num_tc
;
2775 ch
= priv
->channels
.num
;
2776 num_tc
= priv
->channels
.params
.num_tc
;
2778 for (i
= 0; i
< ch
; i
++) {
2779 for (tc
= 0; tc
< num_tc
; tc
++) {
2780 struct mlx5e_channel
*c
= priv
->channels
.c
[i
];
2781 struct mlx5e_txqsq
*sq
= &c
->sq
[tc
];
2783 priv
->txq2sq
[sq
->txq_ix
] = sq
;
2784 priv
->channel_tc2realtxq
[i
][tc
] = i
+ tc
* ch
;
2788 if (!priv
->channels
.ptp
)
2791 if (!test_bit(MLX5E_PTP_STATE_TX
, priv
->channels
.ptp
->state
))
2794 for (tc
= 0; tc
< num_tc
; tc
++) {
2795 struct mlx5e_ptp
*c
= priv
->channels
.ptp
;
2796 struct mlx5e_txqsq
*sq
= &c
->ptpsq
[tc
].txqsq
;
2798 priv
->txq2sq
[sq
->txq_ix
] = sq
;
2799 priv
->port_ptp_tc2realtxq
[tc
] = priv
->num_tc_x_num_ch
+ tc
;
2803 static void mlx5e_update_num_tc_x_num_ch(struct mlx5e_priv
*priv
)
2805 /* Sync with mlx5e_select_queue. */
2806 WRITE_ONCE(priv
->num_tc_x_num_ch
,
2807 priv
->channels
.params
.num_tc
* priv
->channels
.num
);
2810 void mlx5e_activate_priv_channels(struct mlx5e_priv
*priv
)
2812 mlx5e_update_num_tc_x_num_ch(priv
);
2813 mlx5e_build_txq_maps(priv
);
2814 mlx5e_activate_channels(&priv
->channels
);
2815 mlx5e_qos_activate_queues(priv
);
2816 mlx5e_xdp_tx_enable(priv
);
2817 netif_tx_start_all_queues(priv
->netdev
);
2819 if (mlx5e_is_vport_rep(priv
))
2820 mlx5e_add_sqs_fwd_rules(priv
);
2822 mlx5e_wait_channels_min_rx_wqes(&priv
->channels
);
2823 mlx5e_redirect_rqts_to_channels(priv
, &priv
->channels
);
2825 mlx5e_xsk_redirect_rqts_to_channels(priv
, &priv
->channels
);
2828 void mlx5e_deactivate_priv_channels(struct mlx5e_priv
*priv
)
2830 mlx5e_xsk_redirect_rqts_to_drop(priv
, &priv
->channels
);
2832 mlx5e_redirect_rqts_to_drop(priv
);
2834 if (mlx5e_is_vport_rep(priv
))
2835 mlx5e_remove_sqs_fwd_rules(priv
);
2837 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2838 * polling for inactive tx queues.
2840 netif_tx_stop_all_queues(priv
->netdev
);
2841 netif_tx_disable(priv
->netdev
);
2842 mlx5e_xdp_tx_disable(priv
);
2843 mlx5e_deactivate_channels(&priv
->channels
);
2846 static int mlx5e_switch_priv_params(struct mlx5e_priv
*priv
,
2847 struct mlx5e_params
*new_params
,
2848 mlx5e_fp_preactivate preactivate
,
2851 struct mlx5e_params old_params
;
2853 old_params
= priv
->channels
.params
;
2854 priv
->channels
.params
= *new_params
;
2859 err
= preactivate(priv
, context
);
2861 priv
->channels
.params
= old_params
;
2869 static int mlx5e_switch_priv_channels(struct mlx5e_priv
*priv
,
2870 struct mlx5e_channels
*new_chs
,
2871 mlx5e_fp_preactivate preactivate
,
2874 struct net_device
*netdev
= priv
->netdev
;
2875 struct mlx5e_channels old_chs
;
2879 carrier_ok
= netif_carrier_ok(netdev
);
2880 netif_carrier_off(netdev
);
2882 mlx5e_deactivate_priv_channels(priv
);
2884 old_chs
= priv
->channels
;
2885 priv
->channels
= *new_chs
;
2887 /* New channels are ready to roll, call the preactivate hook if needed
2888 * to modify HW settings or update kernel parameters.
2891 err
= preactivate(priv
, context
);
2893 priv
->channels
= old_chs
;
2898 mlx5e_close_channels(&old_chs
);
2899 priv
->profile
->update_rx(priv
);
2902 mlx5e_activate_priv_channels(priv
);
2904 /* return carrier back if needed */
2906 netif_carrier_on(netdev
);
2911 int mlx5e_safe_switch_params(struct mlx5e_priv
*priv
,
2912 struct mlx5e_params
*params
,
2913 mlx5e_fp_preactivate preactivate
,
2914 void *context
, bool reset
)
2916 struct mlx5e_channels new_chs
= {};
2919 reset
&= test_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2921 return mlx5e_switch_priv_params(priv
, params
, preactivate
, context
);
2923 new_chs
.params
= *params
;
2924 err
= mlx5e_open_channels(priv
, &new_chs
);
2927 err
= mlx5e_switch_priv_channels(priv
, &new_chs
, preactivate
, context
);
2929 mlx5e_close_channels(&new_chs
);
2934 int mlx5e_safe_reopen_channels(struct mlx5e_priv
*priv
)
2936 return mlx5e_safe_switch_params(priv
, &priv
->channels
.params
, NULL
, NULL
, true);
2939 void mlx5e_timestamp_init(struct mlx5e_priv
*priv
)
2941 priv
->tstamp
.tx_type
= HWTSTAMP_TX_OFF
;
2942 priv
->tstamp
.rx_filter
= HWTSTAMP_FILTER_NONE
;
2945 static void mlx5e_modify_admin_state(struct mlx5_core_dev
*mdev
,
2946 enum mlx5_port_status state
)
2948 struct mlx5_eswitch
*esw
= mdev
->priv
.eswitch
;
2949 int vport_admin_state
;
2951 mlx5_set_port_admin_status(mdev
, state
);
2953 if (mlx5_eswitch_mode(mdev
) == MLX5_ESWITCH_OFFLOADS
||
2954 !MLX5_CAP_GEN(mdev
, uplink_follow
))
2957 if (state
== MLX5_PORT_UP
)
2958 vport_admin_state
= MLX5_VPORT_ADMIN_STATE_AUTO
;
2960 vport_admin_state
= MLX5_VPORT_ADMIN_STATE_DOWN
;
2962 mlx5_eswitch_set_vport_state(esw
, MLX5_VPORT_UPLINK
, vport_admin_state
);
2965 int mlx5e_open_locked(struct net_device
*netdev
)
2967 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2970 set_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2972 err
= mlx5e_open_channels(priv
, &priv
->channels
);
2974 goto err_clear_state_opened_flag
;
2976 priv
->profile
->update_rx(priv
);
2977 mlx5e_activate_priv_channels(priv
);
2978 mlx5e_apply_traps(priv
, true);
2979 if (priv
->profile
->update_carrier
)
2980 priv
->profile
->update_carrier(priv
);
2982 mlx5e_queue_update_stats(priv
);
2985 err_clear_state_opened_flag
:
2986 clear_bit(MLX5E_STATE_OPENED
, &priv
->state
);
2990 int mlx5e_open(struct net_device
*netdev
)
2992 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
2995 mutex_lock(&priv
->state_lock
);
2996 err
= mlx5e_open_locked(netdev
);
2998 mlx5e_modify_admin_state(priv
->mdev
, MLX5_PORT_UP
);
2999 mutex_unlock(&priv
->state_lock
);
3004 int mlx5e_close_locked(struct net_device
*netdev
)
3006 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3008 /* May already be CLOSED in case a previous configuration operation
3009 * (e.g RX/TX queue size change) that involves close&open failed.
3011 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
3014 mlx5e_apply_traps(priv
, false);
3015 clear_bit(MLX5E_STATE_OPENED
, &priv
->state
);
3017 netif_carrier_off(priv
->netdev
);
3018 mlx5e_deactivate_priv_channels(priv
);
3019 mlx5e_close_channels(&priv
->channels
);
3024 int mlx5e_close(struct net_device
*netdev
)
3026 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3029 if (!netif_device_present(netdev
))
3032 mutex_lock(&priv
->state_lock
);
3033 mlx5e_modify_admin_state(priv
->mdev
, MLX5_PORT_DOWN
);
3034 err
= mlx5e_close_locked(netdev
);
3035 mutex_unlock(&priv
->state_lock
);
3040 static void mlx5e_free_drop_rq(struct mlx5e_rq
*rq
)
3042 mlx5_wq_destroy(&rq
->wq_ctrl
);
3045 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev
*mdev
,
3046 struct mlx5e_rq
*rq
,
3047 struct mlx5e_rq_param
*param
)
3049 void *rqc
= param
->rqc
;
3050 void *rqc_wq
= MLX5_ADDR_OF(rqc
, rqc
, wq
);
3053 param
->wq
.db_numa_node
= param
->wq
.buf_numa_node
;
3055 err
= mlx5_wq_cyc_create(mdev
, ¶m
->wq
, rqc_wq
, &rq
->wqe
.wq
,
3060 /* Mark as unused given "Drop-RQ" packets never reach XDP */
3061 xdp_rxq_info_unused(&rq
->xdp_rxq
);
3068 static int mlx5e_alloc_drop_cq(struct mlx5e_priv
*priv
,
3069 struct mlx5e_cq
*cq
,
3070 struct mlx5e_cq_param
*param
)
3072 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3074 param
->wq
.buf_numa_node
= dev_to_node(mlx5_core_dma_dev(mdev
));
3075 param
->wq
.db_numa_node
= dev_to_node(mlx5_core_dma_dev(mdev
));
3077 return mlx5e_alloc_cq_common(priv
, param
, cq
);
3080 int mlx5e_open_drop_rq(struct mlx5e_priv
*priv
,
3081 struct mlx5e_rq
*drop_rq
)
3083 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3084 struct mlx5e_cq_param cq_param
= {};
3085 struct mlx5e_rq_param rq_param
= {};
3086 struct mlx5e_cq
*cq
= &drop_rq
->cq
;
3089 mlx5e_build_drop_rq_param(mdev
, priv
->drop_rq_q_counter
, &rq_param
);
3091 err
= mlx5e_alloc_drop_cq(priv
, cq
, &cq_param
);
3095 err
= mlx5e_create_cq(cq
, &cq_param
);
3099 err
= mlx5e_alloc_drop_rq(mdev
, drop_rq
, &rq_param
);
3101 goto err_destroy_cq
;
3103 err
= mlx5e_create_rq(drop_rq
, &rq_param
);
3107 err
= mlx5e_modify_rq_state(drop_rq
, MLX5_RQC_STATE_RST
, MLX5_RQC_STATE_RDY
);
3109 mlx5_core_warn(priv
->mdev
, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err
);
3114 mlx5e_free_drop_rq(drop_rq
);
3117 mlx5e_destroy_cq(cq
);
3125 void mlx5e_close_drop_rq(struct mlx5e_rq
*drop_rq
)
3127 mlx5e_destroy_rq(drop_rq
);
3128 mlx5e_free_drop_rq(drop_rq
);
3129 mlx5e_destroy_cq(&drop_rq
->cq
);
3130 mlx5e_free_cq(&drop_rq
->cq
);
3133 int mlx5e_create_tis(struct mlx5_core_dev
*mdev
, void *in
, u32
*tisn
)
3135 void *tisc
= MLX5_ADDR_OF(create_tis_in
, in
, ctx
);
3137 MLX5_SET(tisc
, tisc
, transport_domain
, mdev
->mlx5e_res
.hw_objs
.td
.tdn
);
3139 if (MLX5_GET(tisc
, tisc
, tls_en
))
3140 MLX5_SET(tisc
, tisc
, pd
, mdev
->mlx5e_res
.hw_objs
.pdn
);
3142 if (mlx5_lag_is_lacp_owner(mdev
))
3143 MLX5_SET(tisc
, tisc
, strict_lag_tx_port_affinity
, 1);
3145 return mlx5_core_create_tis(mdev
, in
, tisn
);
3148 void mlx5e_destroy_tis(struct mlx5_core_dev
*mdev
, u32 tisn
)
3150 mlx5_core_destroy_tis(mdev
, tisn
);
3153 void mlx5e_destroy_tises(struct mlx5e_priv
*priv
)
3157 for (i
= 0; i
< mlx5e_get_num_lag_ports(priv
->mdev
); i
++)
3158 for (tc
= 0; tc
< priv
->profile
->max_tc
; tc
++)
3159 mlx5e_destroy_tis(priv
->mdev
, priv
->tisn
[i
][tc
]);
3162 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev
*mdev
)
3164 return MLX5_CAP_GEN(mdev
, lag_tx_port_affinity
) && mlx5e_get_num_lag_ports(mdev
) > 1;
3167 int mlx5e_create_tises(struct mlx5e_priv
*priv
)
3172 for (i
= 0; i
< mlx5e_get_num_lag_ports(priv
->mdev
); i
++) {
3173 for (tc
= 0; tc
< priv
->profile
->max_tc
; tc
++) {
3174 u32 in
[MLX5_ST_SZ_DW(create_tis_in
)] = {};
3177 tisc
= MLX5_ADDR_OF(create_tis_in
, in
, ctx
);
3179 MLX5_SET(tisc
, tisc
, prio
, tc
<< 1);
3181 if (mlx5e_lag_should_assign_affinity(priv
->mdev
))
3182 MLX5_SET(tisc
, tisc
, lag_tx_port_affinity
, i
+ 1);
3184 err
= mlx5e_create_tis(priv
->mdev
, in
, &priv
->tisn
[i
][tc
]);
3186 goto err_close_tises
;
3193 for (; i
>= 0; i
--) {
3194 for (tc
--; tc
>= 0; tc
--)
3195 mlx5e_destroy_tis(priv
->mdev
, priv
->tisn
[i
][tc
]);
3196 tc
= priv
->profile
->max_tc
;
3202 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv
*priv
)
3204 mlx5e_destroy_tises(priv
);
3207 static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv
*priv
,
3208 u32 rqtn
, u32
*tirc
)
3210 MLX5_SET(tirc
, tirc
, transport_domain
, priv
->mdev
->mlx5e_res
.hw_objs
.td
.tdn
);
3211 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_INDIRECT
);
3212 MLX5_SET(tirc
, tirc
, indirect_table
, rqtn
);
3213 MLX5_SET(tirc
, tirc
, tunneled_offload_en
,
3214 priv
->channels
.params
.tunneled_offload_en
);
3216 mlx5e_build_tir_ctx_lro(&priv
->channels
.params
, tirc
);
3219 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv
*priv
,
3220 enum mlx5e_traffic_types tt
,
3223 mlx5e_build_indir_tir_ctx_common(priv
, priv
->indir_rqt
.rqtn
, tirc
);
3224 mlx5e_build_indir_tir_ctx_hash(&priv
->rss_params
,
3225 &tirc_default_config
[tt
], tirc
, false);
3228 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv
*priv
, u32 rqtn
, u32
*tirc
)
3230 mlx5e_build_indir_tir_ctx_common(priv
, rqtn
, tirc
);
3231 MLX5_SET(tirc
, tirc
, rx_hash_fn
, MLX5_RX_HASH_FN_INVERTED_XOR8
);
3234 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv
*priv
,
3235 enum mlx5e_traffic_types tt
,
3238 mlx5e_build_indir_tir_ctx_common(priv
, priv
->indir_rqt
.rqtn
, tirc
);
3239 mlx5e_build_indir_tir_ctx_hash(&priv
->rss_params
,
3240 &tirc_default_config
[tt
], tirc
, true);
3243 int mlx5e_create_indirect_tirs(struct mlx5e_priv
*priv
, bool inner_ttc
)
3245 struct mlx5e_tir
*tir
;
3253 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
3254 in
= kvzalloc(inlen
, GFP_KERNEL
);
3258 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
3259 memset(in
, 0, inlen
);
3260 tir
= &priv
->indir_tir
[tt
];
3261 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
3262 mlx5e_build_indir_tir_ctx(priv
, tt
, tirc
);
3263 err
= mlx5e_create_tir(priv
->mdev
, tir
, in
);
3265 mlx5_core_warn(priv
->mdev
, "create indirect tirs failed, %d\n", err
);
3266 goto err_destroy_inner_tirs
;
3270 if (!inner_ttc
|| !mlx5e_tunnel_inner_ft_supported(priv
->mdev
))
3273 for (i
= 0; i
< MLX5E_NUM_INDIR_TIRS
; i
++) {
3274 memset(in
, 0, inlen
);
3275 tir
= &priv
->inner_indir_tir
[i
];
3276 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
3277 mlx5e_build_inner_indir_tir_ctx(priv
, i
, tirc
);
3278 err
= mlx5e_create_tir(priv
->mdev
, tir
, in
);
3280 mlx5_core_warn(priv
->mdev
, "create inner indirect tirs failed, %d\n", err
);
3281 goto err_destroy_inner_tirs
;
3290 err_destroy_inner_tirs
:
3291 for (i
--; i
>= 0; i
--)
3292 mlx5e_destroy_tir(priv
->mdev
, &priv
->inner_indir_tir
[i
]);
3294 for (tt
--; tt
>= 0; tt
--)
3295 mlx5e_destroy_tir(priv
->mdev
, &priv
->indir_tir
[tt
]);
3302 int mlx5e_create_direct_tirs(struct mlx5e_priv
*priv
, struct mlx5e_tir
*tirs
, int n
)
3304 struct mlx5e_tir
*tir
;
3311 inlen
= MLX5_ST_SZ_BYTES(create_tir_in
);
3312 in
= kvzalloc(inlen
, GFP_KERNEL
);
3316 for (ix
= 0; ix
< n
; ix
++) {
3317 memset(in
, 0, inlen
);
3319 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
3320 mlx5e_build_direct_tir_ctx(priv
, tir
->rqt
.rqtn
, tirc
);
3321 err
= mlx5e_create_tir(priv
->mdev
, tir
, in
);
3323 goto err_destroy_ch_tirs
;
3328 err_destroy_ch_tirs
:
3329 mlx5_core_warn(priv
->mdev
, "create tirs failed, %d\n", err
);
3330 for (ix
--; ix
>= 0; ix
--)
3331 mlx5e_destroy_tir(priv
->mdev
, &tirs
[ix
]);
3339 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv
*priv
)
3343 for (i
= 0; i
< MLX5E_NUM_INDIR_TIRS
; i
++)
3344 mlx5e_destroy_tir(priv
->mdev
, &priv
->indir_tir
[i
]);
3346 /* Verify inner tirs resources allocated */
3347 if (!priv
->inner_indir_tir
[0].tirn
)
3350 for (i
= 0; i
< MLX5E_NUM_INDIR_TIRS
; i
++)
3351 mlx5e_destroy_tir(priv
->mdev
, &priv
->inner_indir_tir
[i
]);
3354 void mlx5e_destroy_direct_tirs(struct mlx5e_priv
*priv
, struct mlx5e_tir
*tirs
, int n
)
3358 for (i
= 0; i
< n
; i
++)
3359 mlx5e_destroy_tir(priv
->mdev
, &tirs
[i
]);
3362 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels
*chs
, bool enable
)
3367 for (i
= 0; i
< chs
->num
; i
++) {
3368 err
= mlx5e_modify_rq_scatter_fcs(&chs
->c
[i
]->rq
, enable
);
3376 static int mlx5e_modify_channels_vsd(struct mlx5e_channels
*chs
, bool vsd
)
3381 for (i
= 0; i
< chs
->num
; i
++) {
3382 err
= mlx5e_modify_rq_vsd(&chs
->c
[i
]->rq
, vsd
);
3386 if (chs
->ptp
&& test_bit(MLX5E_PTP_STATE_RX
, chs
->ptp
->state
))
3387 return mlx5e_modify_rq_vsd(&chs
->ptp
->rq
, vsd
);
3392 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv
*priv
,
3393 struct tc_mqprio_qopt
*mqprio
)
3395 struct mlx5e_params new_params
;
3396 u8 tc
= mqprio
->num_tc
;
3399 mqprio
->hw
= TC_MQPRIO_HW_OFFLOAD_TCS
;
3401 if (tc
&& tc
!= MLX5E_MAX_NUM_TC
)
3404 mutex_lock(&priv
->state_lock
);
3406 /* MQPRIO is another toplevel qdisc that can't be attached
3407 * simultaneously with the offloaded HTB.
3409 if (WARN_ON(priv
->htb
.maj_id
)) {
3414 new_params
= priv
->channels
.params
;
3415 new_params
.num_tc
= tc
? tc
: 1;
3417 err
= mlx5e_safe_switch_params(priv
, &new_params
,
3418 mlx5e_num_channels_changed_ctx
, NULL
, true);
3421 priv
->max_opened_tc
= max_t(u8
, priv
->max_opened_tc
,
3422 priv
->channels
.params
.num_tc
);
3423 mutex_unlock(&priv
->state_lock
);
3427 static int mlx5e_setup_tc_htb(struct mlx5e_priv
*priv
, struct tc_htb_qopt_offload
*htb
)
3431 switch (htb
->command
) {
3433 return mlx5e_htb_root_add(priv
, htb
->parent_classid
, htb
->classid
,
3435 case TC_HTB_DESTROY
:
3436 return mlx5e_htb_root_del(priv
);
3437 case TC_HTB_LEAF_ALLOC_QUEUE
:
3438 res
= mlx5e_htb_leaf_alloc_queue(priv
, htb
->classid
, htb
->parent_classid
,
3439 htb
->rate
, htb
->ceil
, htb
->extack
);
3444 case TC_HTB_LEAF_TO_INNER
:
3445 return mlx5e_htb_leaf_to_inner(priv
, htb
->parent_classid
, htb
->classid
,
3446 htb
->rate
, htb
->ceil
, htb
->extack
);
3447 case TC_HTB_LEAF_DEL
:
3448 return mlx5e_htb_leaf_del(priv
, htb
->classid
, &htb
->moved_qid
, &htb
->qid
,
3450 case TC_HTB_LEAF_DEL_LAST
:
3451 case TC_HTB_LEAF_DEL_LAST_FORCE
:
3452 return mlx5e_htb_leaf_del_last(priv
, htb
->classid
,
3453 htb
->command
== TC_HTB_LEAF_DEL_LAST_FORCE
,
3455 case TC_HTB_NODE_MODIFY
:
3456 return mlx5e_htb_node_modify(priv
, htb
->classid
, htb
->rate
, htb
->ceil
,
3458 case TC_HTB_LEAF_QUERY_QUEUE
:
3459 res
= mlx5e_get_txq_by_classid(priv
, htb
->classid
);
3469 static LIST_HEAD(mlx5e_block_cb_list
);
3471 static int mlx5e_setup_tc(struct net_device
*dev
, enum tc_setup_type type
,
3474 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3475 bool tc_unbind
= false;
3478 if (type
== TC_SETUP_BLOCK
&&
3479 ((struct flow_block_offload
*)type_data
)->command
== FLOW_BLOCK_UNBIND
)
3482 if (!netif_device_present(dev
) && !tc_unbind
)
3486 case TC_SETUP_BLOCK
: {
3487 struct flow_block_offload
*f
= type_data
;
3489 f
->unlocked_driver_cb
= true;
3490 return flow_block_cb_setup_simple(type_data
,
3491 &mlx5e_block_cb_list
,
3492 mlx5e_setup_tc_block_cb
,
3495 case TC_SETUP_QDISC_MQPRIO
:
3496 return mlx5e_setup_tc_mqprio(priv
, type_data
);
3497 case TC_SETUP_QDISC_HTB
:
3498 mutex_lock(&priv
->state_lock
);
3499 err
= mlx5e_setup_tc_htb(priv
, type_data
);
3500 mutex_unlock(&priv
->state_lock
);
3507 void mlx5e_fold_sw_stats64(struct mlx5e_priv
*priv
, struct rtnl_link_stats64
*s
)
3511 for (i
= 0; i
< priv
->max_nch
; i
++) {
3512 struct mlx5e_channel_stats
*channel_stats
= &priv
->channel_stats
[i
];
3513 struct mlx5e_rq_stats
*xskrq_stats
= &channel_stats
->xskrq
;
3514 struct mlx5e_rq_stats
*rq_stats
= &channel_stats
->rq
;
3517 s
->rx_packets
+= rq_stats
->packets
+ xskrq_stats
->packets
;
3518 s
->rx_bytes
+= rq_stats
->bytes
+ xskrq_stats
->bytes
;
3519 s
->multicast
+= rq_stats
->mcast_packets
+ xskrq_stats
->mcast_packets
;
3521 for (j
= 0; j
< priv
->max_opened_tc
; j
++) {
3522 struct mlx5e_sq_stats
*sq_stats
= &channel_stats
->sq
[j
];
3524 s
->tx_packets
+= sq_stats
->packets
;
3525 s
->tx_bytes
+= sq_stats
->bytes
;
3526 s
->tx_dropped
+= sq_stats
->dropped
;
3529 if (priv
->tx_ptp_opened
) {
3530 for (i
= 0; i
< priv
->max_opened_tc
; i
++) {
3531 struct mlx5e_sq_stats
*sq_stats
= &priv
->ptp_stats
.sq
[i
];
3533 s
->tx_packets
+= sq_stats
->packets
;
3534 s
->tx_bytes
+= sq_stats
->bytes
;
3535 s
->tx_dropped
+= sq_stats
->dropped
;
3538 if (priv
->rx_ptp_opened
) {
3539 struct mlx5e_rq_stats
*rq_stats
= &priv
->ptp_stats
.rq
;
3541 s
->rx_packets
+= rq_stats
->packets
;
3542 s
->rx_bytes
+= rq_stats
->bytes
;
3543 s
->multicast
+= rq_stats
->mcast_packets
;
3548 mlx5e_get_stats(struct net_device
*dev
, struct rtnl_link_stats64
*stats
)
3550 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3551 struct mlx5e_pport_stats
*pstats
= &priv
->stats
.pport
;
3553 if (!netif_device_present(dev
))
3556 /* In switchdev mode, monitor counters doesn't monitor
3557 * rx/tx stats of 802_3. The update stats mechanism
3558 * should keep the 802_3 layout counters updated
3560 if (!mlx5e_monitor_counter_supported(priv
) ||
3561 mlx5e_is_uplink_rep(priv
)) {
3562 /* update HW stats in background for next time */
3563 mlx5e_queue_update_stats(priv
);
3566 if (mlx5e_is_uplink_rep(priv
)) {
3567 struct mlx5e_vport_stats
*vstats
= &priv
->stats
.vport
;
3569 stats
->rx_packets
= PPORT_802_3_GET(pstats
, a_frames_received_ok
);
3570 stats
->rx_bytes
= PPORT_802_3_GET(pstats
, a_octets_received_ok
);
3571 stats
->tx_packets
= PPORT_802_3_GET(pstats
, a_frames_transmitted_ok
);
3572 stats
->tx_bytes
= PPORT_802_3_GET(pstats
, a_octets_transmitted_ok
);
3574 /* vport multicast also counts packets that are dropped due to steering
3575 * or rx out of buffer
3577 stats
->multicast
= VPORT_COUNTER_GET(vstats
, received_eth_multicast
.packets
);
3579 mlx5e_fold_sw_stats64(priv
, stats
);
3582 stats
->rx_dropped
= priv
->stats
.qcnt
.rx_out_of_buffer
;
3584 stats
->rx_length_errors
=
3585 PPORT_802_3_GET(pstats
, a_in_range_length_errors
) +
3586 PPORT_802_3_GET(pstats
, a_out_of_range_length_field
) +
3587 PPORT_802_3_GET(pstats
, a_frame_too_long_errors
);
3588 stats
->rx_crc_errors
=
3589 PPORT_802_3_GET(pstats
, a_frame_check_sequence_errors
);
3590 stats
->rx_frame_errors
= PPORT_802_3_GET(pstats
, a_alignment_errors
);
3591 stats
->tx_aborted_errors
= PPORT_2863_GET(pstats
, if_out_discards
);
3592 stats
->rx_errors
= stats
->rx_length_errors
+ stats
->rx_crc_errors
+
3593 stats
->rx_frame_errors
;
3594 stats
->tx_errors
= stats
->tx_aborted_errors
+ stats
->tx_carrier_errors
;
3597 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv
*priv
)
3599 if (mlx5e_is_uplink_rep(priv
))
3600 return; /* no rx mode for uplink rep */
3602 queue_work(priv
->wq
, &priv
->set_rx_mode_work
);
3605 static void mlx5e_set_rx_mode(struct net_device
*dev
)
3607 struct mlx5e_priv
*priv
= netdev_priv(dev
);
3609 mlx5e_nic_set_rx_mode(priv
);
3612 static int mlx5e_set_mac(struct net_device
*netdev
, void *addr
)
3614 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3615 struct sockaddr
*saddr
= addr
;
3617 if (!is_valid_ether_addr(saddr
->sa_data
))
3618 return -EADDRNOTAVAIL
;
3620 netif_addr_lock_bh(netdev
);
3621 ether_addr_copy(netdev
->dev_addr
, saddr
->sa_data
);
3622 netif_addr_unlock_bh(netdev
);
3624 mlx5e_nic_set_rx_mode(priv
);
3629 #define MLX5E_SET_FEATURE(features, feature, enable) \
3632 *features |= feature; \
3634 *features &= ~feature; \
3637 typedef int (*mlx5e_feature_handler
)(struct net_device
*netdev
, bool enable
);
3639 static int set_feature_lro(struct net_device
*netdev
, bool enable
)
3641 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3642 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3643 struct mlx5e_params
*cur_params
;
3644 struct mlx5e_params new_params
;
3648 mutex_lock(&priv
->state_lock
);
3650 if (enable
&& priv
->xsk
.refcnt
) {
3651 netdev_warn(netdev
, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
3657 cur_params
= &priv
->channels
.params
;
3658 if (enable
&& !MLX5E_GET_PFLAG(cur_params
, MLX5E_PFLAG_RX_STRIDING_RQ
)) {
3659 netdev_warn(netdev
, "can't set LRO with legacy RQ\n");
3664 new_params
= *cur_params
;
3665 new_params
.lro_en
= enable
;
3667 if (cur_params
->rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
) {
3668 if (mlx5e_rx_mpwqe_is_linear_skb(mdev
, cur_params
, NULL
) ==
3669 mlx5e_rx_mpwqe_is_linear_skb(mdev
, &new_params
, NULL
))
3673 err
= mlx5e_safe_switch_params(priv
, &new_params
,
3674 mlx5e_modify_tirs_lro_ctx
, NULL
, reset
);
3676 mutex_unlock(&priv
->state_lock
);
3680 static int set_feature_cvlan_filter(struct net_device
*netdev
, bool enable
)
3682 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3685 mlx5e_enable_cvlan_filter(priv
);
3687 mlx5e_disable_cvlan_filter(priv
);
3692 static int set_feature_hw_tc(struct net_device
*netdev
, bool enable
)
3694 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3696 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
3697 if (!enable
&& mlx5e_tc_num_filters(priv
, MLX5_TC_FLAG(NIC_OFFLOAD
))) {
3699 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3704 if (!enable
&& priv
->htb
.maj_id
) {
3705 netdev_err(netdev
, "Active HTB offload, can't turn hw_tc_offload off\n");
3712 static int set_feature_rx_all(struct net_device
*netdev
, bool enable
)
3714 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3715 struct mlx5_core_dev
*mdev
= priv
->mdev
;
3717 return mlx5_set_port_fcs(mdev
, !enable
);
3720 static int set_feature_rx_fcs(struct net_device
*netdev
, bool enable
)
3722 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3725 mutex_lock(&priv
->state_lock
);
3727 priv
->channels
.params
.scatter_fcs_en
= enable
;
3728 err
= mlx5e_modify_channels_scatter_fcs(&priv
->channels
, enable
);
3730 priv
->channels
.params
.scatter_fcs_en
= !enable
;
3732 mutex_unlock(&priv
->state_lock
);
3737 static int set_feature_rx_vlan(struct net_device
*netdev
, bool enable
)
3739 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3742 mutex_lock(&priv
->state_lock
);
3744 priv
->channels
.params
.vlan_strip_disable
= !enable
;
3745 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
3748 err
= mlx5e_modify_channels_vsd(&priv
->channels
, !enable
);
3750 priv
->channels
.params
.vlan_strip_disable
= enable
;
3753 mutex_unlock(&priv
->state_lock
);
3758 #ifdef CONFIG_MLX5_EN_ARFS
3759 static int set_feature_arfs(struct net_device
*netdev
, bool enable
)
3761 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3765 err
= mlx5e_arfs_enable(priv
);
3767 err
= mlx5e_arfs_disable(priv
);
3773 static int mlx5e_handle_feature(struct net_device
*netdev
,
3774 netdev_features_t
*features
,
3775 netdev_features_t wanted_features
,
3776 netdev_features_t feature
,
3777 mlx5e_feature_handler feature_handler
)
3779 netdev_features_t changes
= wanted_features
^ netdev
->features
;
3780 bool enable
= !!(wanted_features
& feature
);
3783 if (!(changes
& feature
))
3786 err
= feature_handler(netdev
, enable
);
3788 netdev_err(netdev
, "%s feature %pNF failed, err %d\n",
3789 enable
? "Enable" : "Disable", &feature
, err
);
3793 MLX5E_SET_FEATURE(features
, feature
, enable
);
3797 int mlx5e_set_features(struct net_device
*netdev
, netdev_features_t features
)
3799 netdev_features_t oper_features
= netdev
->features
;
3802 #define MLX5E_HANDLE_FEATURE(feature, handler) \
3803 mlx5e_handle_feature(netdev, &oper_features, features, feature, handler)
3805 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_LRO
, set_feature_lro
);
3806 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER
,
3807 set_feature_cvlan_filter
);
3808 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC
, set_feature_hw_tc
);
3809 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL
, set_feature_rx_all
);
3810 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS
, set_feature_rx_fcs
);
3811 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX
, set_feature_rx_vlan
);
3812 #ifdef CONFIG_MLX5_EN_ARFS
3813 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE
, set_feature_arfs
);
3815 err
|= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX
, mlx5e_ktls_set_feature_rx
);
3818 netdev
->features
= oper_features
;
3825 static netdev_features_t
mlx5e_fix_uplink_rep_features(struct net_device
*netdev
,
3826 netdev_features_t features
)
3828 features
&= ~NETIF_F_HW_TLS_RX
;
3829 if (netdev
->features
& NETIF_F_HW_TLS_RX
)
3830 netdev_warn(netdev
, "Disabling hw_tls_rx, not supported in switchdev mode\n");
3832 features
&= ~NETIF_F_HW_TLS_TX
;
3833 if (netdev
->features
& NETIF_F_HW_TLS_TX
)
3834 netdev_warn(netdev
, "Disabling hw_tls_tx, not supported in switchdev mode\n");
3836 features
&= ~NETIF_F_NTUPLE
;
3837 if (netdev
->features
& NETIF_F_NTUPLE
)
3838 netdev_warn(netdev
, "Disabling ntuple, not supported in switchdev mode\n");
3843 static netdev_features_t
mlx5e_fix_features(struct net_device
*netdev
,
3844 netdev_features_t features
)
3846 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3847 struct mlx5e_params
*params
;
3849 mutex_lock(&priv
->state_lock
);
3850 params
= &priv
->channels
.params
;
3851 if (!priv
->fs
.vlan
||
3852 !bitmap_empty(mlx5e_vlan_get_active_svlans(priv
->fs
.vlan
), VLAN_N_VID
)) {
3853 /* HW strips the outer C-tag header, this is a problem
3854 * for S-tag traffic.
3856 features
&= ~NETIF_F_HW_VLAN_CTAG_RX
;
3857 if (!params
->vlan_strip_disable
)
3858 netdev_warn(netdev
, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
3861 if (!MLX5E_GET_PFLAG(params
, MLX5E_PFLAG_RX_STRIDING_RQ
)) {
3862 if (features
& NETIF_F_LRO
) {
3863 netdev_warn(netdev
, "Disabling LRO, not supported in legacy RQ\n");
3864 features
&= ~NETIF_F_LRO
;
3868 if (MLX5E_GET_PFLAG(params
, MLX5E_PFLAG_RX_CQE_COMPRESS
)) {
3869 features
&= ~NETIF_F_RXHASH
;
3870 if (netdev
->features
& NETIF_F_RXHASH
)
3871 netdev_warn(netdev
, "Disabling rxhash, not supported when CQE compress is active\n");
3874 if (mlx5e_is_uplink_rep(priv
))
3875 features
= mlx5e_fix_uplink_rep_features(netdev
, features
);
3877 mutex_unlock(&priv
->state_lock
);
3882 static bool mlx5e_xsk_validate_mtu(struct net_device
*netdev
,
3883 struct mlx5e_channels
*chs
,
3884 struct mlx5e_params
*new_params
,
3885 struct mlx5_core_dev
*mdev
)
3889 for (ix
= 0; ix
< chs
->params
.num_channels
; ix
++) {
3890 struct xsk_buff_pool
*xsk_pool
=
3891 mlx5e_xsk_get_pool(&chs
->params
, chs
->params
.xsk
, ix
);
3892 struct mlx5e_xsk_param xsk
;
3897 mlx5e_build_xsk_param(xsk_pool
, &xsk
);
3899 if (!mlx5e_validate_xsk_param(new_params
, &xsk
, mdev
)) {
3900 u32 hr
= mlx5e_get_linear_rq_headroom(new_params
, &xsk
);
3901 int max_mtu_frame
, max_mtu_page
, max_mtu
;
3903 /* Two criteria must be met:
3904 * 1. HW MTU + all headrooms <= XSK frame size.
3905 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
3907 max_mtu_frame
= MLX5E_HW2SW_MTU(new_params
, xsk
.chunk_size
- hr
);
3908 max_mtu_page
= mlx5e_xdp_max_mtu(new_params
, &xsk
);
3909 max_mtu
= min(max_mtu_frame
, max_mtu_page
);
3911 netdev_err(netdev
, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n",
3912 new_params
->sw_mtu
, ix
, max_mtu
);
3920 int mlx5e_change_mtu(struct net_device
*netdev
, int new_mtu
,
3921 mlx5e_fp_preactivate preactivate
)
3923 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
3924 struct mlx5e_params new_params
;
3925 struct mlx5e_params
*params
;
3929 mutex_lock(&priv
->state_lock
);
3931 params
= &priv
->channels
.params
;
3933 new_params
= *params
;
3934 new_params
.sw_mtu
= new_mtu
;
3935 err
= mlx5e_validate_params(priv
->mdev
, &new_params
);
3939 if (params
->xdp_prog
&&
3940 !mlx5e_rx_is_linear_skb(&new_params
, NULL
)) {
3941 netdev_err(netdev
, "MTU(%d) > %d is not allowed while XDP enabled\n",
3942 new_mtu
, mlx5e_xdp_max_mtu(params
, NULL
));
3947 if (priv
->xsk
.refcnt
&&
3948 !mlx5e_xsk_validate_mtu(netdev
, &priv
->channels
,
3949 &new_params
, priv
->mdev
)) {
3957 if (params
->rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
) {
3958 bool is_linear_old
= mlx5e_rx_mpwqe_is_linear_skb(priv
->mdev
, params
, NULL
);
3959 bool is_linear_new
= mlx5e_rx_mpwqe_is_linear_skb(priv
->mdev
,
3961 u8 ppw_old
= mlx5e_mpwqe_log_pkts_per_wqe(params
, NULL
);
3962 u8 ppw_new
= mlx5e_mpwqe_log_pkts_per_wqe(&new_params
, NULL
);
3964 /* Always reset in linear mode - hw_mtu is used in data path.
3965 * Check that the mode was non-linear and didn't change.
3966 * If XSK is active, XSK RQs are linear.
3968 if (!is_linear_old
&& !is_linear_new
&& !priv
->xsk
.refcnt
&&
3973 err
= mlx5e_safe_switch_params(priv
, &new_params
, preactivate
, NULL
, reset
);
3976 netdev
->mtu
= params
->sw_mtu
;
3977 mutex_unlock(&priv
->state_lock
);
3981 static int mlx5e_change_nic_mtu(struct net_device
*netdev
, int new_mtu
)
3983 return mlx5e_change_mtu(netdev
, new_mtu
, mlx5e_set_dev_port_mtu_ctx
);
3986 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv
*priv
, void *ctx
)
3988 bool set
= *(bool *)ctx
;
3990 return mlx5e_ptp_rx_manage_fs(priv
, set
);
3993 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv
*priv
, bool rx_filter
)
3995 bool rx_cqe_compress_def
= priv
->channels
.params
.rx_cqe_compress_def
;
3999 /* Reset CQE compression to Admin default */
4000 return mlx5e_modify_rx_cqe_compression_locked(priv
, rx_cqe_compress_def
);
4002 if (!MLX5E_GET_PFLAG(&priv
->channels
.params
, MLX5E_PFLAG_RX_CQE_COMPRESS
))
4005 /* Disable CQE compression */
4006 netdev_warn(priv
->netdev
, "Disabling RX cqe compression\n");
4007 err
= mlx5e_modify_rx_cqe_compression_locked(priv
, false);
4009 netdev_err(priv
->netdev
, "Failed disabling cqe compression err=%d\n", err
);
4014 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv
*priv
, bool ptp_rx
)
4016 struct mlx5e_params new_params
;
4018 if (ptp_rx
== priv
->channels
.params
.ptp_rx
)
4021 new_params
= priv
->channels
.params
;
4022 new_params
.ptp_rx
= ptp_rx
;
4023 return mlx5e_safe_switch_params(priv
, &new_params
, mlx5e_ptp_rx_manage_fs_ctx
,
4024 &new_params
.ptp_rx
, true);
4027 int mlx5e_hwstamp_set(struct mlx5e_priv
*priv
, struct ifreq
*ifr
)
4029 struct hwtstamp_config config
;
4030 bool rx_cqe_compress_def
;
4034 if (!MLX5_CAP_GEN(priv
->mdev
, device_frequency_khz
) ||
4035 (mlx5_clock_get_ptp_index(priv
->mdev
) == -1))
4038 if (copy_from_user(&config
, ifr
->ifr_data
, sizeof(config
)))
4041 /* TX HW timestamp */
4042 switch (config
.tx_type
) {
4043 case HWTSTAMP_TX_OFF
:
4044 case HWTSTAMP_TX_ON
:
4050 mutex_lock(&priv
->state_lock
);
4051 rx_cqe_compress_def
= priv
->channels
.params
.rx_cqe_compress_def
;
4053 /* RX HW timestamp */
4054 switch (config
.rx_filter
) {
4055 case HWTSTAMP_FILTER_NONE
:
4058 case HWTSTAMP_FILTER_ALL
:
4059 case HWTSTAMP_FILTER_SOME
:
4060 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT
:
4061 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC
:
4062 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ
:
4063 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT
:
4064 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC
:
4065 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ
:
4066 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT
:
4067 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC
:
4068 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ
:
4069 case HWTSTAMP_FILTER_PTP_V2_EVENT
:
4070 case HWTSTAMP_FILTER_PTP_V2_SYNC
:
4071 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ
:
4072 case HWTSTAMP_FILTER_NTP_ALL
:
4073 config
.rx_filter
= HWTSTAMP_FILTER_ALL
;
4074 /* ptp_rx is set if both HW TS is set and CQE
4075 * compression is set
4077 ptp_rx
= rx_cqe_compress_def
;
4084 if (!priv
->profile
->rx_ptp_support
)
4085 err
= mlx5e_hwstamp_config_no_ptp_rx(priv
,
4086 config
.rx_filter
!= HWTSTAMP_FILTER_NONE
);
4088 err
= mlx5e_hwstamp_config_ptp_rx(priv
, ptp_rx
);
4092 memcpy(&priv
->tstamp
, &config
, sizeof(config
));
4093 mutex_unlock(&priv
->state_lock
);
4095 /* might need to fix some features */
4096 netdev_update_features(priv
->netdev
);
4098 return copy_to_user(ifr
->ifr_data
, &config
,
4099 sizeof(config
)) ? -EFAULT
: 0;
4101 mutex_unlock(&priv
->state_lock
);
4105 int mlx5e_hwstamp_get(struct mlx5e_priv
*priv
, struct ifreq
*ifr
)
4107 struct hwtstamp_config
*cfg
= &priv
->tstamp
;
4109 if (!MLX5_CAP_GEN(priv
->mdev
, device_frequency_khz
))
4112 return copy_to_user(ifr
->ifr_data
, cfg
, sizeof(*cfg
)) ? -EFAULT
: 0;
4115 static int mlx5e_ioctl(struct net_device
*dev
, struct ifreq
*ifr
, int cmd
)
4117 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4121 return mlx5e_hwstamp_set(priv
, ifr
);
4123 return mlx5e_hwstamp_get(priv
, ifr
);
4129 #ifdef CONFIG_MLX5_ESWITCH
4130 int mlx5e_set_vf_mac(struct net_device
*dev
, int vf
, u8
*mac
)
4132 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4133 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4135 return mlx5_eswitch_set_vport_mac(mdev
->priv
.eswitch
, vf
+ 1, mac
);
4138 static int mlx5e_set_vf_vlan(struct net_device
*dev
, int vf
, u16 vlan
, u8 qos
,
4141 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4142 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4144 if (vlan_proto
!= htons(ETH_P_8021Q
))
4145 return -EPROTONOSUPPORT
;
4147 return mlx5_eswitch_set_vport_vlan(mdev
->priv
.eswitch
, vf
+ 1,
4151 static int mlx5e_set_vf_spoofchk(struct net_device
*dev
, int vf
, bool setting
)
4153 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4154 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4156 return mlx5_eswitch_set_vport_spoofchk(mdev
->priv
.eswitch
, vf
+ 1, setting
);
4159 static int mlx5e_set_vf_trust(struct net_device
*dev
, int vf
, bool setting
)
4161 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4162 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4164 return mlx5_eswitch_set_vport_trust(mdev
->priv
.eswitch
, vf
+ 1, setting
);
4167 int mlx5e_set_vf_rate(struct net_device
*dev
, int vf
, int min_tx_rate
,
4170 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4171 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4173 return mlx5_eswitch_set_vport_rate(mdev
->priv
.eswitch
, vf
+ 1,
4174 max_tx_rate
, min_tx_rate
);
4177 static int mlx5_vport_link2ifla(u8 esw_link
)
4180 case MLX5_VPORT_ADMIN_STATE_DOWN
:
4181 return IFLA_VF_LINK_STATE_DISABLE
;
4182 case MLX5_VPORT_ADMIN_STATE_UP
:
4183 return IFLA_VF_LINK_STATE_ENABLE
;
4185 return IFLA_VF_LINK_STATE_AUTO
;
4188 static int mlx5_ifla_link2vport(u8 ifla_link
)
4190 switch (ifla_link
) {
4191 case IFLA_VF_LINK_STATE_DISABLE
:
4192 return MLX5_VPORT_ADMIN_STATE_DOWN
;
4193 case IFLA_VF_LINK_STATE_ENABLE
:
4194 return MLX5_VPORT_ADMIN_STATE_UP
;
4196 return MLX5_VPORT_ADMIN_STATE_AUTO
;
4199 static int mlx5e_set_vf_link_state(struct net_device
*dev
, int vf
,
4202 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4203 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4205 if (mlx5e_is_uplink_rep(priv
))
4208 return mlx5_eswitch_set_vport_state(mdev
->priv
.eswitch
, vf
+ 1,
4209 mlx5_ifla_link2vport(link_state
));
4212 int mlx5e_get_vf_config(struct net_device
*dev
,
4213 int vf
, struct ifla_vf_info
*ivi
)
4215 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4216 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4219 if (!netif_device_present(dev
))
4222 err
= mlx5_eswitch_get_vport_config(mdev
->priv
.eswitch
, vf
+ 1, ivi
);
4225 ivi
->linkstate
= mlx5_vport_link2ifla(ivi
->linkstate
);
4229 int mlx5e_get_vf_stats(struct net_device
*dev
,
4230 int vf
, struct ifla_vf_stats
*vf_stats
)
4232 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4233 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4235 return mlx5_eswitch_get_vport_stats(mdev
->priv
.eswitch
, vf
+ 1,
4240 mlx5e_has_offload_stats(const struct net_device
*dev
, int attr_id
)
4242 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4244 if (!netif_device_present(dev
))
4247 if (!mlx5e_is_uplink_rep(priv
))
4250 return mlx5e_rep_has_offload_stats(dev
, attr_id
);
4254 mlx5e_get_offload_stats(int attr_id
, const struct net_device
*dev
,
4257 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4259 if (!mlx5e_is_uplink_rep(priv
))
4262 return mlx5e_rep_get_offload_stats(attr_id
, dev
, sp
);
4266 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev
*mdev
, u8 proto_type
)
4268 switch (proto_type
) {
4270 return MLX5_CAP_ETH(mdev
, tunnel_stateless_gre
);
4273 return (MLX5_CAP_ETH(mdev
, tunnel_stateless_ip_over_ip
) ||
4274 MLX5_CAP_ETH(mdev
, tunnel_stateless_ip_over_ip_tx
));
4280 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev
*mdev
,
4281 struct sk_buff
*skb
)
4283 switch (skb
->inner_protocol
) {
4284 case htons(ETH_P_IP
):
4285 case htons(ETH_P_IPV6
):
4286 case htons(ETH_P_TEB
):
4288 case htons(ETH_P_MPLS_UC
):
4289 case htons(ETH_P_MPLS_MC
):
4290 return MLX5_CAP_ETH(mdev
, tunnel_stateless_mpls_over_gre
);
4295 static netdev_features_t
mlx5e_tunnel_features_check(struct mlx5e_priv
*priv
,
4296 struct sk_buff
*skb
,
4297 netdev_features_t features
)
4299 unsigned int offset
= 0;
4300 struct udphdr
*udph
;
4304 switch (vlan_get_protocol(skb
)) {
4305 case htons(ETH_P_IP
):
4306 proto
= ip_hdr(skb
)->protocol
;
4308 case htons(ETH_P_IPV6
):
4309 proto
= ipv6_find_hdr(skb
, &offset
, -1, NULL
, NULL
);
4317 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv
->mdev
, skb
))
4322 if (mlx5e_tunnel_proto_supported_tx(priv
->mdev
, IPPROTO_IPIP
))
4326 udph
= udp_hdr(skb
);
4327 port
= be16_to_cpu(udph
->dest
);
4329 /* Verify if UDP port is being offloaded by HW */
4330 if (mlx5_vxlan_lookup_port(priv
->mdev
->vxlan
, port
))
4333 #if IS_ENABLED(CONFIG_GENEVE)
4334 /* Support Geneve offload for default UDP port */
4335 if (port
== GENEVE_UDP_PORT
&& mlx5_geneve_tx_allowed(priv
->mdev
))
4339 #ifdef CONFIG_MLX5_EN_IPSEC
4341 return mlx5e_ipsec_feature_check(skb
, features
);
4346 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4347 return features
& ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
4350 netdev_features_t
mlx5e_features_check(struct sk_buff
*skb
,
4351 struct net_device
*netdev
,
4352 netdev_features_t features
)
4354 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4356 features
= vlan_features_check(skb
, features
);
4357 features
= vxlan_features_check(skb
, features
);
4359 /* Validate if the tunneled packet is being offloaded by HW */
4360 if (skb
->encapsulation
&&
4361 (features
& NETIF_F_CSUM_MASK
|| features
& NETIF_F_GSO_MASK
))
4362 return mlx5e_tunnel_features_check(priv
, skb
, features
);
4367 static void mlx5e_tx_timeout_work(struct work_struct
*work
)
4369 struct mlx5e_priv
*priv
= container_of(work
, struct mlx5e_priv
,
4371 struct net_device
*netdev
= priv
->netdev
;
4375 mutex_lock(&priv
->state_lock
);
4377 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
))
4380 for (i
= 0; i
< netdev
->real_num_tx_queues
; i
++) {
4381 struct netdev_queue
*dev_queue
=
4382 netdev_get_tx_queue(netdev
, i
);
4383 struct mlx5e_txqsq
*sq
= priv
->txq2sq
[i
];
4385 if (!netif_xmit_stopped(dev_queue
))
4388 if (mlx5e_reporter_tx_timeout(sq
))
4389 /* break if tried to reopened channels */
4394 mutex_unlock(&priv
->state_lock
);
4398 static void mlx5e_tx_timeout(struct net_device
*dev
, unsigned int txqueue
)
4400 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4402 netdev_err(dev
, "TX timeout detected\n");
4403 queue_work(priv
->wq
, &priv
->tx_timeout_work
);
4406 static int mlx5e_xdp_allowed(struct mlx5e_priv
*priv
, struct bpf_prog
*prog
)
4408 struct net_device
*netdev
= priv
->netdev
;
4409 struct mlx5e_params new_params
;
4411 if (priv
->channels
.params
.lro_en
) {
4412 netdev_warn(netdev
, "can't set XDP while LRO is on, disable LRO first\n");
4416 if (mlx5_fpga_is_ipsec_device(priv
->mdev
)) {
4418 "XDP is not available on Innova cards with IPsec support\n");
4422 new_params
= priv
->channels
.params
;
4423 new_params
.xdp_prog
= prog
;
4425 /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4428 if (!mlx5e_rx_is_linear_skb(&new_params
, NULL
)) {
4429 netdev_warn(netdev
, "XDP is not allowed with MTU(%d) > %d\n",
4431 mlx5e_xdp_max_mtu(&new_params
, NULL
));
4438 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq
*rq
, struct bpf_prog
*prog
)
4440 struct bpf_prog
*old_prog
;
4442 old_prog
= rcu_replace_pointer(rq
->xdp_prog
, prog
,
4443 lockdep_is_held(&rq
->priv
->state_lock
));
4445 bpf_prog_put(old_prog
);
4448 static int mlx5e_xdp_set(struct net_device
*netdev
, struct bpf_prog
*prog
)
4450 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4451 struct mlx5e_params new_params
;
4452 struct bpf_prog
*old_prog
;
4457 mutex_lock(&priv
->state_lock
);
4460 err
= mlx5e_xdp_allowed(priv
, prog
);
4465 /* no need for full reset when exchanging programs */
4466 reset
= (!priv
->channels
.params
.xdp_prog
|| !prog
);
4468 new_params
= priv
->channels
.params
;
4469 new_params
.xdp_prog
= prog
;
4471 mlx5e_set_rq_type(priv
->mdev
, &new_params
);
4472 old_prog
= priv
->channels
.params
.xdp_prog
;
4474 err
= mlx5e_safe_switch_params(priv
, &new_params
, NULL
, NULL
, reset
);
4479 bpf_prog_put(old_prog
);
4481 if (!test_bit(MLX5E_STATE_OPENED
, &priv
->state
) || reset
)
4484 /* exchanging programs w/o reset, we update ref counts on behalf
4485 * of the channels RQs here.
4487 bpf_prog_add(prog
, priv
->channels
.num
);
4488 for (i
= 0; i
< priv
->channels
.num
; i
++) {
4489 struct mlx5e_channel
*c
= priv
->channels
.c
[i
];
4491 mlx5e_rq_replace_xdp_prog(&c
->rq
, prog
);
4492 if (test_bit(MLX5E_CHANNEL_STATE_XSK
, c
->state
)) {
4494 mlx5e_rq_replace_xdp_prog(&c
->xskrq
, prog
);
4499 mutex_unlock(&priv
->state_lock
);
4503 static int mlx5e_xdp(struct net_device
*dev
, struct netdev_bpf
*xdp
)
4505 switch (xdp
->command
) {
4506 case XDP_SETUP_PROG
:
4507 return mlx5e_xdp_set(dev
, xdp
->prog
);
4508 case XDP_SETUP_XSK_POOL
:
4509 return mlx5e_xsk_setup_pool(dev
, xdp
->xsk
.pool
,
4516 #ifdef CONFIG_MLX5_ESWITCH
4517 static int mlx5e_bridge_getlink(struct sk_buff
*skb
, u32 pid
, u32 seq
,
4518 struct net_device
*dev
, u32 filter_mask
,
4521 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4522 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4526 err
= mlx5_eswitch_get_vepa(mdev
->priv
.eswitch
, &setting
);
4529 mode
= setting
? BRIDGE_MODE_VEPA
: BRIDGE_MODE_VEB
;
4530 return ndo_dflt_bridge_getlink(skb
, pid
, seq
, dev
,
4532 0, 0, nlflags
, filter_mask
, NULL
);
4535 static int mlx5e_bridge_setlink(struct net_device
*dev
, struct nlmsghdr
*nlh
,
4536 u16 flags
, struct netlink_ext_ack
*extack
)
4538 struct mlx5e_priv
*priv
= netdev_priv(dev
);
4539 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4540 struct nlattr
*attr
, *br_spec
;
4541 u16 mode
= BRIDGE_MODE_UNDEF
;
4545 br_spec
= nlmsg_find_attr(nlh
, sizeof(struct ifinfomsg
), IFLA_AF_SPEC
);
4549 nla_for_each_nested(attr
, br_spec
, rem
) {
4550 if (nla_type(attr
) != IFLA_BRIDGE_MODE
)
4553 if (nla_len(attr
) < sizeof(mode
))
4556 mode
= nla_get_u16(attr
);
4557 if (mode
> BRIDGE_MODE_VEPA
)
4563 if (mode
== BRIDGE_MODE_UNDEF
)
4566 setting
= (mode
== BRIDGE_MODE_VEPA
) ? 1 : 0;
4567 return mlx5_eswitch_set_vepa(mdev
->priv
.eswitch
, setting
);
4571 const struct net_device_ops mlx5e_netdev_ops
= {
4572 .ndo_open
= mlx5e_open
,
4573 .ndo_stop
= mlx5e_close
,
4574 .ndo_start_xmit
= mlx5e_xmit
,
4575 .ndo_setup_tc
= mlx5e_setup_tc
,
4576 .ndo_select_queue
= mlx5e_select_queue
,
4577 .ndo_get_stats64
= mlx5e_get_stats
,
4578 .ndo_set_rx_mode
= mlx5e_set_rx_mode
,
4579 .ndo_set_mac_address
= mlx5e_set_mac
,
4580 .ndo_vlan_rx_add_vid
= mlx5e_vlan_rx_add_vid
,
4581 .ndo_vlan_rx_kill_vid
= mlx5e_vlan_rx_kill_vid
,
4582 .ndo_set_features
= mlx5e_set_features
,
4583 .ndo_fix_features
= mlx5e_fix_features
,
4584 .ndo_change_mtu
= mlx5e_change_nic_mtu
,
4585 .ndo_do_ioctl
= mlx5e_ioctl
,
4586 .ndo_set_tx_maxrate
= mlx5e_set_tx_maxrate
,
4587 .ndo_features_check
= mlx5e_features_check
,
4588 .ndo_tx_timeout
= mlx5e_tx_timeout
,
4589 .ndo_bpf
= mlx5e_xdp
,
4590 .ndo_xdp_xmit
= mlx5e_xdp_xmit
,
4591 .ndo_xsk_wakeup
= mlx5e_xsk_wakeup
,
4592 #ifdef CONFIG_MLX5_EN_ARFS
4593 .ndo_rx_flow_steer
= mlx5e_rx_flow_steer
,
4595 #ifdef CONFIG_MLX5_ESWITCH
4596 .ndo_bridge_setlink
= mlx5e_bridge_setlink
,
4597 .ndo_bridge_getlink
= mlx5e_bridge_getlink
,
4599 /* SRIOV E-Switch NDOs */
4600 .ndo_set_vf_mac
= mlx5e_set_vf_mac
,
4601 .ndo_set_vf_vlan
= mlx5e_set_vf_vlan
,
4602 .ndo_set_vf_spoofchk
= mlx5e_set_vf_spoofchk
,
4603 .ndo_set_vf_trust
= mlx5e_set_vf_trust
,
4604 .ndo_set_vf_rate
= mlx5e_set_vf_rate
,
4605 .ndo_get_vf_config
= mlx5e_get_vf_config
,
4606 .ndo_set_vf_link_state
= mlx5e_set_vf_link_state
,
4607 .ndo_get_vf_stats
= mlx5e_get_vf_stats
,
4608 .ndo_has_offload_stats
= mlx5e_has_offload_stats
,
4609 .ndo_get_offload_stats
= mlx5e_get_offload_stats
,
4611 .ndo_get_devlink_port
= mlx5e_get_devlink_port
,
4614 void mlx5e_build_default_indir_rqt(u32
*indirection_rqt
, int len
,
4619 for (i
= 0; i
< len
; i
++)
4620 indirection_rqt
[i
] = i
% num_channels
;
4623 static u32
mlx5e_choose_lro_timeout(struct mlx5_core_dev
*mdev
, u32 wanted_timeout
)
4627 /* The supported periods are organized in ascending order */
4628 for (i
= 0; i
< MLX5E_LRO_TIMEOUT_ARR_SIZE
- 1; i
++)
4629 if (MLX5_CAP_ETH(mdev
, lro_timer_supported_periods
[i
]) >= wanted_timeout
)
4632 return MLX5_CAP_ETH(mdev
, lro_timer_supported_periods
[i
]);
4635 void mlx5e_build_rss_params(struct mlx5e_rss_params
*rss_params
,
4638 enum mlx5e_traffic_types tt
;
4640 rss_params
->hfunc
= ETH_RSS_HASH_TOP
;
4641 netdev_rss_key_fill(rss_params
->toeplitz_hash_key
,
4642 sizeof(rss_params
->toeplitz_hash_key
));
4643 mlx5e_build_default_indir_rqt(rss_params
->indirection_rqt
,
4644 MLX5E_INDIR_RQT_SIZE
, num_channels
);
4645 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++)
4646 rss_params
->rx_hash_fields
[tt
] =
4647 tirc_default_config
[tt
].rx_hash_fields
;
4650 void mlx5e_build_nic_params(struct mlx5e_priv
*priv
, struct mlx5e_xsk
*xsk
, u16 mtu
)
4652 struct mlx5e_rss_params
*rss_params
= &priv
->rss_params
;
4653 struct mlx5e_params
*params
= &priv
->channels
.params
;
4654 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4655 u8 rx_cq_period_mode
;
4657 priv
->max_nch
= mlx5e_calc_max_nch(priv
, priv
->profile
);
4659 params
->sw_mtu
= mtu
;
4660 params
->hard_mtu
= MLX5E_ETH_HARD_MTU
;
4661 params
->num_channels
= min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS
/ 2,
4665 /* Set an initial non-zero value, so that mlx5e_select_queue won't
4666 * divide by zero if called before first activating channels.
4668 priv
->num_tc_x_num_ch
= params
->num_channels
* params
->num_tc
;
4671 params
->log_sq_size
= is_kdump_kernel() ?
4672 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE
:
4673 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE
;
4674 MLX5E_SET_PFLAG(params
, MLX5E_PFLAG_SKB_TX_MPWQE
, mlx5e_tx_mpwqe_supported(mdev
));
4677 MLX5E_SET_PFLAG(params
, MLX5E_PFLAG_XDP_TX_MPWQE
, mlx5e_tx_mpwqe_supported(mdev
));
4679 /* set CQE compression */
4680 params
->rx_cqe_compress_def
= false;
4681 if (MLX5_CAP_GEN(mdev
, cqe_compression
) &&
4682 MLX5_CAP_GEN(mdev
, vport_group_manager
))
4683 params
->rx_cqe_compress_def
= slow_pci_heuristic(mdev
);
4685 MLX5E_SET_PFLAG(params
, MLX5E_PFLAG_RX_CQE_COMPRESS
, params
->rx_cqe_compress_def
);
4686 MLX5E_SET_PFLAG(params
, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE
, false);
4689 mlx5e_build_rq_params(mdev
, params
);
4692 if (MLX5_CAP_ETH(mdev
, lro_cap
) &&
4693 params
->rq_wq_type
== MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ
) {
4694 /* No XSK params: checking the availability of striding RQ in general. */
4695 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev
, params
, NULL
))
4696 params
->lro_en
= !slow_pci_heuristic(mdev
);
4698 params
->lro_timeout
= mlx5e_choose_lro_timeout(mdev
, MLX5E_DEFAULT_LRO_TIMEOUT
);
4700 /* CQ moderation params */
4701 rx_cq_period_mode
= MLX5_CAP_GEN(mdev
, cq_period_start_from_cqe
) ?
4702 MLX5_CQ_PERIOD_MODE_START_FROM_CQE
:
4703 MLX5_CQ_PERIOD_MODE_START_FROM_EQE
;
4704 params
->rx_dim_enabled
= MLX5_CAP_GEN(mdev
, cq_moderation
);
4705 params
->tx_dim_enabled
= MLX5_CAP_GEN(mdev
, cq_moderation
);
4706 mlx5e_set_rx_cq_mode_params(params
, rx_cq_period_mode
);
4707 mlx5e_set_tx_cq_mode_params(params
, MLX5_CQ_PERIOD_MODE_START_FROM_EQE
);
4710 mlx5_query_min_inline(mdev
, ¶ms
->tx_min_inline_mode
);
4713 mlx5e_build_rss_params(rss_params
, params
->num_channels
);
4714 params
->tunneled_offload_en
=
4715 mlx5e_tunnel_inner_ft_supported(mdev
);
4720 /* Do not update netdev->features directly in here
4721 * on mlx5e_attach_netdev() we will call mlx5e_update_features()
4722 * To update netdev->features please modify mlx5e_fix_features()
4726 static void mlx5e_set_netdev_dev_addr(struct net_device
*netdev
)
4728 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4730 mlx5_query_mac_address(priv
->mdev
, netdev
->dev_addr
);
4731 if (is_zero_ether_addr(netdev
->dev_addr
) &&
4732 !MLX5_CAP_GEN(priv
->mdev
, vport_group_manager
)) {
4733 eth_hw_addr_random(netdev
);
4734 mlx5_core_info(priv
->mdev
, "Assigned random MAC address %pM\n", netdev
->dev_addr
);
4738 static int mlx5e_vxlan_set_port(struct net_device
*netdev
, unsigned int table
,
4739 unsigned int entry
, struct udp_tunnel_info
*ti
)
4741 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4743 return mlx5_vxlan_add_port(priv
->mdev
->vxlan
, ntohs(ti
->port
));
4746 static int mlx5e_vxlan_unset_port(struct net_device
*netdev
, unsigned int table
,
4747 unsigned int entry
, struct udp_tunnel_info
*ti
)
4749 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4751 return mlx5_vxlan_del_port(priv
->mdev
->vxlan
, ntohs(ti
->port
));
4754 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv
*priv
)
4756 if (!mlx5_vxlan_allowed(priv
->mdev
->vxlan
))
4759 priv
->nic_info
.set_port
= mlx5e_vxlan_set_port
;
4760 priv
->nic_info
.unset_port
= mlx5e_vxlan_unset_port
;
4761 priv
->nic_info
.flags
= UDP_TUNNEL_NIC_INFO_MAY_SLEEP
|
4762 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN
;
4763 priv
->nic_info
.tables
[0].tunnel_types
= UDP_TUNNEL_TYPE_VXLAN
;
4764 /* Don't count the space hard-coded to the IANA port */
4765 priv
->nic_info
.tables
[0].n_entries
=
4766 mlx5_vxlan_max_udp_ports(priv
->mdev
) - 1;
4768 priv
->netdev
->udp_tunnel_nic_info
= &priv
->nic_info
;
4771 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev
*mdev
)
4775 for (tt
= 0; tt
< MLX5E_NUM_TUNNEL_TT
; tt
++) {
4776 if (mlx5e_tunnel_proto_supported_tx(mdev
, mlx5e_get_proto_by_tunnel_type(tt
)))
4779 return (mlx5_vxlan_allowed(mdev
->vxlan
) || mlx5_geneve_tx_allowed(mdev
));
4782 static void mlx5e_build_nic_netdev(struct net_device
*netdev
)
4784 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4785 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4789 SET_NETDEV_DEV(netdev
, mdev
->device
);
4791 netdev
->netdev_ops
= &mlx5e_netdev_ops
;
4793 mlx5e_dcbnl_build_netdev(netdev
);
4795 netdev
->watchdog_timeo
= 15 * HZ
;
4797 netdev
->ethtool_ops
= &mlx5e_ethtool_ops
;
4799 netdev
->vlan_features
|= NETIF_F_SG
;
4800 netdev
->vlan_features
|= NETIF_F_HW_CSUM
;
4801 netdev
->vlan_features
|= NETIF_F_GRO
;
4802 netdev
->vlan_features
|= NETIF_F_TSO
;
4803 netdev
->vlan_features
|= NETIF_F_TSO6
;
4804 netdev
->vlan_features
|= NETIF_F_RXCSUM
;
4805 netdev
->vlan_features
|= NETIF_F_RXHASH
;
4807 netdev
->mpls_features
|= NETIF_F_SG
;
4808 netdev
->mpls_features
|= NETIF_F_HW_CSUM
;
4809 netdev
->mpls_features
|= NETIF_F_TSO
;
4810 netdev
->mpls_features
|= NETIF_F_TSO6
;
4812 netdev
->hw_enc_features
|= NETIF_F_HW_VLAN_CTAG_TX
;
4813 netdev
->hw_enc_features
|= NETIF_F_HW_VLAN_CTAG_RX
;
4815 if (!!MLX5_CAP_ETH(mdev
, lro_cap
) &&
4816 mlx5e_check_fragmented_striding_rq_cap(mdev
))
4817 netdev
->vlan_features
|= NETIF_F_LRO
;
4819 netdev
->hw_features
= netdev
->vlan_features
;
4820 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_TX
;
4821 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_RX
;
4822 netdev
->hw_features
|= NETIF_F_HW_VLAN_CTAG_FILTER
;
4823 netdev
->hw_features
|= NETIF_F_HW_VLAN_STAG_TX
;
4825 if (mlx5e_tunnel_any_tx_proto_supported(mdev
)) {
4826 netdev
->hw_enc_features
|= NETIF_F_HW_CSUM
;
4827 netdev
->hw_enc_features
|= NETIF_F_TSO
;
4828 netdev
->hw_enc_features
|= NETIF_F_TSO6
;
4829 netdev
->hw_enc_features
|= NETIF_F_GSO_PARTIAL
;
4832 if (mlx5_vxlan_allowed(mdev
->vxlan
) || mlx5_geneve_tx_allowed(mdev
)) {
4833 netdev
->hw_features
|= NETIF_F_GSO_UDP_TUNNEL
;
4834 netdev
->hw_enc_features
|= NETIF_F_GSO_UDP_TUNNEL
;
4835 netdev
->vlan_features
|= NETIF_F_GSO_UDP_TUNNEL
;
4838 if (mlx5e_tunnel_proto_supported_tx(mdev
, IPPROTO_GRE
)) {
4839 netdev
->hw_features
|= NETIF_F_GSO_GRE
;
4840 netdev
->hw_enc_features
|= NETIF_F_GSO_GRE
;
4841 netdev
->gso_partial_features
|= NETIF_F_GSO_GRE
;
4844 if (mlx5e_tunnel_proto_supported_tx(mdev
, IPPROTO_IPIP
)) {
4845 netdev
->hw_features
|= NETIF_F_GSO_IPXIP4
|
4847 netdev
->hw_enc_features
|= NETIF_F_GSO_IPXIP4
|
4849 netdev
->gso_partial_features
|= NETIF_F_GSO_IPXIP4
|
4853 netdev
->hw_features
|= NETIF_F_GSO_PARTIAL
;
4854 netdev
->gso_partial_features
|= NETIF_F_GSO_UDP_L4
;
4855 netdev
->hw_features
|= NETIF_F_GSO_UDP_L4
;
4856 netdev
->features
|= NETIF_F_GSO_UDP_L4
;
4858 mlx5_query_port_fcs(mdev
, &fcs_supported
, &fcs_enabled
);
4861 netdev
->hw_features
|= NETIF_F_RXALL
;
4863 if (MLX5_CAP_ETH(mdev
, scatter_fcs
))
4864 netdev
->hw_features
|= NETIF_F_RXFCS
;
4866 if (mlx5_qos_is_supported(mdev
))
4867 netdev
->hw_features
|= NETIF_F_HW_TC
;
4869 netdev
->features
= netdev
->hw_features
;
4873 netdev
->features
&= ~NETIF_F_RXALL
;
4874 netdev
->features
&= ~NETIF_F_LRO
;
4875 netdev
->features
&= ~NETIF_F_RXFCS
;
4877 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
4878 if (FT_CAP(flow_modify_en
) &&
4879 FT_CAP(modify_root
) &&
4880 FT_CAP(identified_miss_table_mode
) &&
4881 FT_CAP(flow_table_modify
)) {
4882 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
4883 netdev
->hw_features
|= NETIF_F_HW_TC
;
4885 #ifdef CONFIG_MLX5_EN_ARFS
4886 netdev
->hw_features
|= NETIF_F_NTUPLE
;
4890 netdev
->features
|= NETIF_F_HIGHDMA
;
4891 netdev
->features
|= NETIF_F_HW_VLAN_STAG_FILTER
;
4893 netdev
->priv_flags
|= IFF_UNICAST_FLT
;
4895 mlx5e_set_netdev_dev_addr(netdev
);
4896 mlx5e_ipsec_build_netdev(priv
);
4897 mlx5e_tls_build_netdev(priv
);
4900 void mlx5e_create_q_counters(struct mlx5e_priv
*priv
)
4902 u32 out
[MLX5_ST_SZ_DW(alloc_q_counter_out
)] = {};
4903 u32 in
[MLX5_ST_SZ_DW(alloc_q_counter_in
)] = {};
4904 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4907 MLX5_SET(alloc_q_counter_in
, in
, opcode
, MLX5_CMD_OP_ALLOC_Q_COUNTER
);
4908 err
= mlx5_cmd_exec_inout(mdev
, alloc_q_counter
, in
, out
);
4911 MLX5_GET(alloc_q_counter_out
, out
, counter_set_id
);
4913 err
= mlx5_cmd_exec_inout(mdev
, alloc_q_counter
, in
, out
);
4915 priv
->drop_rq_q_counter
=
4916 MLX5_GET(alloc_q_counter_out
, out
, counter_set_id
);
4919 void mlx5e_destroy_q_counters(struct mlx5e_priv
*priv
)
4921 u32 in
[MLX5_ST_SZ_DW(dealloc_q_counter_in
)] = {};
4923 MLX5_SET(dealloc_q_counter_in
, in
, opcode
,
4924 MLX5_CMD_OP_DEALLOC_Q_COUNTER
);
4925 if (priv
->q_counter
) {
4926 MLX5_SET(dealloc_q_counter_in
, in
, counter_set_id
,
4928 mlx5_cmd_exec_in(priv
->mdev
, dealloc_q_counter
, in
);
4931 if (priv
->drop_rq_q_counter
) {
4932 MLX5_SET(dealloc_q_counter_in
, in
, counter_set_id
,
4933 priv
->drop_rq_q_counter
);
4934 mlx5_cmd_exec_in(priv
->mdev
, dealloc_q_counter
, in
);
4938 static int mlx5e_nic_init(struct mlx5_core_dev
*mdev
,
4939 struct net_device
*netdev
)
4941 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
4942 struct devlink_port
*dl_port
;
4945 mlx5e_build_nic_params(priv
, &priv
->xsk
, netdev
->mtu
);
4946 mlx5e_vxlan_set_netdev_info(priv
);
4948 mlx5e_timestamp_init(priv
);
4950 err
= mlx5e_ipsec_init(priv
);
4952 mlx5_core_err(mdev
, "IPSec initialization failed, %d\n", err
);
4954 err
= mlx5e_tls_init(priv
);
4956 mlx5_core_err(mdev
, "TLS initialization failed, %d\n", err
);
4958 dl_port
= mlx5e_devlink_get_dl_port(priv
);
4959 if (dl_port
->registered
)
4960 mlx5e_health_create_reporters(priv
);
4965 static void mlx5e_nic_cleanup(struct mlx5e_priv
*priv
)
4967 struct devlink_port
*dl_port
= mlx5e_devlink_get_dl_port(priv
);
4969 if (dl_port
->registered
)
4970 mlx5e_health_destroy_reporters(priv
);
4971 mlx5e_tls_cleanup(priv
);
4972 mlx5e_ipsec_cleanup(priv
);
4975 static int mlx5e_init_nic_rx(struct mlx5e_priv
*priv
)
4977 struct mlx5_core_dev
*mdev
= priv
->mdev
;
4978 u16 max_nch
= priv
->max_nch
;
4981 mlx5e_create_q_counters(priv
);
4983 err
= mlx5e_open_drop_rq(priv
, &priv
->drop_rq
);
4985 mlx5_core_err(mdev
, "open drop rq failed, %d\n", err
);
4986 goto err_destroy_q_counters
;
4989 err
= mlx5e_create_indirect_rqt(priv
);
4991 goto err_close_drop_rq
;
4993 err
= mlx5e_create_direct_rqts(priv
, priv
->direct_tir
, max_nch
);
4995 goto err_destroy_indirect_rqts
;
4997 err
= mlx5e_create_indirect_tirs(priv
, true);
4999 goto err_destroy_direct_rqts
;
5001 err
= mlx5e_create_direct_tirs(priv
, priv
->direct_tir
, max_nch
);
5003 goto err_destroy_indirect_tirs
;
5005 err
= mlx5e_create_direct_rqts(priv
, priv
->xsk_tir
, max_nch
);
5007 goto err_destroy_direct_tirs
;
5009 err
= mlx5e_create_direct_tirs(priv
, priv
->xsk_tir
, max_nch
);
5011 goto err_destroy_xsk_rqts
;
5013 err
= mlx5e_create_direct_rqts(priv
, &priv
->ptp_tir
, 1);
5015 goto err_destroy_xsk_tirs
;
5017 err
= mlx5e_create_direct_tirs(priv
, &priv
->ptp_tir
, 1);
5019 goto err_destroy_ptp_rqt
;
5021 err
= mlx5e_create_flow_steering(priv
);
5023 mlx5_core_warn(mdev
, "create flow steering failed, %d\n", err
);
5024 goto err_destroy_ptp_direct_tir
;
5027 err
= mlx5e_tc_nic_init(priv
);
5029 goto err_destroy_flow_steering
;
5031 err
= mlx5e_accel_init_rx(priv
);
5033 goto err_tc_nic_cleanup
;
5035 #ifdef CONFIG_MLX5_EN_ARFS
5036 priv
->netdev
->rx_cpu_rmap
= mlx5_eq_table_get_rmap(priv
->mdev
);
5042 mlx5e_tc_nic_cleanup(priv
);
5043 err_destroy_flow_steering
:
5044 mlx5e_destroy_flow_steering(priv
);
5045 err_destroy_ptp_direct_tir
:
5046 mlx5e_destroy_direct_tirs(priv
, &priv
->ptp_tir
, 1);
5047 err_destroy_ptp_rqt
:
5048 mlx5e_destroy_direct_rqts(priv
, &priv
->ptp_tir
, 1);
5049 err_destroy_xsk_tirs
:
5050 mlx5e_destroy_direct_tirs(priv
, priv
->xsk_tir
, max_nch
);
5051 err_destroy_xsk_rqts
:
5052 mlx5e_destroy_direct_rqts(priv
, priv
->xsk_tir
, max_nch
);
5053 err_destroy_direct_tirs
:
5054 mlx5e_destroy_direct_tirs(priv
, priv
->direct_tir
, max_nch
);
5055 err_destroy_indirect_tirs
:
5056 mlx5e_destroy_indirect_tirs(priv
);
5057 err_destroy_direct_rqts
:
5058 mlx5e_destroy_direct_rqts(priv
, priv
->direct_tir
, max_nch
);
5059 err_destroy_indirect_rqts
:
5060 mlx5e_destroy_rqt(priv
, &priv
->indir_rqt
);
5062 mlx5e_close_drop_rq(&priv
->drop_rq
);
5063 err_destroy_q_counters
:
5064 mlx5e_destroy_q_counters(priv
);
5068 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv
*priv
)
5070 u16 max_nch
= priv
->max_nch
;
5072 mlx5e_accel_cleanup_rx(priv
);
5073 mlx5e_tc_nic_cleanup(priv
);
5074 mlx5e_destroy_flow_steering(priv
);
5075 mlx5e_destroy_direct_tirs(priv
, &priv
->ptp_tir
, 1);
5076 mlx5e_destroy_direct_rqts(priv
, &priv
->ptp_tir
, 1);
5077 mlx5e_destroy_direct_tirs(priv
, priv
->xsk_tir
, max_nch
);
5078 mlx5e_destroy_direct_rqts(priv
, priv
->xsk_tir
, max_nch
);
5079 mlx5e_destroy_direct_tirs(priv
, priv
->direct_tir
, max_nch
);
5080 mlx5e_destroy_indirect_tirs(priv
);
5081 mlx5e_destroy_direct_rqts(priv
, priv
->direct_tir
, max_nch
);
5082 mlx5e_destroy_rqt(priv
, &priv
->indir_rqt
);
5083 mlx5e_close_drop_rq(&priv
->drop_rq
);
5084 mlx5e_destroy_q_counters(priv
);
5087 static int mlx5e_init_nic_tx(struct mlx5e_priv
*priv
)
5091 err
= mlx5e_create_tises(priv
);
5093 mlx5_core_warn(priv
->mdev
, "create tises failed, %d\n", err
);
5097 mlx5e_dcbnl_initialize(priv
);
5101 static void mlx5e_nic_enable(struct mlx5e_priv
*priv
)
5103 struct net_device
*netdev
= priv
->netdev
;
5104 struct mlx5_core_dev
*mdev
= priv
->mdev
;
5106 mlx5e_init_l2_addr(priv
);
5108 /* Marking the link as currently not needed by the Driver */
5109 if (!netif_running(netdev
))
5110 mlx5e_modify_admin_state(mdev
, MLX5_PORT_DOWN
);
5112 mlx5e_set_netdev_mtu_boundaries(priv
);
5113 mlx5e_set_dev_port_mtu(priv
);
5115 mlx5_lag_add_netdev(mdev
, netdev
);
5117 mlx5e_enable_async_events(priv
);
5118 mlx5e_enable_blocking_events(priv
);
5119 if (mlx5e_monitor_counter_supported(priv
))
5120 mlx5e_monitor_counter_init(priv
);
5122 mlx5e_hv_vhca_stats_create(priv
);
5123 if (netdev
->reg_state
!= NETREG_REGISTERED
)
5125 mlx5e_dcbnl_init_app(priv
);
5127 mlx5e_nic_set_rx_mode(priv
);
5130 if (netif_running(netdev
))
5132 udp_tunnel_nic_reset_ntf(priv
->netdev
);
5133 netif_device_attach(netdev
);
5137 static void mlx5e_nic_disable(struct mlx5e_priv
*priv
)
5139 struct mlx5_core_dev
*mdev
= priv
->mdev
;
5141 if (priv
->netdev
->reg_state
== NETREG_REGISTERED
)
5142 mlx5e_dcbnl_delete_app(priv
);
5145 if (netif_running(priv
->netdev
))
5146 mlx5e_close(priv
->netdev
);
5147 netif_device_detach(priv
->netdev
);
5150 mlx5e_nic_set_rx_mode(priv
);
5152 mlx5e_hv_vhca_stats_destroy(priv
);
5153 if (mlx5e_monitor_counter_supported(priv
))
5154 mlx5e_monitor_counter_cleanup(priv
);
5156 mlx5e_disable_blocking_events(priv
);
5157 if (priv
->en_trap
) {
5158 mlx5e_deactivate_trap(priv
);
5159 mlx5e_close_trap(priv
->en_trap
);
5160 priv
->en_trap
= NULL
;
5162 mlx5e_disable_async_events(priv
);
5163 mlx5_lag_remove_netdev(mdev
, priv
->netdev
);
5164 mlx5_vxlan_reset_to_default(mdev
->vxlan
);
5167 int mlx5e_update_nic_rx(struct mlx5e_priv
*priv
)
5169 return mlx5e_refresh_tirs(priv
, false, false);
5172 static const struct mlx5e_profile mlx5e_nic_profile
= {
5173 .init
= mlx5e_nic_init
,
5174 .cleanup
= mlx5e_nic_cleanup
,
5175 .init_rx
= mlx5e_init_nic_rx
,
5176 .cleanup_rx
= mlx5e_cleanup_nic_rx
,
5177 .init_tx
= mlx5e_init_nic_tx
,
5178 .cleanup_tx
= mlx5e_cleanup_nic_tx
,
5179 .enable
= mlx5e_nic_enable
,
5180 .disable
= mlx5e_nic_disable
,
5181 .update_rx
= mlx5e_update_nic_rx
,
5182 .update_stats
= mlx5e_stats_update_ndo_stats
,
5183 .update_carrier
= mlx5e_update_carrier
,
5184 .rx_handlers
= &mlx5e_rx_handlers_nic
,
5185 .max_tc
= MLX5E_MAX_NUM_TC
,
5186 .rq_groups
= MLX5E_NUM_RQ_GROUPS(XSK
),
5187 .stats_grps
= mlx5e_nic_stats_grps
,
5188 .stats_grps_num
= mlx5e_nic_stats_grps_num
,
5189 .rx_ptp_support
= true,
5192 /* mlx5e generic netdev management API (move to en_common.c) */
5193 int mlx5e_priv_init(struct mlx5e_priv
*priv
,
5194 struct net_device
*netdev
,
5195 struct mlx5_core_dev
*mdev
)
5199 priv
->netdev
= netdev
;
5200 priv
->msglevel
= MLX5E_MSG_LEVEL
;
5201 priv
->max_opened_tc
= 1;
5203 if (!alloc_cpumask_var(&priv
->scratchpad
.cpumask
, GFP_KERNEL
))
5206 mutex_init(&priv
->state_lock
);
5207 hash_init(priv
->htb
.qos_tc2node
);
5208 INIT_WORK(&priv
->update_carrier_work
, mlx5e_update_carrier_work
);
5209 INIT_WORK(&priv
->set_rx_mode_work
, mlx5e_set_rx_mode_work
);
5210 INIT_WORK(&priv
->tx_timeout_work
, mlx5e_tx_timeout_work
);
5211 INIT_WORK(&priv
->update_stats_work
, mlx5e_update_stats_work
);
5213 priv
->wq
= create_singlethread_workqueue("mlx5e");
5215 goto err_free_cpumask
;
5220 free_cpumask_var(priv
->scratchpad
.cpumask
);
5225 void mlx5e_priv_cleanup(struct mlx5e_priv
*priv
)
5229 /* bail if change profile failed and also rollback failed */
5233 destroy_workqueue(priv
->wq
);
5234 free_cpumask_var(priv
->scratchpad
.cpumask
);
5236 for (i
= 0; i
< priv
->htb
.max_qos_sqs
; i
++)
5237 kfree(priv
->htb
.qos_sq_stats
[i
]);
5238 kvfree(priv
->htb
.qos_sq_stats
);
5240 memset(priv
, 0, sizeof(*priv
));
5244 mlx5e_create_netdev(struct mlx5_core_dev
*mdev
, unsigned int txqs
, unsigned int rxqs
)
5246 struct net_device
*netdev
;
5249 netdev
= alloc_etherdev_mqs(sizeof(struct mlx5e_priv
), txqs
, rxqs
);
5251 mlx5_core_err(mdev
, "alloc_etherdev_mqs() failed\n");
5255 err
= mlx5e_priv_init(netdev_priv(netdev
), netdev
, mdev
);
5257 mlx5_core_err(mdev
, "mlx5e_priv_init failed, err=%d\n", err
);
5258 goto err_free_netdev
;
5261 netif_carrier_off(netdev
);
5262 dev_net_set(netdev
, mlx5_core_net(mdev
));
5267 free_netdev(netdev
);
5272 static void mlx5e_update_features(struct net_device
*netdev
)
5274 if (netdev
->reg_state
!= NETREG_REGISTERED
)
5275 return; /* features will be updated on netdev registration */
5278 netdev_update_features(netdev
);
5282 static void mlx5e_reset_channels(struct net_device
*netdev
)
5284 netdev_reset_tc(netdev
);
5287 int mlx5e_attach_netdev(struct mlx5e_priv
*priv
)
5289 const bool take_rtnl
= priv
->netdev
->reg_state
== NETREG_REGISTERED
;
5290 const struct mlx5e_profile
*profile
= priv
->profile
;
5294 clear_bit(MLX5E_STATE_DESTROYING
, &priv
->state
);
5296 /* max number of channels may have changed */
5297 max_nch
= mlx5e_get_max_num_channels(priv
->mdev
);
5298 if (priv
->channels
.params
.num_channels
> max_nch
) {
5299 mlx5_core_warn(priv
->mdev
, "MLX5E: Reducing number of channels to %d\n", max_nch
);
5300 /* Reducing the number of channels - RXFH has to be reset, and
5301 * mlx5e_num_channels_changed below will build the RQT.
5303 priv
->netdev
->priv_flags
&= ~IFF_RXFH_CONFIGURED
;
5304 priv
->channels
.params
.num_channels
= max_nch
;
5306 /* 1. Set the real number of queues in the kernel the first time.
5307 * 2. Set our default XPS cpumask.
5310 * rtnl_lock is required by netif_set_real_num_*_queues in case the
5311 * netdev has been registered by this point (if this function was called
5312 * in the reload or resume flow).
5316 err
= mlx5e_num_channels_changed(priv
);
5322 err
= profile
->init_tx(priv
);
5326 err
= profile
->init_rx(priv
);
5328 goto err_cleanup_tx
;
5330 if (profile
->enable
)
5331 profile
->enable(priv
);
5333 mlx5e_update_features(priv
->netdev
);
5338 profile
->cleanup_tx(priv
);
5341 mlx5e_reset_channels(priv
->netdev
);
5342 set_bit(MLX5E_STATE_DESTROYING
, &priv
->state
);
5343 cancel_work_sync(&priv
->update_stats_work
);
5347 void mlx5e_detach_netdev(struct mlx5e_priv
*priv
)
5349 const struct mlx5e_profile
*profile
= priv
->profile
;
5351 set_bit(MLX5E_STATE_DESTROYING
, &priv
->state
);
5353 if (profile
->disable
)
5354 profile
->disable(priv
);
5355 flush_workqueue(priv
->wq
);
5357 profile
->cleanup_rx(priv
);
5358 profile
->cleanup_tx(priv
);
5359 mlx5e_reset_channels(priv
->netdev
);
5360 cancel_work_sync(&priv
->update_stats_work
);
5364 mlx5e_netdev_attach_profile(struct net_device
*netdev
, struct mlx5_core_dev
*mdev
,
5365 const struct mlx5e_profile
*new_profile
, void *new_ppriv
)
5367 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
5370 err
= mlx5e_priv_init(priv
, netdev
, mdev
);
5372 mlx5_core_err(mdev
, "mlx5e_priv_init failed, err=%d\n", err
);
5375 netif_carrier_off(netdev
);
5376 priv
->profile
= new_profile
;
5377 priv
->ppriv
= new_ppriv
;
5378 err
= new_profile
->init(priv
->mdev
, priv
->netdev
);
5381 err
= mlx5e_attach_netdev(priv
);
5383 goto profile_cleanup
;
5387 new_profile
->cleanup(priv
);
5389 mlx5e_priv_cleanup(priv
);
5393 int mlx5e_netdev_change_profile(struct mlx5e_priv
*priv
,
5394 const struct mlx5e_profile
*new_profile
, void *new_ppriv
)
5396 unsigned int new_max_nch
= mlx5e_calc_max_nch(priv
, new_profile
);
5397 const struct mlx5e_profile
*orig_profile
= priv
->profile
;
5398 struct net_device
*netdev
= priv
->netdev
;
5399 struct mlx5_core_dev
*mdev
= priv
->mdev
;
5400 void *orig_ppriv
= priv
->ppriv
;
5401 int err
, rollback_err
;
5404 if (new_max_nch
!= priv
->max_nch
) {
5405 netdev_warn(netdev
, "%s: Replacing profile with different max channels\n",
5410 /* cleanup old profile */
5411 mlx5e_detach_netdev(priv
);
5412 priv
->profile
->cleanup(priv
);
5413 mlx5e_priv_cleanup(priv
);
5415 err
= mlx5e_netdev_attach_profile(netdev
, mdev
, new_profile
, new_ppriv
);
5416 if (err
) { /* roll back to original profile */
5417 netdev_warn(netdev
, "%s: new profile init failed, %d\n", __func__
, err
);
5424 rollback_err
= mlx5e_netdev_attach_profile(netdev
, mdev
, orig_profile
, orig_ppriv
);
5426 netdev_err(netdev
, "%s: failed to rollback to orig profile, %d\n",
5427 __func__
, rollback_err
);
5431 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv
*priv
)
5433 mlx5e_netdev_change_profile(priv
, &mlx5e_nic_profile
, NULL
);
5436 void mlx5e_destroy_netdev(struct mlx5e_priv
*priv
)
5438 struct net_device
*netdev
= priv
->netdev
;
5440 mlx5e_priv_cleanup(priv
);
5441 free_netdev(netdev
);
5444 static int mlx5e_resume(struct auxiliary_device
*adev
)
5446 struct mlx5_adev
*edev
= container_of(adev
, struct mlx5_adev
, adev
);
5447 struct mlx5e_priv
*priv
= dev_get_drvdata(&adev
->dev
);
5448 struct net_device
*netdev
= priv
->netdev
;
5449 struct mlx5_core_dev
*mdev
= edev
->mdev
;
5452 if (netif_device_present(netdev
))
5455 err
= mlx5e_create_mdev_resources(mdev
);
5459 err
= mlx5e_attach_netdev(priv
);
5461 mlx5e_destroy_mdev_resources(mdev
);
5468 static int mlx5e_suspend(struct auxiliary_device
*adev
, pm_message_t state
)
5470 struct mlx5e_priv
*priv
= dev_get_drvdata(&adev
->dev
);
5471 struct net_device
*netdev
= priv
->netdev
;
5472 struct mlx5_core_dev
*mdev
= priv
->mdev
;
5474 if (!netif_device_present(netdev
))
5477 mlx5e_detach_netdev(priv
);
5478 mlx5e_destroy_mdev_resources(mdev
);
5482 static int mlx5e_probe(struct auxiliary_device
*adev
,
5483 const struct auxiliary_device_id
*id
)
5485 struct mlx5_adev
*edev
= container_of(adev
, struct mlx5_adev
, adev
);
5486 const struct mlx5e_profile
*profile
= &mlx5e_nic_profile
;
5487 struct mlx5_core_dev
*mdev
= edev
->mdev
;
5488 struct net_device
*netdev
;
5489 pm_message_t state
= {};
5490 unsigned int txqs
, rxqs
, ptp_txqs
= 0;
5491 struct mlx5e_priv
*priv
;
5496 if (MLX5_CAP_GEN(mdev
, ts_cqe_to_dest_cqn
))
5497 ptp_txqs
= profile
->max_tc
;
5499 if (mlx5_qos_is_supported(mdev
))
5500 qos_sqs
= mlx5e_qos_max_leaf_nodes(mdev
);
5502 nch
= mlx5e_get_max_num_channels(mdev
);
5503 txqs
= nch
* profile
->max_tc
+ ptp_txqs
+ qos_sqs
;
5504 rxqs
= nch
* profile
->rq_groups
;
5505 netdev
= mlx5e_create_netdev(mdev
, txqs
, rxqs
);
5507 mlx5_core_err(mdev
, "mlx5e_create_netdev failed\n");
5511 mlx5e_build_nic_netdev(netdev
);
5513 priv
= netdev_priv(netdev
);
5514 dev_set_drvdata(&adev
->dev
, priv
);
5516 priv
->profile
= profile
;
5519 err
= mlx5e_devlink_port_register(priv
);
5521 mlx5_core_err(mdev
, "mlx5e_devlink_port_register failed, %d\n", err
);
5522 goto err_destroy_netdev
;
5525 err
= profile
->init(mdev
, netdev
);
5527 mlx5_core_err(mdev
, "mlx5e_nic_profile init failed, %d\n", err
);
5528 goto err_devlink_cleanup
;
5531 err
= mlx5e_resume(adev
);
5533 mlx5_core_err(mdev
, "mlx5e_resume failed, %d\n", err
);
5534 goto err_profile_cleanup
;
5537 err
= register_netdev(netdev
);
5539 mlx5_core_err(mdev
, "register_netdev failed, %d\n", err
);
5543 mlx5e_devlink_port_type_eth_set(priv
);
5545 mlx5e_dcbnl_init_app(priv
);
5546 mlx5_uplink_netdev_set(mdev
, netdev
);
5550 mlx5e_suspend(adev
, state
);
5551 err_profile_cleanup
:
5552 profile
->cleanup(priv
);
5553 err_devlink_cleanup
:
5554 mlx5e_devlink_port_unregister(priv
);
5556 mlx5e_destroy_netdev(priv
);
5560 static void mlx5e_remove(struct auxiliary_device
*adev
)
5562 struct mlx5e_priv
*priv
= dev_get_drvdata(&adev
->dev
);
5563 pm_message_t state
= {};
5565 mlx5e_dcbnl_delete_app(priv
);
5566 unregister_netdev(priv
->netdev
);
5567 mlx5e_suspend(adev
, state
);
5568 priv
->profile
->cleanup(priv
);
5569 mlx5e_devlink_port_unregister(priv
);
5570 mlx5e_destroy_netdev(priv
);
5573 static const struct auxiliary_device_id mlx5e_id_table
[] = {
5574 { .name
= MLX5_ADEV_NAME
".eth", },
5578 MODULE_DEVICE_TABLE(auxiliary
, mlx5e_id_table
);
5580 static struct auxiliary_driver mlx5e_driver
= {
5582 .probe
= mlx5e_probe
,
5583 .remove
= mlx5e_remove
,
5584 .suspend
= mlx5e_suspend
,
5585 .resume
= mlx5e_resume
,
5586 .id_table
= mlx5e_id_table
,
5589 int mlx5e_init(void)
5593 mlx5e_ipsec_build_inverse_table();
5594 mlx5e_build_ptys2ethtool_map();
5595 ret
= auxiliary_driver_register(&mlx5e_driver
);
5599 ret
= mlx5e_rep_init();
5601 auxiliary_driver_unregister(&mlx5e_driver
);
5605 void mlx5e_cleanup(void)
5607 mlx5e_rep_cleanup();
5608 auxiliary_driver_unregister(&mlx5e_driver
);