2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/debugfs.h>
34 #include <linux/mlx5/fs.h>
35 #include <net/switchdev.h>
36 #include <net/pkt_cls.h>
37 #include <net/act_api.h>
38 #include <net/devlink.h>
39 #include <net/ipv6_stubs.h>
44 #include "en/params.h"
47 #include "en/rep/tc.h"
48 #include "en/rep/neigh.h"
49 #include "en/rep/bridge.h"
50 #include "en/devlink.h"
53 #include "lib/devcom.h"
54 #include "lib/vxlan.h"
55 #define CREATE_TRACE_POINTS
56 #include "diag/en_rep_tracepoint.h"
57 #include "diag/reporter_vnic.h"
58 #include "en_accel/ipsec.h"
59 #include "en/tc/int_port.h"
61 #include "en/fs_ethtool.h"
63 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \
64 max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
65 #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1
67 static const char mlx5e_rep_driver_name
[] = "mlx5e_rep";
69 static void mlx5e_rep_get_drvinfo(struct net_device
*dev
,
70 struct ethtool_drvinfo
*drvinfo
)
72 struct mlx5e_priv
*priv
= netdev_priv(dev
);
73 struct mlx5_core_dev
*mdev
= priv
->mdev
;
75 strscpy(drvinfo
->driver
, mlx5e_rep_driver_name
,
76 sizeof(drvinfo
->driver
));
77 snprintf(drvinfo
->fw_version
, sizeof(drvinfo
->fw_version
),
79 fw_rev_maj(mdev
), fw_rev_min(mdev
),
80 fw_rev_sub(mdev
), mdev
->board_id
);
83 static const struct counter_desc sw_rep_stats_desc
[] = {
84 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats
, rx_packets
) },
85 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats
, rx_bytes
) },
86 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats
, tx_packets
) },
87 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats
, tx_bytes
) },
90 static const struct counter_desc vport_rep_stats_desc
[] = {
91 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, vport_rx_packets
) },
92 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, vport_rx_bytes
) },
93 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, vport_tx_packets
) },
94 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, vport_tx_bytes
) },
95 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
96 rx_vport_rdma_unicast_packets
) },
97 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, rx_vport_rdma_unicast_bytes
) },
98 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
99 tx_vport_rdma_unicast_packets
) },
100 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
, tx_vport_rdma_unicast_bytes
) },
101 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
102 rx_vport_rdma_multicast_packets
) },
103 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
104 rx_vport_rdma_multicast_bytes
) },
105 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
106 tx_vport_rdma_multicast_packets
) },
107 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats
,
108 tx_vport_rdma_multicast_bytes
) },
111 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
112 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
114 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep
)
116 return NUM_VPORT_REP_SW_COUNTERS
;
119 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep
)
123 for (i
= 0; i
< NUM_VPORT_REP_SW_COUNTERS
; i
++)
124 strcpy(data
+ (idx
++) * ETH_GSTRING_LEN
,
125 sw_rep_stats_desc
[i
].format
);
129 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep
)
133 for (i
= 0; i
< NUM_VPORT_REP_SW_COUNTERS
; i
++)
134 data
[idx
++] = MLX5E_READ_CTR64_CPU(&priv
->stats
.sw
,
135 sw_rep_stats_desc
, i
);
139 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep
)
141 struct mlx5e_sw_stats
*s
= &priv
->stats
.sw
;
142 struct rtnl_link_stats64 stats64
= {};
144 memset(s
, 0, sizeof(*s
));
145 mlx5e_fold_sw_stats64(priv
, &stats64
);
147 s
->rx_packets
= stats64
.rx_packets
;
148 s
->rx_bytes
= stats64
.rx_bytes
;
149 s
->tx_packets
= stats64
.tx_packets
;
150 s
->tx_bytes
= stats64
.tx_bytes
;
151 s
->tx_queue_dropped
= stats64
.tx_dropped
;
154 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep
)
156 return NUM_VPORT_REP_HW_COUNTERS
;
159 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep
)
163 for (i
= 0; i
< NUM_VPORT_REP_HW_COUNTERS
; i
++)
164 strcpy(data
+ (idx
++) * ETH_GSTRING_LEN
, vport_rep_stats_desc
[i
].format
);
168 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep
)
172 for (i
= 0; i
< NUM_VPORT_REP_HW_COUNTERS
; i
++)
173 data
[idx
++] = MLX5E_READ_CTR64_CPU(&priv
->stats
.rep_stats
,
174 vport_rep_stats_desc
, i
);
178 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep
)
180 struct mlx5e_rep_stats
*rep_stats
= &priv
->stats
.rep_stats
;
181 int outlen
= MLX5_ST_SZ_BYTES(query_vport_counter_out
);
182 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
183 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
184 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
188 out
= kvzalloc(outlen
, GFP_KERNEL
);
192 err
= mlx5_core_query_vport_counter(esw
->dev
, 1, rep
->vport
- 1, 0, out
);
194 netdev_warn(priv
->netdev
, "vport %d error %d reading stats\n",
199 #define MLX5_GET_CTR(p, x) \
200 MLX5_GET64(query_vport_counter_out, p, x)
201 /* flip tx/rx as we are reporting the counters for the switch vport */
202 rep_stats
->vport_rx_packets
=
203 MLX5_GET_CTR(out
, transmitted_ib_unicast
.packets
) +
204 MLX5_GET_CTR(out
, transmitted_eth_unicast
.packets
) +
205 MLX5_GET_CTR(out
, transmitted_ib_multicast
.packets
) +
206 MLX5_GET_CTR(out
, transmitted_eth_multicast
.packets
) +
207 MLX5_GET_CTR(out
, transmitted_eth_broadcast
.packets
);
209 rep_stats
->vport_tx_packets
=
210 MLX5_GET_CTR(out
, received_ib_unicast
.packets
) +
211 MLX5_GET_CTR(out
, received_eth_unicast
.packets
) +
212 MLX5_GET_CTR(out
, received_ib_multicast
.packets
) +
213 MLX5_GET_CTR(out
, received_eth_multicast
.packets
) +
214 MLX5_GET_CTR(out
, received_eth_broadcast
.packets
);
216 rep_stats
->vport_rx_bytes
=
217 MLX5_GET_CTR(out
, transmitted_ib_unicast
.octets
) +
218 MLX5_GET_CTR(out
, transmitted_eth_unicast
.octets
) +
219 MLX5_GET_CTR(out
, transmitted_ib_multicast
.octets
) +
220 MLX5_GET_CTR(out
, transmitted_eth_broadcast
.octets
);
222 rep_stats
->vport_tx_bytes
=
223 MLX5_GET_CTR(out
, received_ib_unicast
.octets
) +
224 MLX5_GET_CTR(out
, received_eth_unicast
.octets
) +
225 MLX5_GET_CTR(out
, received_ib_multicast
.octets
) +
226 MLX5_GET_CTR(out
, received_eth_multicast
.octets
) +
227 MLX5_GET_CTR(out
, received_eth_broadcast
.octets
);
229 rep_stats
->rx_vport_rdma_unicast_packets
=
230 MLX5_GET_CTR(out
, transmitted_ib_unicast
.packets
);
231 rep_stats
->tx_vport_rdma_unicast_packets
=
232 MLX5_GET_CTR(out
, received_ib_unicast
.packets
);
233 rep_stats
->rx_vport_rdma_unicast_bytes
=
234 MLX5_GET_CTR(out
, transmitted_ib_unicast
.octets
);
235 rep_stats
->tx_vport_rdma_unicast_bytes
=
236 MLX5_GET_CTR(out
, received_ib_unicast
.octets
);
237 rep_stats
->rx_vport_rdma_multicast_packets
=
238 MLX5_GET_CTR(out
, transmitted_ib_multicast
.packets
);
239 rep_stats
->tx_vport_rdma_multicast_packets
=
240 MLX5_GET_CTR(out
, received_ib_multicast
.packets
);
241 rep_stats
->rx_vport_rdma_multicast_bytes
=
242 MLX5_GET_CTR(out
, transmitted_ib_multicast
.octets
);
243 rep_stats
->tx_vport_rdma_multicast_bytes
=
244 MLX5_GET_CTR(out
, received_ib_multicast
.octets
);
250 static void mlx5e_rep_get_strings(struct net_device
*dev
,
251 u32 stringset
, uint8_t *data
)
253 struct mlx5e_priv
*priv
= netdev_priv(dev
);
257 mlx5e_stats_fill_strings(priv
, data
);
262 static void mlx5e_rep_get_ethtool_stats(struct net_device
*dev
,
263 struct ethtool_stats
*stats
, u64
*data
)
265 struct mlx5e_priv
*priv
= netdev_priv(dev
);
267 mlx5e_ethtool_get_ethtool_stats(priv
, stats
, data
);
270 static int mlx5e_rep_get_sset_count(struct net_device
*dev
, int sset
)
272 struct mlx5e_priv
*priv
= netdev_priv(dev
);
276 return mlx5e_stats_total_num(priv
);
283 mlx5e_rep_get_ringparam(struct net_device
*dev
,
284 struct ethtool_ringparam
*param
,
285 struct kernel_ethtool_ringparam
*kernel_param
,
286 struct netlink_ext_ack
*extack
)
288 struct mlx5e_priv
*priv
= netdev_priv(dev
);
290 mlx5e_ethtool_get_ringparam(priv
, param
, kernel_param
);
294 mlx5e_rep_set_ringparam(struct net_device
*dev
,
295 struct ethtool_ringparam
*param
,
296 struct kernel_ethtool_ringparam
*kernel_param
,
297 struct netlink_ext_ack
*extack
)
299 struct mlx5e_priv
*priv
= netdev_priv(dev
);
301 return mlx5e_ethtool_set_ringparam(priv
, param
);
304 static void mlx5e_rep_get_channels(struct net_device
*dev
,
305 struct ethtool_channels
*ch
)
307 struct mlx5e_priv
*priv
= netdev_priv(dev
);
309 mlx5e_ethtool_get_channels(priv
, ch
);
312 static int mlx5e_rep_set_channels(struct net_device
*dev
,
313 struct ethtool_channels
*ch
)
315 struct mlx5e_priv
*priv
= netdev_priv(dev
);
317 return mlx5e_ethtool_set_channels(priv
, ch
);
320 static int mlx5e_rep_get_coalesce(struct net_device
*netdev
,
321 struct ethtool_coalesce
*coal
,
322 struct kernel_ethtool_coalesce
*kernel_coal
,
323 struct netlink_ext_ack
*extack
)
325 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
327 return mlx5e_ethtool_get_coalesce(priv
, coal
, kernel_coal
);
330 static int mlx5e_rep_set_coalesce(struct net_device
*netdev
,
331 struct ethtool_coalesce
*coal
,
332 struct kernel_ethtool_coalesce
*kernel_coal
,
333 struct netlink_ext_ack
*extack
)
335 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
337 return mlx5e_ethtool_set_coalesce(priv
, coal
, kernel_coal
, extack
);
340 static u32
mlx5e_rep_get_rxfh_key_size(struct net_device
*netdev
)
342 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
344 return mlx5e_ethtool_get_rxfh_key_size(priv
);
347 static u32
mlx5e_rep_get_rxfh_indir_size(struct net_device
*netdev
)
349 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
351 return mlx5e_ethtool_get_rxfh_indir_size(priv
);
354 static const struct ethtool_ops mlx5e_rep_ethtool_ops
= {
355 .supported_coalesce_params
= ETHTOOL_COALESCE_USECS
|
356 ETHTOOL_COALESCE_MAX_FRAMES
|
357 ETHTOOL_COALESCE_USE_ADAPTIVE
,
358 .get_drvinfo
= mlx5e_rep_get_drvinfo
,
359 .get_link
= ethtool_op_get_link
,
360 .get_strings
= mlx5e_rep_get_strings
,
361 .get_sset_count
= mlx5e_rep_get_sset_count
,
362 .get_ethtool_stats
= mlx5e_rep_get_ethtool_stats
,
363 .get_ringparam
= mlx5e_rep_get_ringparam
,
364 .set_ringparam
= mlx5e_rep_set_ringparam
,
365 .get_channels
= mlx5e_rep_get_channels
,
366 .set_channels
= mlx5e_rep_set_channels
,
367 .get_coalesce
= mlx5e_rep_get_coalesce
,
368 .set_coalesce
= mlx5e_rep_set_coalesce
,
369 .get_rxfh_key_size
= mlx5e_rep_get_rxfh_key_size
,
370 .get_rxfh_indir_size
= mlx5e_rep_get_rxfh_indir_size
,
373 static void mlx5e_sqs2vport_stop(struct mlx5_eswitch
*esw
,
374 struct mlx5_eswitch_rep
*rep
)
376 struct mlx5e_rep_sq
*rep_sq
, *tmp
;
377 struct mlx5e_rep_sq_peer
*sq_peer
;
378 struct mlx5e_rep_priv
*rpriv
;
381 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
384 rpriv
= mlx5e_rep_to_rep_priv(rep
);
385 list_for_each_entry_safe(rep_sq
, tmp
, &rpriv
->vport_sqs_list
, list
) {
386 mlx5_eswitch_del_send_to_vport_rule(rep_sq
->send_to_vport_rule
);
387 xa_for_each(&rep_sq
->sq_peer
, i
, sq_peer
) {
389 mlx5_eswitch_del_send_to_vport_rule(sq_peer
->rule
);
391 xa_erase(&rep_sq
->sq_peer
, i
);
395 xa_destroy(&rep_sq
->sq_peer
);
396 list_del(&rep_sq
->list
);
401 static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch
*esw
, struct mlx5_eswitch_rep
*rep
,
402 struct mlx5e_rep_sq
*rep_sq
, int i
)
404 struct mlx5_flow_handle
*flow_rule
;
405 struct mlx5_devcom_comp_dev
*tmp
;
406 struct mlx5_eswitch
*peer_esw
;
408 mlx5_devcom_for_each_peer_entry(esw
->devcom
, peer_esw
, tmp
) {
409 u16 peer_rule_idx
= MLX5_CAP_GEN(peer_esw
->dev
, vhca_id
);
410 struct mlx5e_rep_sq_peer
*sq_peer
;
413 sq_peer
= kzalloc(sizeof(*sq_peer
), GFP_KERNEL
);
417 flow_rule
= mlx5_eswitch_add_send_to_vport_rule(peer_esw
, esw
,
419 if (IS_ERR(flow_rule
)) {
421 return PTR_ERR(flow_rule
);
424 sq_peer
->rule
= flow_rule
;
425 sq_peer
->peer
= peer_esw
;
426 err
= xa_insert(&rep_sq
->sq_peer
, peer_rule_idx
, sq_peer
, GFP_KERNEL
);
429 mlx5_eswitch_del_send_to_vport_rule(flow_rule
);
437 static int mlx5e_sqs2vport_start(struct mlx5_eswitch
*esw
,
438 struct mlx5_eswitch_rep
*rep
,
439 u32
*sqns_array
, int sqns_num
)
441 struct mlx5_flow_handle
*flow_rule
;
442 struct mlx5e_rep_priv
*rpriv
;
443 struct mlx5e_rep_sq
*rep_sq
;
444 bool devcom_locked
= false;
448 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
451 rpriv
= mlx5e_rep_to_rep_priv(rep
);
453 if (mlx5_devcom_comp_is_ready(esw
->devcom
) &&
454 mlx5_devcom_for_each_peer_begin(esw
->devcom
))
455 devcom_locked
= true;
457 for (i
= 0; i
< sqns_num
; i
++) {
458 rep_sq
= kzalloc(sizeof(*rep_sq
), GFP_KERNEL
);
464 /* Add re-inject rule to the PF/representor sqs */
465 flow_rule
= mlx5_eswitch_add_send_to_vport_rule(esw
, esw
, rep
,
467 if (IS_ERR(flow_rule
)) {
468 err
= PTR_ERR(flow_rule
);
472 rep_sq
->send_to_vport_rule
= flow_rule
;
473 rep_sq
->sqn
= sqns_array
[i
];
475 xa_init(&rep_sq
->sq_peer
);
477 err
= mlx5e_sqs2vport_add_peers_rules(esw
, rep
, rep_sq
, i
);
479 mlx5_eswitch_del_send_to_vport_rule(rep_sq
->send_to_vport_rule
);
480 xa_destroy(&rep_sq
->sq_peer
);
486 list_add(&rep_sq
->list
, &rpriv
->vport_sqs_list
);
490 mlx5_devcom_for_each_peer_end(esw
->devcom
);
495 mlx5e_sqs2vport_stop(esw
, rep
);
498 mlx5_devcom_for_each_peer_end(esw
->devcom
);
504 mlx5e_add_sqs_fwd_rules(struct mlx5e_priv
*priv
)
506 int sqs_per_channel
= mlx5e_get_dcb_num_tc(&priv
->channels
.params
);
507 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
508 bool is_uplink_rep
= mlx5e_is_uplink_rep(priv
);
509 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
510 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
511 int n
, tc
, nch
, num_sqs
= 0;
512 struct mlx5e_channel
*c
;
517 ptp_sq
= !!(priv
->channels
.ptp
&&
518 MLX5E_GET_PFLAG(&priv
->channels
.params
, MLX5E_PFLAG_TX_PORT_TS
));
519 nch
= priv
->channels
.num
+ ptp_sq
;
520 /* +2 for xdpsqs, they don't exist on the ptp channel but will not be
521 * counted for by num_sqs.
524 sqs_per_channel
+= 2;
526 sqs
= kvcalloc(nch
* sqs_per_channel
, sizeof(*sqs
), GFP_KERNEL
);
530 for (n
= 0; n
< priv
->channels
.num
; n
++) {
531 c
= priv
->channels
.c
[n
];
532 for (tc
= 0; tc
< c
->num_tc
; tc
++)
533 sqs
[num_sqs
++] = c
->sq
[tc
].sqn
;
537 sqs
[num_sqs
++] = c
->rq_xdpsq
.sqn
;
539 sqs
[num_sqs
++] = c
->xdpsq
.sqn
;
543 struct mlx5e_ptp
*ptp_ch
= priv
->channels
.ptp
;
545 for (tc
= 0; tc
< ptp_ch
->num_tc
; tc
++)
546 sqs
[num_sqs
++] = ptp_ch
->ptpsq
[tc
].txqsq
.sqn
;
549 err
= mlx5e_sqs2vport_start(esw
, rep
, sqs
, num_sqs
);
554 netdev_warn(priv
->netdev
, "Failed to add SQs FWD rules %d\n", err
);
559 mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv
*priv
)
561 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
562 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
563 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
565 mlx5e_sqs2vport_stop(esw
, rep
);
569 mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv
*priv
)
571 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
572 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
573 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
574 struct mlx5_flow_handle
*flow_rule
;
575 struct mlx5_flow_group
*g
;
577 g
= esw
->fdb_table
.offloads
.send_to_vport_meta_grp
;
581 flow_rule
= mlx5_eswitch_add_send_to_vport_meta_rule(esw
, rep
->vport
);
582 if (IS_ERR(flow_rule
))
583 return PTR_ERR(flow_rule
);
585 rpriv
->send_to_vport_meta_rule
= flow_rule
;
591 mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv
*priv
)
593 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
595 if (rpriv
->send_to_vport_meta_rule
)
596 mlx5_eswitch_del_send_to_vport_meta_rule(rpriv
->send_to_vport_meta_rule
);
599 void mlx5e_rep_activate_channels(struct mlx5e_priv
*priv
)
601 mlx5e_add_sqs_fwd_rules(priv
);
602 mlx5e_rep_add_meta_tunnel_rule(priv
);
605 void mlx5e_rep_deactivate_channels(struct mlx5e_priv
*priv
)
607 mlx5e_rep_del_meta_tunnel_rule(priv
);
608 mlx5e_remove_sqs_fwd_rules(priv
);
611 static int mlx5e_rep_open(struct net_device
*dev
)
613 struct mlx5e_priv
*priv
= netdev_priv(dev
);
614 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
615 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
618 mutex_lock(&priv
->state_lock
);
619 err
= mlx5e_open_locked(dev
);
623 if (!mlx5_modify_vport_admin_state(priv
->mdev
,
624 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT
,
626 MLX5_VPORT_ADMIN_STATE_UP
))
627 netif_carrier_on(dev
);
630 mutex_unlock(&priv
->state_lock
);
634 static int mlx5e_rep_close(struct net_device
*dev
)
636 struct mlx5e_priv
*priv
= netdev_priv(dev
);
637 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
638 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
641 mutex_lock(&priv
->state_lock
);
642 mlx5_modify_vport_admin_state(priv
->mdev
,
643 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT
,
645 MLX5_VPORT_ADMIN_STATE_DOWN
);
646 ret
= mlx5e_close_locked(dev
);
647 mutex_unlock(&priv
->state_lock
);
651 bool mlx5e_is_uplink_rep(struct mlx5e_priv
*priv
)
653 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
654 struct mlx5_eswitch_rep
*rep
;
656 if (!MLX5_ESWITCH_MANAGER(priv
->mdev
))
659 if (!rpriv
) /* non vport rep mlx5e instances don't use this field */
663 return (rep
->vport
== MLX5_VPORT_UPLINK
);
666 bool mlx5e_rep_has_offload_stats(const struct net_device
*dev
, int attr_id
)
669 case IFLA_OFFLOAD_XSTATS_CPU_HIT
:
677 mlx5e_get_sw_stats64(const struct net_device
*dev
,
678 struct rtnl_link_stats64
*stats
)
680 struct mlx5e_priv
*priv
= netdev_priv(dev
);
682 mlx5e_fold_sw_stats64(priv
, stats
);
686 int mlx5e_rep_get_offload_stats(int attr_id
, const struct net_device
*dev
,
690 case IFLA_OFFLOAD_XSTATS_CPU_HIT
:
691 return mlx5e_get_sw_stats64(dev
, sp
);
698 mlx5e_rep_get_stats(struct net_device
*dev
, struct rtnl_link_stats64
*stats
)
700 struct mlx5e_priv
*priv
= netdev_priv(dev
);
702 /* update HW stats in background for next time */
703 mlx5e_queue_update_stats(priv
);
704 memcpy(stats
, &priv
->stats
.vf_vport
, sizeof(*stats
));
707 static int mlx5e_rep_change_mtu(struct net_device
*netdev
, int new_mtu
)
709 return mlx5e_change_mtu(netdev
, new_mtu
, NULL
);
712 static int mlx5e_rep_change_carrier(struct net_device
*dev
, bool new_carrier
)
714 struct mlx5e_priv
*priv
= netdev_priv(dev
);
715 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
716 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
720 err
= mlx5_modify_vport_admin_state(priv
->mdev
, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT
,
721 rep
->vport
, 1, MLX5_VPORT_ADMIN_STATE_UP
);
724 netif_carrier_on(dev
);
726 err
= mlx5_modify_vport_admin_state(priv
->mdev
, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT
,
727 rep
->vport
, 1, MLX5_VPORT_ADMIN_STATE_DOWN
);
730 netif_carrier_off(dev
);
735 static const struct net_device_ops mlx5e_netdev_ops_rep
= {
736 .ndo_open
= mlx5e_rep_open
,
737 .ndo_stop
= mlx5e_rep_close
,
738 .ndo_start_xmit
= mlx5e_xmit
,
739 .ndo_setup_tc
= mlx5e_rep_setup_tc
,
740 .ndo_get_stats64
= mlx5e_rep_get_stats
,
741 .ndo_has_offload_stats
= mlx5e_rep_has_offload_stats
,
742 .ndo_get_offload_stats
= mlx5e_rep_get_offload_stats
,
743 .ndo_change_mtu
= mlx5e_rep_change_mtu
,
744 .ndo_change_carrier
= mlx5e_rep_change_carrier
,
747 bool mlx5e_eswitch_uplink_rep(const struct net_device
*netdev
)
749 return netdev
->netdev_ops
== &mlx5e_netdev_ops
&&
750 mlx5e_is_uplink_rep(netdev_priv(netdev
));
753 bool mlx5e_eswitch_vf_rep(const struct net_device
*netdev
)
755 return netdev
->netdev_ops
== &mlx5e_netdev_ops_rep
;
758 /* One indirect TIR set for outer. Inner not supported in reps. */
759 #define REP_NUM_INDIR_TIRS MLX5E_NUM_INDIR_TIRS
761 static int mlx5e_rep_max_nch_limit(struct mlx5_core_dev
*mdev
)
763 int max_tir_num
= 1 << MLX5_CAP_GEN(mdev
, log_max_tir
);
764 int num_vports
= mlx5_eswitch_get_total_vports(mdev
);
766 return (max_tir_num
- mlx5e_get_pf_num_tirs(mdev
)
767 - (num_vports
* REP_NUM_INDIR_TIRS
)) / num_vports
;
770 static void mlx5e_build_rep_params(struct net_device
*netdev
)
772 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
773 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
774 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
775 struct mlx5_core_dev
*mdev
= priv
->mdev
;
776 struct mlx5e_params
*params
;
778 u8 cq_period_mode
= MLX5_CAP_GEN(mdev
, cq_period_start_from_cqe
) ?
779 MLX5_CQ_PERIOD_MODE_START_FROM_CQE
:
780 MLX5_CQ_PERIOD_MODE_START_FROM_EQE
;
782 params
= &priv
->channels
.params
;
784 params
->num_channels
= MLX5E_REP_PARAMS_DEF_NUM_CHANNELS
;
785 params
->hard_mtu
= MLX5E_ETH_HARD_MTU
;
786 params
->sw_mtu
= netdev
->mtu
;
789 if (rep
->vport
== MLX5_VPORT_UPLINK
)
790 params
->log_sq_size
= MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE
;
792 params
->log_sq_size
= MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE
;
795 mlx5e_build_rq_params(mdev
, params
);
797 /* update XDP supported features */
798 mlx5e_set_xdp_feature(netdev
);
800 /* CQ moderation params */
801 params
->rx_dim_enabled
= MLX5_CAP_GEN(mdev
, cq_moderation
);
802 mlx5e_set_rx_cq_mode_params(params
, cq_period_mode
);
804 params
->mqprio
.num_tc
= 1;
805 if (rep
->vport
!= MLX5_VPORT_UPLINK
)
806 params
->vlan_strip_disable
= true;
808 mlx5_query_min_inline(mdev
, ¶ms
->tx_min_inline_mode
);
811 static void mlx5e_build_rep_netdev(struct net_device
*netdev
,
812 struct mlx5_core_dev
*mdev
)
814 SET_NETDEV_DEV(netdev
, mdev
->device
);
815 netdev
->netdev_ops
= &mlx5e_netdev_ops_rep
;
816 eth_hw_addr_random(netdev
);
817 netdev
->ethtool_ops
= &mlx5e_rep_ethtool_ops
;
819 netdev
->watchdog_timeo
= 15 * HZ
;
821 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
822 netdev
->hw_features
|= NETIF_F_HW_TC
;
824 netdev
->hw_features
|= NETIF_F_SG
;
825 netdev
->hw_features
|= NETIF_F_IP_CSUM
;
826 netdev
->hw_features
|= NETIF_F_IPV6_CSUM
;
827 netdev
->hw_features
|= NETIF_F_GRO
;
828 netdev
->hw_features
|= NETIF_F_TSO
;
829 netdev
->hw_features
|= NETIF_F_TSO6
;
830 netdev
->hw_features
|= NETIF_F_RXCSUM
;
832 netdev
->features
|= netdev
->hw_features
;
833 netdev
->features
|= NETIF_F_NETNS_LOCAL
;
836 static int mlx5e_init_rep(struct mlx5_core_dev
*mdev
,
837 struct net_device
*netdev
)
839 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
842 mlx5e_fs_init(priv
->profile
, mdev
,
843 !test_bit(MLX5E_STATE_DESTROYING
, &priv
->state
),
846 netdev_err(priv
->netdev
, "FS allocation failed\n");
850 mlx5e_build_rep_params(netdev
);
851 mlx5e_timestamp_init(priv
);
856 static int mlx5e_init_ul_rep(struct mlx5_core_dev
*mdev
,
857 struct net_device
*netdev
)
859 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
861 priv
->dfs_root
= debugfs_create_dir("nic",
862 mlx5_debugfs_get_dev_root(mdev
));
864 priv
->fs
= mlx5e_fs_init(priv
->profile
, mdev
,
865 !test_bit(MLX5E_STATE_DESTROYING
, &priv
->state
),
868 netdev_err(priv
->netdev
, "FS allocation failed\n");
869 debugfs_remove_recursive(priv
->dfs_root
);
873 mlx5e_vxlan_set_netdev_info(priv
);
874 mlx5e_build_rep_params(netdev
);
875 mlx5e_timestamp_init(priv
);
879 static void mlx5e_cleanup_rep(struct mlx5e_priv
*priv
)
881 mlx5e_fs_cleanup(priv
->fs
);
882 debugfs_remove_recursive(priv
->dfs_root
);
886 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv
*priv
)
888 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
889 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
890 struct ttc_params ttc_params
= {};
893 mlx5e_fs_set_ns(priv
->fs
,
894 mlx5_get_flow_namespace(priv
->mdev
,
895 MLX5_FLOW_NAMESPACE_KERNEL
), false);
897 /* The inner_ttc in the ttc params is intentionally not set */
898 mlx5e_set_ttc_params(priv
->fs
, priv
->rx_res
, &ttc_params
, false);
900 if (rep
->vport
!= MLX5_VPORT_UPLINK
)
901 /* To give uplik rep TTC a lower level for chaining from root ft */
902 ttc_params
.ft_attr
.level
= MLX5E_TTC_FT_LEVEL
+ 1;
904 mlx5e_fs_set_ttc(priv
->fs
, mlx5_create_ttc_table(priv
->mdev
, &ttc_params
), false);
905 if (IS_ERR(mlx5e_fs_get_ttc(priv
->fs
, false))) {
906 err
= PTR_ERR(mlx5e_fs_get_ttc(priv
->fs
, false));
907 netdev_err(priv
->netdev
, "Failed to create rep ttc table, err=%d\n",
914 static int mlx5e_create_rep_root_ft(struct mlx5e_priv
*priv
)
916 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
917 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
918 struct mlx5_flow_table_attr ft_attr
= {};
919 struct mlx5_flow_namespace
*ns
;
922 if (rep
->vport
!= MLX5_VPORT_UPLINK
) {
923 /* non uplik reps will skip any bypass tables and go directly to
926 rpriv
->root_ft
= mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv
->fs
, false));
930 /* uplink root ft will be used to auto chain, to ethtool or ttc tables */
931 ns
= mlx5_get_flow_namespace(priv
->mdev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
933 netdev_err(priv
->netdev
, "Failed to get reps offloads namespace\n");
937 ft_attr
.max_fte
= 0; /* Empty table, miss rule will always point to next table */
941 rpriv
->root_ft
= mlx5_create_flow_table(ns
, &ft_attr
);
942 if (IS_ERR(rpriv
->root_ft
)) {
943 err
= PTR_ERR(rpriv
->root_ft
);
944 rpriv
->root_ft
= NULL
;
950 static void mlx5e_destroy_rep_root_ft(struct mlx5e_priv
*priv
)
952 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
953 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
955 if (rep
->vport
!= MLX5_VPORT_UPLINK
)
957 mlx5_destroy_flow_table(rpriv
->root_ft
);
960 static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv
*priv
)
962 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
963 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
964 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
965 struct mlx5_flow_handle
*flow_rule
;
966 struct mlx5_flow_destination dest
;
968 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
969 dest
.ft
= rpriv
->root_ft
;
971 flow_rule
= mlx5_eswitch_create_vport_rx_rule(esw
, rep
->vport
, &dest
);
972 if (IS_ERR(flow_rule
))
973 return PTR_ERR(flow_rule
);
974 rpriv
->vport_rx_rule
= flow_rule
;
978 static void rep_vport_rx_rule_destroy(struct mlx5e_priv
*priv
)
980 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
982 if (!rpriv
->vport_rx_rule
)
985 mlx5_del_flow_rules(rpriv
->vport_rx_rule
);
986 rpriv
->vport_rx_rule
= NULL
;
989 int mlx5e_rep_bond_update(struct mlx5e_priv
*priv
, bool cleanup
)
991 rep_vport_rx_rule_destroy(priv
);
993 return cleanup
? 0 : mlx5e_create_rep_vport_rx_rule(priv
);
996 static int mlx5e_init_rep_rx(struct mlx5e_priv
*priv
)
998 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1001 priv
->rx_res
= mlx5e_rx_res_alloc();
1002 if (!priv
->rx_res
) {
1007 mlx5e_fs_init_l2_addr(priv
->fs
, priv
->netdev
);
1009 err
= mlx5e_open_drop_rq(priv
, &priv
->drop_rq
);
1011 mlx5_core_err(mdev
, "open drop rq failed, %d\n", err
);
1012 goto err_rx_res_free
;
1015 err
= mlx5e_rx_res_init(priv
->rx_res
, priv
->mdev
, 0,
1016 priv
->max_nch
, priv
->drop_rq
.rqn
,
1017 &priv
->channels
.params
.packet_merge
,
1018 priv
->channels
.params
.num_channels
);
1020 goto err_close_drop_rq
;
1022 err
= mlx5e_create_rep_ttc_table(priv
);
1024 goto err_destroy_rx_res
;
1026 err
= mlx5e_create_rep_root_ft(priv
);
1028 goto err_destroy_ttc_table
;
1030 err
= mlx5e_create_rep_vport_rx_rule(priv
);
1032 goto err_destroy_root_ft
;
1034 mlx5e_ethtool_init_steering(priv
->fs
);
1038 err_destroy_root_ft
:
1039 mlx5e_destroy_rep_root_ft(priv
);
1040 err_destroy_ttc_table
:
1041 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv
->fs
, false));
1043 mlx5e_rx_res_destroy(priv
->rx_res
);
1045 mlx5e_close_drop_rq(&priv
->drop_rq
);
1047 mlx5e_rx_res_free(priv
->rx_res
);
1048 priv
->rx_res
= NULL
;
1050 mlx5e_fs_cleanup(priv
->fs
);
1055 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv
*priv
)
1057 mlx5e_ethtool_cleanup_steering(priv
->fs
);
1058 rep_vport_rx_rule_destroy(priv
);
1059 mlx5e_destroy_rep_root_ft(priv
);
1060 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv
->fs
, false));
1061 mlx5e_rx_res_destroy(priv
->rx_res
);
1062 mlx5e_close_drop_rq(&priv
->drop_rq
);
1063 mlx5e_rx_res_free(priv
->rx_res
);
1064 priv
->rx_res
= NULL
;
1067 static void mlx5e_rep_mpesw_work(struct work_struct
*work
)
1069 struct mlx5_rep_uplink_priv
*uplink_priv
=
1070 container_of(work
, struct mlx5_rep_uplink_priv
,
1072 struct mlx5e_rep_priv
*rpriv
=
1073 container_of(uplink_priv
, struct mlx5e_rep_priv
,
1075 struct mlx5e_priv
*priv
= netdev_priv(rpriv
->netdev
);
1077 rep_vport_rx_rule_destroy(priv
);
1078 mlx5e_create_rep_vport_rx_rule(priv
);
1081 static int mlx5e_init_ul_rep_rx(struct mlx5e_priv
*priv
)
1083 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1086 mlx5e_create_q_counters(priv
);
1087 err
= mlx5e_init_rep_rx(priv
);
1091 mlx5e_tc_int_port_init_rep_rx(priv
);
1093 INIT_WORK(&rpriv
->uplink_priv
.mpesw_work
, mlx5e_rep_mpesw_work
);
1099 static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv
*priv
)
1101 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1103 cancel_work_sync(&rpriv
->uplink_priv
.mpesw_work
);
1104 mlx5e_tc_int_port_cleanup_rep_rx(priv
);
1105 mlx5e_cleanup_rep_rx(priv
);
1106 mlx5e_destroy_q_counters(priv
);
1109 static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv
*rpriv
)
1111 struct mlx5_rep_uplink_priv
*uplink_priv
;
1112 struct net_device
*netdev
;
1113 struct mlx5e_priv
*priv
;
1116 netdev
= rpriv
->netdev
;
1117 priv
= netdev_priv(netdev
);
1118 uplink_priv
= &rpriv
->uplink_priv
;
1120 err
= mlx5e_rep_tc_init(rpriv
);
1124 mlx5_init_port_tun_entropy(&uplink_priv
->tun_entropy
, priv
->mdev
);
1126 mlx5e_rep_bond_init(rpriv
);
1127 err
= mlx5e_rep_tc_netdevice_event_register(rpriv
);
1129 mlx5_core_err(priv
->mdev
, "Failed to register netdev notifier, err: %d\n",
1137 mlx5e_rep_bond_cleanup(rpriv
);
1138 mlx5e_rep_tc_cleanup(rpriv
);
1142 static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv
*rpriv
)
1144 mlx5e_rep_tc_netdevice_event_unregister(rpriv
);
1145 mlx5e_rep_bond_cleanup(rpriv
);
1146 mlx5e_rep_tc_cleanup(rpriv
);
1149 static int mlx5e_init_rep_tx(struct mlx5e_priv
*priv
)
1151 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1154 err
= mlx5e_create_tises(priv
);
1156 mlx5_core_warn(priv
->mdev
, "create tises failed, %d\n", err
);
1160 err
= mlx5e_rep_neigh_init(rpriv
);
1162 goto err_neigh_init
;
1164 if (rpriv
->rep
->vport
== MLX5_VPORT_UPLINK
) {
1165 err
= mlx5e_init_uplink_rep_tx(rpriv
);
1170 err
= mlx5e_tc_ht_init(&rpriv
->tc_ht
);
1177 if (rpriv
->rep
->vport
== MLX5_VPORT_UPLINK
)
1178 mlx5e_cleanup_uplink_rep_tx(rpriv
);
1180 mlx5e_rep_neigh_cleanup(rpriv
);
1182 mlx5e_destroy_tises(priv
);
1186 static void mlx5e_cleanup_rep_tx(struct mlx5e_priv
*priv
)
1188 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1190 mlx5e_tc_ht_cleanup(&rpriv
->tc_ht
);
1192 if (rpriv
->rep
->vport
== MLX5_VPORT_UPLINK
)
1193 mlx5e_cleanup_uplink_rep_tx(rpriv
);
1195 mlx5e_rep_neigh_cleanup(rpriv
);
1196 mlx5e_destroy_tises(priv
);
1199 static void mlx5e_rep_enable(struct mlx5e_priv
*priv
)
1201 mlx5e_set_netdev_mtu_boundaries(priv
);
1204 static void mlx5e_rep_disable(struct mlx5e_priv
*priv
)
1208 static int mlx5e_update_rep_rx(struct mlx5e_priv
*priv
)
1213 static int mlx5e_rep_event_mpesw(struct mlx5e_priv
*priv
)
1215 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1216 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
1218 if (rep
->vport
!= MLX5_VPORT_UPLINK
)
1221 queue_work(priv
->wq
, &rpriv
->uplink_priv
.mpesw_work
);
1226 static int uplink_rep_async_event(struct notifier_block
*nb
, unsigned long event
, void *data
)
1228 struct mlx5e_priv
*priv
= container_of(nb
, struct mlx5e_priv
, events_nb
);
1230 if (event
== MLX5_EVENT_TYPE_PORT_CHANGE
) {
1231 struct mlx5_eqe
*eqe
= data
;
1233 switch (eqe
->sub_type
) {
1234 case MLX5_PORT_CHANGE_SUBTYPE_DOWN
:
1235 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE
:
1236 queue_work(priv
->wq
, &priv
->update_carrier_work
);
1245 if (event
== MLX5_DEV_EVENT_PORT_AFFINITY
)
1246 return mlx5e_rep_tc_event_port_affinity(priv
);
1247 else if (event
== MLX5_DEV_EVENT_MULTIPORT_ESW
)
1248 return mlx5e_rep_event_mpesw(priv
);
1253 static void mlx5e_uplink_rep_enable(struct mlx5e_priv
*priv
)
1255 struct net_device
*netdev
= priv
->netdev
;
1256 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1259 mlx5e_ipsec_init(priv
);
1261 netdev
->min_mtu
= ETH_MIN_MTU
;
1262 mlx5_query_port_max_mtu(priv
->mdev
, &max_mtu
, 1);
1263 netdev
->max_mtu
= MLX5E_HW2SW_MTU(&priv
->channels
.params
, max_mtu
);
1264 mlx5e_set_dev_port_mtu(priv
);
1266 mlx5e_rep_tc_enable(priv
);
1268 if (MLX5_CAP_GEN(mdev
, uplink_follow
))
1269 mlx5_modify_vport_admin_state(mdev
, MLX5_VPORT_STATE_OP_MOD_UPLINK
,
1270 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO
);
1271 mlx5_lag_add_netdev(mdev
, netdev
);
1272 priv
->events_nb
.notifier_call
= uplink_rep_async_event
;
1273 mlx5_notifier_register(mdev
, &priv
->events_nb
);
1274 mlx5e_dcbnl_initialize(priv
);
1275 mlx5e_dcbnl_init_app(priv
);
1276 mlx5e_rep_bridge_init(priv
);
1278 netdev
->wanted_features
|= NETIF_F_HW_TC
;
1281 if (netif_running(netdev
))
1283 udp_tunnel_nic_reset_ntf(priv
->netdev
);
1284 netif_device_attach(netdev
);
1288 static void mlx5e_uplink_rep_disable(struct mlx5e_priv
*priv
)
1290 struct mlx5_core_dev
*mdev
= priv
->mdev
;
1293 if (netif_running(priv
->netdev
))
1294 mlx5e_close(priv
->netdev
);
1295 netif_device_detach(priv
->netdev
);
1298 mlx5e_rep_bridge_cleanup(priv
);
1299 mlx5e_dcbnl_delete_app(priv
);
1300 mlx5_notifier_unregister(mdev
, &priv
->events_nb
);
1301 mlx5e_rep_tc_disable(priv
);
1302 mlx5_lag_remove_netdev(mdev
, priv
->netdev
);
1303 mlx5_vxlan_reset_to_default(mdev
->vxlan
);
1305 mlx5e_ipsec_cleanup(priv
);
1308 static MLX5E_DEFINE_STATS_GRP(sw_rep
, 0);
1309 static MLX5E_DEFINE_STATS_GRP(vport_rep
, MLX5E_NDO_UPDATE_STATS
);
1311 /* The stats groups order is opposite to the update_stats() order calls */
1312 static mlx5e_stats_grp_t mlx5e_rep_stats_grps
[] = {
1313 &MLX5E_STATS_GRP(sw_rep
),
1314 &MLX5E_STATS_GRP(vport_rep
),
1317 static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv
*priv
)
1319 return ARRAY_SIZE(mlx5e_rep_stats_grps
);
1322 /* The stats groups order is opposite to the update_stats() order calls */
1323 static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps
[] = {
1324 &MLX5E_STATS_GRP(sw
),
1325 &MLX5E_STATS_GRP(qcnt
),
1326 &MLX5E_STATS_GRP(vnic_env
),
1327 &MLX5E_STATS_GRP(vport
),
1328 &MLX5E_STATS_GRP(802_3
),
1329 &MLX5E_STATS_GRP(2863),
1330 &MLX5E_STATS_GRP(2819),
1331 &MLX5E_STATS_GRP(phy
),
1332 &MLX5E_STATS_GRP(eth_ext
),
1333 &MLX5E_STATS_GRP(pcie
),
1334 &MLX5E_STATS_GRP(per_prio
),
1335 &MLX5E_STATS_GRP(pme
),
1336 &MLX5E_STATS_GRP(channels
),
1337 &MLX5E_STATS_GRP(per_port_buff_congest
),
1338 #ifdef CONFIG_MLX5_EN_IPSEC
1339 &MLX5E_STATS_GRP(ipsec_hw
),
1340 &MLX5E_STATS_GRP(ipsec_sw
),
1342 &MLX5E_STATS_GRP(ptp
),
1345 static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv
*priv
)
1347 return ARRAY_SIZE(mlx5e_ul_rep_stats_grps
);
1351 mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter
*reporter
,
1352 struct devlink_fmsg
*fmsg
,
1353 struct netlink_ext_ack
*extack
)
1355 struct mlx5e_rep_priv
*rpriv
= devlink_health_reporter_priv(reporter
);
1356 struct mlx5_eswitch_rep
*rep
= rpriv
->rep
;
1358 return mlx5_reporter_vnic_diagnose_counters(rep
->esw
->dev
, fmsg
,
1362 static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops
= {
1364 .diagnose
= mlx5e_rep_vnic_reporter_diagnose
,
1367 static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv
*priv
,
1368 struct devlink_port
*dl_port
)
1370 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1371 struct devlink_health_reporter
*reporter
;
1373 reporter
= devl_port_health_reporter_create(dl_port
,
1374 &mlx5_rep_vnic_reporter_ops
,
1376 if (IS_ERR(reporter
)) {
1377 mlx5_core_err(priv
->mdev
,
1378 "Failed to create representor vnic reporter, err = %ld\n",
1383 rpriv
->rep_vnic_reporter
= reporter
;
1386 static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv
*priv
)
1388 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1390 if (!IS_ERR_OR_NULL(rpriv
->rep_vnic_reporter
))
1391 devl_health_reporter_destroy(rpriv
->rep_vnic_reporter
);
1394 static const struct mlx5e_profile mlx5e_rep_profile
= {
1395 .init
= mlx5e_init_rep
,
1396 .cleanup
= mlx5e_cleanup_rep
,
1397 .init_rx
= mlx5e_init_rep_rx
,
1398 .cleanup_rx
= mlx5e_cleanup_rep_rx
,
1399 .init_tx
= mlx5e_init_rep_tx
,
1400 .cleanup_tx
= mlx5e_cleanup_rep_tx
,
1401 .enable
= mlx5e_rep_enable
,
1402 .disable
= mlx5e_rep_disable
,
1403 .update_rx
= mlx5e_update_rep_rx
,
1404 .update_stats
= mlx5e_stats_update_ndo_stats
,
1405 .rx_handlers
= &mlx5e_rx_handlers_rep
,
1407 .stats_grps
= mlx5e_rep_stats_grps
,
1408 .stats_grps_num
= mlx5e_rep_stats_grps_num
,
1409 .max_nch_limit
= mlx5e_rep_max_nch_limit
,
1412 static const struct mlx5e_profile mlx5e_uplink_rep_profile
= {
1413 .init
= mlx5e_init_ul_rep
,
1414 .cleanup
= mlx5e_cleanup_rep
,
1415 .init_rx
= mlx5e_init_ul_rep_rx
,
1416 .cleanup_rx
= mlx5e_cleanup_ul_rep_rx
,
1417 .init_tx
= mlx5e_init_rep_tx
,
1418 .cleanup_tx
= mlx5e_cleanup_rep_tx
,
1419 .enable
= mlx5e_uplink_rep_enable
,
1420 .disable
= mlx5e_uplink_rep_disable
,
1421 .update_rx
= mlx5e_update_rep_rx
,
1422 .update_stats
= mlx5e_stats_update_ndo_stats
,
1423 .update_carrier
= mlx5e_update_carrier
,
1424 .rx_handlers
= &mlx5e_rx_handlers_rep
,
1425 .max_tc
= MLX5E_MAX_NUM_TC
,
1426 .stats_grps
= mlx5e_ul_rep_stats_grps
,
1427 .stats_grps_num
= mlx5e_ul_rep_stats_grps_num
,
1430 /* e-Switch vport representors */
1432 mlx5e_vport_uplink_rep_load(struct mlx5_core_dev
*dev
, struct mlx5_eswitch_rep
*rep
)
1434 struct mlx5e_priv
*priv
= netdev_priv(mlx5_uplink_netdev_get(dev
));
1435 struct mlx5e_rep_priv
*rpriv
= mlx5e_rep_to_rep_priv(rep
);
1437 rpriv
->netdev
= priv
->netdev
;
1438 return mlx5e_netdev_change_profile(priv
, &mlx5e_uplink_rep_profile
,
1443 mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv
*rpriv
)
1445 struct net_device
*netdev
= rpriv
->netdev
;
1446 struct mlx5e_priv
*priv
;
1448 priv
= netdev_priv(netdev
);
1450 mlx5e_netdev_attach_nic_profile(priv
);
1454 mlx5e_vport_vf_rep_load(struct mlx5_core_dev
*dev
, struct mlx5_eswitch_rep
*rep
)
1456 struct mlx5e_rep_priv
*rpriv
= mlx5e_rep_to_rep_priv(rep
);
1457 const struct mlx5e_profile
*profile
;
1458 struct devlink_port
*dl_port
;
1459 struct net_device
*netdev
;
1460 struct mlx5e_priv
*priv
;
1463 profile
= &mlx5e_rep_profile
;
1464 netdev
= mlx5e_create_netdev(dev
, profile
);
1467 "Failed to create representor netdev for vport %d\n",
1472 mlx5e_build_rep_netdev(netdev
, dev
);
1473 rpriv
->netdev
= netdev
;
1475 priv
= netdev_priv(netdev
);
1476 priv
->profile
= profile
;
1477 priv
->ppriv
= rpriv
;
1478 err
= profile
->init(dev
, netdev
);
1480 netdev_warn(netdev
, "rep profile init failed, %d\n", err
);
1481 goto err_destroy_netdev
;
1484 err
= mlx5e_attach_netdev(netdev_priv(netdev
));
1487 "Failed to attach representor netdev for vport %d\n",
1489 goto err_cleanup_profile
;
1492 dl_port
= mlx5_esw_offloads_devlink_port(dev
->priv
.eswitch
,
1495 SET_NETDEV_DEVLINK_PORT(netdev
, dl_port
);
1496 mlx5e_rep_vnic_reporter_create(priv
, dl_port
);
1499 err
= register_netdev(netdev
);
1502 "Failed to register representor netdev for vport %d\n",
1504 goto err_detach_netdev
;
1510 mlx5e_rep_vnic_reporter_destroy(priv
);
1511 mlx5e_detach_netdev(netdev_priv(netdev
));
1512 err_cleanup_profile
:
1513 priv
->profile
->cleanup(priv
);
1516 mlx5e_destroy_netdev(netdev_priv(netdev
));
1521 mlx5e_vport_rep_load(struct mlx5_core_dev
*dev
, struct mlx5_eswitch_rep
*rep
)
1523 struct mlx5e_rep_priv
*rpriv
;
1526 rpriv
= kvzalloc(sizeof(*rpriv
), GFP_KERNEL
);
1530 /* rpriv->rep to be looked up when profile->init() is called */
1532 rep
->rep_data
[REP_ETH
].priv
= rpriv
;
1533 INIT_LIST_HEAD(&rpriv
->vport_sqs_list
);
1535 if (rep
->vport
== MLX5_VPORT_UPLINK
)
1536 err
= mlx5e_vport_uplink_rep_load(dev
, rep
);
1538 err
= mlx5e_vport_vf_rep_load(dev
, rep
);
1547 mlx5e_vport_rep_unload(struct mlx5_eswitch_rep
*rep
)
1549 struct mlx5e_rep_priv
*rpriv
= mlx5e_rep_to_rep_priv(rep
);
1550 struct net_device
*netdev
= rpriv
->netdev
;
1551 struct mlx5e_priv
*priv
= netdev_priv(netdev
);
1552 void *ppriv
= priv
->ppriv
;
1554 if (rep
->vport
== MLX5_VPORT_UPLINK
) {
1555 mlx5e_vport_uplink_rep_unload(rpriv
);
1559 unregister_netdev(netdev
);
1560 mlx5e_rep_vnic_reporter_destroy(priv
);
1561 mlx5e_detach_netdev(priv
);
1562 priv
->profile
->cleanup(priv
);
1563 mlx5e_destroy_netdev(priv
);
1565 kvfree(ppriv
); /* mlx5e_rep_priv */
1568 static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep
*rep
)
1570 struct mlx5e_rep_priv
*rpriv
;
1572 rpriv
= mlx5e_rep_to_rep_priv(rep
);
1574 return rpriv
->netdev
;
1577 static void mlx5e_vport_rep_event_unpair(struct mlx5_eswitch_rep
*rep
,
1578 struct mlx5_eswitch
*peer_esw
)
1580 u16 i
= MLX5_CAP_GEN(peer_esw
->dev
, vhca_id
);
1581 struct mlx5e_rep_priv
*rpriv
;
1582 struct mlx5e_rep_sq
*rep_sq
;
1584 WARN_ON_ONCE(!peer_esw
);
1585 rpriv
= mlx5e_rep_to_rep_priv(rep
);
1586 list_for_each_entry(rep_sq
, &rpriv
->vport_sqs_list
, list
) {
1587 struct mlx5e_rep_sq_peer
*sq_peer
= xa_load(&rep_sq
->sq_peer
, i
);
1589 if (!sq_peer
|| sq_peer
->peer
!= peer_esw
)
1592 mlx5_eswitch_del_send_to_vport_rule(sq_peer
->rule
);
1593 xa_erase(&rep_sq
->sq_peer
, i
);
1598 static int mlx5e_vport_rep_event_pair(struct mlx5_eswitch
*esw
,
1599 struct mlx5_eswitch_rep
*rep
,
1600 struct mlx5_eswitch
*peer_esw
)
1602 u16 i
= MLX5_CAP_GEN(peer_esw
->dev
, vhca_id
);
1603 struct mlx5_flow_handle
*flow_rule
;
1604 struct mlx5e_rep_sq_peer
*sq_peer
;
1605 struct mlx5e_rep_priv
*rpriv
;
1606 struct mlx5e_rep_sq
*rep_sq
;
1609 rpriv
= mlx5e_rep_to_rep_priv(rep
);
1610 list_for_each_entry(rep_sq
, &rpriv
->vport_sqs_list
, list
) {
1611 sq_peer
= xa_load(&rep_sq
->sq_peer
, i
);
1613 if (sq_peer
&& sq_peer
->peer
)
1616 flow_rule
= mlx5_eswitch_add_send_to_vport_rule(peer_esw
, esw
, rep
,
1618 if (IS_ERR(flow_rule
)) {
1619 err
= PTR_ERR(flow_rule
);
1624 sq_peer
->rule
= flow_rule
;
1625 sq_peer
->peer
= peer_esw
;
1628 sq_peer
= kzalloc(sizeof(*sq_peer
), GFP_KERNEL
);
1633 err
= xa_insert(&rep_sq
->sq_peer
, i
, sq_peer
, GFP_KERNEL
);
1636 sq_peer
->rule
= flow_rule
;
1637 sq_peer
->peer
= peer_esw
;
1644 mlx5_eswitch_del_send_to_vport_rule(flow_rule
);
1646 mlx5e_vport_rep_event_unpair(rep
, peer_esw
);
1650 static int mlx5e_vport_rep_event(struct mlx5_eswitch
*esw
,
1651 struct mlx5_eswitch_rep
*rep
,
1652 enum mlx5_switchdev_event event
,
1657 if (event
== MLX5_SWITCHDEV_EVENT_PAIR
)
1658 err
= mlx5e_vport_rep_event_pair(esw
, rep
, data
);
1659 else if (event
== MLX5_SWITCHDEV_EVENT_UNPAIR
)
1660 mlx5e_vport_rep_event_unpair(rep
, data
);
1665 static const struct mlx5_eswitch_rep_ops rep_ops
= {
1666 .load
= mlx5e_vport_rep_load
,
1667 .unload
= mlx5e_vport_rep_unload
,
1668 .get_proto_dev
= mlx5e_vport_rep_get_proto_dev
,
1669 .event
= mlx5e_vport_rep_event
,
1672 static int mlx5e_rep_probe(struct auxiliary_device
*adev
,
1673 const struct auxiliary_device_id
*id
)
1675 struct mlx5_adev
*edev
= container_of(adev
, struct mlx5_adev
, adev
);
1676 struct mlx5_core_dev
*mdev
= edev
->mdev
;
1677 struct mlx5_eswitch
*esw
;
1679 esw
= mdev
->priv
.eswitch
;
1680 mlx5_eswitch_register_vport_reps(esw
, &rep_ops
, REP_ETH
);
1684 static void mlx5e_rep_remove(struct auxiliary_device
*adev
)
1686 struct mlx5_adev
*vdev
= container_of(adev
, struct mlx5_adev
, adev
);
1687 struct mlx5_core_dev
*mdev
= vdev
->mdev
;
1688 struct mlx5_eswitch
*esw
;
1690 esw
= mdev
->priv
.eswitch
;
1691 mlx5_eswitch_unregister_vport_reps(esw
, REP_ETH
);
1694 static const struct auxiliary_device_id mlx5e_rep_id_table
[] = {
1695 { .name
= MLX5_ADEV_NAME
".eth-rep", },
1699 MODULE_DEVICE_TABLE(auxiliary
, mlx5e_rep_id_table
);
1701 static struct auxiliary_driver mlx5e_rep_driver
= {
1703 .probe
= mlx5e_rep_probe
,
1704 .remove
= mlx5e_rep_remove
,
1705 .id_table
= mlx5e_rep_id_table
,
1708 int mlx5e_rep_init(void)
1710 return auxiliary_driver_register(&mlx5e_rep_driver
);
1713 void mlx5e_rep_cleanup(void)
1715 auxiliary_driver_unregister(&mlx5e_rep_driver
);