2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
46 #include "lib/devcom.h"
48 #include "lib/fs_chains.h"
50 #include "en/mapping.h"
53 #include "en/tc/post_meter.h"
55 #define mlx5_esw_for_each_rep(esw, i, rep) \
56 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
58 /* There are two match-all miss flows, one for unicast dst mac and
61 #define MLX5_ESW_MISS_FLOWS (2)
62 #define UPLINK_REP_INDEX 0
64 #define MLX5_ESW_VPORT_TBL_SIZE 128
65 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
67 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
69 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns
= {
70 .max_fte
= MLX5_ESW_VPORT_TBL_SIZE
,
71 .max_num_groups
= MLX5_ESW_VPORT_TBL_NUM_GROUPS
,
75 static struct mlx5_eswitch_rep
*mlx5_eswitch_get_rep(struct mlx5_eswitch
*esw
,
78 return xa_load(&esw
->offloads
.vport_reps
, vport_num
);
82 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch
*esw
,
83 struct mlx5_flow_spec
*spec
,
84 struct mlx5_esw_flow_attr
*attr
)
86 if (!MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, flow_source
) || !attr
|| !attr
->in_rep
)
90 spec
->flow_context
.flow_source
= mlx5e_tc_int_port_get_flow_source(attr
->int_port
);
95 spec
->flow_context
.flow_source
= (attr
->in_rep
->vport
== MLX5_VPORT_UPLINK
) ?
96 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK
:
97 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT
;
100 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
101 * are not needed as well in the following process. So clear them all for simplicity.
104 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch
*esw
, struct mlx5_flow_spec
*spec
)
106 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
109 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
110 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
, 0);
112 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
113 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
, 0);
115 if (!memchr_inv(misc2
, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2
)))
116 spec
->match_criteria_enable
&= ~MLX5_MATCH_MISC_PARAMETERS_2
;
121 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch
*esw
,
122 struct mlx5_flow_spec
*spec
,
123 struct mlx5_flow_attr
*attr
,
124 struct mlx5_eswitch
*src_esw
,
127 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
132 /* Use metadata matching because vport is not represented by single
133 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
135 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
136 if (mlx5_esw_indir_table_decap_vport(attr
))
137 vport
= mlx5_esw_indir_table_decap_vport(attr
);
139 if (!attr
->chain
&& esw_attr
&& esw_attr
->int_port
)
141 mlx5e_tc_int_port_get_metadata_for_match(esw_attr
->int_port
);
144 mlx5_eswitch_get_vport_metadata_for_match(src_esw
, vport
);
146 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
147 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
, metadata
);
149 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
150 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
,
151 mlx5_eswitch_get_vport_metadata_mask());
153 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS_2
;
155 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
156 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
158 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
159 MLX5_SET(fte_match_set_misc
, misc
,
160 source_eswitch_owner_vhca_id
,
161 MLX5_CAP_GEN(src_esw
->dev
, vhca_id
));
163 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
164 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
165 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
166 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
167 source_eswitch_owner_vhca_id
);
169 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS
;
174 esw_setup_decap_indir(struct mlx5_eswitch
*esw
,
175 struct mlx5_flow_attr
*attr
)
177 struct mlx5_flow_table
*ft
;
179 if (!(attr
->flags
& MLX5_ATTR_FLAG_SRC_REWRITE
))
182 ft
= mlx5_esw_indir_table_get(esw
, attr
,
183 mlx5_esw_indir_table_decap_vport(attr
), true);
184 return PTR_ERR_OR_ZERO(ft
);
188 esw_cleanup_decap_indir(struct mlx5_eswitch
*esw
,
189 struct mlx5_flow_attr
*attr
)
191 if (mlx5_esw_indir_table_decap_vport(attr
))
192 mlx5_esw_indir_table_put(esw
,
193 mlx5_esw_indir_table_decap_vport(attr
),
198 esw_setup_mtu_dest(struct mlx5_flow_destination
*dest
,
199 struct mlx5e_meter_attr
*meter
,
202 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_RANGE
;
203 dest
[i
].range
.field
= MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN
;
204 dest
[i
].range
.min
= 0;
205 dest
[i
].range
.max
= meter
->params
.mtu
;
206 dest
[i
].range
.hit_ft
= mlx5e_post_meter_get_mtu_true_ft(meter
->post_meter
);
207 dest
[i
].range
.miss_ft
= mlx5e_post_meter_get_mtu_false_ft(meter
->post_meter
);
213 esw_setup_sampler_dest(struct mlx5_flow_destination
*dest
,
214 struct mlx5_flow_act
*flow_act
,
218 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
219 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER
;
220 dest
[i
].sampler_id
= sampler_id
;
226 esw_setup_ft_dest(struct mlx5_flow_destination
*dest
,
227 struct mlx5_flow_act
*flow_act
,
228 struct mlx5_eswitch
*esw
,
229 struct mlx5_flow_attr
*attr
,
232 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
233 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
234 dest
[i
].ft
= attr
->dest_ft
;
236 if (mlx5_esw_indir_table_decap_vport(attr
))
237 return esw_setup_decap_indir(esw
, attr
);
242 esw_setup_accept_dest(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
243 struct mlx5_fs_chains
*chains
, int i
)
245 if (mlx5_chains_ignore_flow_level_supported(chains
))
246 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
247 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
248 dest
[i
].ft
= mlx5_chains_get_tc_end_ft(chains
);
252 esw_setup_slow_path_dest(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
253 struct mlx5_eswitch
*esw
, int i
)
255 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw
->dev
, ignore_flow_level
))
256 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
257 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
258 dest
[i
].ft
= mlx5_eswitch_get_slow_fdb(esw
);
262 esw_setup_chain_dest(struct mlx5_flow_destination
*dest
,
263 struct mlx5_flow_act
*flow_act
,
264 struct mlx5_fs_chains
*chains
,
265 u32 chain
, u32 prio
, u32 level
,
268 struct mlx5_flow_table
*ft
;
270 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
271 ft
= mlx5_chains_get_table(chains
, chain
, prio
, level
);
275 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
280 static void esw_put_dest_tables_loop(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
,
283 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
284 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
287 for (i
= from
; i
< to
; i
++)
288 if (esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE
)
289 mlx5_chains_put_table(chains
, 0, 1, 0);
290 else if (mlx5_esw_indir_table_needed(esw
, attr
, esw_attr
->dests
[i
].rep
->vport
,
291 esw_attr
->dests
[i
].mdev
))
292 mlx5_esw_indir_table_put(esw
, esw_attr
->dests
[i
].rep
->vport
,
297 esw_is_chain_src_port_rewrite(struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
)
301 for (i
= esw_attr
->split_count
; i
< esw_attr
->out_count
; i
++)
302 if (esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE
)
308 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination
*dest
,
309 struct mlx5_flow_act
*flow_act
,
310 struct mlx5_eswitch
*esw
,
311 struct mlx5_fs_chains
*chains
,
312 struct mlx5_flow_attr
*attr
,
315 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
318 if (!(attr
->flags
& MLX5_ATTR_FLAG_SRC_REWRITE
))
321 /* flow steering cannot handle more than one dest with the same ft
324 if (esw_attr
->out_count
- esw_attr
->split_count
> 1)
327 err
= esw_setup_chain_dest(dest
, flow_act
, chains
, attr
->dest_chain
, 1, 0, *i
);
331 if (esw_attr
->dests
[esw_attr
->split_count
].pkt_reformat
) {
332 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
333 flow_act
->pkt_reformat
= esw_attr
->dests
[esw_attr
->split_count
].pkt_reformat
;
340 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch
*esw
,
341 struct mlx5_flow_attr
*attr
)
343 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
345 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, esw_attr
->out_count
);
349 esw_is_indir_table(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
)
351 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
355 /* Indirect table is supported only for flows with in_port uplink
356 * and the destination is vport on the same eswitch as the uplink,
357 * return false in case at least one of destinations doesn't meet
360 for (i
= esw_attr
->split_count
; i
< esw_attr
->out_count
; i
++) {
361 if (esw_attr
->dests
[i
].rep
&&
362 mlx5_esw_indir_table_needed(esw
, attr
, esw_attr
->dests
[i
].rep
->vport
,
363 esw_attr
->dests
[i
].mdev
)) {
374 esw_setup_indir_table(struct mlx5_flow_destination
*dest
,
375 struct mlx5_flow_act
*flow_act
,
376 struct mlx5_eswitch
*esw
,
377 struct mlx5_flow_attr
*attr
,
380 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
383 if (!(attr
->flags
& MLX5_ATTR_FLAG_SRC_REWRITE
))
386 for (j
= esw_attr
->split_count
; j
< esw_attr
->out_count
; j
++, (*i
)++) {
387 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
388 dest
[*i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
390 dest
[*i
].ft
= mlx5_esw_indir_table_get(esw
, attr
,
391 esw_attr
->dests
[j
].rep
->vport
, false);
392 if (IS_ERR(dest
[*i
].ft
)) {
393 err
= PTR_ERR(dest
[*i
].ft
);
394 goto err_indir_tbl_get
;
398 if (mlx5_esw_indir_table_decap_vport(attr
)) {
399 err
= esw_setup_decap_indir(esw
, attr
);
401 goto err_indir_tbl_get
;
407 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, j
);
411 static void esw_cleanup_indir_table(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
)
413 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
415 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, esw_attr
->out_count
);
416 esw_cleanup_decap_indir(esw
, attr
);
420 esw_cleanup_chain_dest(struct mlx5_fs_chains
*chains
, u32 chain
, u32 prio
, u32 level
)
422 mlx5_chains_put_table(chains
, chain
, prio
, level
);
425 static bool esw_same_vhca_id(struct mlx5_core_dev
*mdev1
, struct mlx5_core_dev
*mdev2
)
427 return MLX5_CAP_GEN(mdev1
, vhca_id
) == MLX5_CAP_GEN(mdev2
, vhca_id
);
430 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch
*esw
,
431 struct mlx5_esw_flow_attr
*esw_attr
,
434 if (esw
->offloads
.ft_ipsec_tx_pol
&&
435 esw_attr
->dests
[attr_idx
].rep
&&
436 esw_attr
->dests
[attr_idx
].rep
->vport
== MLX5_VPORT_UPLINK
&&
437 /* To be aligned with software, encryption is needed only for tunnel device */
438 (esw_attr
->dests
[attr_idx
].flags
& MLX5_ESW_DEST_ENCAP_VALID
) &&
439 esw_attr
->dests
[attr_idx
].rep
!= esw_attr
->in_rep
&&
440 esw_same_vhca_id(esw_attr
->dests
[attr_idx
].mdev
, esw
->dev
))
446 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch
*esw
,
447 struct mlx5_esw_flow_attr
*esw_attr
)
451 if (!esw
->offloads
.ft_ipsec_tx_pol
)
454 for (i
= 0; i
< esw_attr
->split_count
; i
++)
455 if (esw_setup_uplink_fwd_ipsec_needed(esw
, esw_attr
, i
))
458 for (i
= esw_attr
->split_count
; i
< esw_attr
->out_count
; i
++)
459 if (esw_setup_uplink_fwd_ipsec_needed(esw
, esw_attr
, i
) &&
460 (esw_attr
->out_count
- esw_attr
->split_count
> 1))
467 esw_setup_dest_fwd_vport(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
468 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
469 int attr_idx
, int dest_idx
, bool pkt_reformat
)
471 dest
[dest_idx
].type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
472 dest
[dest_idx
].vport
.num
= esw_attr
->dests
[attr_idx
].rep
->vport
;
473 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
)) {
474 dest
[dest_idx
].vport
.vhca_id
=
475 MLX5_CAP_GEN(esw_attr
->dests
[attr_idx
].mdev
, vhca_id
);
476 dest
[dest_idx
].vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
477 if (dest
[dest_idx
].vport
.num
== MLX5_VPORT_UPLINK
&&
478 mlx5_lag_is_mpesw(esw
->dev
))
479 dest
[dest_idx
].type
= MLX5_FLOW_DESTINATION_TYPE_UPLINK
;
481 if (esw_attr
->dests
[attr_idx
].flags
& MLX5_ESW_DEST_ENCAP_VALID
) {
483 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
484 flow_act
->pkt_reformat
= esw_attr
->dests
[attr_idx
].pkt_reformat
;
486 dest
[dest_idx
].vport
.flags
|= MLX5_FLOW_DEST_VPORT_REFORMAT_ID
;
487 dest
[dest_idx
].vport
.pkt_reformat
= esw_attr
->dests
[attr_idx
].pkt_reformat
;
492 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
493 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
494 int attr_idx
, int dest_idx
, bool pkt_reformat
)
496 dest
[dest_idx
].ft
= esw
->offloads
.ft_ipsec_tx_pol
;
497 dest
[dest_idx
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
499 esw_attr
->dests
[attr_idx
].flags
& MLX5_ESW_DEST_ENCAP_VALID
) {
500 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
501 flow_act
->pkt_reformat
= esw_attr
->dests
[attr_idx
].pkt_reformat
;
506 esw_setup_vport_dest(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
507 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
508 int attr_idx
, int dest_idx
, bool pkt_reformat
)
510 if (esw_setup_uplink_fwd_ipsec_needed(esw
, esw_attr
, attr_idx
))
511 esw_setup_dest_fwd_ipsec(dest
, flow_act
, esw
, esw_attr
,
512 attr_idx
, dest_idx
, pkt_reformat
);
514 esw_setup_dest_fwd_vport(dest
, flow_act
, esw
, esw_attr
,
515 attr_idx
, dest_idx
, pkt_reformat
);
519 esw_setup_vport_dests(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
520 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
525 for (j
= esw_attr
->split_count
; j
< esw_attr
->out_count
; j
++, i
++)
526 esw_setup_vport_dest(dest
, flow_act
, esw
, esw_attr
, j
, i
, true);
531 esw_src_port_rewrite_supported(struct mlx5_eswitch
*esw
)
533 return MLX5_CAP_GEN(esw
->dev
, reg_c_preserve
) &&
534 mlx5_eswitch_vport_match_metadata_enabled(esw
) &&
535 MLX5_CAP_ESW_FLOWTABLE_FDB(esw
->dev
, ignore_flow_level
);
539 esw_dests_to_vf_pf_vports(struct mlx5_flow_destination
*dests
, int max_dest
)
541 bool vf_dest
= false, pf_dest
= false;
544 for (i
= 0; i
< max_dest
; i
++) {
545 if (dests
[i
].type
!= MLX5_FLOW_DESTINATION_TYPE_VPORT
)
548 if (dests
[i
].vport
.num
== MLX5_VPORT_UPLINK
)
553 if (vf_dest
&& pf_dest
)
561 esw_setup_dests(struct mlx5_flow_destination
*dest
,
562 struct mlx5_flow_act
*flow_act
,
563 struct mlx5_eswitch
*esw
,
564 struct mlx5_flow_attr
*attr
,
565 struct mlx5_flow_spec
*spec
,
568 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
569 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
572 if (!mlx5_eswitch_termtbl_required(esw
, attr
, flow_act
, spec
) &&
573 esw_src_port_rewrite_supported(esw
))
574 attr
->flags
|= MLX5_ATTR_FLAG_SRC_REWRITE
;
576 if (attr
->flags
& MLX5_ATTR_FLAG_SLOW_PATH
) {
577 esw_setup_slow_path_dest(dest
, flow_act
, esw
, *i
);
582 if (attr
->flags
& MLX5_ATTR_FLAG_SAMPLE
) {
583 esw_setup_sampler_dest(dest
, flow_act
, attr
->sample_attr
.sampler_id
, *i
);
585 } else if (attr
->flags
& MLX5_ATTR_FLAG_ACCEPT
) {
586 esw_setup_accept_dest(dest
, flow_act
, chains
, *i
);
588 } else if (attr
->flags
& MLX5_ATTR_FLAG_MTU
) {
589 err
= esw_setup_mtu_dest(dest
, &attr
->meter_attr
, *i
);
591 } else if (esw_is_indir_table(esw
, attr
)) {
592 err
= esw_setup_indir_table(dest
, flow_act
, esw
, attr
, i
);
593 } else if (esw_is_chain_src_port_rewrite(esw
, esw_attr
)) {
594 err
= esw_setup_chain_src_port_rewrite(dest
, flow_act
, esw
, chains
, attr
, i
);
596 *i
= esw_setup_vport_dests(dest
, flow_act
, esw
, esw_attr
, *i
);
599 err
= esw_setup_ft_dest(dest
, flow_act
, esw
, attr
, *i
);
601 } else if (attr
->dest_chain
) {
602 err
= esw_setup_chain_dest(dest
, flow_act
, chains
, attr
->dest_chain
,
613 esw_cleanup_dests(struct mlx5_eswitch
*esw
,
614 struct mlx5_flow_attr
*attr
)
616 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
617 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
620 esw_cleanup_decap_indir(esw
, attr
);
621 } else if (!mlx5e_tc_attr_flags_skip(attr
->flags
)) {
622 if (attr
->dest_chain
)
623 esw_cleanup_chain_dest(chains
, attr
->dest_chain
, 1, 0);
624 else if (esw_is_indir_table(esw
, attr
))
625 esw_cleanup_indir_table(esw
, attr
);
626 else if (esw_is_chain_src_port_rewrite(esw
, esw_attr
))
627 esw_cleanup_chain_src_port_rewrite(esw
, attr
);
632 esw_setup_meter(struct mlx5_flow_attr
*attr
, struct mlx5_flow_act
*flow_act
)
634 struct mlx5e_flow_meter_handle
*meter
;
636 meter
= attr
->meter_attr
.meter
;
637 flow_act
->exe_aso
.type
= attr
->exe_aso_type
;
638 flow_act
->exe_aso
.object_id
= meter
->obj_id
;
639 flow_act
->exe_aso
.flow_meter
.meter_idx
= meter
->idx
;
640 flow_act
->exe_aso
.flow_meter
.init_color
= MLX5_FLOW_METER_COLOR_GREEN
;
641 /* use metadata reg 5 for packet color */
642 flow_act
->exe_aso
.return_reg_id
= 5;
645 struct mlx5_flow_handle
*
646 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch
*esw
,
647 struct mlx5_flow_spec
*spec
,
648 struct mlx5_flow_attr
*attr
)
650 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
651 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
652 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
653 bool split
= !!(esw_attr
->split_count
);
654 struct mlx5_vport_tbl_attr fwd_attr
;
655 struct mlx5_flow_destination
*dest
;
656 struct mlx5_flow_handle
*rule
;
657 struct mlx5_flow_table
*fdb
;
660 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
661 return ERR_PTR(-EOPNOTSUPP
);
663 if (!mlx5_eswitch_vlan_actions_supported(esw
->dev
, 1))
664 return ERR_PTR(-EOPNOTSUPP
);
666 if (!esw_flow_dests_fwd_ipsec_check(esw
, esw_attr
))
667 return ERR_PTR(-EOPNOTSUPP
);
669 dest
= kcalloc(MLX5_MAX_FLOW_FWD_VPORTS
+ 1, sizeof(*dest
), GFP_KERNEL
);
671 return ERR_PTR(-ENOMEM
);
673 flow_act
.action
= attr
->action
;
675 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
) {
676 flow_act
.vlan
[0].ethtype
= ntohs(esw_attr
->vlan_proto
[0]);
677 flow_act
.vlan
[0].vid
= esw_attr
->vlan_vid
[0];
678 flow_act
.vlan
[0].prio
= esw_attr
->vlan_prio
[0];
679 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2
) {
680 flow_act
.vlan
[1].ethtype
= ntohs(esw_attr
->vlan_proto
[1]);
681 flow_act
.vlan
[1].vid
= esw_attr
->vlan_vid
[1];
682 flow_act
.vlan
[1].prio
= esw_attr
->vlan_prio
[1];
686 mlx5_eswitch_set_rule_flow_source(esw
, spec
, esw_attr
);
688 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
691 err
= esw_setup_dests(dest
, &flow_act
, esw
, attr
, spec
, &i
);
694 goto err_create_goto_table
;
697 /* Header rewrite with combined wire+loopback in FDB is not allowed */
698 if ((flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
) &&
699 esw_dests_to_vf_pf_vports(dest
, i
)) {
701 "FDB: Header rewrite with forwarding to both PF and VF is not allowed\n");
702 rule
= ERR_PTR(-EINVAL
);
707 if (esw_attr
->decap_pkt_reformat
)
708 flow_act
.pkt_reformat
= esw_attr
->decap_pkt_reformat
;
710 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
711 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
712 dest
[i
].counter_id
= mlx5_fc_id(attr
->counter
);
716 if (attr
->outer_match_level
!= MLX5_MATCH_NONE
)
717 spec
->match_criteria_enable
|= MLX5_MATCH_OUTER_HEADERS
;
718 if (attr
->inner_match_level
!= MLX5_MATCH_NONE
)
719 spec
->match_criteria_enable
|= MLX5_MATCH_INNER_HEADERS
;
721 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
722 flow_act
.modify_hdr
= attr
->modify_hdr
;
724 if ((flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO
) &&
725 attr
->exe_aso_type
== MLX5_EXE_ASO_FLOW_METER
)
726 esw_setup_meter(attr
, &flow_act
);
729 fwd_attr
.chain
= attr
->chain
;
730 fwd_attr
.prio
= attr
->prio
;
731 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
732 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
734 fdb
= mlx5_esw_vporttbl_get(esw
, &fwd_attr
);
736 if (attr
->chain
|| attr
->prio
)
737 fdb
= mlx5_chains_get_table(chains
, attr
->chain
,
742 if (!(attr
->flags
& MLX5_ATTR_FLAG_NO_IN_PORT
))
743 mlx5_eswitch_set_rule_source_port(esw
, spec
, attr
,
744 esw_attr
->in_mdev
->priv
.eswitch
,
745 esw_attr
->in_rep
->vport
);
748 rule
= ERR_CAST(fdb
);
757 if (mlx5_eswitch_termtbl_required(esw
, attr
, &flow_act
, spec
))
758 rule
= mlx5_eswitch_add_termtbl_rule(esw
, fdb
, spec
, esw_attr
,
761 rule
= mlx5_add_flow_rules(fdb
, spec
, &flow_act
, dest
, i
);
765 atomic64_inc(&esw
->offloads
.num_flows
);
772 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
773 else if (attr
->chain
|| attr
->prio
)
774 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
776 esw_cleanup_dests(esw
, attr
);
777 err_create_goto_table
:
782 struct mlx5_flow_handle
*
783 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch
*esw
,
784 struct mlx5_flow_spec
*spec
,
785 struct mlx5_flow_attr
*attr
)
787 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
788 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
789 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
790 struct mlx5_vport_tbl_attr fwd_attr
;
791 struct mlx5_flow_destination
*dest
;
792 struct mlx5_flow_table
*fast_fdb
;
793 struct mlx5_flow_table
*fwd_fdb
;
794 struct mlx5_flow_handle
*rule
;
797 dest
= kcalloc(MLX5_MAX_FLOW_FWD_VPORTS
+ 1, sizeof(*dest
), GFP_KERNEL
);
799 return ERR_PTR(-ENOMEM
);
801 fast_fdb
= mlx5_chains_get_table(chains
, attr
->chain
, attr
->prio
, 0);
802 if (IS_ERR(fast_fdb
)) {
803 rule
= ERR_CAST(fast_fdb
);
807 fwd_attr
.chain
= attr
->chain
;
808 fwd_attr
.prio
= attr
->prio
;
809 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
810 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
811 fwd_fdb
= mlx5_esw_vporttbl_get(esw
, &fwd_attr
);
812 if (IS_ERR(fwd_fdb
)) {
813 rule
= ERR_CAST(fwd_fdb
);
817 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
818 for (i
= 0; i
< esw_attr
->split_count
; i
++) {
819 if (esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE
)
820 /* Source port rewrite (forward to ovs internal port or statck device) isn't
821 * supported in the rule of split action.
825 esw_setup_vport_dest(dest
, &flow_act
, esw
, esw_attr
, i
, i
, false);
829 goto err_chain_src_rewrite
;
832 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
833 dest
[i
].ft
= fwd_fdb
;
836 mlx5_eswitch_set_rule_source_port(esw
, spec
, attr
,
837 esw_attr
->in_mdev
->priv
.eswitch
,
838 esw_attr
->in_rep
->vport
);
840 if (attr
->outer_match_level
!= MLX5_MATCH_NONE
)
841 spec
->match_criteria_enable
|= MLX5_MATCH_OUTER_HEADERS
;
843 flow_act
.flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
844 rule
= mlx5_add_flow_rules(fast_fdb
, spec
, &flow_act
, dest
, i
);
847 i
= esw_attr
->split_count
;
848 goto err_chain_src_rewrite
;
851 atomic64_inc(&esw
->offloads
.num_flows
);
855 err_chain_src_rewrite
:
856 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
858 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
865 __mlx5_eswitch_del_rule(struct mlx5_eswitch
*esw
,
866 struct mlx5_flow_handle
*rule
,
867 struct mlx5_flow_attr
*attr
,
870 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
871 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
872 bool split
= (esw_attr
->split_count
> 0);
873 struct mlx5_vport_tbl_attr fwd_attr
;
876 mlx5_del_flow_rules(rule
);
878 if (!mlx5e_tc_attr_flags_skip(attr
->flags
)) {
879 /* unref the term table */
880 for (i
= 0; i
< MLX5_MAX_FLOW_FWD_VPORTS
; i
++) {
881 if (esw_attr
->dests
[i
].termtbl
)
882 mlx5_eswitch_termtbl_put(esw
, esw_attr
->dests
[i
].termtbl
);
886 atomic64_dec(&esw
->offloads
.num_flows
);
888 if (fwd_rule
|| split
) {
889 fwd_attr
.chain
= attr
->chain
;
890 fwd_attr
.prio
= attr
->prio
;
891 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
892 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
896 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
897 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
900 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
901 else if (attr
->chain
|| attr
->prio
)
902 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
903 esw_cleanup_dests(esw
, attr
);
908 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch
*esw
,
909 struct mlx5_flow_handle
*rule
,
910 struct mlx5_flow_attr
*attr
)
912 __mlx5_eswitch_del_rule(esw
, rule
, attr
, false);
916 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch
*esw
,
917 struct mlx5_flow_handle
*rule
,
918 struct mlx5_flow_attr
*attr
)
920 __mlx5_eswitch_del_rule(esw
, rule
, attr
, true);
923 struct mlx5_flow_handle
*
924 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch
*on_esw
,
925 struct mlx5_eswitch
*from_esw
,
926 struct mlx5_eswitch_rep
*rep
,
929 struct mlx5_flow_act flow_act
= {0};
930 struct mlx5_flow_destination dest
= {};
931 struct mlx5_flow_handle
*flow_rule
;
932 struct mlx5_flow_spec
*spec
;
936 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
938 flow_rule
= ERR_PTR(-ENOMEM
);
942 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
943 MLX5_SET(fte_match_set_misc
, misc
, source_sqn
, sqn
);
945 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
946 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_sqn
);
948 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
950 /* source vport is the esw manager */
951 vport
= from_esw
->manager_vport
;
953 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw
)) {
954 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
955 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
956 mlx5_eswitch_get_vport_metadata_for_match(from_esw
, vport
));
958 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
959 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
960 mlx5_eswitch_get_vport_metadata_mask());
962 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS_2
;
964 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
965 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
967 if (MLX5_CAP_ESW(on_esw
->dev
, merged_eswitch
))
968 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
,
969 MLX5_CAP_GEN(from_esw
->dev
, vhca_id
));
971 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
972 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
974 if (MLX5_CAP_ESW(on_esw
->dev
, merged_eswitch
))
975 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
976 source_eswitch_owner_vhca_id
);
978 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS
;
981 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
982 dest
.vport
.num
= rep
->vport
;
983 dest
.vport
.vhca_id
= MLX5_CAP_GEN(rep
->esw
->dev
, vhca_id
);
984 dest
.vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
985 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
987 if (rep
->vport
== MLX5_VPORT_UPLINK
&&
988 on_esw
== from_esw
&& on_esw
->offloads
.ft_ipsec_tx_pol
) {
989 dest
.ft
= on_esw
->offloads
.ft_ipsec_tx_pol
;
990 flow_act
.flags
= FLOW_ACT_IGNORE_FLOW_LEVEL
;
991 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
993 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
994 dest
.vport
.num
= rep
->vport
;
995 dest
.vport
.vhca_id
= MLX5_CAP_GEN(rep
->esw
->dev
, vhca_id
);
996 dest
.vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
999 if (MLX5_CAP_ESW_FLOWTABLE(on_esw
->dev
, flow_source
) &&
1000 rep
->vport
== MLX5_VPORT_UPLINK
)
1001 spec
->flow_context
.flow_source
= MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT
;
1003 flow_rule
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw
),
1004 spec
, &flow_act
, &dest
, 1);
1005 if (IS_ERR(flow_rule
))
1006 esw_warn(on_esw
->dev
, "FDB: Failed to add send to vport rule err %ld\n",
1007 PTR_ERR(flow_rule
));
1012 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule
);
1014 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle
*rule
)
1016 mlx5_del_flow_rules(rule
);
1019 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle
*rule
)
1022 mlx5_del_flow_rules(rule
);
1025 struct mlx5_flow_handle
*
1026 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch
*esw
, u16 vport_num
)
1028 struct mlx5_flow_destination dest
= {};
1029 struct mlx5_flow_act flow_act
= {0};
1030 struct mlx5_flow_handle
*flow_rule
;
1031 struct mlx5_flow_spec
*spec
;
1033 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1035 return ERR_PTR(-ENOMEM
);
1037 MLX5_SET(fte_match_param
, spec
->match_criteria
,
1038 misc_parameters_2
.metadata_reg_c_0
, mlx5_eswitch_get_vport_metadata_mask());
1039 MLX5_SET(fte_match_param
, spec
->match_criteria
,
1040 misc_parameters_2
.metadata_reg_c_1
, ESW_TUN_MASK
);
1041 MLX5_SET(fte_match_param
, spec
->match_value
, misc_parameters_2
.metadata_reg_c_1
,
1042 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK
);
1044 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1045 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1046 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1048 MLX5_SET(fte_match_param
, spec
->match_value
, misc_parameters_2
.metadata_reg_c_0
,
1049 mlx5_eswitch_get_vport_metadata_for_match(esw
, vport_num
));
1050 dest
.vport
.num
= vport_num
;
1052 flow_rule
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1053 spec
, &flow_act
, &dest
, 1);
1054 if (IS_ERR(flow_rule
))
1055 esw_warn(esw
->dev
, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1056 vport_num
, PTR_ERR(flow_rule
));
1062 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch
*esw
)
1064 return MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, fdb_to_vport_reg_c_id
) &
1065 MLX5_FDB_TO_VPORT_REG_C_1
;
1068 static int esw_set_passing_vport_metadata(struct mlx5_eswitch
*esw
, bool enable
)
1070 u32 out
[MLX5_ST_SZ_DW(query_esw_vport_context_out
)] = {};
1071 u32 min
[MLX5_ST_SZ_DW(modify_esw_vport_context_in
)] = {};
1072 u32 in
[MLX5_ST_SZ_DW(query_esw_vport_context_in
)] = {};
1076 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
) &&
1077 !mlx5_eswitch_vport_match_metadata_enabled(esw
))
1080 MLX5_SET(query_esw_vport_context_in
, in
, opcode
,
1081 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT
);
1082 err
= mlx5_cmd_exec_inout(esw
->dev
, query_esw_vport_context
, in
, out
);
1086 curr
= MLX5_GET(query_esw_vport_context_out
, out
,
1087 esw_vport_context
.fdb_to_vport_reg_c_id
);
1088 wanted
= MLX5_FDB_TO_VPORT_REG_C_0
;
1089 if (mlx5_eswitch_reg_c1_loopback_supported(esw
))
1090 wanted
|= MLX5_FDB_TO_VPORT_REG_C_1
;
1097 MLX5_SET(modify_esw_vport_context_in
, min
,
1098 esw_vport_context
.fdb_to_vport_reg_c_id
, curr
);
1099 MLX5_SET(modify_esw_vport_context_in
, min
,
1100 field_select
.fdb_to_vport_reg_c_id
, 1);
1102 err
= mlx5_eswitch_modify_esw_vport_context(esw
->dev
, 0, false, min
);
1104 if (enable
&& (curr
& MLX5_FDB_TO_VPORT_REG_C_1
))
1105 esw
->flags
|= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
;
1107 esw
->flags
&= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
;
1113 static void peer_miss_rules_setup(struct mlx5_eswitch
*esw
,
1114 struct mlx5_core_dev
*peer_dev
,
1115 struct mlx5_flow_spec
*spec
,
1116 struct mlx5_flow_destination
*dest
)
1120 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1121 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1123 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1124 mlx5_eswitch_get_vport_metadata_mask());
1126 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1128 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1131 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
,
1132 MLX5_CAP_GEN(peer_dev
, vhca_id
));
1134 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
1136 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1138 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
1139 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
1140 source_eswitch_owner_vhca_id
);
1143 dest
->type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1144 dest
->vport
.num
= peer_dev
->priv
.eswitch
->manager_vport
;
1145 dest
->vport
.vhca_id
= MLX5_CAP_GEN(peer_dev
, vhca_id
);
1146 dest
->vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
1149 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch
*esw
,
1150 struct mlx5_eswitch
*peer_esw
,
1151 struct mlx5_flow_spec
*spec
,
1156 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1157 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1159 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1160 mlx5_eswitch_get_vport_metadata_for_match(peer_esw
,
1163 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1165 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
1169 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch
*esw
,
1170 struct mlx5_core_dev
*peer_dev
)
1172 struct mlx5_flow_destination dest
= {};
1173 struct mlx5_flow_act flow_act
= {0};
1174 struct mlx5_flow_handle
**flows
;
1175 /* total vports is the same for both e-switches */
1176 int nvports
= esw
->total_vports
;
1177 struct mlx5_flow_handle
*flow
;
1178 struct mlx5_flow_spec
*spec
;
1179 struct mlx5_vport
*vport
;
1184 if (!MLX5_VPORT_MANAGER(esw
->dev
) && !mlx5_core_is_ecpf_esw_manager(esw
->dev
))
1187 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1191 peer_miss_rules_setup(esw
, peer_dev
, spec
, &dest
);
1193 flows
= kvcalloc(nvports
, sizeof(*flows
), GFP_KERNEL
);
1196 goto alloc_flows_err
;
1199 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1200 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1203 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1204 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1205 esw_set_peer_miss_rule_source_port(esw
, peer_dev
->priv
.eswitch
,
1206 spec
, MLX5_VPORT_PF
);
1208 flow
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1209 spec
, &flow_act
, &dest
, 1);
1211 err
= PTR_ERR(flow
);
1212 goto add_pf_flow_err
;
1214 flows
[vport
->index
] = flow
;
1217 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1218 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1219 MLX5_SET(fte_match_set_misc
, misc
, source_port
, MLX5_VPORT_ECPF
);
1220 flow
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1221 spec
, &flow_act
, &dest
, 1);
1223 err
= PTR_ERR(flow
);
1224 goto add_ecpf_flow_err
;
1226 flows
[vport
->index
] = flow
;
1229 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
)) {
1230 esw_set_peer_miss_rule_source_port(esw
,
1231 peer_dev
->priv
.eswitch
,
1232 spec
, vport
->vport
);
1234 flow
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1235 spec
, &flow_act
, &dest
, 1);
1237 err
= PTR_ERR(flow
);
1238 goto add_vf_flow_err
;
1240 flows
[vport
->index
] = flow
;
1243 if (mlx5_core_ec_sriov_enabled(esw
->dev
)) {
1244 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, mlx5_core_max_ec_vfs(esw
->dev
)) {
1245 if (i
>= mlx5_core_max_ec_vfs(peer_dev
))
1247 esw_set_peer_miss_rule_source_port(esw
, peer_dev
->priv
.eswitch
,
1248 spec
, vport
->vport
);
1249 flow
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1250 spec
, &flow_act
, &dest
, 1);
1252 err
= PTR_ERR(flow
);
1253 goto add_ec_vf_flow_err
;
1255 flows
[vport
->index
] = flow
;
1258 esw
->fdb_table
.offloads
.peer_miss_rules
[mlx5_get_dev_index(peer_dev
)] = flows
;
1264 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, mlx5_core_max_ec_vfs(esw
->dev
)) {
1265 if (!flows
[vport
->index
])
1267 mlx5_del_flow_rules(flows
[vport
->index
]);
1270 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
)) {
1271 if (!flows
[vport
->index
])
1273 mlx5_del_flow_rules(flows
[vport
->index
]);
1275 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1276 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1277 mlx5_del_flow_rules(flows
[vport
->index
]);
1280 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1281 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1282 mlx5_del_flow_rules(flows
[vport
->index
]);
1285 esw_warn(esw
->dev
, "FDB: Failed to add peer miss flow rule err %d\n", err
);
1292 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch
*esw
,
1293 struct mlx5_core_dev
*peer_dev
)
1295 u16 peer_index
= mlx5_get_dev_index(peer_dev
);
1296 struct mlx5_flow_handle
**flows
;
1297 struct mlx5_vport
*vport
;
1300 flows
= esw
->fdb_table
.offloads
.peer_miss_rules
[peer_index
];
1304 if (mlx5_core_ec_sriov_enabled(esw
->dev
)) {
1305 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, mlx5_core_max_ec_vfs(esw
->dev
)) {
1306 /* The flow for a particular vport could be NULL if the other ECPF
1307 * has fewer or no VFs enabled
1309 if (!flows
[vport
->index
])
1311 mlx5_del_flow_rules(flows
[vport
->index
]);
1315 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
))
1316 mlx5_del_flow_rules(flows
[vport
->index
]);
1318 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1319 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1320 mlx5_del_flow_rules(flows
[vport
->index
]);
1323 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1324 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1325 mlx5_del_flow_rules(flows
[vport
->index
]);
1329 esw
->fdb_table
.offloads
.peer_miss_rules
[peer_index
] = NULL
;
1332 static int esw_add_fdb_miss_rule(struct mlx5_eswitch
*esw
)
1334 struct mlx5_flow_act flow_act
= {0};
1335 struct mlx5_flow_destination dest
= {};
1336 struct mlx5_flow_handle
*flow_rule
= NULL
;
1337 struct mlx5_flow_spec
*spec
;
1344 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1350 spec
->match_criteria_enable
= MLX5_MATCH_OUTER_HEADERS
;
1351 headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1353 dmac_c
= MLX5_ADDR_OF(fte_match_param
, headers_c
,
1354 outer_headers
.dmac_47_16
);
1357 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1358 dest
.vport
.num
= esw
->manager_vport
;
1359 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1361 flow_rule
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1362 spec
, &flow_act
, &dest
, 1);
1363 if (IS_ERR(flow_rule
)) {
1364 err
= PTR_ERR(flow_rule
);
1365 esw_warn(esw
->dev
, "FDB: Failed to add unicast miss flow rule err %d\n", err
);
1369 esw
->fdb_table
.offloads
.miss_rule_uni
= flow_rule
;
1371 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1373 dmac_v
= MLX5_ADDR_OF(fte_match_param
, headers_v
,
1374 outer_headers
.dmac_47_16
);
1376 flow_rule
= mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw
),
1377 spec
, &flow_act
, &dest
, 1);
1378 if (IS_ERR(flow_rule
)) {
1379 err
= PTR_ERR(flow_rule
);
1380 esw_warn(esw
->dev
, "FDB: Failed to add multicast miss flow rule err %d\n", err
);
1381 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_uni
);
1385 esw
->fdb_table
.offloads
.miss_rule_multi
= flow_rule
;
1392 struct mlx5_flow_handle
*
1393 esw_add_restore_rule(struct mlx5_eswitch
*esw
, u32 tag
)
1395 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
1396 struct mlx5_flow_table
*ft
= esw
->offloads
.ft_offloads_restore
;
1397 struct mlx5_flow_context
*flow_context
;
1398 struct mlx5_flow_handle
*flow_rule
;
1399 struct mlx5_flow_destination dest
;
1400 struct mlx5_flow_spec
*spec
;
1403 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
1404 return ERR_PTR(-EOPNOTSUPP
);
1406 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1408 return ERR_PTR(-ENOMEM
);
1410 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1412 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1413 ESW_REG_C0_USER_DATA_METADATA_MASK
);
1414 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1416 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
, tag
);
1417 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1418 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
1419 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
1420 flow_act
.modify_hdr
= esw
->offloads
.restore_copy_hdr_id
;
1422 flow_context
= &spec
->flow_context
;
1423 flow_context
->flags
|= FLOW_CONTEXT_HAS_TAG
;
1424 flow_context
->flow_tag
= tag
;
1425 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
1426 dest
.ft
= esw
->offloads
.ft_offloads
;
1428 flow_rule
= mlx5_add_flow_rules(ft
, spec
, &flow_act
, &dest
, 1);
1431 if (IS_ERR(flow_rule
))
1433 "Failed to create restore rule for tag: %d, err(%d)\n",
1434 tag
, (int)PTR_ERR(flow_rule
));
1439 #define MAX_PF_SQ 256
1440 #define MAX_SQ_NVPORTS 32
1443 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch
*esw
,
1447 void *match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
1451 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1452 MLX5_SET(create_flow_group_in
, flow_group_in
,
1453 match_criteria_enable
,
1454 MLX5_MATCH_MISC_PARAMETERS_2
| match_params
);
1456 MLX5_SET(fte_match_param
, match_criteria
,
1457 misc_parameters_2
.metadata_reg_c_0
,
1458 mlx5_eswitch_get_vport_metadata_mask());
1460 MLX5_SET(create_flow_group_in
, flow_group_in
,
1461 match_criteria_enable
,
1462 MLX5_MATCH_MISC_PARAMETERS
| match_params
);
1464 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1465 misc_parameters
.source_port
);
1469 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1470 static void esw_vport_tbl_put(struct mlx5_eswitch
*esw
)
1472 struct mlx5_vport_tbl_attr attr
;
1473 struct mlx5_vport
*vport
;
1478 mlx5_esw_for_each_vport(esw
, i
, vport
) {
1479 attr
.vport
= vport
->vport
;
1480 attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
1481 mlx5_esw_vporttbl_put(esw
, &attr
);
1485 static int esw_vport_tbl_get(struct mlx5_eswitch
*esw
)
1487 struct mlx5_vport_tbl_attr attr
;
1488 struct mlx5_flow_table
*fdb
;
1489 struct mlx5_vport
*vport
;
1494 mlx5_esw_for_each_vport(esw
, i
, vport
) {
1495 attr
.vport
= vport
->vport
;
1496 attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
1497 fdb
= mlx5_esw_vporttbl_get(esw
, &attr
);
1504 esw_vport_tbl_put(esw
);
1505 return PTR_ERR(fdb
);
1508 #define fdb_modify_header_fwd_to_table_supported(esw) \
1509 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1510 static void esw_init_chains_offload_flags(struct mlx5_eswitch
*esw
, u32
*flags
)
1512 struct mlx5_core_dev
*dev
= esw
->dev
;
1514 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, ignore_flow_level
))
1515 *flags
|= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED
;
1517 if (!MLX5_CAP_ESW_FLOWTABLE(dev
, multi_fdb_encap
) &&
1518 esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
) {
1519 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1520 esw_warn(dev
, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1521 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw
)) {
1522 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1523 esw_warn(dev
, "Tc chains and priorities offload aren't supported\n");
1524 } else if (!fdb_modify_header_fwd_to_table_supported(esw
)) {
1525 /* Disabled when ttl workaround is needed, e.g
1526 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1529 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1530 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1532 *flags
|= MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1533 esw_info(dev
, "Supported tc chains and prios offload\n");
1536 if (esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
)
1537 *flags
|= MLX5_CHAINS_FT_TUNNEL_SUPPORTED
;
1541 esw_chains_create(struct mlx5_eswitch
*esw
, struct mlx5_flow_table
*miss_fdb
)
1543 struct mlx5_core_dev
*dev
= esw
->dev
;
1544 struct mlx5_flow_table
*nf_ft
, *ft
;
1545 struct mlx5_chains_attr attr
= {};
1546 struct mlx5_fs_chains
*chains
;
1549 esw_init_chains_offload_flags(esw
, &attr
.flags
);
1550 attr
.ns
= MLX5_FLOW_NAMESPACE_FDB
;
1551 attr
.max_grp_num
= esw
->params
.large_group_num
;
1552 attr
.default_ft
= miss_fdb
;
1553 attr
.mapping
= esw
->offloads
.reg_c0_obj_pool
;
1555 chains
= mlx5_chains_create(dev
, &attr
);
1556 if (IS_ERR(chains
)) {
1557 err
= PTR_ERR(chains
);
1558 esw_warn(dev
, "Failed to create fdb chains err(%d)\n", err
);
1561 mlx5_chains_print_info(chains
);
1563 esw
->fdb_table
.offloads
.esw_chains_priv
= chains
;
1565 /* Create tc_end_ft which is the always created ft chain */
1566 nf_ft
= mlx5_chains_get_table(chains
, mlx5_chains_get_nf_ft_chain(chains
),
1568 if (IS_ERR(nf_ft
)) {
1569 err
= PTR_ERR(nf_ft
);
1573 /* Always open the root for fast path */
1574 ft
= mlx5_chains_get_table(chains
, 0, 1, 0);
1580 /* Open level 1 for split fdb rules now if prios isn't supported */
1581 if (!mlx5_chains_prios_supported(chains
)) {
1582 err
= esw_vport_tbl_get(esw
);
1587 mlx5_chains_set_end_ft(chains
, nf_ft
);
1592 mlx5_chains_put_table(chains
, 0, 1, 0);
1594 mlx5_chains_put_table(chains
, mlx5_chains_get_nf_ft_chain(chains
), 1, 0);
1596 mlx5_chains_destroy(chains
);
1597 esw
->fdb_table
.offloads
.esw_chains_priv
= NULL
;
1603 esw_chains_destroy(struct mlx5_eswitch
*esw
, struct mlx5_fs_chains
*chains
)
1605 if (!mlx5_chains_prios_supported(chains
))
1606 esw_vport_tbl_put(esw
);
1607 mlx5_chains_put_table(chains
, 0, 1, 0);
1608 mlx5_chains_put_table(chains
, mlx5_chains_get_nf_ft_chain(chains
), 1, 0);
1609 mlx5_chains_destroy(chains
);
1612 #else /* CONFIG_MLX5_CLS_ACT */
1615 esw_chains_create(struct mlx5_eswitch
*esw
, struct mlx5_flow_table
*miss_fdb
)
1619 esw_chains_destroy(struct mlx5_eswitch
*esw
, struct mlx5_fs_chains
*chains
)
1625 esw_create_send_to_vport_group(struct mlx5_eswitch
*esw
,
1626 struct mlx5_flow_table
*fdb
,
1630 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1631 struct mlx5_flow_group
*g
;
1632 void *match_criteria
;
1635 memset(flow_group_in
, 0, inlen
);
1637 mlx5_esw_set_flow_group_source_port(esw
, flow_group_in
, MLX5_MATCH_MISC_PARAMETERS
);
1639 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
1640 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_sqn
);
1642 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
) &&
1643 MLX5_CAP_ESW(esw
->dev
, merged_eswitch
)) {
1644 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1645 misc_parameters
.source_eswitch_owner_vhca_id
);
1646 MLX5_SET(create_flow_group_in
, flow_group_in
,
1647 source_eswitch_owner_vhca_id_valid
, 1);
1650 /* See comment at table_size calculation */
1651 count
= MLX5_MAX_PORTS
* (esw
->total_vports
* MAX_SQ_NVPORTS
+ MAX_PF_SQ
);
1652 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
1653 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, *ix
+ count
- 1);
1656 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1659 esw_warn(esw
->dev
, "Failed to create send-to-vport flow group err(%d)\n", err
);
1662 esw
->fdb_table
.offloads
.send_to_vport_grp
= g
;
1669 esw_create_meta_send_to_vport_group(struct mlx5_eswitch
*esw
,
1670 struct mlx5_flow_table
*fdb
,
1674 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1675 struct mlx5_flow_group
*g
;
1676 void *match_criteria
;
1679 if (!esw_src_port_rewrite_supported(esw
))
1682 memset(flow_group_in
, 0, inlen
);
1684 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
1685 MLX5_MATCH_MISC_PARAMETERS_2
);
1687 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
1689 MLX5_SET(fte_match_param
, match_criteria
,
1690 misc_parameters_2
.metadata_reg_c_0
,
1691 mlx5_eswitch_get_vport_metadata_mask());
1692 MLX5_SET(fte_match_param
, match_criteria
,
1693 misc_parameters_2
.metadata_reg_c_1
, ESW_TUN_MASK
);
1695 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, *ix
);
1696 MLX5_SET(create_flow_group_in
, flow_group_in
,
1697 end_flow_index
, *ix
+ esw
->total_vports
- 1);
1698 *ix
+= esw
->total_vports
;
1700 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1704 "Failed to create send-to-vport meta flow group err(%d)\n", err
);
1705 goto send_vport_meta_err
;
1707 esw
->fdb_table
.offloads
.send_to_vport_meta_grp
= g
;
1711 send_vport_meta_err
:
1716 esw_create_peer_esw_miss_group(struct mlx5_eswitch
*esw
,
1717 struct mlx5_flow_table
*fdb
,
1721 int max_peer_ports
= (esw
->total_vports
- 1) * (MLX5_MAX_PORTS
- 1);
1722 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1723 struct mlx5_flow_group
*g
;
1724 void *match_criteria
;
1727 if (!MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
1730 memset(flow_group_in
, 0, inlen
);
1732 mlx5_esw_set_flow_group_source_port(esw
, flow_group_in
, 0);
1734 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1735 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
1739 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1740 misc_parameters
.source_eswitch_owner_vhca_id
);
1742 MLX5_SET(create_flow_group_in
, flow_group_in
,
1743 source_eswitch_owner_vhca_id_valid
, 1);
1746 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, *ix
);
1747 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
1748 *ix
+ max_peer_ports
);
1749 *ix
+= max_peer_ports
+ 1;
1751 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1754 esw_warn(esw
->dev
, "Failed to create peer miss flow group err(%d)\n", err
);
1757 esw
->fdb_table
.offloads
.peer_miss_grp
= g
;
1764 esw_create_miss_group(struct mlx5_eswitch
*esw
,
1765 struct mlx5_flow_table
*fdb
,
1769 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1770 struct mlx5_flow_group
*g
;
1771 void *match_criteria
;
1775 memset(flow_group_in
, 0, inlen
);
1777 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
1778 MLX5_MATCH_OUTER_HEADERS
);
1779 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
1781 dmac
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
1782 outer_headers
.dmac_47_16
);
1785 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, *ix
);
1786 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
1787 *ix
+ MLX5_ESW_MISS_FLOWS
);
1789 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1792 esw_warn(esw
->dev
, "Failed to create miss flow group err(%d)\n", err
);
1795 esw
->fdb_table
.offloads
.miss_grp
= g
;
1797 err
= esw_add_fdb_miss_rule(esw
);
1804 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
1809 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch
*esw
)
1811 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1812 struct mlx5_flow_table_attr ft_attr
= {};
1813 struct mlx5_core_dev
*dev
= esw
->dev
;
1814 struct mlx5_flow_namespace
*root_ns
;
1815 struct mlx5_flow_table
*fdb
= NULL
;
1816 int table_size
, ix
= 0, err
= 0;
1817 u32 flags
= 0, *flow_group_in
;
1819 esw_debug(esw
->dev
, "Create offloads FDB Tables\n");
1821 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
1825 root_ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_FDB
);
1827 esw_warn(dev
, "Failed to get FDB flow namespace\n");
1831 esw
->fdb_table
.offloads
.ns
= root_ns
;
1832 err
= mlx5_flow_namespace_set_mode(root_ns
,
1833 esw
->dev
->priv
.steering
->mode
);
1835 esw_warn(dev
, "Failed to set FDB namespace steering mode\n");
1839 /* To be strictly correct:
1840 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1842 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1843 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1844 * but as the peer device might not be in switchdev mode it's not
1845 * possible. We use the fact that by default FW sets max vfs and max sfs
1846 * to the same value on both devices. If it needs to be changed in the future note
1847 * the peer miss group should also be created based on the number of
1848 * total vports of the peer (currently is also uses esw->total_vports).
1850 table_size
= MLX5_MAX_PORTS
* (esw
->total_vports
* MAX_SQ_NVPORTS
+ MAX_PF_SQ
) +
1851 esw
->total_vports
* MLX5_MAX_PORTS
+ MLX5_ESW_MISS_FLOWS
;
1853 /* create the slow path fdb with encap set, so further table instances
1854 * can be created at run time while VFs are probed if the FW allows that.
1856 if (esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
)
1857 flags
|= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
|
1858 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP
);
1860 ft_attr
.flags
= flags
;
1861 ft_attr
.max_fte
= table_size
;
1862 ft_attr
.prio
= FDB_SLOW_PATH
;
1864 fdb
= mlx5_create_flow_table(root_ns
, &ft_attr
);
1867 esw_warn(dev
, "Failed to create slow path FDB Table err %d\n", err
);
1870 esw
->fdb_table
.offloads
.slow_fdb
= fdb
;
1872 /* Create empty TC-miss managed table. This allows plugging in following
1873 * priorities without directly exposing their level 0 table to
1874 * eswitch_offloads and passing it as miss_fdb to following call to
1875 * esw_chains_create().
1877 memset(&ft_attr
, 0, sizeof(ft_attr
));
1878 ft_attr
.prio
= FDB_TC_MISS
;
1879 esw
->fdb_table
.offloads
.tc_miss_table
= mlx5_create_flow_table(root_ns
, &ft_attr
);
1880 if (IS_ERR(esw
->fdb_table
.offloads
.tc_miss_table
)) {
1881 err
= PTR_ERR(esw
->fdb_table
.offloads
.tc_miss_table
);
1882 esw_warn(dev
, "Failed to create TC miss FDB Table err %d\n", err
);
1883 goto tc_miss_table_err
;
1886 err
= esw_chains_create(esw
, esw
->fdb_table
.offloads
.tc_miss_table
);
1888 esw_warn(dev
, "Failed to open fdb chains err(%d)\n", err
);
1889 goto fdb_chains_err
;
1892 err
= esw_create_send_to_vport_group(esw
, fdb
, flow_group_in
, &ix
);
1894 goto send_vport_err
;
1896 err
= esw_create_meta_send_to_vport_group(esw
, fdb
, flow_group_in
, &ix
);
1898 goto send_vport_meta_err
;
1900 err
= esw_create_peer_esw_miss_group(esw
, fdb
, flow_group_in
, &ix
);
1904 err
= esw_create_miss_group(esw
, fdb
, flow_group_in
, &ix
);
1908 kvfree(flow_group_in
);
1912 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
1913 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.peer_miss_grp
);
1915 if (esw
->fdb_table
.offloads
.send_to_vport_meta_grp
)
1916 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_meta_grp
);
1917 send_vport_meta_err
:
1918 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
1920 esw_chains_destroy(esw
, esw_chains(esw
));
1922 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.tc_miss_table
);
1924 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw
));
1926 /* Holds true only as long as DMFS is the default */
1927 mlx5_flow_namespace_set_mode(root_ns
, MLX5_FLOW_STEERING_MODE_DMFS
);
1929 kvfree(flow_group_in
);
1933 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch
*esw
)
1935 if (!mlx5_eswitch_get_slow_fdb(esw
))
1938 esw_debug(esw
->dev
, "Destroy offloads FDB Tables\n");
1939 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_multi
);
1940 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_uni
);
1941 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
1942 if (esw
->fdb_table
.offloads
.send_to_vport_meta_grp
)
1943 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_meta_grp
);
1944 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
1945 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.peer_miss_grp
);
1946 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
1948 esw_chains_destroy(esw
, esw_chains(esw
));
1950 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.tc_miss_table
);
1951 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw
));
1952 /* Holds true only as long as DMFS is the default */
1953 mlx5_flow_namespace_set_mode(esw
->fdb_table
.offloads
.ns
,
1954 MLX5_FLOW_STEERING_MODE_DMFS
);
1955 atomic64_set(&esw
->user_count
, 0);
1958 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch
*esw
)
1962 nvports
= esw
->total_vports
+ MLX5_ESW_MISS_FLOWS
;
1963 if (mlx5e_tc_int_port_supported(esw
))
1964 nvports
+= MLX5E_TC_MAX_INT_PORT_NUM
;
1969 static int esw_create_offloads_table(struct mlx5_eswitch
*esw
)
1971 struct mlx5_flow_table_attr ft_attr
= {};
1972 struct mlx5_core_dev
*dev
= esw
->dev
;
1973 struct mlx5_flow_table
*ft_offloads
;
1974 struct mlx5_flow_namespace
*ns
;
1977 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
1979 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
1983 ft_attr
.max_fte
= esw_get_nr_ft_offloads_steering_src_ports(esw
) +
1984 MLX5_ESW_FT_OFFLOADS_DROP_RULE
;
1987 ft_offloads
= mlx5_create_flow_table(ns
, &ft_attr
);
1988 if (IS_ERR(ft_offloads
)) {
1989 err
= PTR_ERR(ft_offloads
);
1990 esw_warn(esw
->dev
, "Failed to create offloads table, err %d\n", err
);
1994 esw
->offloads
.ft_offloads
= ft_offloads
;
1998 static void esw_destroy_offloads_table(struct mlx5_eswitch
*esw
)
2000 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
2002 mlx5_destroy_flow_table(offloads
->ft_offloads
);
2005 static int esw_create_vport_rx_group(struct mlx5_eswitch
*esw
)
2007 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2008 struct mlx5_flow_group
*g
;
2013 nvports
= esw_get_nr_ft_offloads_steering_src_ports(esw
);
2014 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2018 mlx5_esw_set_flow_group_source_port(esw
, flow_group_in
, 0);
2020 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
2021 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, nvports
- 1);
2023 g
= mlx5_create_flow_group(esw
->offloads
.ft_offloads
, flow_group_in
);
2027 mlx5_core_warn(esw
->dev
, "Failed to create vport rx group err %d\n", err
);
2031 esw
->offloads
.vport_rx_group
= g
;
2033 kvfree(flow_group_in
);
2037 static void esw_destroy_vport_rx_group(struct mlx5_eswitch
*esw
)
2039 mlx5_destroy_flow_group(esw
->offloads
.vport_rx_group
);
2042 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch
*esw
)
2044 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2045 * for the drop rule, which is placed at the end of the table.
2046 * So return the total of vport and int_port as rule index.
2048 return esw_get_nr_ft_offloads_steering_src_ports(esw
);
2051 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch
*esw
)
2053 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2054 struct mlx5_flow_group
*g
;
2059 flow_index
= esw_create_vport_rx_drop_rule_index(esw
);
2061 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2065 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, flow_index
);
2066 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, flow_index
);
2068 g
= mlx5_create_flow_group(esw
->offloads
.ft_offloads
, flow_group_in
);
2072 mlx5_core_warn(esw
->dev
, "Failed to create vport rx drop group err %d\n", err
);
2076 esw
->offloads
.vport_rx_drop_group
= g
;
2078 kvfree(flow_group_in
);
2082 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch
*esw
)
2084 if (esw
->offloads
.vport_rx_drop_group
)
2085 mlx5_destroy_flow_group(esw
->offloads
.vport_rx_drop_group
);
2089 mlx5_esw_set_spec_source_port(struct mlx5_eswitch
*esw
,
2091 struct mlx5_flow_spec
*spec
)
2095 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
2096 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
2097 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
2098 mlx5_eswitch_get_vport_metadata_for_match(esw
, vport
));
2100 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
2101 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
2102 mlx5_eswitch_get_vport_metadata_mask());
2104 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
2106 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
2107 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
2109 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
2110 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
2112 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
2116 struct mlx5_flow_handle
*
2117 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch
*esw
, u16 vport
,
2118 struct mlx5_flow_destination
*dest
)
2120 struct mlx5_flow_act flow_act
= {0};
2121 struct mlx5_flow_handle
*flow_rule
;
2122 struct mlx5_flow_spec
*spec
;
2124 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
2126 flow_rule
= ERR_PTR(-ENOMEM
);
2130 mlx5_esw_set_spec_source_port(esw
, vport
, spec
);
2132 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2133 flow_rule
= mlx5_add_flow_rules(esw
->offloads
.ft_offloads
, spec
,
2134 &flow_act
, dest
, 1);
2135 if (IS_ERR(flow_rule
)) {
2136 esw_warn(esw
->dev
, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule
));
2145 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch
*esw
)
2147 struct mlx5_flow_act flow_act
= {};
2148 struct mlx5_flow_handle
*flow_rule
;
2150 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_DROP
;
2151 flow_rule
= mlx5_add_flow_rules(esw
->offloads
.ft_offloads
, NULL
,
2152 &flow_act
, NULL
, 0);
2153 if (IS_ERR(flow_rule
)) {
2155 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2156 PTR_ERR(flow_rule
));
2157 return PTR_ERR(flow_rule
);
2160 esw
->offloads
.vport_rx_drop_rule
= flow_rule
;
2165 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch
*esw
)
2167 if (esw
->offloads
.vport_rx_drop_rule
)
2168 mlx5_del_flow_rules(esw
->offloads
.vport_rx_drop_rule
);
2171 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch
*esw
, u8
*mode
)
2173 u8 prev_mlx5_mode
, mlx5_mode
= MLX5_INLINE_MODE_L2
;
2174 struct mlx5_core_dev
*dev
= esw
->dev
;
2175 struct mlx5_vport
*vport
;
2178 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
2181 if (!mlx5_esw_is_fdb_created(esw
))
2184 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
2185 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
2186 mlx5_mode
= MLX5_INLINE_MODE_NONE
;
2188 case MLX5_CAP_INLINE_MODE_L2
:
2189 mlx5_mode
= MLX5_INLINE_MODE_L2
;
2191 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
2196 mlx5_query_nic_vport_min_inline(dev
, esw
->first_host_vport
, &prev_mlx5_mode
);
2197 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
2198 mlx5_query_nic_vport_min_inline(dev
, vport
->vport
, &mlx5_mode
);
2199 if (prev_mlx5_mode
!= mlx5_mode
)
2201 prev_mlx5_mode
= mlx5_mode
;
2209 static void esw_destroy_restore_table(struct mlx5_eswitch
*esw
)
2211 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
2213 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
2216 mlx5_modify_header_dealloc(esw
->dev
, offloads
->restore_copy_hdr_id
);
2217 mlx5_destroy_flow_group(offloads
->restore_group
);
2218 mlx5_destroy_flow_table(offloads
->ft_offloads_restore
);
2221 static int esw_create_restore_table(struct mlx5_eswitch
*esw
)
2223 u8 modact
[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto
)] = {};
2224 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2225 struct mlx5_flow_table_attr ft_attr
= {};
2226 struct mlx5_core_dev
*dev
= esw
->dev
;
2227 struct mlx5_flow_namespace
*ns
;
2228 struct mlx5_modify_hdr
*mod_hdr
;
2229 void *match_criteria
, *misc
;
2230 struct mlx5_flow_table
*ft
;
2231 struct mlx5_flow_group
*g
;
2235 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
2238 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
2240 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
2244 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2245 if (!flow_group_in
) {
2250 ft_attr
.max_fte
= 1 << ESW_REG_C0_USER_DATA_METADATA_BITS
;
2251 ft
= mlx5_create_flow_table(ns
, &ft_attr
);
2254 esw_warn(esw
->dev
, "Failed to create restore table, err %d\n",
2259 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
2261 misc
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
2264 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
2265 ESW_REG_C0_USER_DATA_METADATA_MASK
);
2266 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
2267 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
2268 ft_attr
.max_fte
- 1);
2269 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
2270 MLX5_MATCH_MISC_PARAMETERS_2
);
2271 g
= mlx5_create_flow_group(ft
, flow_group_in
);
2274 esw_warn(dev
, "Failed to create restore flow group, err: %d\n",
2279 MLX5_SET(copy_action_in
, modact
, action_type
, MLX5_ACTION_TYPE_COPY
);
2280 MLX5_SET(copy_action_in
, modact
, src_field
,
2281 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1
);
2282 MLX5_SET(copy_action_in
, modact
, dst_field
,
2283 MLX5_ACTION_IN_FIELD_METADATA_REG_B
);
2284 mod_hdr
= mlx5_modify_header_alloc(esw
->dev
,
2285 MLX5_FLOW_NAMESPACE_KERNEL
, 1,
2287 if (IS_ERR(mod_hdr
)) {
2288 err
= PTR_ERR(mod_hdr
);
2289 esw_warn(dev
, "Failed to create restore mod header, err: %d\n",
2294 esw
->offloads
.ft_offloads_restore
= ft
;
2295 esw
->offloads
.restore_group
= g
;
2296 esw
->offloads
.restore_copy_hdr_id
= mod_hdr
;
2298 kvfree(flow_group_in
);
2303 mlx5_destroy_flow_group(g
);
2305 mlx5_destroy_flow_table(ft
);
2307 kvfree(flow_group_in
);
2312 static int esw_offloads_start(struct mlx5_eswitch
*esw
,
2313 struct netlink_ext_ack
*extack
)
2317 esw
->mode
= MLX5_ESWITCH_OFFLOADS
;
2318 err
= mlx5_eswitch_enable_locked(esw
, esw
->dev
->priv
.sriov
.num_vfs
);
2320 NL_SET_ERR_MSG_MOD(extack
,
2321 "Failed setting eswitch to offloads");
2322 esw
->mode
= MLX5_ESWITCH_LEGACY
;
2323 mlx5_rescan_drivers(esw
->dev
);
2326 if (esw
->offloads
.inline_mode
== MLX5_INLINE_MODE_NONE
) {
2327 if (mlx5_eswitch_inline_mode_get(esw
,
2328 &esw
->offloads
.inline_mode
)) {
2329 esw
->offloads
.inline_mode
= MLX5_INLINE_MODE_L2
;
2330 NL_SET_ERR_MSG_MOD(extack
,
2331 "Inline mode is different between vports");
2337 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch
*esw
, const struct mlx5_vport
*vport
)
2339 struct mlx5_eswitch_rep
*rep
;
2343 rep
= kzalloc(sizeof(*rep
), GFP_KERNEL
);
2347 rep
->vport
= vport
->vport
;
2348 rep
->vport_index
= vport
->index
;
2349 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++)
2350 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_UNREGISTERED
);
2352 err
= xa_insert(&esw
->offloads
.vport_reps
, rep
->vport
, rep
, GFP_KERNEL
);
2363 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch
*esw
,
2364 struct mlx5_eswitch_rep
*rep
)
2366 xa_erase(&esw
->offloads
.vport_reps
, rep
->vport
);
2370 static void esw_offloads_cleanup_reps(struct mlx5_eswitch
*esw
)
2372 struct mlx5_eswitch_rep
*rep
;
2375 mlx5_esw_for_each_rep(esw
, i
, rep
)
2376 mlx5_esw_offloads_rep_cleanup(esw
, rep
);
2377 xa_destroy(&esw
->offloads
.vport_reps
);
2380 static int esw_offloads_init_reps(struct mlx5_eswitch
*esw
)
2382 struct mlx5_vport
*vport
;
2386 xa_init(&esw
->offloads
.vport_reps
);
2388 mlx5_esw_for_each_vport(esw
, i
, vport
) {
2389 err
= mlx5_esw_offloads_rep_init(esw
, vport
);
2396 esw_offloads_cleanup_reps(esw
);
2400 static int esw_port_metadata_set(struct devlink
*devlink
, u32 id
,
2401 struct devlink_param_gset_ctx
*ctx
)
2403 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
2404 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
2407 down_write(&esw
->mode_lock
);
2408 if (mlx5_esw_is_fdb_created(esw
)) {
2412 if (!mlx5_esw_vport_match_metadata_supported(esw
)) {
2417 esw
->flags
|= MLX5_ESWITCH_VPORT_MATCH_METADATA
;
2419 esw
->flags
&= ~MLX5_ESWITCH_VPORT_MATCH_METADATA
;
2421 up_write(&esw
->mode_lock
);
2425 static int esw_port_metadata_get(struct devlink
*devlink
, u32 id
,
2426 struct devlink_param_gset_ctx
*ctx
)
2428 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
2430 ctx
->val
.vbool
= mlx5_eswitch_vport_match_metadata_enabled(dev
->priv
.eswitch
);
2434 static int esw_port_metadata_validate(struct devlink
*devlink
, u32 id
,
2435 union devlink_param_value val
,
2436 struct netlink_ext_ack
*extack
)
2438 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
2441 esw_mode
= mlx5_eswitch_mode(dev
);
2442 if (esw_mode
== MLX5_ESWITCH_OFFLOADS
) {
2443 NL_SET_ERR_MSG_MOD(extack
,
2444 "E-Switch must either disabled or non switchdev mode");
2450 static const struct devlink_param esw_devlink_params
[] = {
2451 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA
,
2452 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL
,
2453 BIT(DEVLINK_PARAM_CMODE_RUNTIME
),
2454 esw_port_metadata_get
,
2455 esw_port_metadata_set
,
2456 esw_port_metadata_validate
),
2459 int esw_offloads_init(struct mlx5_eswitch
*esw
)
2463 err
= esw_offloads_init_reps(esw
);
2467 err
= devl_params_register(priv_to_devlink(esw
->dev
),
2469 ARRAY_SIZE(esw_devlink_params
));
2476 esw_offloads_cleanup_reps(esw
);
2480 void esw_offloads_cleanup(struct mlx5_eswitch
*esw
)
2482 devl_params_unregister(priv_to_devlink(esw
->dev
),
2484 ARRAY_SIZE(esw_devlink_params
));
2485 esw_offloads_cleanup_reps(esw
);
2488 static void __esw_offloads_unload_rep(struct mlx5_eswitch
*esw
,
2489 struct mlx5_eswitch_rep
*rep
, u8 rep_type
)
2491 if (atomic_cmpxchg(&rep
->rep_data
[rep_type
].state
,
2492 REP_LOADED
, REP_REGISTERED
) == REP_LOADED
)
2493 esw
->offloads
.rep_ops
[rep_type
]->unload(rep
);
2496 static void __unload_reps_all_vport(struct mlx5_eswitch
*esw
, u8 rep_type
)
2498 struct mlx5_eswitch_rep
*rep
;
2501 mlx5_esw_for_each_rep(esw
, i
, rep
)
2502 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2505 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch
*esw
, u16 vport_num
)
2507 struct mlx5_eswitch_rep
*rep
;
2511 rep
= mlx5_eswitch_get_rep(esw
, vport_num
);
2512 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++)
2513 if (atomic_cmpxchg(&rep
->rep_data
[rep_type
].state
,
2514 REP_REGISTERED
, REP_LOADED
) == REP_REGISTERED
) {
2515 err
= esw
->offloads
.rep_ops
[rep_type
]->load(esw
->dev
, rep
);
2523 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_REGISTERED
);
2524 for (--rep_type
; rep_type
>= 0; rep_type
--)
2525 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2529 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch
*esw
, u16 vport_num
)
2531 struct mlx5_eswitch_rep
*rep
;
2534 rep
= mlx5_eswitch_get_rep(esw
, vport_num
);
2535 for (rep_type
= NUM_REP_TYPES
- 1; rep_type
>= 0; rep_type
--)
2536 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2539 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
)
2541 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2544 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw
, vport
);
2547 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
)
2549 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2552 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw
, vport
);
2555 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
,
2556 struct mlx5_devlink_port
*dl_port
,
2557 u32 controller
, u32 sfnum
)
2559 return mlx5_esw_offloads_sf_devlink_port_init(esw
, vport
, dl_port
, controller
, sfnum
);
2562 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
)
2564 mlx5_esw_offloads_sf_devlink_port_cleanup(esw
, vport
);
2567 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
)
2571 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2574 err
= mlx5_esw_offloads_devlink_port_register(esw
, vport
);
2578 err
= mlx5_esw_offloads_rep_load(esw
, vport
->vport
);
2584 mlx5_esw_offloads_devlink_port_unregister(esw
, vport
);
2588 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch
*esw
, struct mlx5_vport
*vport
)
2590 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2593 mlx5_esw_offloads_rep_unload(esw
, vport
->vport
);
2595 mlx5_esw_offloads_devlink_port_unregister(esw
, vport
);
2598 static int esw_set_slave_root_fdb(struct mlx5_core_dev
*master
,
2599 struct mlx5_core_dev
*slave
)
2601 u32 in
[MLX5_ST_SZ_DW(set_flow_table_root_in
)] = {};
2602 u32 out
[MLX5_ST_SZ_DW(set_flow_table_root_out
)] = {};
2603 struct mlx5_flow_root_namespace
*root
;
2604 struct mlx5_flow_namespace
*ns
;
2607 MLX5_SET(set_flow_table_root_in
, in
, opcode
,
2608 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT
);
2609 MLX5_SET(set_flow_table_root_in
, in
, table_type
,
2613 ns
= mlx5_get_flow_namespace(master
,
2614 MLX5_FLOW_NAMESPACE_FDB
);
2615 root
= find_root(&ns
->node
);
2616 mutex_lock(&root
->chain_lock
);
2617 MLX5_SET(set_flow_table_root_in
, in
,
2618 table_eswitch_owner_vhca_id_valid
, 1);
2619 MLX5_SET(set_flow_table_root_in
, in
,
2620 table_eswitch_owner_vhca_id
,
2621 MLX5_CAP_GEN(master
, vhca_id
));
2622 MLX5_SET(set_flow_table_root_in
, in
, table_id
,
2625 ns
= mlx5_get_flow_namespace(slave
,
2626 MLX5_FLOW_NAMESPACE_FDB
);
2627 root
= find_root(&ns
->node
);
2628 mutex_lock(&root
->chain_lock
);
2629 MLX5_SET(set_flow_table_root_in
, in
, table_id
,
2633 err
= mlx5_cmd_exec(slave
, in
, sizeof(in
), out
, sizeof(out
));
2634 mutex_unlock(&root
->chain_lock
);
2639 static int __esw_set_master_egress_rule(struct mlx5_core_dev
*master
,
2640 struct mlx5_core_dev
*slave
,
2641 struct mlx5_vport
*vport
,
2642 struct mlx5_flow_table
*acl
)
2644 u16 slave_index
= MLX5_CAP_GEN(slave
, vhca_id
);
2645 struct mlx5_flow_handle
*flow_rule
= NULL
;
2646 struct mlx5_flow_destination dest
= {};
2647 struct mlx5_flow_act flow_act
= {};
2648 struct mlx5_flow_spec
*spec
;
2652 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
2656 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
2657 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
2659 MLX5_SET(fte_match_set_misc
, misc
, source_port
, MLX5_VPORT_UPLINK
);
2660 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
, slave_index
);
2662 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
2663 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
2664 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
2665 source_eswitch_owner_vhca_id
);
2667 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2668 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
2669 dest
.vport
.num
= slave
->priv
.eswitch
->manager_vport
;
2670 dest
.vport
.vhca_id
= MLX5_CAP_GEN(slave
, vhca_id
);
2671 dest
.vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
2673 flow_rule
= mlx5_add_flow_rules(acl
, spec
, &flow_act
,
2675 if (IS_ERR(flow_rule
)) {
2676 err
= PTR_ERR(flow_rule
);
2678 err
= xa_insert(&vport
->egress
.offloads
.bounce_rules
,
2679 slave_index
, flow_rule
, GFP_KERNEL
);
2681 mlx5_del_flow_rules(flow_rule
);
2688 static int esw_master_egress_create_resources(struct mlx5_eswitch
*esw
,
2689 struct mlx5_flow_namespace
*egress_ns
,
2690 struct mlx5_vport
*vport
, size_t count
)
2692 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2693 struct mlx5_flow_table_attr ft_attr
= {
2694 .max_fte
= count
, .prio
= 0, .level
= 0,
2696 struct mlx5_flow_table
*acl
;
2697 struct mlx5_flow_group
*g
;
2698 void *match_criteria
;
2702 if (vport
->egress
.acl
)
2705 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2709 if (vport
->vport
|| mlx5_core_is_ecpf(esw
->dev
))
2710 ft_attr
.flags
= MLX5_FLOW_TABLE_OTHER_VPORT
;
2712 acl
= mlx5_create_vport_flow_table(egress_ns
, &ft_attr
, vport
->vport
);
2718 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
2720 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
2721 misc_parameters
.source_port
);
2722 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
2723 misc_parameters
.source_eswitch_owner_vhca_id
);
2724 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
2725 MLX5_MATCH_MISC_PARAMETERS
);
2727 MLX5_SET(create_flow_group_in
, flow_group_in
,
2728 source_eswitch_owner_vhca_id_valid
, 1);
2729 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
2730 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, count
);
2732 g
= mlx5_create_flow_group(acl
, flow_group_in
);
2738 vport
->egress
.acl
= acl
;
2739 vport
->egress
.offloads
.bounce_grp
= g
;
2740 vport
->egress
.type
= VPORT_EGRESS_ACL_TYPE_SHARED_FDB
;
2741 xa_init_flags(&vport
->egress
.offloads
.bounce_rules
, XA_FLAGS_ALLOC
);
2743 kvfree(flow_group_in
);
2748 mlx5_destroy_flow_table(acl
);
2750 kvfree(flow_group_in
);
2754 static void esw_master_egress_destroy_resources(struct mlx5_vport
*vport
)
2756 if (!xa_empty(&vport
->egress
.offloads
.bounce_rules
))
2758 mlx5_destroy_flow_group(vport
->egress
.offloads
.bounce_grp
);
2759 vport
->egress
.offloads
.bounce_grp
= NULL
;
2760 mlx5_destroy_flow_table(vport
->egress
.acl
);
2761 vport
->egress
.acl
= NULL
;
2764 static int esw_set_master_egress_rule(struct mlx5_core_dev
*master
,
2765 struct mlx5_core_dev
*slave
, size_t count
)
2767 struct mlx5_eswitch
*esw
= master
->priv
.eswitch
;
2768 u16 slave_index
= MLX5_CAP_GEN(slave
, vhca_id
);
2769 struct mlx5_flow_namespace
*egress_ns
;
2770 struct mlx5_vport
*vport
;
2773 vport
= mlx5_eswitch_get_vport(esw
, esw
->manager_vport
);
2775 return PTR_ERR(vport
);
2777 egress_ns
= mlx5_get_flow_vport_acl_namespace(master
,
2778 MLX5_FLOW_NAMESPACE_ESW_EGRESS
,
2783 if (vport
->egress
.acl
&& vport
->egress
.type
!= VPORT_EGRESS_ACL_TYPE_SHARED_FDB
)
2786 err
= esw_master_egress_create_resources(esw
, egress_ns
, vport
, count
);
2790 if (xa_load(&vport
->egress
.offloads
.bounce_rules
, slave_index
))
2793 err
= __esw_set_master_egress_rule(master
, slave
, vport
, vport
->egress
.acl
);
2800 esw_master_egress_destroy_resources(vport
);
2804 static void esw_unset_master_egress_rule(struct mlx5_core_dev
*dev
,
2805 struct mlx5_core_dev
*slave_dev
)
2807 struct mlx5_vport
*vport
;
2809 vport
= mlx5_eswitch_get_vport(dev
->priv
.eswitch
,
2810 dev
->priv
.eswitch
->manager_vport
);
2812 esw_acl_egress_ofld_bounce_rule_destroy(vport
, MLX5_CAP_GEN(slave_dev
, vhca_id
));
2814 if (xa_empty(&vport
->egress
.offloads
.bounce_rules
)) {
2815 esw_acl_egress_ofld_cleanup(vport
);
2816 xa_destroy(&vport
->egress
.offloads
.bounce_rules
);
2820 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch
*master_esw
,
2821 struct mlx5_eswitch
*slave_esw
, int max_slaves
)
2825 err
= esw_set_slave_root_fdb(master_esw
->dev
,
2830 err
= esw_set_master_egress_rule(master_esw
->dev
,
2831 slave_esw
->dev
, max_slaves
);
2838 esw_set_slave_root_fdb(NULL
, slave_esw
->dev
);
2842 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch
*master_esw
,
2843 struct mlx5_eswitch
*slave_esw
)
2845 esw_set_slave_root_fdb(NULL
, slave_esw
->dev
);
2846 esw_unset_master_egress_rule(master_esw
->dev
, slave_esw
->dev
);
2849 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2850 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2852 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch
*esw
,
2853 struct mlx5_eswitch
*peer_esw
)
2855 const struct mlx5_eswitch_rep_ops
*ops
;
2856 struct mlx5_eswitch_rep
*rep
;
2860 mlx5_esw_for_each_rep(esw
, i
, rep
) {
2861 rep_type
= NUM_REP_TYPES
;
2862 while (rep_type
--) {
2863 ops
= esw
->offloads
.rep_ops
[rep_type
];
2864 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
2866 ops
->event(esw
, rep
, MLX5_SWITCHDEV_EVENT_UNPAIR
, peer_esw
);
2871 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch
*esw
,
2872 struct mlx5_eswitch
*peer_esw
)
2874 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2875 mlx5e_tc_clean_fdb_peer_flows(esw
);
2877 mlx5_esw_offloads_rep_event_unpair(esw
, peer_esw
);
2878 esw_del_fdb_peer_miss_rules(esw
, peer_esw
->dev
);
2881 static int mlx5_esw_offloads_pair(struct mlx5_eswitch
*esw
,
2882 struct mlx5_eswitch
*peer_esw
)
2884 const struct mlx5_eswitch_rep_ops
*ops
;
2885 struct mlx5_eswitch_rep
*rep
;
2890 err
= esw_add_fdb_peer_miss_rules(esw
, peer_esw
->dev
);
2894 mlx5_esw_for_each_rep(esw
, i
, rep
) {
2895 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++) {
2896 ops
= esw
->offloads
.rep_ops
[rep_type
];
2897 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
2899 err
= ops
->event(esw
, rep
, MLX5_SWITCHDEV_EVENT_PAIR
, peer_esw
);
2909 mlx5_esw_offloads_unpair(esw
, peer_esw
);
2913 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch
*esw
,
2914 struct mlx5_eswitch
*peer_esw
,
2917 u16 peer_vhca_id
= MLX5_CAP_GEN(peer_esw
->dev
, vhca_id
);
2918 u16 vhca_id
= MLX5_CAP_GEN(esw
->dev
, vhca_id
);
2919 struct mlx5_flow_root_namespace
*peer_ns
;
2920 struct mlx5_flow_root_namespace
*ns
;
2923 peer_ns
= peer_esw
->dev
->priv
.steering
->fdb_root_ns
;
2924 ns
= esw
->dev
->priv
.steering
->fdb_root_ns
;
2927 err
= mlx5_flow_namespace_set_peer(ns
, peer_ns
, peer_vhca_id
);
2931 err
= mlx5_flow_namespace_set_peer(peer_ns
, ns
, vhca_id
);
2933 mlx5_flow_namespace_set_peer(ns
, NULL
, peer_vhca_id
);
2937 mlx5_flow_namespace_set_peer(ns
, NULL
, peer_vhca_id
);
2938 mlx5_flow_namespace_set_peer(peer_ns
, NULL
, vhca_id
);
2944 static int mlx5_esw_offloads_devcom_event(int event
,
2948 struct mlx5_eswitch
*esw
= my_data
;
2949 struct mlx5_eswitch
*peer_esw
= event_data
;
2950 u16 esw_i
, peer_esw_i
;
2954 peer_esw_i
= MLX5_CAP_GEN(peer_esw
->dev
, vhca_id
);
2955 esw_i
= MLX5_CAP_GEN(esw
->dev
, vhca_id
);
2956 esw_paired
= !!xa_load(&esw
->paired
, peer_esw_i
);
2959 case ESW_OFFLOADS_DEVCOM_PAIR
:
2960 if (mlx5_eswitch_vport_match_metadata_enabled(esw
) !=
2961 mlx5_eswitch_vport_match_metadata_enabled(peer_esw
))
2967 err
= mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, true);
2971 err
= mlx5_esw_offloads_pair(esw
, peer_esw
);
2975 err
= mlx5_esw_offloads_pair(peer_esw
, esw
);
2979 err
= xa_insert(&esw
->paired
, peer_esw_i
, peer_esw
, GFP_KERNEL
);
2983 err
= xa_insert(&peer_esw
->paired
, esw_i
, esw
, GFP_KERNEL
);
2988 peer_esw
->num_peers
++;
2989 mlx5_devcom_comp_set_ready(esw
->devcom
, true);
2992 case ESW_OFFLOADS_DEVCOM_UNPAIR
:
2996 peer_esw
->num_peers
--;
2998 if (!esw
->num_peers
&& !peer_esw
->num_peers
)
2999 mlx5_devcom_comp_set_ready(esw
->devcom
, false);
3000 xa_erase(&peer_esw
->paired
, esw_i
);
3001 xa_erase(&esw
->paired
, peer_esw_i
);
3002 mlx5_esw_offloads_unpair(peer_esw
, esw
);
3003 mlx5_esw_offloads_unpair(esw
, peer_esw
);
3004 mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, false);
3011 xa_erase(&esw
->paired
, peer_esw_i
);
3013 mlx5_esw_offloads_unpair(peer_esw
, esw
);
3015 mlx5_esw_offloads_unpair(esw
, peer_esw
);
3017 mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, false);
3019 mlx5_core_err(esw
->dev
, "esw offloads devcom event failure, event %u err %d",
3024 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch
*esw
, u64 key
)
3028 for (i
= 0; i
< MLX5_MAX_PORTS
; i
++)
3029 INIT_LIST_HEAD(&esw
->offloads
.peer_flows
[i
]);
3030 mutex_init(&esw
->offloads
.peer_mutex
);
3032 if (!MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
3035 if ((MLX5_VPORT_MANAGER(esw
->dev
) || mlx5_core_is_ecpf_esw_manager(esw
->dev
)) &&
3036 !mlx5_lag_is_supported(esw
->dev
))
3039 xa_init(&esw
->paired
);
3041 esw
->devcom
= mlx5_devcom_register_component(esw
->dev
->priv
.devc
,
3042 MLX5_DEVCOM_ESW_OFFLOADS
,
3044 mlx5_esw_offloads_devcom_event
,
3046 if (IS_ERR_OR_NULL(esw
->devcom
))
3049 mlx5_devcom_send_event(esw
->devcom
,
3050 ESW_OFFLOADS_DEVCOM_PAIR
,
3051 ESW_OFFLOADS_DEVCOM_UNPAIR
,
3055 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch
*esw
)
3057 if (IS_ERR_OR_NULL(esw
->devcom
))
3060 mlx5_devcom_send_event(esw
->devcom
,
3061 ESW_OFFLOADS_DEVCOM_UNPAIR
,
3062 ESW_OFFLOADS_DEVCOM_UNPAIR
,
3065 mlx5_devcom_unregister_component(esw
->devcom
);
3066 xa_destroy(&esw
->paired
);
3070 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch
*esw
)
3072 return mlx5_devcom_comp_is_ready(esw
->devcom
);
3075 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch
*esw
)
3077 if (!MLX5_CAP_ESW(esw
->dev
, esw_uplink_ingress_acl
))
3080 if (!(MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, fdb_to_vport_reg_c_id
) &
3081 MLX5_FDB_TO_VPORT_REG_C_0
))
3087 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
3089 /* Share the same metadata for uplink's. This is fine because:
3090 * (a) In shared FDB mode (LAG) both uplink's are treated the
3091 * same and tagged with the same metadata.
3092 * (b) In non shared FDB mode, packets from physical port0
3093 * cannot hit eswitch of PF1 and vice versa.
3095 static u32
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch
*esw
)
3097 return MLX5_ESW_METADATA_RSVD_UPLINK
;
3100 u32
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch
*esw
)
3102 u32 vport_end_ida
= (1 << ESW_VPORT_BITS
) - 1;
3103 /* Reserve 0xf for internal port offload */
3104 u32 max_pf_num
= (1 << ESW_PFNUM_BITS
) - 2;
3108 /* Only 4 bits of pf_num */
3109 pf_num
= mlx5_get_dev_index(esw
->dev
);
3110 if (pf_num
> max_pf_num
)
3113 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3114 /* Use only non-zero vport_id (2-4095) for all PF's */
3115 id
= ida_alloc_range(&esw
->offloads
.vport_metadata_ida
,
3116 MLX5_ESW_METADATA_RSVD_UPLINK
+ 1,
3117 vport_end_ida
, GFP_KERNEL
);
3120 id
= (pf_num
<< ESW_VPORT_BITS
) | id
;
3124 void mlx5_esw_match_metadata_free(struct mlx5_eswitch
*esw
, u32 metadata
)
3126 u32 vport_bit_mask
= (1 << ESW_VPORT_BITS
) - 1;
3128 /* Metadata contains only 12 bits of actual ida id */
3129 ida_free(&esw
->offloads
.vport_metadata_ida
, metadata
& vport_bit_mask
);
3132 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch
*esw
,
3133 struct mlx5_vport
*vport
)
3135 if (vport
->vport
== MLX5_VPORT_UPLINK
)
3136 vport
->default_metadata
= mlx5_esw_match_metadata_reserved(esw
);
3138 vport
->default_metadata
= mlx5_esw_match_metadata_alloc(esw
);
3140 vport
->metadata
= vport
->default_metadata
;
3141 return vport
->metadata
? 0 : -ENOSPC
;
3144 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch
*esw
,
3145 struct mlx5_vport
*vport
)
3147 if (!vport
->default_metadata
)
3150 if (vport
->vport
== MLX5_VPORT_UPLINK
)
3153 WARN_ON(vport
->metadata
!= vport
->default_metadata
);
3154 mlx5_esw_match_metadata_free(esw
, vport
->default_metadata
);
3157 static void esw_offloads_metadata_uninit(struct mlx5_eswitch
*esw
)
3159 struct mlx5_vport
*vport
;
3162 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
))
3165 mlx5_esw_for_each_vport(esw
, i
, vport
)
3166 esw_offloads_vport_metadata_cleanup(esw
, vport
);
3169 static int esw_offloads_metadata_init(struct mlx5_eswitch
*esw
)
3171 struct mlx5_vport
*vport
;
3175 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
))
3178 mlx5_esw_for_each_vport(esw
, i
, vport
) {
3179 err
= esw_offloads_vport_metadata_setup(esw
, vport
);
3187 esw_offloads_metadata_uninit(esw
);
3192 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch
*esw
,
3193 struct mlx5_vport
*vport
)
3197 err
= esw_acl_ingress_ofld_setup(esw
, vport
);
3201 err
= esw_acl_egress_ofld_setup(esw
, vport
);
3208 esw_acl_ingress_ofld_cleanup(esw
, vport
);
3213 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch
*esw
,
3214 struct mlx5_vport
*vport
)
3216 esw_acl_egress_ofld_cleanup(vport
);
3217 esw_acl_ingress_ofld_cleanup(esw
, vport
);
3220 static int esw_create_offloads_acl_tables(struct mlx5_eswitch
*esw
)
3222 struct mlx5_vport
*uplink
, *manager
;
3225 uplink
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
3227 return PTR_ERR(uplink
);
3229 ret
= esw_vport_create_offloads_acl_tables(esw
, uplink
);
3233 manager
= mlx5_eswitch_get_vport(esw
, esw
->manager_vport
);
3234 if (IS_ERR(manager
)) {
3235 ret
= PTR_ERR(manager
);
3239 ret
= esw_vport_create_offloads_acl_tables(esw
, manager
);
3246 esw_vport_destroy_offloads_acl_tables(esw
, uplink
);
3250 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch
*esw
)
3252 struct mlx5_vport
*vport
;
3254 vport
= mlx5_eswitch_get_vport(esw
, esw
->manager_vport
);
3256 esw_vport_destroy_offloads_acl_tables(esw
, vport
);
3258 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
3260 esw_vport_destroy_offloads_acl_tables(esw
, vport
);
3263 int mlx5_eswitch_reload_reps(struct mlx5_eswitch
*esw
)
3265 struct mlx5_eswitch_rep
*rep
;
3269 if (!esw
|| esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
3272 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_UPLINK
);
3273 if (atomic_read(&rep
->rep_data
[REP_ETH
].state
) != REP_LOADED
)
3276 ret
= mlx5_esw_offloads_rep_load(esw
, MLX5_VPORT_UPLINK
);
3280 mlx5_esw_for_each_rep(esw
, i
, rep
) {
3281 if (atomic_read(&rep
->rep_data
[REP_ETH
].state
) == REP_LOADED
)
3282 mlx5_esw_offloads_rep_load(esw
, rep
->vport
);
3288 static int esw_offloads_steering_init(struct mlx5_eswitch
*esw
)
3290 struct mlx5_esw_indir_table
*indir
;
3293 memset(&esw
->fdb_table
.offloads
, 0, sizeof(struct offloads_fdb
));
3294 mutex_init(&esw
->fdb_table
.offloads
.vports
.lock
);
3295 hash_init(esw
->fdb_table
.offloads
.vports
.table
);
3296 atomic64_set(&esw
->user_count
, 0);
3298 indir
= mlx5_esw_indir_table_init();
3299 if (IS_ERR(indir
)) {
3300 err
= PTR_ERR(indir
);
3301 goto create_indir_err
;
3303 esw
->fdb_table
.offloads
.indir
= indir
;
3305 err
= esw_create_offloads_acl_tables(esw
);
3307 goto create_acl_err
;
3309 err
= esw_create_offloads_table(esw
);
3311 goto create_offloads_err
;
3313 err
= esw_create_restore_table(esw
);
3315 goto create_restore_err
;
3317 err
= esw_create_offloads_fdb_tables(esw
);
3319 goto create_fdb_err
;
3321 err
= esw_create_vport_rx_group(esw
);
3325 err
= esw_create_vport_rx_drop_group(esw
);
3327 goto create_rx_drop_fg_err
;
3329 err
= esw_create_vport_rx_drop_rule(esw
);
3331 goto create_rx_drop_rule_err
;
3335 create_rx_drop_rule_err
:
3336 esw_destroy_vport_rx_drop_group(esw
);
3337 create_rx_drop_fg_err
:
3338 esw_destroy_vport_rx_group(esw
);
3340 esw_destroy_offloads_fdb_tables(esw
);
3342 esw_destroy_restore_table(esw
);
3344 esw_destroy_offloads_table(esw
);
3345 create_offloads_err
:
3346 esw_destroy_offloads_acl_tables(esw
);
3348 mlx5_esw_indir_table_destroy(esw
->fdb_table
.offloads
.indir
);
3350 mutex_destroy(&esw
->fdb_table
.offloads
.vports
.lock
);
3354 static void esw_offloads_steering_cleanup(struct mlx5_eswitch
*esw
)
3356 esw_destroy_vport_rx_drop_rule(esw
);
3357 esw_destroy_vport_rx_drop_group(esw
);
3358 esw_destroy_vport_rx_group(esw
);
3359 esw_destroy_offloads_fdb_tables(esw
);
3360 esw_destroy_restore_table(esw
);
3361 esw_destroy_offloads_table(esw
);
3362 esw_destroy_offloads_acl_tables(esw
);
3363 mlx5_esw_indir_table_destroy(esw
->fdb_table
.offloads
.indir
);
3364 mutex_destroy(&esw
->fdb_table
.offloads
.vports
.lock
);
3368 esw_vfs_changed_event_handler(struct mlx5_eswitch
*esw
, const u32
*out
)
3370 struct devlink
*devlink
;
3371 bool host_pf_disabled
;
3374 new_num_vfs
= MLX5_GET(query_esw_functions_out
, out
,
3375 host_params_context
.host_num_of_vfs
);
3376 host_pf_disabled
= MLX5_GET(query_esw_functions_out
, out
,
3377 host_params_context
.host_pf_disabled
);
3379 if (new_num_vfs
== esw
->esw_funcs
.num_vfs
|| host_pf_disabled
)
3382 devlink
= priv_to_devlink(esw
->dev
);
3384 /* Number of VFs can only change from "0 to x" or "x to 0". */
3385 if (esw
->esw_funcs
.num_vfs
> 0) {
3386 mlx5_eswitch_unload_vf_vports(esw
, esw
->esw_funcs
.num_vfs
);
3390 err
= mlx5_eswitch_load_vf_vports(esw
, new_num_vfs
,
3391 MLX5_VPORT_UC_ADDR_CHANGE
);
3393 devl_unlock(devlink
);
3397 esw
->esw_funcs
.num_vfs
= new_num_vfs
;
3398 devl_unlock(devlink
);
3401 static void esw_functions_changed_event_handler(struct work_struct
*work
)
3403 struct mlx5_host_work
*host_work
;
3404 struct mlx5_eswitch
*esw
;
3407 host_work
= container_of(work
, struct mlx5_host_work
, work
);
3408 esw
= host_work
->esw
;
3410 out
= mlx5_esw_query_functions(esw
->dev
);
3414 esw_vfs_changed_event_handler(esw
, out
);
3420 int mlx5_esw_funcs_changed_handler(struct notifier_block
*nb
, unsigned long type
, void *data
)
3422 struct mlx5_esw_functions
*esw_funcs
;
3423 struct mlx5_host_work
*host_work
;
3424 struct mlx5_eswitch
*esw
;
3426 host_work
= kzalloc(sizeof(*host_work
), GFP_ATOMIC
);
3430 esw_funcs
= mlx5_nb_cof(nb
, struct mlx5_esw_functions
, nb
);
3431 esw
= container_of(esw_funcs
, struct mlx5_eswitch
, esw_funcs
);
3433 host_work
->esw
= esw
;
3435 INIT_WORK(&host_work
->work
, esw_functions_changed_event_handler
);
3436 queue_work(esw
->work_queue
, &host_work
->work
);
3441 static int mlx5_esw_host_number_init(struct mlx5_eswitch
*esw
)
3443 const u32
*query_host_out
;
3445 if (!mlx5_core_is_ecpf_esw_manager(esw
->dev
))
3448 query_host_out
= mlx5_esw_query_functions(esw
->dev
);
3449 if (IS_ERR(query_host_out
))
3450 return PTR_ERR(query_host_out
);
3452 /* Mark non local controller with non zero controller number. */
3453 esw
->offloads
.host_number
= MLX5_GET(query_esw_functions_out
, query_host_out
,
3454 host_params_context
.host_number
);
3455 kvfree(query_host_out
);
3459 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch
*esw
, u32 controller
)
3461 /* Local controller is always valid */
3462 if (controller
== 0)
3465 if (!mlx5_core_is_ecpf_esw_manager(esw
->dev
))
3468 /* External host number starts with zero in device */
3469 return (controller
== esw
->offloads
.host_number
+ 1);
3472 int esw_offloads_enable(struct mlx5_eswitch
*esw
)
3474 struct mapping_ctx
*reg_c0_obj_pool
;
3475 struct mlx5_vport
*vport
;
3480 mutex_init(&esw
->offloads
.termtbl_mutex
);
3481 mlx5_rdma_enable_roce(esw
->dev
);
3483 err
= mlx5_esw_host_number_init(esw
);
3487 err
= esw_offloads_metadata_init(esw
);
3491 err
= esw_set_passing_vport_metadata(esw
, true);
3493 goto err_vport_metadata
;
3495 mapping_id
= mlx5_query_nic_system_image_guid(esw
->dev
);
3497 reg_c0_obj_pool
= mapping_create_for_id(mapping_id
, MAPPING_TYPE_CHAIN
,
3498 sizeof(struct mlx5_mapped_obj
),
3499 ESW_REG_C0_USER_DATA_METADATA_MASK
,
3502 if (IS_ERR(reg_c0_obj_pool
)) {
3503 err
= PTR_ERR(reg_c0_obj_pool
);
3506 esw
->offloads
.reg_c0_obj_pool
= reg_c0_obj_pool
;
3508 err
= esw_offloads_steering_init(esw
);
3510 goto err_steering_init
;
3512 /* Representor will control the vport link state */
3513 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
)
3514 vport
->info
.link_state
= MLX5_VPORT_ADMIN_STATE_DOWN
;
3515 if (mlx5_core_ec_sriov_enabled(esw
->dev
))
3516 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, esw
->esw_funcs
.num_ec_vfs
)
3517 vport
->info
.link_state
= MLX5_VPORT_ADMIN_STATE_DOWN
;
3519 /* Uplink vport rep must load first. */
3520 err
= mlx5_esw_offloads_rep_load(esw
, MLX5_VPORT_UPLINK
);
3524 err
= mlx5_eswitch_enable_pf_vf_vports(esw
, MLX5_VPORT_UC_ADDR_CHANGE
);
3531 mlx5_esw_offloads_rep_unload(esw
, MLX5_VPORT_UPLINK
);
3533 esw_offloads_steering_cleanup(esw
);
3535 mapping_destroy(reg_c0_obj_pool
);
3537 esw_set_passing_vport_metadata(esw
, false);
3539 esw_offloads_metadata_uninit(esw
);
3541 mlx5_rdma_disable_roce(esw
->dev
);
3542 mutex_destroy(&esw
->offloads
.termtbl_mutex
);
3546 static int esw_offloads_stop(struct mlx5_eswitch
*esw
,
3547 struct netlink_ext_ack
*extack
)
3551 esw
->mode
= MLX5_ESWITCH_LEGACY
;
3553 /* If changing from switchdev to legacy mode without sriov enabled,
3554 * no need to create legacy fdb.
3556 if (!mlx5_core_is_pf(esw
->dev
) || !mlx5_sriov_is_enabled(esw
->dev
))
3559 err
= mlx5_eswitch_enable_locked(esw
, MLX5_ESWITCH_IGNORE_NUM_VFS
);
3561 NL_SET_ERR_MSG_MOD(extack
, "Failed setting eswitch to legacy");
3566 void esw_offloads_disable(struct mlx5_eswitch
*esw
)
3568 mlx5_eswitch_disable_pf_vf_vports(esw
);
3569 mlx5_esw_offloads_rep_unload(esw
, MLX5_VPORT_UPLINK
);
3570 esw_set_passing_vport_metadata(esw
, false);
3571 esw_offloads_steering_cleanup(esw
);
3572 mapping_destroy(esw
->offloads
.reg_c0_obj_pool
);
3573 esw_offloads_metadata_uninit(esw
);
3574 mlx5_rdma_disable_roce(esw
->dev
);
3575 mutex_destroy(&esw
->offloads
.termtbl_mutex
);
3578 static int esw_mode_from_devlink(u16 mode
, u16
*mlx5_mode
)
3581 case DEVLINK_ESWITCH_MODE_LEGACY
:
3582 *mlx5_mode
= MLX5_ESWITCH_LEGACY
;
3584 case DEVLINK_ESWITCH_MODE_SWITCHDEV
:
3585 *mlx5_mode
= MLX5_ESWITCH_OFFLOADS
;
3594 static int esw_mode_to_devlink(u16 mlx5_mode
, u16
*mode
)
3596 switch (mlx5_mode
) {
3597 case MLX5_ESWITCH_LEGACY
:
3598 *mode
= DEVLINK_ESWITCH_MODE_LEGACY
;
3600 case MLX5_ESWITCH_OFFLOADS
:
3601 *mode
= DEVLINK_ESWITCH_MODE_SWITCHDEV
;
3610 static int esw_inline_mode_from_devlink(u8 mode
, u8
*mlx5_mode
)
3613 case DEVLINK_ESWITCH_INLINE_MODE_NONE
:
3614 *mlx5_mode
= MLX5_INLINE_MODE_NONE
;
3616 case DEVLINK_ESWITCH_INLINE_MODE_LINK
:
3617 *mlx5_mode
= MLX5_INLINE_MODE_L2
;
3619 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK
:
3620 *mlx5_mode
= MLX5_INLINE_MODE_IP
;
3622 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
:
3623 *mlx5_mode
= MLX5_INLINE_MODE_TCP_UDP
;
3632 static int esw_inline_mode_to_devlink(u8 mlx5_mode
, u8
*mode
)
3634 switch (mlx5_mode
) {
3635 case MLX5_INLINE_MODE_NONE
:
3636 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NONE
;
3638 case MLX5_INLINE_MODE_L2
:
3639 *mode
= DEVLINK_ESWITCH_INLINE_MODE_LINK
;
3641 case MLX5_INLINE_MODE_IP
:
3642 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NETWORK
;
3644 case MLX5_INLINE_MODE_TCP_UDP
:
3645 *mode
= DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
;
3654 static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink
*devlink
)
3656 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
3657 struct net
*devl_net
, *netdev_net
;
3660 mutex_lock(&dev
->mlx5e_res
.uplink_netdev_lock
);
3661 if (dev
->mlx5e_res
.uplink_netdev
) {
3662 netdev_net
= dev_net(dev
->mlx5e_res
.uplink_netdev
);
3663 devl_net
= devlink_net(devlink
);
3664 ret
= net_eq(devl_net
, netdev_net
);
3666 mutex_unlock(&dev
->mlx5e_res
.uplink_netdev_lock
);
3670 int mlx5_eswitch_block_mode(struct mlx5_core_dev
*dev
)
3672 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
3675 if (!mlx5_esw_allowed(esw
))
3678 /* Take TC into account */
3679 err
= mlx5_esw_try_lock(esw
);
3683 esw
->offloads
.num_block_mode
++;
3684 mlx5_esw_unlock(esw
);
3688 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev
*dev
)
3690 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
3692 if (!mlx5_esw_allowed(esw
))
3695 down_write(&esw
->mode_lock
);
3696 esw
->offloads
.num_block_mode
--;
3697 up_write(&esw
->mode_lock
);
3700 int mlx5_devlink_eswitch_mode_set(struct devlink
*devlink
, u16 mode
,
3701 struct netlink_ext_ack
*extack
)
3703 u16 cur_mlx5_mode
, mlx5_mode
= 0;
3704 struct mlx5_eswitch
*esw
;
3707 esw
= mlx5_devlink_eswitch_get(devlink
);
3709 return PTR_ERR(esw
);
3711 if (esw_mode_from_devlink(mode
, &mlx5_mode
))
3714 if (mode
== DEVLINK_ESWITCH_MODE_SWITCHDEV
&&
3715 !esw_offloads_devlink_ns_eq_netdev_ns(devlink
)) {
3716 NL_SET_ERR_MSG_MOD(extack
,
3717 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's.");
3721 mlx5_lag_disable_change(esw
->dev
);
3722 err
= mlx5_esw_try_lock(esw
);
3724 NL_SET_ERR_MSG_MOD(extack
, "Can't change mode, E-Switch is busy");
3727 cur_mlx5_mode
= err
;
3730 if (cur_mlx5_mode
== mlx5_mode
)
3733 if (esw
->offloads
.num_block_mode
) {
3734 NL_SET_ERR_MSG_MOD(extack
,
3735 "Can't change eswitch mode when IPsec SA and/or policies are configured");
3740 esw
->eswitch_operation_in_progress
= true;
3741 up_write(&esw
->mode_lock
);
3743 mlx5_eswitch_disable_locked(esw
);
3744 if (mode
== DEVLINK_ESWITCH_MODE_SWITCHDEV
) {
3745 if (mlx5_devlink_trap_get_num_active(esw
->dev
)) {
3746 NL_SET_ERR_MSG_MOD(extack
,
3747 "Can't change mode while devlink traps are active");
3751 err
= esw_offloads_start(esw
, extack
);
3752 } else if (mode
== DEVLINK_ESWITCH_MODE_LEGACY
) {
3753 err
= esw_offloads_stop(esw
, extack
);
3754 mlx5_rescan_drivers(esw
->dev
);
3760 down_write(&esw
->mode_lock
);
3761 esw
->eswitch_operation_in_progress
= false;
3763 mlx5_esw_unlock(esw
);
3765 mlx5_lag_enable_change(esw
->dev
);
3769 int mlx5_devlink_eswitch_mode_get(struct devlink
*devlink
, u16
*mode
)
3771 struct mlx5_eswitch
*esw
;
3773 esw
= mlx5_devlink_eswitch_get(devlink
);
3775 return PTR_ERR(esw
);
3777 return esw_mode_to_devlink(esw
->mode
, mode
);
3780 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch
*esw
, u8 mlx5_mode
,
3781 struct netlink_ext_ack
*extack
)
3783 struct mlx5_core_dev
*dev
= esw
->dev
;
3784 struct mlx5_vport
*vport
;
3785 u16 err_vport_num
= 0;
3789 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
3790 err
= mlx5_modify_nic_vport_min_inline(dev
, vport
->vport
, mlx5_mode
);
3792 err_vport_num
= vport
->vport
;
3793 NL_SET_ERR_MSG_MOD(extack
,
3794 "Failed to set min inline on vport");
3795 goto revert_inline_mode
;
3798 if (mlx5_core_ec_sriov_enabled(esw
->dev
)) {
3799 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, esw
->esw_funcs
.num_ec_vfs
) {
3800 err
= mlx5_modify_nic_vport_min_inline(dev
, vport
->vport
, mlx5_mode
);
3802 err_vport_num
= vport
->vport
;
3803 NL_SET_ERR_MSG_MOD(extack
,
3804 "Failed to set min inline on vport");
3805 goto revert_ec_vf_inline_mode
;
3811 revert_ec_vf_inline_mode
:
3812 mlx5_esw_for_each_ec_vf_vport(esw
, i
, vport
, esw
->esw_funcs
.num_ec_vfs
) {
3813 if (vport
->vport
== err_vport_num
)
3815 mlx5_modify_nic_vport_min_inline(dev
,
3817 esw
->offloads
.inline_mode
);
3820 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
3821 if (vport
->vport
== err_vport_num
)
3823 mlx5_modify_nic_vport_min_inline(dev
,
3825 esw
->offloads
.inline_mode
);
3830 int mlx5_devlink_eswitch_inline_mode_set(struct devlink
*devlink
, u8 mode
,
3831 struct netlink_ext_ack
*extack
)
3833 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
3834 struct mlx5_eswitch
*esw
;
3838 esw
= mlx5_devlink_eswitch_get(devlink
);
3840 return PTR_ERR(esw
);
3842 down_write(&esw
->mode_lock
);
3844 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
3845 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
3846 if (mode
== DEVLINK_ESWITCH_INLINE_MODE_NONE
) {
3852 case MLX5_CAP_INLINE_MODE_L2
:
3853 NL_SET_ERR_MSG_MOD(extack
, "Inline mode can't be set");
3856 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
3860 if (atomic64_read(&esw
->offloads
.num_flows
) > 0) {
3861 NL_SET_ERR_MSG_MOD(extack
,
3862 "Can't set inline mode when flows are configured");
3867 err
= esw_inline_mode_from_devlink(mode
, &mlx5_mode
);
3871 esw
->eswitch_operation_in_progress
= true;
3872 up_write(&esw
->mode_lock
);
3874 err
= mlx5_esw_vports_inline_set(esw
, mlx5_mode
, extack
);
3876 esw
->offloads
.inline_mode
= mlx5_mode
;
3878 down_write(&esw
->mode_lock
);
3879 esw
->eswitch_operation_in_progress
= false;
3880 up_write(&esw
->mode_lock
);
3884 up_write(&esw
->mode_lock
);
3888 int mlx5_devlink_eswitch_inline_mode_get(struct devlink
*devlink
, u8
*mode
)
3890 struct mlx5_eswitch
*esw
;
3892 esw
= mlx5_devlink_eswitch_get(devlink
);
3894 return PTR_ERR(esw
);
3896 return esw_inline_mode_to_devlink(esw
->offloads
.inline_mode
, mode
);
3899 bool mlx5_eswitch_block_encap(struct mlx5_core_dev
*dev
)
3901 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
3903 if (!mlx5_esw_allowed(esw
))
3906 down_write(&esw
->mode_lock
);
3907 if (esw
->mode
!= MLX5_ESWITCH_LEGACY
&&
3908 esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
) {
3909 up_write(&esw
->mode_lock
);
3913 esw
->offloads
.num_block_encap
++;
3914 up_write(&esw
->mode_lock
);
3918 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev
*dev
)
3920 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
3922 if (!mlx5_esw_allowed(esw
))
3925 down_write(&esw
->mode_lock
);
3926 esw
->offloads
.num_block_encap
--;
3927 up_write(&esw
->mode_lock
);
3930 int mlx5_devlink_eswitch_encap_mode_set(struct devlink
*devlink
,
3931 enum devlink_eswitch_encap_mode encap
,
3932 struct netlink_ext_ack
*extack
)
3934 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
3935 struct mlx5_eswitch
*esw
;
3938 esw
= mlx5_devlink_eswitch_get(devlink
);
3940 return PTR_ERR(esw
);
3942 down_write(&esw
->mode_lock
);
3944 if (encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
&&
3945 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, reformat
) ||
3946 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, decap
))) {
3951 if (encap
&& encap
!= DEVLINK_ESWITCH_ENCAP_MODE_BASIC
) {
3956 if (esw
->mode
== MLX5_ESWITCH_LEGACY
) {
3957 esw
->offloads
.encap
= encap
;
3961 if (esw
->offloads
.encap
== encap
)
3964 if (atomic64_read(&esw
->offloads
.num_flows
) > 0) {
3965 NL_SET_ERR_MSG_MOD(extack
,
3966 "Can't set encapsulation when flows are configured");
3971 if (esw
->offloads
.num_block_encap
) {
3972 NL_SET_ERR_MSG_MOD(extack
,
3973 "Can't set encapsulation when IPsec SA and/or policies are configured");
3978 esw
->eswitch_operation_in_progress
= true;
3979 up_write(&esw
->mode_lock
);
3981 esw_destroy_offloads_fdb_tables(esw
);
3983 esw
->offloads
.encap
= encap
;
3985 err
= esw_create_offloads_fdb_tables(esw
);
3988 NL_SET_ERR_MSG_MOD(extack
,
3989 "Failed re-creating fast FDB table");
3990 esw
->offloads
.encap
= !encap
;
3991 (void)esw_create_offloads_fdb_tables(esw
);
3994 down_write(&esw
->mode_lock
);
3995 esw
->eswitch_operation_in_progress
= false;
3998 up_write(&esw
->mode_lock
);
4002 int mlx5_devlink_eswitch_encap_mode_get(struct devlink
*devlink
,
4003 enum devlink_eswitch_encap_mode
*encap
)
4005 struct mlx5_eswitch
*esw
;
4007 esw
= mlx5_devlink_eswitch_get(devlink
);
4009 return PTR_ERR(esw
);
4011 *encap
= esw
->offloads
.encap
;
4016 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch
*esw
, u16 vport_num
)
4018 /* Currently, only ECPF based device has representor for host PF. */
4019 if (vport_num
== MLX5_VPORT_PF
&&
4020 !mlx5_core_is_ecpf_esw_manager(esw
->dev
))
4023 if (vport_num
== MLX5_VPORT_ECPF
&&
4024 !mlx5_ecpf_vport_exists(esw
->dev
))
4030 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch
*esw
,
4031 const struct mlx5_eswitch_rep_ops
*ops
,
4034 struct mlx5_eswitch_rep_data
*rep_data
;
4035 struct mlx5_eswitch_rep
*rep
;
4038 esw
->offloads
.rep_ops
[rep_type
] = ops
;
4039 mlx5_esw_for_each_rep(esw
, i
, rep
) {
4040 if (likely(mlx5_eswitch_vport_has_rep(esw
, rep
->vport
))) {
4042 rep_data
= &rep
->rep_data
[rep_type
];
4043 atomic_set(&rep_data
->state
, REP_REGISTERED
);
4047 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps
);
4049 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch
*esw
, u8 rep_type
)
4051 struct mlx5_eswitch_rep
*rep
;
4054 if (esw
->mode
== MLX5_ESWITCH_OFFLOADS
)
4055 __unload_reps_all_vport(esw
, rep_type
);
4057 mlx5_esw_for_each_rep(esw
, i
, rep
)
4058 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_UNREGISTERED
);
4060 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps
);
4062 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch
*esw
, u8 rep_type
)
4064 struct mlx5_eswitch_rep
*rep
;
4066 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_UPLINK
);
4067 return rep
->rep_data
[rep_type
].priv
;
4070 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch
*esw
,
4074 struct mlx5_eswitch_rep
*rep
;
4076 rep
= mlx5_eswitch_get_rep(esw
, vport
);
4078 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
4079 esw
->offloads
.rep_ops
[rep_type
]->get_proto_dev
)
4080 return esw
->offloads
.rep_ops
[rep_type
]->get_proto_dev(rep
);
4083 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev
);
4085 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch
*esw
, u8 rep_type
)
4087 return mlx5_eswitch_get_proto_dev(esw
, MLX5_VPORT_UPLINK
, rep_type
);
4089 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev
);
4091 struct mlx5_eswitch_rep
*mlx5_eswitch_vport_rep(struct mlx5_eswitch
*esw
,
4094 return mlx5_eswitch_get_rep(esw
, vport
);
4096 EXPORT_SYMBOL(mlx5_eswitch_vport_rep
);
4098 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch
*esw
)
4100 return !!(esw
->flags
& MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
);
4102 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled
);
4104 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch
*esw
)
4106 return !!(esw
->flags
& MLX5_ESWITCH_VPORT_MATCH_METADATA
);
4108 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled
);
4110 u32
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch
*esw
,
4113 struct mlx5_vport
*vport
= mlx5_eswitch_get_vport(esw
, vport_num
);
4115 if (WARN_ON_ONCE(IS_ERR(vport
)))
4118 return vport
->metadata
<< (32 - ESW_SOURCE_PORT_METADATA_BITS
);
4120 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match
);
4122 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch
*esw
, u16 vport_num
, u16
*vhca_id
)
4124 int query_out_sz
= MLX5_ST_SZ_BYTES(query_hca_cap_out
);
4131 query_ctx
= kzalloc(query_out_sz
, GFP_KERNEL
);
4135 err
= mlx5_vport_get_other_func_general_cap(esw
->dev
, vport_num
, query_ctx
);
4139 hca_caps
= MLX5_ADDR_OF(query_hca_cap_out
, query_ctx
, capability
);
4140 *vhca_id
= MLX5_GET(cmd_hca_cap
, hca_caps
, vhca_id
);
4147 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch
*esw
, u16 vport_num
)
4149 u16
*old_entry
, *vhca_map_entry
, vhca_id
;
4152 err
= mlx5_esw_query_vport_vhca_id(esw
, vport_num
, &vhca_id
);
4154 esw_warn(esw
->dev
, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
4159 vhca_map_entry
= kmalloc(sizeof(*vhca_map_entry
), GFP_KERNEL
);
4160 if (!vhca_map_entry
)
4163 *vhca_map_entry
= vport_num
;
4164 old_entry
= xa_store(&esw
->offloads
.vhca_map
, vhca_id
, vhca_map_entry
, GFP_KERNEL
);
4165 if (xa_is_err(old_entry
)) {
4166 kfree(vhca_map_entry
);
4167 return xa_err(old_entry
);
4173 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch
*esw
, u16 vport_num
)
4175 u16
*vhca_map_entry
, vhca_id
;
4178 err
= mlx5_esw_query_vport_vhca_id(esw
, vport_num
, &vhca_id
);
4180 esw_warn(esw
->dev
, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
4183 vhca_map_entry
= xa_erase(&esw
->offloads
.vhca_map
, vhca_id
);
4184 kfree(vhca_map_entry
);
4187 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch
*esw
, u16 vhca_id
, u16
*vport_num
)
4189 u16
*res
= xa_load(&esw
->offloads
.vhca_map
, vhca_id
);
4198 u32
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch
*esw
,
4201 struct mlx5_vport
*vport
= mlx5_eswitch_get_vport(esw
, vport_num
);
4203 if (WARN_ON_ONCE(IS_ERR(vport
)))
4206 return vport
->metadata
;
4208 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set
);
4210 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port
*port
,
4211 u8
*hw_addr
, int *hw_addr_len
,
4212 struct netlink_ext_ack
*extack
)
4214 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4215 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4217 mutex_lock(&esw
->state_lock
);
4218 ether_addr_copy(hw_addr
, vport
->info
.mac
);
4219 *hw_addr_len
= ETH_ALEN
;
4220 mutex_unlock(&esw
->state_lock
);
4224 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port
*port
,
4225 const u8
*hw_addr
, int hw_addr_len
,
4226 struct netlink_ext_ack
*extack
)
4228 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4229 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4231 return mlx5_eswitch_set_vport_mac(esw
, vport
->vport
, hw_addr
);
4234 int mlx5_devlink_port_fn_migratable_get(struct devlink_port
*port
, bool *is_enabled
,
4235 struct netlink_ext_ack
*extack
)
4237 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4238 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4240 if (!MLX5_CAP_GEN(esw
->dev
, migration
)) {
4241 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support migration");
4245 if (!MLX5_CAP_GEN(esw
->dev
, vhca_resource_manager
)) {
4246 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support VHCA management");
4250 mutex_lock(&esw
->state_lock
);
4251 *is_enabled
= vport
->info
.mig_enabled
;
4252 mutex_unlock(&esw
->state_lock
);
4256 int mlx5_devlink_port_fn_migratable_set(struct devlink_port
*port
, bool enable
,
4257 struct netlink_ext_ack
*extack
)
4259 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4260 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4261 int query_out_sz
= MLX5_ST_SZ_BYTES(query_hca_cap_out
);
4266 if (!MLX5_CAP_GEN(esw
->dev
, migration
)) {
4267 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support migration");
4271 if (!MLX5_CAP_GEN(esw
->dev
, vhca_resource_manager
)) {
4272 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support VHCA management");
4276 mutex_lock(&esw
->state_lock
);
4278 if (vport
->info
.mig_enabled
== enable
) {
4283 query_ctx
= kzalloc(query_out_sz
, GFP_KERNEL
);
4289 err
= mlx5_vport_get_other_func_cap(esw
->dev
, vport
->vport
, query_ctx
,
4290 MLX5_CAP_GENERAL_2
);
4292 NL_SET_ERR_MSG_MOD(extack
, "Failed getting HCA caps");
4296 hca_caps
= MLX5_ADDR_OF(query_hca_cap_out
, query_ctx
, capability
);
4297 MLX5_SET(cmd_hca_cap_2
, hca_caps
, migratable
, enable
);
4299 err
= mlx5_vport_set_other_func_cap(esw
->dev
, hca_caps
, vport
->vport
,
4300 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2
);
4302 NL_SET_ERR_MSG_MOD(extack
, "Failed setting HCA migratable cap");
4306 vport
->info
.mig_enabled
= enable
;
4311 mutex_unlock(&esw
->state_lock
);
4315 int mlx5_devlink_port_fn_roce_get(struct devlink_port
*port
, bool *is_enabled
,
4316 struct netlink_ext_ack
*extack
)
4318 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4319 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4321 if (!MLX5_CAP_GEN(esw
->dev
, vhca_resource_manager
)) {
4322 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support VHCA management");
4326 mutex_lock(&esw
->state_lock
);
4327 *is_enabled
= vport
->info
.roce_enabled
;
4328 mutex_unlock(&esw
->state_lock
);
4332 int mlx5_devlink_port_fn_roce_set(struct devlink_port
*port
, bool enable
,
4333 struct netlink_ext_ack
*extack
)
4335 struct mlx5_eswitch
*esw
= mlx5_devlink_eswitch_nocheck_get(port
->devlink
);
4336 struct mlx5_vport
*vport
= mlx5_devlink_port_vport_get(port
);
4337 int query_out_sz
= MLX5_ST_SZ_BYTES(query_hca_cap_out
);
4338 u16 vport_num
= vport
->vport
;
4343 if (!MLX5_CAP_GEN(esw
->dev
, vhca_resource_manager
)) {
4344 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support VHCA management");
4348 mutex_lock(&esw
->state_lock
);
4350 if (vport
->info
.roce_enabled
== enable
) {
4355 query_ctx
= kzalloc(query_out_sz
, GFP_KERNEL
);
4361 err
= mlx5_vport_get_other_func_cap(esw
->dev
, vport_num
, query_ctx
,
4364 NL_SET_ERR_MSG_MOD(extack
, "Failed getting HCA caps");
4368 hca_caps
= MLX5_ADDR_OF(query_hca_cap_out
, query_ctx
, capability
);
4369 MLX5_SET(cmd_hca_cap
, hca_caps
, roce
, enable
);
4371 err
= mlx5_vport_set_other_func_cap(esw
->dev
, hca_caps
, vport_num
,
4372 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE
);
4374 NL_SET_ERR_MSG_MOD(extack
, "Failed setting HCA roce cap");
4378 vport
->info
.roce_enabled
= enable
;
4383 mutex_unlock(&esw
->state_lock
);
4388 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch
*esw
, struct mlx5_flow_handle
*rule
,
4389 struct mlx5_esw_flow_attr
*esw_attr
, int attr_idx
)
4391 struct mlx5_flow_destination new_dest
= {};
4392 struct mlx5_flow_destination old_dest
= {};
4394 if (!esw_setup_uplink_fwd_ipsec_needed(esw
, esw_attr
, attr_idx
))
4397 esw_setup_dest_fwd_ipsec(&old_dest
, NULL
, esw
, esw_attr
, attr_idx
, 0, false);
4398 esw_setup_dest_fwd_vport(&new_dest
, NULL
, esw
, esw_attr
, attr_idx
, 0, false);
4400 return mlx5_modify_rule_destination(rule
, &new_dest
, &old_dest
);
4403 #ifdef CONFIG_XFRM_OFFLOAD
4404 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port
*port
, bool *is_enabled
,
4405 struct netlink_ext_ack
*extack
)
4407 struct mlx5_eswitch
*esw
;
4408 struct mlx5_vport
*vport
;
4411 esw
= mlx5_devlink_eswitch_get(port
->devlink
);
4413 return PTR_ERR(esw
);
4415 if (!mlx5_esw_ipsec_vf_offload_supported(esw
->dev
)) {
4416 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support IPSec crypto");
4420 vport
= mlx5_devlink_port_vport_get(port
);
4422 mutex_lock(&esw
->state_lock
);
4423 if (!vport
->enabled
) {
4428 *is_enabled
= vport
->info
.ipsec_crypto_enabled
;
4430 mutex_unlock(&esw
->state_lock
);
4434 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port
*port
, bool enable
,
4435 struct netlink_ext_ack
*extack
)
4437 struct mlx5_eswitch
*esw
;
4438 struct mlx5_vport
*vport
;
4442 esw
= mlx5_devlink_eswitch_get(port
->devlink
);
4444 return PTR_ERR(esw
);
4446 vport_num
= mlx5_esw_devlink_port_index_to_vport_num(port
->index
);
4447 err
= mlx5_esw_ipsec_vf_crypto_offload_supported(esw
->dev
, vport_num
);
4449 NL_SET_ERR_MSG_MOD(extack
,
4450 "Device doesn't support IPsec crypto");
4454 vport
= mlx5_devlink_port_vport_get(port
);
4456 mutex_lock(&esw
->state_lock
);
4457 if (!vport
->enabled
) {
4459 NL_SET_ERR_MSG_MOD(extack
, "Eswitch vport is disabled");
4463 if (vport
->info
.ipsec_crypto_enabled
== enable
)
4466 if (!esw
->enabled_ipsec_vf_count
&& esw
->dev
->num_ipsec_offloads
) {
4471 err
= mlx5_esw_ipsec_vf_crypto_offload_set(esw
, vport
, enable
);
4473 NL_SET_ERR_MSG_MOD(extack
, "Failed to set IPsec crypto");
4477 vport
->info
.ipsec_crypto_enabled
= enable
;
4479 esw
->enabled_ipsec_vf_count
++;
4481 esw
->enabled_ipsec_vf_count
--;
4483 mutex_unlock(&esw
->state_lock
);
4487 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port
*port
, bool *is_enabled
,
4488 struct netlink_ext_ack
*extack
)
4490 struct mlx5_eswitch
*esw
;
4491 struct mlx5_vport
*vport
;
4494 esw
= mlx5_devlink_eswitch_get(port
->devlink
);
4496 return PTR_ERR(esw
);
4498 if (!mlx5_esw_ipsec_vf_offload_supported(esw
->dev
)) {
4499 NL_SET_ERR_MSG_MOD(extack
, "Device doesn't support IPsec packet");
4503 vport
= mlx5_devlink_port_vport_get(port
);
4505 mutex_lock(&esw
->state_lock
);
4506 if (!vport
->enabled
) {
4511 *is_enabled
= vport
->info
.ipsec_packet_enabled
;
4513 mutex_unlock(&esw
->state_lock
);
4517 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port
*port
,
4519 struct netlink_ext_ack
*extack
)
4521 struct mlx5_eswitch
*esw
;
4522 struct mlx5_vport
*vport
;
4526 esw
= mlx5_devlink_eswitch_get(port
->devlink
);
4528 return PTR_ERR(esw
);
4530 vport_num
= mlx5_esw_devlink_port_index_to_vport_num(port
->index
);
4531 err
= mlx5_esw_ipsec_vf_packet_offload_supported(esw
->dev
, vport_num
);
4533 NL_SET_ERR_MSG_MOD(extack
,
4534 "Device doesn't support IPsec packet mode");
4538 vport
= mlx5_devlink_port_vport_get(port
);
4539 mutex_lock(&esw
->state_lock
);
4540 if (!vport
->enabled
) {
4542 NL_SET_ERR_MSG_MOD(extack
, "Eswitch vport is disabled");
4546 if (vport
->info
.ipsec_packet_enabled
== enable
)
4549 if (!esw
->enabled_ipsec_vf_count
&& esw
->dev
->num_ipsec_offloads
) {
4554 err
= mlx5_esw_ipsec_vf_packet_offload_set(esw
, vport
, enable
);
4556 NL_SET_ERR_MSG_MOD(extack
,
4557 "Failed to set IPsec packet mode");
4561 vport
->info
.ipsec_packet_enabled
= enable
;
4563 esw
->enabled_ipsec_vf_count
++;
4565 esw
->enabled_ipsec_vf_count
--;
4567 mutex_unlock(&esw
->state_lock
);
4570 #endif /* CONFIG_XFRM_OFFLOAD */