1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
6 #include <rdma/ib_user_verbs.h>
7 #include <rdma/ib_verbs.h>
8 #include <rdma/uverbs_types.h>
9 #include <rdma/uverbs_ioctl.h>
10 #include <rdma/uverbs_std_types.h>
11 #include <rdma/mlx5_user_ioctl_cmds.h>
12 #include <rdma/mlx5_user_ioctl_verbs.h>
13 #include <rdma/ib_hdrs.h>
14 #include <rdma/ib_umem.h>
15 #include <linux/mlx5/driver.h>
16 #include <linux/mlx5/fs.h>
17 #include <linux/mlx5/fs_helpers.h>
18 #include <linux/mlx5/eswitch.h>
19 #include <net/inet_ecn.h>
25 #define UVERBS_MODULE_NAME mlx5_ib
26 #include <rdma/uverbs_named_ioctl.h>
29 MATCH_CRITERIA_ENABLE_OUTER_BIT
,
30 MATCH_CRITERIA_ENABLE_MISC_BIT
,
31 MATCH_CRITERIA_ENABLE_INNER_BIT
,
32 MATCH_CRITERIA_ENABLE_MISC2_BIT
35 #define HEADER_IS_ZERO(match_criteria, headers) \
36 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
37 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
39 static u8 get_match_criteria_enable(u32 *match_criteria)
41 u8 match_criteria_enable
;
43 match_criteria_enable
=
44 (!HEADER_IS_ZERO(match_criteria
, outer_headers
)) <<
45 MATCH_CRITERIA_ENABLE_OUTER_BIT
;
46 match_criteria_enable
|=
47 (!HEADER_IS_ZERO(match_criteria
, misc_parameters
)) <<
48 MATCH_CRITERIA_ENABLE_MISC_BIT
;
49 match_criteria_enable
|=
50 (!HEADER_IS_ZERO(match_criteria
, inner_headers
)) <<
51 MATCH_CRITERIA_ENABLE_INNER_BIT
;
52 match_criteria_enable
|=
53 (!HEADER_IS_ZERO(match_criteria
, misc_parameters_2
)) <<
54 MATCH_CRITERIA_ENABLE_MISC2_BIT
;
56 return match_criteria_enable
;
59 static int set_proto(void *outer_c
, void *outer_v
, u8 mask
, u8 val
)
68 entry_mask
= MLX5_GET(fte_match_set_lyr_2_4
, outer_c
,
70 entry_val
= MLX5_GET(fte_match_set_lyr_2_4
, outer_v
,
73 MLX5_SET(fte_match_set_lyr_2_4
, outer_c
, ip_protocol
, mask
);
74 MLX5_SET(fte_match_set_lyr_2_4
, outer_v
, ip_protocol
, val
);
77 /* Don't override existing ip protocol */
78 if (mask
!= entry_mask
|| val
!= entry_val
)
84 static void set_flow_label(void *misc_c
, void *misc_v
, u32 mask
, u32 val
,
88 MLX5_SET(fte_match_set_misc
,
89 misc_c
, inner_ipv6_flow_label
, mask
);
90 MLX5_SET(fte_match_set_misc
,
91 misc_v
, inner_ipv6_flow_label
, val
);
93 MLX5_SET(fte_match_set_misc
,
94 misc_c
, outer_ipv6_flow_label
, mask
);
95 MLX5_SET(fte_match_set_misc
,
96 misc_v
, outer_ipv6_flow_label
, val
);
100 static void set_tos(void *outer_c
, void *outer_v
, u8 mask
, u8 val
)
102 MLX5_SET(fte_match_set_lyr_2_4
, outer_c
, ip_ecn
, mask
);
103 MLX5_SET(fte_match_set_lyr_2_4
, outer_v
, ip_ecn
, val
);
104 MLX5_SET(fte_match_set_lyr_2_4
, outer_c
, ip_dscp
, mask
>> 2);
105 MLX5_SET(fte_match_set_lyr_2_4
, outer_v
, ip_dscp
, val
>> 2);
108 static int check_mpls_supp_fields(u32 field_support
, const __be32
*set_mask
)
110 if (MLX5_GET(fte_match_mpls
, set_mask
, mpls_label
) &&
111 !(field_support
& MLX5_FIELD_SUPPORT_MPLS_LABEL
))
114 if (MLX5_GET(fte_match_mpls
, set_mask
, mpls_exp
) &&
115 !(field_support
& MLX5_FIELD_SUPPORT_MPLS_EXP
))
118 if (MLX5_GET(fte_match_mpls
, set_mask
, mpls_s_bos
) &&
119 !(field_support
& MLX5_FIELD_SUPPORT_MPLS_S_BOS
))
122 if (MLX5_GET(fte_match_mpls
, set_mask
, mpls_ttl
) &&
123 !(field_support
& MLX5_FIELD_SUPPORT_MPLS_TTL
))
129 #define LAST_ETH_FIELD vlan_tag
130 #define LAST_IPV4_FIELD tos
131 #define LAST_IPV6_FIELD traffic_class
132 #define LAST_TCP_UDP_FIELD src_port
133 #define LAST_TUNNEL_FIELD tunnel_id
134 #define LAST_FLOW_TAG_FIELD tag_id
135 #define LAST_DROP_FIELD size
136 #define LAST_COUNTERS_FIELD counters
138 /* Field is the last supported field */
139 #define FIELDS_NOT_SUPPORTED(filter, field) \
140 memchr_inv((void *)&filter.field + sizeof(filter.field), 0, \
141 sizeof(filter) - offsetofend(typeof(filter), field))
143 int parse_flow_flow_action(struct mlx5_ib_flow_action
*maction
,
145 struct mlx5_flow_act
*action
)
148 switch (maction
->ib_action
.type
) {
149 case IB_FLOW_ACTION_UNSPECIFIED
:
150 if (maction
->flow_action_raw
.sub_type
==
151 MLX5_IB_FLOW_ACTION_MODIFY_HEADER
) {
152 if (action
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
154 action
->action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
156 maction
->flow_action_raw
.modify_hdr
;
159 if (maction
->flow_action_raw
.sub_type
==
160 MLX5_IB_FLOW_ACTION_DECAP
) {
161 if (action
->action
& MLX5_FLOW_CONTEXT_ACTION_DECAP
)
163 action
->action
|= MLX5_FLOW_CONTEXT_ACTION_DECAP
;
166 if (maction
->flow_action_raw
.sub_type
==
167 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT
) {
169 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
)
172 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
173 action
->pkt_reformat
=
174 maction
->flow_action_raw
.pkt_reformat
;
183 static int parse_flow_attr(struct mlx5_core_dev
*mdev
,
184 struct mlx5_flow_spec
*spec
,
185 const union ib_flow_spec
*ib_spec
,
186 const struct ib_flow_attr
*flow_attr
,
187 struct mlx5_flow_act
*action
, u32 prev_type
)
189 struct mlx5_flow_context
*flow_context
= &spec
->flow_context
;
190 u32
*match_c
= spec
->match_criteria
;
191 u32
*match_v
= spec
->match_value
;
192 void *misc_params_c
= MLX5_ADDR_OF(fte_match_param
, match_c
,
194 void *misc_params_v
= MLX5_ADDR_OF(fte_match_param
, match_v
,
196 void *misc_params2_c
= MLX5_ADDR_OF(fte_match_param
, match_c
,
198 void *misc_params2_v
= MLX5_ADDR_OF(fte_match_param
, match_v
,
205 if (ib_spec
->type
& IB_FLOW_SPEC_INNER
) {
206 headers_c
= MLX5_ADDR_OF(fte_match_param
, match_c
,
208 headers_v
= MLX5_ADDR_OF(fte_match_param
, match_v
,
210 match_ipv
= MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
211 ft_field_support
.inner_ip_version
);
213 headers_c
= MLX5_ADDR_OF(fte_match_param
, match_c
,
215 headers_v
= MLX5_ADDR_OF(fte_match_param
, match_v
,
217 match_ipv
= MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
218 ft_field_support
.outer_ip_version
);
221 switch (ib_spec
->type
& ~IB_FLOW_SPEC_INNER
) {
222 case IB_FLOW_SPEC_ETH
:
223 if (FIELDS_NOT_SUPPORTED(ib_spec
->eth
.mask
, LAST_ETH_FIELD
))
226 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
228 ib_spec
->eth
.mask
.dst_mac
);
229 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
231 ib_spec
->eth
.val
.dst_mac
);
233 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
235 ib_spec
->eth
.mask
.src_mac
);
236 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
238 ib_spec
->eth
.val
.src_mac
);
240 if (ib_spec
->eth
.mask
.vlan_tag
) {
241 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
243 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
246 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
247 first_vid
, ntohs(ib_spec
->eth
.mask
.vlan_tag
));
248 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
249 first_vid
, ntohs(ib_spec
->eth
.val
.vlan_tag
));
251 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
253 ntohs(ib_spec
->eth
.mask
.vlan_tag
) >> 12);
254 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
256 ntohs(ib_spec
->eth
.val
.vlan_tag
) >> 12);
258 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
260 ntohs(ib_spec
->eth
.mask
.vlan_tag
) >> 13);
261 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
263 ntohs(ib_spec
->eth
.val
.vlan_tag
) >> 13);
265 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
266 ethertype
, ntohs(ib_spec
->eth
.mask
.ether_type
));
267 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
268 ethertype
, ntohs(ib_spec
->eth
.val
.ether_type
));
270 case IB_FLOW_SPEC_IPV4
:
271 if (FIELDS_NOT_SUPPORTED(ib_spec
->ipv4
.mask
, LAST_IPV4_FIELD
))
275 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
277 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
278 ip_version
, MLX5_FS_IPV4_VERSION
);
280 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
282 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
283 ethertype
, ETH_P_IP
);
286 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
287 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
288 &ib_spec
->ipv4
.mask
.src_ip
,
289 sizeof(ib_spec
->ipv4
.mask
.src_ip
));
290 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
291 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
292 &ib_spec
->ipv4
.val
.src_ip
,
293 sizeof(ib_spec
->ipv4
.val
.src_ip
));
294 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
295 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
296 &ib_spec
->ipv4
.mask
.dst_ip
,
297 sizeof(ib_spec
->ipv4
.mask
.dst_ip
));
298 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
299 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
300 &ib_spec
->ipv4
.val
.dst_ip
,
301 sizeof(ib_spec
->ipv4
.val
.dst_ip
));
303 set_tos(headers_c
, headers_v
,
304 ib_spec
->ipv4
.mask
.tos
, ib_spec
->ipv4
.val
.tos
);
306 if (set_proto(headers_c
, headers_v
,
307 ib_spec
->ipv4
.mask
.proto
,
308 ib_spec
->ipv4
.val
.proto
))
311 case IB_FLOW_SPEC_IPV6
:
312 if (FIELDS_NOT_SUPPORTED(ib_spec
->ipv6
.mask
, LAST_IPV6_FIELD
))
316 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
318 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
319 ip_version
, MLX5_FS_IPV6_VERSION
);
321 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
323 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
324 ethertype
, ETH_P_IPV6
);
327 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
328 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
329 &ib_spec
->ipv6
.mask
.src_ip
,
330 sizeof(ib_spec
->ipv6
.mask
.src_ip
));
331 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
332 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
333 &ib_spec
->ipv6
.val
.src_ip
,
334 sizeof(ib_spec
->ipv6
.val
.src_ip
));
335 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
336 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
337 &ib_spec
->ipv6
.mask
.dst_ip
,
338 sizeof(ib_spec
->ipv6
.mask
.dst_ip
));
339 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
340 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
341 &ib_spec
->ipv6
.val
.dst_ip
,
342 sizeof(ib_spec
->ipv6
.val
.dst_ip
));
344 set_tos(headers_c
, headers_v
,
345 ib_spec
->ipv6
.mask
.traffic_class
,
346 ib_spec
->ipv6
.val
.traffic_class
);
348 if (set_proto(headers_c
, headers_v
,
349 ib_spec
->ipv6
.mask
.next_hdr
,
350 ib_spec
->ipv6
.val
.next_hdr
))
353 set_flow_label(misc_params_c
, misc_params_v
,
354 ntohl(ib_spec
->ipv6
.mask
.flow_label
),
355 ntohl(ib_spec
->ipv6
.val
.flow_label
),
356 ib_spec
->type
& IB_FLOW_SPEC_INNER
);
358 case IB_FLOW_SPEC_ESP
:
360 case IB_FLOW_SPEC_TCP
:
361 if (FIELDS_NOT_SUPPORTED(ib_spec
->tcp_udp
.mask
,
365 if (set_proto(headers_c
, headers_v
, 0xff, IPPROTO_TCP
))
368 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, tcp_sport
,
369 ntohs(ib_spec
->tcp_udp
.mask
.src_port
));
370 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, tcp_sport
,
371 ntohs(ib_spec
->tcp_udp
.val
.src_port
));
373 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, tcp_dport
,
374 ntohs(ib_spec
->tcp_udp
.mask
.dst_port
));
375 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, tcp_dport
,
376 ntohs(ib_spec
->tcp_udp
.val
.dst_port
));
378 case IB_FLOW_SPEC_UDP
:
379 if (FIELDS_NOT_SUPPORTED(ib_spec
->tcp_udp
.mask
,
383 if (set_proto(headers_c
, headers_v
, 0xff, IPPROTO_UDP
))
386 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, udp_sport
,
387 ntohs(ib_spec
->tcp_udp
.mask
.src_port
));
388 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, udp_sport
,
389 ntohs(ib_spec
->tcp_udp
.val
.src_port
));
391 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, udp_dport
,
392 ntohs(ib_spec
->tcp_udp
.mask
.dst_port
));
393 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, udp_dport
,
394 ntohs(ib_spec
->tcp_udp
.val
.dst_port
));
396 case IB_FLOW_SPEC_GRE
:
397 if (ib_spec
->gre
.mask
.c_ks_res0_ver
)
400 if (set_proto(headers_c
, headers_v
, 0xff, IPPROTO_GRE
))
403 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_protocol
,
405 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
,
408 MLX5_SET(fte_match_set_misc
, misc_params_c
, gre_protocol
,
409 ntohs(ib_spec
->gre
.mask
.protocol
));
410 MLX5_SET(fte_match_set_misc
, misc_params_v
, gre_protocol
,
411 ntohs(ib_spec
->gre
.val
.protocol
));
413 memcpy(MLX5_ADDR_OF(fte_match_set_misc
, misc_params_c
,
415 &ib_spec
->gre
.mask
.key
,
416 sizeof(ib_spec
->gre
.mask
.key
));
417 memcpy(MLX5_ADDR_OF(fte_match_set_misc
, misc_params_v
,
419 &ib_spec
->gre
.val
.key
,
420 sizeof(ib_spec
->gre
.val
.key
));
422 case IB_FLOW_SPEC_MPLS
:
424 case IB_FLOW_SPEC_UDP
:
425 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
426 ft_field_support
.outer_first_mpls_over_udp
),
427 &ib_spec
->mpls
.mask
.tag
))
430 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_v
,
431 outer_first_mpls_over_udp
),
432 &ib_spec
->mpls
.val
.tag
,
433 sizeof(ib_spec
->mpls
.val
.tag
));
434 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_c
,
435 outer_first_mpls_over_udp
),
436 &ib_spec
->mpls
.mask
.tag
,
437 sizeof(ib_spec
->mpls
.mask
.tag
));
439 case IB_FLOW_SPEC_GRE
:
440 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
441 ft_field_support
.outer_first_mpls_over_gre
),
442 &ib_spec
->mpls
.mask
.tag
))
445 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_v
,
446 outer_first_mpls_over_gre
),
447 &ib_spec
->mpls
.val
.tag
,
448 sizeof(ib_spec
->mpls
.val
.tag
));
449 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_c
,
450 outer_first_mpls_over_gre
),
451 &ib_spec
->mpls
.mask
.tag
,
452 sizeof(ib_spec
->mpls
.mask
.tag
));
455 if (ib_spec
->type
& IB_FLOW_SPEC_INNER
) {
456 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
457 ft_field_support
.inner_first_mpls
),
458 &ib_spec
->mpls
.mask
.tag
))
461 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_v
,
463 &ib_spec
->mpls
.val
.tag
,
464 sizeof(ib_spec
->mpls
.val
.tag
));
465 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_c
,
467 &ib_spec
->mpls
.mask
.tag
,
468 sizeof(ib_spec
->mpls
.mask
.tag
));
470 if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
471 ft_field_support
.outer_first_mpls
),
472 &ib_spec
->mpls
.mask
.tag
))
475 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_v
,
477 &ib_spec
->mpls
.val
.tag
,
478 sizeof(ib_spec
->mpls
.val
.tag
));
479 memcpy(MLX5_ADDR_OF(fte_match_set_misc2
, misc_params2_c
,
481 &ib_spec
->mpls
.mask
.tag
,
482 sizeof(ib_spec
->mpls
.mask
.tag
));
486 case IB_FLOW_SPEC_VXLAN_TUNNEL
:
487 if (FIELDS_NOT_SUPPORTED(ib_spec
->tunnel
.mask
,
491 MLX5_SET(fte_match_set_misc
, misc_params_c
, vxlan_vni
,
492 ntohl(ib_spec
->tunnel
.mask
.tunnel_id
));
493 MLX5_SET(fte_match_set_misc
, misc_params_v
, vxlan_vni
,
494 ntohl(ib_spec
->tunnel
.val
.tunnel_id
));
496 case IB_FLOW_SPEC_ACTION_TAG
:
497 if (FIELDS_NOT_SUPPORTED(ib_spec
->flow_tag
,
498 LAST_FLOW_TAG_FIELD
))
500 if (ib_spec
->flow_tag
.tag_id
>= BIT(24))
503 flow_context
->flow_tag
= ib_spec
->flow_tag
.tag_id
;
504 flow_context
->flags
|= FLOW_CONTEXT_HAS_TAG
;
506 case IB_FLOW_SPEC_ACTION_DROP
:
507 if (FIELDS_NOT_SUPPORTED(ib_spec
->drop
,
510 action
->action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
;
512 case IB_FLOW_SPEC_ACTION_HANDLE
:
513 ret
= parse_flow_flow_action(to_mflow_act(ib_spec
->action
.act
),
514 flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_EGRESS
, action
);
518 case IB_FLOW_SPEC_ACTION_COUNT
:
519 if (FIELDS_NOT_SUPPORTED(ib_spec
->flow_count
,
520 LAST_COUNTERS_FIELD
))
523 /* for now support only one counters spec per flow */
524 if (action
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
)
527 action
->counters
= ib_spec
->flow_count
.counters
;
528 action
->action
|= MLX5_FLOW_CONTEXT_ACTION_COUNT
;
537 /* If a flow could catch both multicast and unicast packets,
538 * it won't fall into the multicast flow steering table and this rule
539 * could steal other multicast packets.
541 static bool flow_is_multicast_only(const struct ib_flow_attr
*ib_attr
)
543 union ib_flow_spec
*flow_spec
;
545 if (ib_attr
->type
!= IB_FLOW_ATTR_NORMAL
||
546 ib_attr
->num_of_specs
< 1)
549 flow_spec
= (union ib_flow_spec
*)(ib_attr
+ 1);
550 if (flow_spec
->type
== IB_FLOW_SPEC_IPV4
) {
551 struct ib_flow_spec_ipv4
*ipv4_spec
;
553 ipv4_spec
= (struct ib_flow_spec_ipv4
*)flow_spec
;
554 if (ipv4_is_multicast(ipv4_spec
->val
.dst_ip
))
560 if (flow_spec
->type
== IB_FLOW_SPEC_ETH
) {
561 struct ib_flow_spec_eth
*eth_spec
;
563 eth_spec
= (struct ib_flow_spec_eth
*)flow_spec
;
564 return is_multicast_ether_addr(eth_spec
->mask
.dst_mac
) &&
565 is_multicast_ether_addr(eth_spec
->val
.dst_mac
);
571 static bool is_valid_ethertype(struct mlx5_core_dev
*mdev
,
572 const struct ib_flow_attr
*flow_attr
,
575 union ib_flow_spec
*ib_spec
= (union ib_flow_spec
*)(flow_attr
+ 1);
576 int match_ipv
= check_inner
?
577 MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
578 ft_field_support
.inner_ip_version
) :
579 MLX5_CAP_FLOWTABLE_NIC_RX(mdev
,
580 ft_field_support
.outer_ip_version
);
581 int inner_bit
= check_inner
? IB_FLOW_SPEC_INNER
: 0;
582 bool ipv4_spec_valid
, ipv6_spec_valid
;
583 unsigned int ip_spec_type
= 0;
584 bool has_ethertype
= false;
585 unsigned int spec_index
;
586 bool mask_valid
= true;
590 /* Validate that ethertype is correct */
591 for (spec_index
= 0; spec_index
< flow_attr
->num_of_specs
; spec_index
++) {
592 if ((ib_spec
->type
== (IB_FLOW_SPEC_ETH
| inner_bit
)) &&
593 ib_spec
->eth
.mask
.ether_type
) {
594 mask_valid
= (ib_spec
->eth
.mask
.ether_type
==
596 has_ethertype
= true;
597 eth_type
= ntohs(ib_spec
->eth
.val
.ether_type
);
598 } else if ((ib_spec
->type
== (IB_FLOW_SPEC_IPV4
| inner_bit
)) ||
599 (ib_spec
->type
== (IB_FLOW_SPEC_IPV6
| inner_bit
))) {
600 ip_spec_type
= ib_spec
->type
;
602 ib_spec
= (void *)ib_spec
+ ib_spec
->size
;
605 type_valid
= (!has_ethertype
) || (!ip_spec_type
);
606 if (!type_valid
&& mask_valid
) {
607 ipv4_spec_valid
= (eth_type
== ETH_P_IP
) &&
608 (ip_spec_type
== (IB_FLOW_SPEC_IPV4
| inner_bit
));
609 ipv6_spec_valid
= (eth_type
== ETH_P_IPV6
) &&
610 (ip_spec_type
== (IB_FLOW_SPEC_IPV6
| inner_bit
));
612 type_valid
= (ipv4_spec_valid
) || (ipv6_spec_valid
) ||
613 (((eth_type
== ETH_P_MPLS_UC
) ||
614 (eth_type
== ETH_P_MPLS_MC
)) && match_ipv
);
620 static bool is_valid_attr(struct mlx5_core_dev
*mdev
,
621 const struct ib_flow_attr
*flow_attr
)
623 return is_valid_ethertype(mdev
, flow_attr
, false) &&
624 is_valid_ethertype(mdev
, flow_attr
, true);
627 static void put_flow_table(struct mlx5_ib_dev
*dev
,
628 struct mlx5_ib_flow_prio
*prio
, bool ft_added
)
630 prio
->refcount
-= !!ft_added
;
631 if (!prio
->refcount
) {
632 mlx5_destroy_flow_table(prio
->flow_table
);
633 prio
->flow_table
= NULL
;
637 static int mlx5_ib_destroy_flow(struct ib_flow
*flow_id
)
639 struct mlx5_ib_flow_handler
*handler
= container_of(flow_id
,
640 struct mlx5_ib_flow_handler
,
642 struct mlx5_ib_flow_handler
*iter
, *tmp
;
643 struct mlx5_ib_dev
*dev
= handler
->dev
;
645 mutex_lock(&dev
->flow_db
->lock
);
647 list_for_each_entry_safe(iter
, tmp
, &handler
->list
, list
) {
648 mlx5_del_flow_rules(iter
->rule
);
649 put_flow_table(dev
, iter
->prio
, true);
650 list_del(&iter
->list
);
654 mlx5_del_flow_rules(handler
->rule
);
655 put_flow_table(dev
, handler
->prio
, true);
656 mlx5_ib_counters_clear_description(handler
->ibcounters
);
657 mutex_unlock(&dev
->flow_db
->lock
);
658 if (handler
->flow_matcher
)
659 atomic_dec(&handler
->flow_matcher
->usecnt
);
665 static int ib_prio_to_core_prio(unsigned int priority
, bool dont_trap
)
673 enum flow_table_type
{
678 #define MLX5_FS_MAX_TYPES 6
679 #define MLX5_FS_MAX_ENTRIES BIT(16)
681 static bool mlx5_ib_shared_ft_allowed(struct ib_device
*device
)
683 struct mlx5_ib_dev
*dev
= to_mdev(device
);
685 return MLX5_CAP_GEN(dev
->mdev
, shared_object_to_user_object_allowed
);
688 static struct mlx5_ib_flow_prio
*_get_prio(struct mlx5_ib_dev
*dev
,
689 struct mlx5_flow_namespace
*ns
,
690 struct mlx5_ib_flow_prio
*prio
,
692 int num_entries
, int num_groups
,
695 struct mlx5_flow_table_attr ft_attr
= {};
696 struct mlx5_flow_table
*ft
;
698 ft_attr
.prio
= priority
;
699 ft_attr
.max_fte
= num_entries
;
700 ft_attr
.flags
= flags
;
701 ft_attr
.autogroup
.max_num_groups
= num_groups
;
702 ft
= mlx5_create_auto_grouped_flow_table(ns
, &ft_attr
);
706 prio
->flow_table
= ft
;
711 static struct mlx5_ib_flow_prio
*get_flow_table(struct mlx5_ib_dev
*dev
,
712 struct ib_flow_attr
*flow_attr
,
713 enum flow_table_type ft_type
)
715 bool dont_trap
= flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
;
716 struct mlx5_flow_namespace
*ns
= NULL
;
717 enum mlx5_flow_namespace_type fn_type
;
718 struct mlx5_ib_flow_prio
*prio
;
719 struct mlx5_flow_table
*ft
;
727 max_table_size
= BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
,
729 esw_encap
= mlx5_eswitch_get_encap_mode(dev
->mdev
) !=
730 DEVLINK_ESWITCH_ENCAP_MODE_NONE
;
731 switch (flow_attr
->type
) {
732 case IB_FLOW_ATTR_NORMAL
:
733 if (flow_is_multicast_only(flow_attr
) && !dont_trap
)
734 priority
= MLX5_IB_FLOW_MCAST_PRIO
;
736 priority
= ib_prio_to_core_prio(flow_attr
->priority
,
738 if (ft_type
== MLX5_IB_FT_RX
) {
739 fn_type
= MLX5_FLOW_NAMESPACE_BYPASS
;
740 prio
= &dev
->flow_db
->prios
[priority
];
741 if (!dev
->is_rep
&& !esw_encap
&&
742 MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
, decap
))
743 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP
;
744 if (!dev
->is_rep
&& !esw_encap
&&
745 MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
,
746 reformat_l3_tunnel_to_l2
))
747 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
;
749 max_table_size
= BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
750 dev
->mdev
, log_max_ft_size
));
751 fn_type
= MLX5_FLOW_NAMESPACE_EGRESS
;
752 prio
= &dev
->flow_db
->egress_prios
[priority
];
753 if (!dev
->is_rep
&& !esw_encap
&&
754 MLX5_CAP_FLOWTABLE_NIC_TX(dev
->mdev
, reformat
))
755 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
;
757 ns
= mlx5_get_flow_namespace(dev
->mdev
, fn_type
);
758 num_entries
= MLX5_FS_MAX_ENTRIES
;
759 num_groups
= MLX5_FS_MAX_TYPES
;
761 case IB_FLOW_ATTR_ALL_DEFAULT
:
762 case IB_FLOW_ATTR_MC_DEFAULT
:
763 ns
= mlx5_get_flow_namespace(dev
->mdev
,
764 MLX5_FLOW_NAMESPACE_LEFTOVERS
);
765 build_leftovers_ft_param(&priority
, &num_entries
, &num_groups
);
766 prio
= &dev
->flow_db
->prios
[MLX5_IB_FLOW_LEFTOVERS_PRIO
];
768 case IB_FLOW_ATTR_SNIFFER
:
769 if (!MLX5_CAP_FLOWTABLE(dev
->mdev
,
770 allow_sniffer_and_nic_rx_shared_tir
))
771 return ERR_PTR(-EOPNOTSUPP
);
773 ns
= mlx5_get_flow_namespace(
774 dev
->mdev
, ft_type
== MLX5_IB_FT_RX
?
775 MLX5_FLOW_NAMESPACE_SNIFFER_RX
:
776 MLX5_FLOW_NAMESPACE_SNIFFER_TX
);
778 prio
= &dev
->flow_db
->sniffer
[ft_type
];
788 return ERR_PTR(-EOPNOTSUPP
);
790 max_table_size
= min_t(int, num_entries
, max_table_size
);
792 ft
= prio
->flow_table
;
794 return _get_prio(dev
, ns
, prio
, priority
, max_table_size
,
801 RDMA_RX_ECN_OPCOUNTER_PRIO
,
802 RDMA_RX_CNP_OPCOUNTER_PRIO
,
806 RDMA_TX_CNP_OPCOUNTER_PRIO
,
809 static int set_vhca_port_spec(struct mlx5_ib_dev
*dev
, u32 port_num
,
810 struct mlx5_flow_spec
*spec
)
812 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev
->mdev
,
813 ft_field_support
.source_vhca_port
) ||
814 !MLX5_CAP_FLOWTABLE_RDMA_TX(dev
->mdev
,
815 ft_field_support
.source_vhca_port
))
818 MLX5_SET_TO_ONES(fte_match_param
, &spec
->match_criteria
,
819 misc_parameters
.source_vhca_port
);
820 MLX5_SET(fte_match_param
, &spec
->match_value
,
821 misc_parameters
.source_vhca_port
, port_num
);
826 static int set_ecn_ce_spec(struct mlx5_ib_dev
*dev
, u32 port_num
,
827 struct mlx5_flow_spec
*spec
, int ipv
)
829 if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev
->mdev
,
830 ft_field_support
.outer_ip_version
))
833 if (mlx5_core_mp_enabled(dev
->mdev
) &&
834 set_vhca_port_spec(dev
, port_num
, spec
))
837 MLX5_SET_TO_ONES(fte_match_param
, spec
->match_criteria
,
838 outer_headers
.ip_ecn
);
839 MLX5_SET(fte_match_param
, spec
->match_value
, outer_headers
.ip_ecn
,
841 MLX5_SET_TO_ONES(fte_match_param
, spec
->match_criteria
,
842 outer_headers
.ip_version
);
843 MLX5_SET(fte_match_param
, spec
->match_value
, outer_headers
.ip_version
,
846 spec
->match_criteria_enable
=
847 get_match_criteria_enable(spec
->match_criteria
);
852 static int set_cnp_spec(struct mlx5_ib_dev
*dev
, u32 port_num
,
853 struct mlx5_flow_spec
*spec
)
855 if (mlx5_core_mp_enabled(dev
->mdev
) &&
856 set_vhca_port_spec(dev
, port_num
, spec
))
859 MLX5_SET_TO_ONES(fte_match_param
, spec
->match_criteria
,
860 misc_parameters
.bth_opcode
);
861 MLX5_SET(fte_match_param
, spec
->match_value
, misc_parameters
.bth_opcode
,
864 spec
->match_criteria_enable
=
865 get_match_criteria_enable(spec
->match_criteria
);
870 int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev
*dev
, u32 port_num
,
871 struct mlx5_ib_op_fc
*opfc
,
872 enum mlx5_ib_optional_counter_type type
)
874 enum mlx5_flow_namespace_type fn_type
;
875 int priority
, i
, err
, spec_num
;
876 struct mlx5_flow_act flow_act
= {};
877 struct mlx5_flow_destination dst
;
878 struct mlx5_flow_namespace
*ns
;
879 struct mlx5_ib_flow_prio
*prio
;
880 struct mlx5_flow_spec
*spec
;
882 spec
= kcalloc(MAX_OPFC_RULES
, sizeof(*spec
), GFP_KERNEL
);
887 case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS
:
888 if (set_ecn_ce_spec(dev
, port_num
, &spec
[0],
889 MLX5_FS_IPV4_VERSION
) ||
890 set_ecn_ce_spec(dev
, port_num
, &spec
[1],
891 MLX5_FS_IPV6_VERSION
)) {
896 fn_type
= MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS
;
897 priority
= RDMA_RX_ECN_OPCOUNTER_PRIO
;
900 case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS
:
901 if (!MLX5_CAP_FLOWTABLE(dev
->mdev
,
902 ft_field_support_2_nic_receive_rdma
.bth_opcode
) ||
903 set_cnp_spec(dev
, port_num
, &spec
[0])) {
908 fn_type
= MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS
;
909 priority
= RDMA_RX_CNP_OPCOUNTER_PRIO
;
912 case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS
:
913 if (!MLX5_CAP_FLOWTABLE(dev
->mdev
,
914 ft_field_support_2_nic_transmit_rdma
.bth_opcode
) ||
915 set_cnp_spec(dev
, port_num
, &spec
[0])) {
920 fn_type
= MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS
;
921 priority
= RDMA_TX_CNP_OPCOUNTER_PRIO
;
929 ns
= mlx5_get_flow_namespace(dev
->mdev
, fn_type
);
935 prio
= &dev
->flow_db
->opfcs
[type
];
936 if (!prio
->flow_table
) {
937 prio
= _get_prio(dev
, ns
, prio
, priority
,
938 dev
->num_ports
* MAX_OPFC_RULES
, 1, 0);
945 dst
.type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
946 dst
.counter_id
= mlx5_fc_id(opfc
->fc
);
949 MLX5_FLOW_CONTEXT_ACTION_COUNT
| MLX5_FLOW_CONTEXT_ACTION_ALLOW
;
951 for (i
= 0; i
< spec_num
; i
++) {
952 opfc
->rule
[i
] = mlx5_add_flow_rules(prio
->flow_table
, &spec
[i
],
954 if (IS_ERR(opfc
->rule
[i
])) {
955 err
= PTR_ERR(opfc
->rule
[i
]);
959 prio
->refcount
+= spec_num
;
965 for (i
-= 1; i
>= 0; i
--)
966 mlx5_del_flow_rules(opfc
->rule
[i
]);
967 put_flow_table(dev
, prio
, false);
973 void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev
*dev
,
974 struct mlx5_ib_op_fc
*opfc
,
975 enum mlx5_ib_optional_counter_type type
)
979 for (i
= 0; i
< MAX_OPFC_RULES
&& opfc
->rule
[i
]; i
++) {
980 mlx5_del_flow_rules(opfc
->rule
[i
]);
981 put_flow_table(dev
, &dev
->flow_db
->opfcs
[type
], true);
985 static void set_underlay_qp(struct mlx5_ib_dev
*dev
,
986 struct mlx5_flow_spec
*spec
,
989 void *misc_params_c
= MLX5_ADDR_OF(fte_match_param
,
990 spec
->match_criteria
,
992 void *misc_params_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
996 MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
,
997 ft_field_support
.bth_dst_qp
)) {
998 MLX5_SET(fte_match_set_misc
,
999 misc_params_v
, bth_dst_qp
, underlay_qpn
);
1000 MLX5_SET(fte_match_set_misc
,
1001 misc_params_c
, bth_dst_qp
, 0xffffff);
1005 static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev
*dev
,
1006 struct mlx5_flow_spec
*spec
,
1007 struct mlx5_eswitch_rep
*rep
)
1009 struct mlx5_eswitch
*esw
= dev
->mdev
->priv
.eswitch
;
1012 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1013 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1016 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1017 mlx5_eswitch_get_vport_metadata_for_match(rep
->esw
,
1019 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1022 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1023 mlx5_eswitch_get_vport_metadata_mask());
1025 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1028 MLX5_SET(fte_match_set_misc
, misc
, source_port
, rep
->vport
);
1030 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1033 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
1037 static struct mlx5_ib_flow_handler
*_create_flow_rule(struct mlx5_ib_dev
*dev
,
1038 struct mlx5_ib_flow_prio
*ft_prio
,
1039 const struct ib_flow_attr
*flow_attr
,
1040 struct mlx5_flow_destination
*dst
,
1042 struct mlx5_ib_create_flow
*ucmd
)
1044 struct mlx5_flow_table
*ft
= ft_prio
->flow_table
;
1045 struct mlx5_ib_flow_handler
*handler
;
1046 struct mlx5_flow_act flow_act
= {};
1047 struct mlx5_flow_spec
*spec
;
1048 struct mlx5_flow_destination dest_arr
[2] = {};
1049 struct mlx5_flow_destination
*rule_dst
= dest_arr
;
1050 const void *ib_flow
= (const void *)flow_attr
+ sizeof(*flow_attr
);
1051 unsigned int spec_index
;
1055 bool is_egress
= flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_EGRESS
;
1057 if (!is_valid_attr(dev
->mdev
, flow_attr
))
1058 return ERR_PTR(-EINVAL
);
1060 if (dev
->is_rep
&& is_egress
)
1061 return ERR_PTR(-EINVAL
);
1063 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1064 handler
= kzalloc(sizeof(*handler
), GFP_KERNEL
);
1065 if (!handler
|| !spec
) {
1070 INIT_LIST_HEAD(&handler
->list
);
1072 for (spec_index
= 0; spec_index
< flow_attr
->num_of_specs
; spec_index
++) {
1073 err
= parse_flow_attr(dev
->mdev
, spec
,
1074 ib_flow
, flow_attr
, &flow_act
,
1079 prev_type
= ((union ib_flow_spec
*)ib_flow
)->type
;
1080 ib_flow
+= ((union ib_flow_spec
*)ib_flow
)->size
;
1083 if (dst
&& !(flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_DROP
)) {
1084 memcpy(&dest_arr
[0], dst
, sizeof(*dst
));
1088 if (!flow_is_multicast_only(flow_attr
))
1089 set_underlay_qp(dev
, spec
, underlay_qpn
);
1091 if (dev
->is_rep
&& flow_attr
->type
!= IB_FLOW_ATTR_SNIFFER
) {
1092 struct mlx5_eswitch_rep
*rep
;
1094 rep
= dev
->port
[flow_attr
->port
- 1].rep
;
1100 mlx5_ib_set_rule_source_port(dev
, spec
, rep
);
1103 spec
->match_criteria_enable
= get_match_criteria_enable(spec
->match_criteria
);
1105 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
1106 struct mlx5_ib_mcounters
*mcounters
;
1108 err
= mlx5_ib_flow_counters_set_data(flow_act
.counters
, ucmd
);
1112 mcounters
= to_mcounters(flow_act
.counters
);
1113 handler
->ibcounters
= flow_act
.counters
;
1114 dest_arr
[dest_num
].type
=
1115 MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
1116 dest_arr
[dest_num
].counter_id
=
1117 mlx5_fc_id(mcounters
->hw_cntrs_hndl
);
1121 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_DROP
) {
1125 if (flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
)
1127 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
;
1129 flow_act
.action
|= MLX5_FLOW_CONTEXT_ACTION_ALLOW
;
1131 flow_act
.action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1134 if ((spec
->flow_context
.flags
& FLOW_CONTEXT_HAS_TAG
) &&
1135 (flow_attr
->type
== IB_FLOW_ATTR_ALL_DEFAULT
||
1136 flow_attr
->type
== IB_FLOW_ATTR_MC_DEFAULT
)) {
1137 mlx5_ib_warn(dev
, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1138 spec
->flow_context
.flow_tag
, flow_attr
->type
);
1142 handler
->rule
= mlx5_add_flow_rules(ft
, spec
,
1144 rule_dst
, dest_num
);
1146 if (IS_ERR(handler
->rule
)) {
1147 err
= PTR_ERR(handler
->rule
);
1151 ft_prio
->refcount
++;
1152 handler
->prio
= ft_prio
;
1155 ft_prio
->flow_table
= ft
;
1157 if (err
&& handler
) {
1158 mlx5_ib_counters_clear_description(handler
->ibcounters
);
1162 return err
? ERR_PTR(err
) : handler
;
1165 static struct mlx5_ib_flow_handler
*create_flow_rule(struct mlx5_ib_dev
*dev
,
1166 struct mlx5_ib_flow_prio
*ft_prio
,
1167 const struct ib_flow_attr
*flow_attr
,
1168 struct mlx5_flow_destination
*dst
)
1170 return _create_flow_rule(dev
, ft_prio
, flow_attr
, dst
, 0, NULL
);
1178 static struct mlx5_ib_flow_handler
*create_leftovers_rule(struct mlx5_ib_dev
*dev
,
1179 struct mlx5_ib_flow_prio
*ft_prio
,
1180 struct ib_flow_attr
*flow_attr
,
1181 struct mlx5_flow_destination
*dst
)
1183 struct mlx5_ib_flow_handler
*handler_ucast
= NULL
;
1184 struct mlx5_ib_flow_handler
*handler
= NULL
;
1187 struct ib_flow_attr flow_attr
;
1188 struct ib_flow_spec_eth eth_flow
;
1189 } leftovers_specs
[] = {
1193 .size
= sizeof(leftovers_specs
[0])
1196 .type
= IB_FLOW_SPEC_ETH
,
1197 .size
= sizeof(struct ib_flow_spec_eth
),
1198 .mask
= {.dst_mac
= {0x1} },
1199 .val
= {.dst_mac
= {0x1} }
1205 .size
= sizeof(leftovers_specs
[0])
1208 .type
= IB_FLOW_SPEC_ETH
,
1209 .size
= sizeof(struct ib_flow_spec_eth
),
1210 .mask
= {.dst_mac
= {0x1} },
1211 .val
= {.dst_mac
= {} }
1216 handler
= create_flow_rule(dev
, ft_prio
,
1217 &leftovers_specs
[LEFTOVERS_MC
].flow_attr
,
1219 if (!IS_ERR(handler
) &&
1220 flow_attr
->type
== IB_FLOW_ATTR_ALL_DEFAULT
) {
1221 handler_ucast
= create_flow_rule(dev
, ft_prio
,
1222 &leftovers_specs
[LEFTOVERS_UC
].flow_attr
,
1224 if (IS_ERR(handler_ucast
)) {
1225 mlx5_del_flow_rules(handler
->rule
);
1226 ft_prio
->refcount
--;
1228 handler
= handler_ucast
;
1230 list_add(&handler_ucast
->list
, &handler
->list
);
1237 static struct mlx5_ib_flow_handler
*create_sniffer_rule(struct mlx5_ib_dev
*dev
,
1238 struct mlx5_ib_flow_prio
*ft_rx
,
1239 struct mlx5_ib_flow_prio
*ft_tx
,
1240 struct mlx5_flow_destination
*dst
)
1242 struct mlx5_ib_flow_handler
*handler_rx
;
1243 struct mlx5_ib_flow_handler
*handler_tx
;
1245 static const struct ib_flow_attr flow_attr
= {
1247 .type
= IB_FLOW_ATTR_SNIFFER
,
1248 .size
= sizeof(flow_attr
)
1251 handler_rx
= create_flow_rule(dev
, ft_rx
, &flow_attr
, dst
);
1252 if (IS_ERR(handler_rx
)) {
1253 err
= PTR_ERR(handler_rx
);
1257 handler_tx
= create_flow_rule(dev
, ft_tx
, &flow_attr
, dst
);
1258 if (IS_ERR(handler_tx
)) {
1259 err
= PTR_ERR(handler_tx
);
1263 list_add(&handler_tx
->list
, &handler_rx
->list
);
1268 mlx5_del_flow_rules(handler_rx
->rule
);
1272 return ERR_PTR(err
);
1275 static struct ib_flow
*mlx5_ib_create_flow(struct ib_qp
*qp
,
1276 struct ib_flow_attr
*flow_attr
,
1277 struct ib_udata
*udata
)
1279 struct mlx5_ib_dev
*dev
= to_mdev(qp
->device
);
1280 struct mlx5_ib_qp
*mqp
= to_mqp(qp
);
1281 struct mlx5_ib_flow_handler
*handler
= NULL
;
1282 struct mlx5_flow_destination
*dst
= NULL
;
1283 struct mlx5_ib_flow_prio
*ft_prio_tx
= NULL
;
1284 struct mlx5_ib_flow_prio
*ft_prio
;
1285 bool is_egress
= flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_EGRESS
;
1286 struct mlx5_ib_create_flow
*ucmd
= NULL
, ucmd_hdr
;
1287 size_t min_ucmd_sz
, required_ucmd_sz
;
1291 if (udata
&& udata
->inlen
) {
1292 min_ucmd_sz
= offsetofend(struct mlx5_ib_create_flow
, reserved
);
1293 if (udata
->inlen
< min_ucmd_sz
)
1294 return ERR_PTR(-EOPNOTSUPP
);
1296 err
= ib_copy_from_udata(&ucmd_hdr
, udata
, min_ucmd_sz
);
1298 return ERR_PTR(err
);
1300 /* currently supports only one counters data */
1301 if (ucmd_hdr
.ncounters_data
> 1)
1302 return ERR_PTR(-EINVAL
);
1304 required_ucmd_sz
= min_ucmd_sz
+
1305 sizeof(struct mlx5_ib_flow_counters_data
) *
1306 ucmd_hdr
.ncounters_data
;
1307 if (udata
->inlen
> required_ucmd_sz
&&
1308 !ib_is_udata_cleared(udata
, required_ucmd_sz
,
1309 udata
->inlen
- required_ucmd_sz
))
1310 return ERR_PTR(-EOPNOTSUPP
);
1312 ucmd
= kzalloc(required_ucmd_sz
, GFP_KERNEL
);
1314 return ERR_PTR(-ENOMEM
);
1316 err
= ib_copy_from_udata(ucmd
, udata
, required_ucmd_sz
);
1321 if (flow_attr
->priority
> MLX5_IB_FLOW_LAST_PRIO
) {
1326 if (flow_attr
->flags
&
1327 ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP
| IB_FLOW_ATTR_FLAGS_EGRESS
)) {
1333 (flow_attr
->type
== IB_FLOW_ATTR_ALL_DEFAULT
||
1334 flow_attr
->type
== IB_FLOW_ATTR_MC_DEFAULT
)) {
1339 dst
= kzalloc(sizeof(*dst
), GFP_KERNEL
);
1345 mutex_lock(&dev
->flow_db
->lock
);
1347 ft_prio
= get_flow_table(dev
, flow_attr
,
1348 is_egress
? MLX5_IB_FT_TX
: MLX5_IB_FT_RX
);
1349 if (IS_ERR(ft_prio
)) {
1350 err
= PTR_ERR(ft_prio
);
1353 if (flow_attr
->type
== IB_FLOW_ATTR_SNIFFER
) {
1354 ft_prio_tx
= get_flow_table(dev
, flow_attr
, MLX5_IB_FT_TX
);
1355 if (IS_ERR(ft_prio_tx
)) {
1356 err
= PTR_ERR(ft_prio_tx
);
1363 dst
->type
= MLX5_FLOW_DESTINATION_TYPE_PORT
;
1365 dst
->type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
1367 dst
->tir_num
= mqp
->rss_qp
.tirn
;
1369 dst
->tir_num
= mqp
->raw_packet_qp
.rq
.tirn
;
1372 switch (flow_attr
->type
) {
1373 case IB_FLOW_ATTR_NORMAL
:
1374 underlay_qpn
= (mqp
->flags
& IB_QP_CREATE_SOURCE_QPN
) ?
1377 handler
= _create_flow_rule(dev
, ft_prio
, flow_attr
, dst
,
1378 underlay_qpn
, ucmd
);
1380 case IB_FLOW_ATTR_ALL_DEFAULT
:
1381 case IB_FLOW_ATTR_MC_DEFAULT
:
1382 handler
= create_leftovers_rule(dev
, ft_prio
, flow_attr
, dst
);
1384 case IB_FLOW_ATTR_SNIFFER
:
1385 handler
= create_sniffer_rule(dev
, ft_prio
, ft_prio_tx
, dst
);
1392 if (IS_ERR(handler
)) {
1393 err
= PTR_ERR(handler
);
1398 mutex_unlock(&dev
->flow_db
->lock
);
1402 return &handler
->ibflow
;
1405 put_flow_table(dev
, ft_prio
, false);
1407 put_flow_table(dev
, ft_prio_tx
, false);
1409 mutex_unlock(&dev
->flow_db
->lock
);
1413 return ERR_PTR(err
);
1416 static struct mlx5_ib_flow_prio
*
1417 _get_flow_table(struct mlx5_ib_dev
*dev
, u16 user_priority
,
1418 enum mlx5_flow_namespace_type ns_type
,
1421 struct mlx5_flow_namespace
*ns
= NULL
;
1422 struct mlx5_ib_flow_prio
*prio
= NULL
;
1423 int max_table_size
= 0;
1429 priority
= MLX5_IB_FLOW_MCAST_PRIO
;
1431 priority
= ib_prio_to_core_prio(user_priority
, false);
1433 esw_encap
= mlx5_eswitch_get_encap_mode(dev
->mdev
) !=
1434 DEVLINK_ESWITCH_ENCAP_MODE_NONE
;
1436 case MLX5_FLOW_NAMESPACE_BYPASS
:
1437 max_table_size
= BIT(
1438 MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
, log_max_ft_size
));
1439 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
, decap
) && !esw_encap
)
1440 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP
;
1441 if (MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
,
1442 reformat_l3_tunnel_to_l2
) &&
1444 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
;
1446 case MLX5_FLOW_NAMESPACE_EGRESS
:
1447 max_table_size
= BIT(
1448 MLX5_CAP_FLOWTABLE_NIC_TX(dev
->mdev
, log_max_ft_size
));
1449 if (MLX5_CAP_FLOWTABLE_NIC_TX(dev
->mdev
, reformat
) &&
1451 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
;
1453 case MLX5_FLOW_NAMESPACE_FDB_BYPASS
:
1454 max_table_size
= BIT(
1455 MLX5_CAP_ESW_FLOWTABLE_FDB(dev
->mdev
, log_max_ft_size
));
1456 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
->mdev
, decap
) && esw_encap
)
1457 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP
;
1458 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
->mdev
,
1459 reformat_l3_tunnel_to_l2
) &&
1461 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
;
1462 priority
= user_priority
;
1464 case MLX5_FLOW_NAMESPACE_RDMA_RX
:
1465 max_table_size
= BIT(
1466 MLX5_CAP_FLOWTABLE_RDMA_RX(dev
->mdev
, log_max_ft_size
));
1467 priority
= user_priority
;
1469 case MLX5_FLOW_NAMESPACE_RDMA_TX
:
1470 max_table_size
= BIT(
1471 MLX5_CAP_FLOWTABLE_RDMA_TX(dev
->mdev
, log_max_ft_size
));
1472 priority
= user_priority
;
1478 max_table_size
= min_t(int, max_table_size
, MLX5_FS_MAX_ENTRIES
);
1480 ns
= mlx5_get_flow_namespace(dev
->mdev
, ns_type
);
1482 return ERR_PTR(-EOPNOTSUPP
);
1485 case MLX5_FLOW_NAMESPACE_BYPASS
:
1486 prio
= &dev
->flow_db
->prios
[priority
];
1488 case MLX5_FLOW_NAMESPACE_EGRESS
:
1489 prio
= &dev
->flow_db
->egress_prios
[priority
];
1491 case MLX5_FLOW_NAMESPACE_FDB_BYPASS
:
1492 prio
= &dev
->flow_db
->fdb
[priority
];
1494 case MLX5_FLOW_NAMESPACE_RDMA_RX
:
1495 prio
= &dev
->flow_db
->rdma_rx
[priority
];
1497 case MLX5_FLOW_NAMESPACE_RDMA_TX
:
1498 prio
= &dev
->flow_db
->rdma_tx
[priority
];
1500 default: return ERR_PTR(-EINVAL
);
1504 return ERR_PTR(-EINVAL
);
1506 if (prio
->flow_table
)
1509 return _get_prio(dev
, ns
, prio
, priority
, max_table_size
,
1510 MLX5_FS_MAX_TYPES
, flags
);
1513 static struct mlx5_ib_flow_handler
*
1514 _create_raw_flow_rule(struct mlx5_ib_dev
*dev
,
1515 struct mlx5_ib_flow_prio
*ft_prio
,
1516 struct mlx5_flow_destination
*dst
,
1517 struct mlx5_ib_flow_matcher
*fs_matcher
,
1518 struct mlx5_flow_context
*flow_context
,
1519 struct mlx5_flow_act
*flow_act
,
1520 void *cmd_in
, int inlen
,
1523 struct mlx5_ib_flow_handler
*handler
;
1524 struct mlx5_flow_spec
*spec
;
1525 struct mlx5_flow_table
*ft
= ft_prio
->flow_table
;
1528 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1529 handler
= kzalloc(sizeof(*handler
), GFP_KERNEL
);
1530 if (!handler
|| !spec
) {
1535 INIT_LIST_HEAD(&handler
->list
);
1537 memcpy(spec
->match_value
, cmd_in
, inlen
);
1538 memcpy(spec
->match_criteria
, fs_matcher
->matcher_mask
.match_params
,
1539 fs_matcher
->mask_len
);
1540 spec
->match_criteria_enable
= fs_matcher
->match_criteria_enable
;
1541 spec
->flow_context
= *flow_context
;
1543 handler
->rule
= mlx5_add_flow_rules(ft
, spec
,
1544 flow_act
, dst
, dst_num
);
1546 if (IS_ERR(handler
->rule
)) {
1547 err
= PTR_ERR(handler
->rule
);
1551 ft_prio
->refcount
++;
1552 handler
->prio
= ft_prio
;
1554 ft_prio
->flow_table
= ft
;
1560 return err
? ERR_PTR(err
) : handler
;
1563 static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher
*fs_matcher
,
1567 void *match_v_set_lyr_2_4
, *match_c_set_lyr_2_4
;
1568 void *dmac
, *dmac_mask
;
1569 void *ipv4
, *ipv4_mask
;
1571 if (!(fs_matcher
->match_criteria_enable
&
1572 (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT
)))
1575 match_c
= fs_matcher
->matcher_mask
.match_params
;
1576 match_v_set_lyr_2_4
= MLX5_ADDR_OF(fte_match_param
, match_v
,
1578 match_c_set_lyr_2_4
= MLX5_ADDR_OF(fte_match_param
, match_c
,
1581 dmac
= MLX5_ADDR_OF(fte_match_set_lyr_2_4
, match_v_set_lyr_2_4
,
1583 dmac_mask
= MLX5_ADDR_OF(fte_match_set_lyr_2_4
, match_c_set_lyr_2_4
,
1586 if (is_multicast_ether_addr(dmac
) &&
1587 is_multicast_ether_addr(dmac_mask
))
1590 ipv4
= MLX5_ADDR_OF(fte_match_set_lyr_2_4
, match_v_set_lyr_2_4
,
1591 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
);
1593 ipv4_mask
= MLX5_ADDR_OF(fte_match_set_lyr_2_4
, match_c_set_lyr_2_4
,
1594 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
);
1596 if (ipv4_is_multicast(*(__be32
*)(ipv4
)) &&
1597 ipv4_is_multicast(*(__be32
*)(ipv4_mask
)))
1603 static struct mlx5_ib_flow_handler
*raw_fs_rule_add(
1604 struct mlx5_ib_dev
*dev
, struct mlx5_ib_flow_matcher
*fs_matcher
,
1605 struct mlx5_flow_context
*flow_context
, struct mlx5_flow_act
*flow_act
,
1606 u32 counter_id
, void *cmd_in
, int inlen
, int dest_id
, int dest_type
)
1608 struct mlx5_flow_destination
*dst
;
1609 struct mlx5_ib_flow_prio
*ft_prio
;
1610 struct mlx5_ib_flow_handler
*handler
;
1615 if (fs_matcher
->flow_type
!= MLX5_IB_FLOW_TYPE_NORMAL
)
1616 return ERR_PTR(-EOPNOTSUPP
);
1618 if (fs_matcher
->priority
> MLX5_IB_FLOW_LAST_PRIO
)
1619 return ERR_PTR(-ENOMEM
);
1621 dst
= kcalloc(2, sizeof(*dst
), GFP_KERNEL
);
1623 return ERR_PTR(-ENOMEM
);
1625 mcast
= raw_fs_is_multicast(fs_matcher
, cmd_in
);
1626 mutex_lock(&dev
->flow_db
->lock
);
1628 ft_prio
= _get_flow_table(dev
, fs_matcher
->priority
,
1629 fs_matcher
->ns_type
, mcast
);
1630 if (IS_ERR(ft_prio
)) {
1631 err
= PTR_ERR(ft_prio
);
1635 switch (dest_type
) {
1636 case MLX5_FLOW_DESTINATION_TYPE_TIR
:
1637 dst
[dst_num
].type
= dest_type
;
1638 dst
[dst_num
++].tir_num
= dest_id
;
1639 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1641 case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
:
1642 dst
[dst_num
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM
;
1643 dst
[dst_num
++].ft_num
= dest_id
;
1644 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1646 case MLX5_FLOW_DESTINATION_TYPE_PORT
:
1647 dst
[dst_num
++].type
= MLX5_FLOW_DESTINATION_TYPE_PORT
;
1648 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_ALLOW
;
1654 if (flow_act
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
1655 dst
[dst_num
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
1656 dst
[dst_num
].counter_id
= counter_id
;
1660 handler
= _create_raw_flow_rule(dev
, ft_prio
, dst_num
? dst
: NULL
,
1661 fs_matcher
, flow_context
, flow_act
,
1662 cmd_in
, inlen
, dst_num
);
1664 if (IS_ERR(handler
)) {
1665 err
= PTR_ERR(handler
);
1669 mutex_unlock(&dev
->flow_db
->lock
);
1670 atomic_inc(&fs_matcher
->usecnt
);
1671 handler
->flow_matcher
= fs_matcher
;
1678 put_flow_table(dev
, ft_prio
, false);
1680 mutex_unlock(&dev
->flow_db
->lock
);
1683 return ERR_PTR(err
);
1686 static void destroy_flow_action_raw(struct mlx5_ib_flow_action
*maction
)
1688 switch (maction
->flow_action_raw
.sub_type
) {
1689 case MLX5_IB_FLOW_ACTION_MODIFY_HEADER
:
1690 mlx5_modify_header_dealloc(maction
->flow_action_raw
.dev
->mdev
,
1691 maction
->flow_action_raw
.modify_hdr
);
1693 case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT
:
1694 mlx5_packet_reformat_dealloc(maction
->flow_action_raw
.dev
->mdev
,
1695 maction
->flow_action_raw
.pkt_reformat
);
1697 case MLX5_IB_FLOW_ACTION_DECAP
:
1704 static int mlx5_ib_destroy_flow_action(struct ib_flow_action
*action
)
1706 struct mlx5_ib_flow_action
*maction
= to_mflow_act(action
);
1708 switch (action
->type
) {
1709 case IB_FLOW_ACTION_UNSPECIFIED
:
1710 destroy_flow_action_raw(maction
);
1722 mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type
,
1723 enum mlx5_flow_namespace_type
*namespace)
1725 switch (table_type
) {
1726 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX
:
1727 *namespace = MLX5_FLOW_NAMESPACE_BYPASS
;
1729 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX
:
1730 *namespace = MLX5_FLOW_NAMESPACE_EGRESS
;
1732 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB
:
1733 *namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS
;
1735 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX
:
1736 *namespace = MLX5_FLOW_NAMESPACE_RDMA_RX
;
1738 case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX
:
1739 *namespace = MLX5_FLOW_NAMESPACE_RDMA_TX
;
1748 static const struct uverbs_attr_spec mlx5_ib_flow_type
[] = {
1749 [MLX5_IB_FLOW_TYPE_NORMAL
] = {
1750 .type
= UVERBS_ATTR_TYPE_PTR_IN
,
1752 .len
= sizeof(u16
), /* data is priority */
1753 .min_len
= sizeof(u16
),
1756 [MLX5_IB_FLOW_TYPE_SNIFFER
] = {
1757 .type
= UVERBS_ATTR_TYPE_PTR_IN
,
1758 UVERBS_ATTR_NO_DATA(),
1760 [MLX5_IB_FLOW_TYPE_ALL_DEFAULT
] = {
1761 .type
= UVERBS_ATTR_TYPE_PTR_IN
,
1762 UVERBS_ATTR_NO_DATA(),
1764 [MLX5_IB_FLOW_TYPE_MC_DEFAULT
] = {
1765 .type
= UVERBS_ATTR_TYPE_PTR_IN
,
1766 UVERBS_ATTR_NO_DATA(),
1770 static bool is_flow_dest(void *obj
, int *dest_id
, int *dest_type
)
1772 struct devx_obj
*devx_obj
= obj
;
1773 u16 opcode
= MLX5_GET(general_obj_in_cmd_hdr
, devx_obj
->dinbox
, opcode
);
1776 case MLX5_CMD_OP_DESTROY_TIR
:
1777 *dest_type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
1778 *dest_id
= MLX5_GET(general_obj_in_cmd_hdr
, devx_obj
->dinbox
,
1782 case MLX5_CMD_OP_DESTROY_FLOW_TABLE
:
1783 *dest_type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
1784 *dest_id
= MLX5_GET(destroy_flow_table_in
, devx_obj
->dinbox
,
1792 static int get_dests(struct uverbs_attr_bundle
*attrs
,
1793 struct mlx5_ib_flow_matcher
*fs_matcher
, int *dest_id
,
1794 int *dest_type
, struct ib_qp
**qp
, u32
*flags
)
1796 bool dest_devx
, dest_qp
;
1800 dest_devx
= uverbs_attr_is_valid(attrs
,
1801 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX
);
1802 dest_qp
= uverbs_attr_is_valid(attrs
,
1803 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP
);
1806 err
= uverbs_get_flags32(flags
, attrs
, MLX5_IB_ATTR_CREATE_FLOW_FLAGS
,
1807 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS
|
1808 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP
);
1812 /* Both flags are not allowed */
1813 if (*flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS
&&
1814 *flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP
)
1817 if (fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_BYPASS
) {
1818 if (dest_devx
&& (dest_qp
|| *flags
))
1820 else if (dest_qp
&& *flags
)
1824 /* Allow only DEVX object, drop as dest for FDB */
1825 if (fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_FDB_BYPASS
&&
1826 !(dest_devx
|| (*flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP
)))
1829 /* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
1830 if ((fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_RDMA_RX
) &&
1831 ((!dest_devx
&& !dest_qp
) || (dest_devx
&& dest_qp
)))
1837 uverbs_attr_get_obj(attrs
,
1838 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX
);
1840 /* Verify that the given DEVX object is a flow
1841 * steering destination.
1843 if (!is_flow_dest(devx_obj
, dest_id
, dest_type
))
1845 /* Allow only flow table as dest when inserting to FDB or RDMA_RX */
1846 if ((fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_FDB_BYPASS
||
1847 fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_RDMA_RX
) &&
1848 *dest_type
!= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
)
1850 } else if (dest_qp
) {
1851 struct mlx5_ib_qp
*mqp
;
1853 *qp
= uverbs_attr_get_obj(attrs
,
1854 MLX5_IB_ATTR_CREATE_FLOW_DEST_QP
);
1856 return PTR_ERR(*qp
);
1858 if ((*qp
)->qp_type
!= IB_QPT_RAW_PACKET
)
1863 *dest_id
= mqp
->rss_qp
.tirn
;
1865 *dest_id
= mqp
->raw_packet_qp
.rq
.tirn
;
1866 *dest_type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
1867 } else if ((fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_EGRESS
||
1868 fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_RDMA_TX
) &&
1869 !(*flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP
)) {
1870 *dest_type
= MLX5_FLOW_DESTINATION_TYPE_PORT
;
1873 if (*dest_type
== MLX5_FLOW_DESTINATION_TYPE_TIR
&&
1874 (fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_EGRESS
||
1875 fs_matcher
->ns_type
== MLX5_FLOW_NAMESPACE_RDMA_TX
))
1881 static bool is_flow_counter(void *obj
, u32 offset
, u32
*counter_id
)
1883 struct devx_obj
*devx_obj
= obj
;
1884 u16 opcode
= MLX5_GET(general_obj_in_cmd_hdr
, devx_obj
->dinbox
, opcode
);
1886 if (opcode
== MLX5_CMD_OP_DEALLOC_FLOW_COUNTER
) {
1888 if (offset
&& offset
>= devx_obj
->flow_counter_bulk_size
)
1891 *counter_id
= MLX5_GET(dealloc_flow_counter_in
,
1894 *counter_id
+= offset
;
1901 #define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
1902 static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW
)(
1903 struct uverbs_attr_bundle
*attrs
)
1905 struct mlx5_flow_context flow_context
= {.flow_tag
=
1906 MLX5_FS_DEFAULT_FLOW_TAG
};
1907 u32
*offset_attr
, offset
= 0, counter_id
= 0;
1908 int dest_id
, dest_type
= -1, inlen
, len
, ret
, i
;
1909 struct mlx5_ib_flow_handler
*flow_handler
;
1910 struct mlx5_ib_flow_matcher
*fs_matcher
;
1911 struct ib_uobject
**arr_flow_actions
;
1912 struct ib_uflow_resources
*uflow_res
;
1913 struct mlx5_flow_act flow_act
= {};
1914 struct ib_qp
*qp
= NULL
;
1915 void *devx_obj
, *cmd_in
;
1916 struct ib_uobject
*uobj
;
1917 struct mlx5_ib_dev
*dev
;
1920 if (!capable(CAP_NET_RAW
))
1923 fs_matcher
= uverbs_attr_get_obj(attrs
,
1924 MLX5_IB_ATTR_CREATE_FLOW_MATCHER
);
1925 uobj
= uverbs_attr_get_uobject(attrs
, MLX5_IB_ATTR_CREATE_FLOW_HANDLE
);
1926 dev
= mlx5_udata_to_mdev(&attrs
->driver_udata
);
1928 if (get_dests(attrs
, fs_matcher
, &dest_id
, &dest_type
, &qp
, &flags
))
1931 if (flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS
)
1932 flow_act
.action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS
;
1934 if (flags
& MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP
)
1935 flow_act
.action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
;
1937 len
= uverbs_attr_get_uobjs_arr(attrs
,
1938 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX
, &arr_flow_actions
);
1940 devx_obj
= arr_flow_actions
[0]->object
;
1942 if (uverbs_attr_is_valid(attrs
,
1943 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET
)) {
1945 int num_offsets
= uverbs_attr_ptr_get_array_size(
1947 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET
,
1950 if (num_offsets
!= 1)
1953 offset_attr
= uverbs_attr_get_alloced_ptr(
1955 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET
);
1956 offset
= *offset_attr
;
1959 if (!is_flow_counter(devx_obj
, offset
, &counter_id
))
1962 flow_act
.action
|= MLX5_FLOW_CONTEXT_ACTION_COUNT
;
1965 cmd_in
= uverbs_attr_get_alloced_ptr(
1966 attrs
, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE
);
1967 inlen
= uverbs_attr_get_len(attrs
,
1968 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE
);
1970 uflow_res
= flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS
);
1974 len
= uverbs_attr_get_uobjs_arr(attrs
,
1975 MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS
, &arr_flow_actions
);
1976 for (i
= 0; i
< len
; i
++) {
1977 struct mlx5_ib_flow_action
*maction
=
1978 to_mflow_act(arr_flow_actions
[i
]->object
);
1980 ret
= parse_flow_flow_action(maction
, false, &flow_act
);
1983 flow_resources_add(uflow_res
, IB_FLOW_SPEC_ACTION_HANDLE
,
1984 arr_flow_actions
[i
]->object
);
1987 ret
= uverbs_copy_from(&flow_context
.flow_tag
, attrs
,
1988 MLX5_IB_ATTR_CREATE_FLOW_TAG
);
1990 if (flow_context
.flow_tag
>= BIT(24)) {
1994 flow_context
.flags
|= FLOW_CONTEXT_HAS_TAG
;
1998 raw_fs_rule_add(dev
, fs_matcher
, &flow_context
, &flow_act
,
1999 counter_id
, cmd_in
, inlen
, dest_id
, dest_type
);
2000 if (IS_ERR(flow_handler
)) {
2001 ret
= PTR_ERR(flow_handler
);
2005 ib_set_flow(uobj
, &flow_handler
->ibflow
, qp
, &dev
->ib_dev
, uflow_res
);
2009 ib_uverbs_flow_resources_free(uflow_res
);
2013 static int flow_matcher_cleanup(struct ib_uobject
*uobject
,
2014 enum rdma_remove_reason why
,
2015 struct uverbs_attr_bundle
*attrs
)
2017 struct mlx5_ib_flow_matcher
*obj
= uobject
->object
;
2019 if (atomic_read(&obj
->usecnt
))
2026 static int steering_anchor_create_ft(struct mlx5_ib_dev
*dev
,
2027 struct mlx5_ib_flow_prio
*ft_prio
,
2028 enum mlx5_flow_namespace_type ns_type
)
2030 struct mlx5_flow_table_attr ft_attr
= {};
2031 struct mlx5_flow_namespace
*ns
;
2032 struct mlx5_flow_table
*ft
;
2034 if (ft_prio
->anchor
.ft
)
2037 ns
= mlx5_get_flow_namespace(dev
->mdev
, ns_type
);
2041 ft_attr
.flags
= MLX5_FLOW_TABLE_UNMANAGED
;
2042 ft_attr
.uid
= MLX5_SHARED_RESOURCE_UID
;
2044 ft_attr
.max_fte
= 2;
2047 ft
= mlx5_create_flow_table(ns
, &ft_attr
);
2051 ft_prio
->anchor
.ft
= ft
;
2056 static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio
*ft_prio
)
2058 if (ft_prio
->anchor
.ft
) {
2059 mlx5_destroy_flow_table(ft_prio
->anchor
.ft
);
2060 ft_prio
->anchor
.ft
= NULL
;
2065 steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio
*ft_prio
)
2067 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2068 struct mlx5_flow_group
*fg
;
2069 void *flow_group_in
;
2072 if (ft_prio
->anchor
.fg_drop
)
2075 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2079 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 1);
2080 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, 1);
2082 fg
= mlx5_create_flow_group(ft_prio
->anchor
.ft
, flow_group_in
);
2088 ft_prio
->anchor
.fg_drop
= fg
;
2091 kvfree(flow_group_in
);
2097 steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio
*ft_prio
)
2099 if (ft_prio
->anchor
.fg_drop
) {
2100 mlx5_destroy_flow_group(ft_prio
->anchor
.fg_drop
);
2101 ft_prio
->anchor
.fg_drop
= NULL
;
2106 steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio
*ft_prio
)
2108 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2109 struct mlx5_flow_group
*fg
;
2110 void *flow_group_in
;
2113 if (ft_prio
->anchor
.fg_goto_table
)
2116 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2120 fg
= mlx5_create_flow_group(ft_prio
->anchor
.ft
, flow_group_in
);
2125 ft_prio
->anchor
.fg_goto_table
= fg
;
2128 kvfree(flow_group_in
);
2134 steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio
*ft_prio
)
2136 if (ft_prio
->anchor
.fg_goto_table
) {
2137 mlx5_destroy_flow_group(ft_prio
->anchor
.fg_goto_table
);
2138 ft_prio
->anchor
.fg_goto_table
= NULL
;
2143 steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio
*ft_prio
)
2145 struct mlx5_flow_act flow_act
= {};
2146 struct mlx5_flow_handle
*handle
;
2148 if (ft_prio
->anchor
.rule_drop
)
2151 flow_act
.fg
= ft_prio
->anchor
.fg_drop
;
2152 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_DROP
;
2154 handle
= mlx5_add_flow_rules(ft_prio
->anchor
.ft
, NULL
, &flow_act
,
2157 return PTR_ERR(handle
);
2159 ft_prio
->anchor
.rule_drop
= handle
;
2164 static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio
*ft_prio
)
2166 if (ft_prio
->anchor
.rule_drop
) {
2167 mlx5_del_flow_rules(ft_prio
->anchor
.rule_drop
);
2168 ft_prio
->anchor
.rule_drop
= NULL
;
2173 steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio
*ft_prio
)
2175 struct mlx5_flow_destination dest
= {};
2176 struct mlx5_flow_act flow_act
= {};
2177 struct mlx5_flow_handle
*handle
;
2179 if (ft_prio
->anchor
.rule_goto_table
)
2182 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2183 flow_act
.flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
2184 flow_act
.fg
= ft_prio
->anchor
.fg_goto_table
;
2186 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
2187 dest
.ft
= ft_prio
->flow_table
;
2189 handle
= mlx5_add_flow_rules(ft_prio
->anchor
.ft
, NULL
, &flow_act
,
2192 return PTR_ERR(handle
);
2194 ft_prio
->anchor
.rule_goto_table
= handle
;
2200 steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio
*ft_prio
)
2202 if (ft_prio
->anchor
.rule_goto_table
) {
2203 mlx5_del_flow_rules(ft_prio
->anchor
.rule_goto_table
);
2204 ft_prio
->anchor
.rule_goto_table
= NULL
;
2208 static int steering_anchor_create_res(struct mlx5_ib_dev
*dev
,
2209 struct mlx5_ib_flow_prio
*ft_prio
,
2210 enum mlx5_flow_namespace_type ns_type
)
2214 err
= steering_anchor_create_ft(dev
, ft_prio
, ns_type
);
2218 err
= steering_anchor_create_fg_drop(ft_prio
);
2222 err
= steering_anchor_create_fg_goto_table(ft_prio
);
2224 goto destroy_fg_drop
;
2226 err
= steering_anchor_create_rule_drop(ft_prio
);
2228 goto destroy_fg_goto_table
;
2230 err
= steering_anchor_create_rule_goto_table(ft_prio
);
2232 goto destroy_rule_drop
;
2237 steering_anchor_destroy_rule_drop(ft_prio
);
2238 destroy_fg_goto_table
:
2239 steering_anchor_destroy_fg_goto_table(ft_prio
);
2241 steering_anchor_destroy_fg_drop(ft_prio
);
2243 steering_anchor_destroy_ft(ft_prio
);
2248 static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio
*ft_prio
)
2250 steering_anchor_destroy_rule_goto_table(ft_prio
);
2251 steering_anchor_destroy_rule_drop(ft_prio
);
2252 steering_anchor_destroy_fg_goto_table(ft_prio
);
2253 steering_anchor_destroy_fg_drop(ft_prio
);
2254 steering_anchor_destroy_ft(ft_prio
);
2257 static int steering_anchor_cleanup(struct ib_uobject
*uobject
,
2258 enum rdma_remove_reason why
,
2259 struct uverbs_attr_bundle
*attrs
)
2261 struct mlx5_ib_steering_anchor
*obj
= uobject
->object
;
2263 if (atomic_read(&obj
->usecnt
))
2266 mutex_lock(&obj
->dev
->flow_db
->lock
);
2267 if (!--obj
->ft_prio
->anchor
.rule_goto_table_ref
)
2268 steering_anchor_destroy_rule_goto_table(obj
->ft_prio
);
2270 put_flow_table(obj
->dev
, obj
->ft_prio
, true);
2271 mutex_unlock(&obj
->dev
->flow_db
->lock
);
2277 static void fs_cleanup_anchor(struct mlx5_ib_flow_prio
*prio
,
2281 mlx5_steering_anchor_destroy_res(&prio
[count
]);
2284 void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev
*dev
)
2286 fs_cleanup_anchor(dev
->flow_db
->prios
, MLX5_IB_NUM_FLOW_FT
);
2287 fs_cleanup_anchor(dev
->flow_db
->egress_prios
, MLX5_IB_NUM_FLOW_FT
);
2288 fs_cleanup_anchor(dev
->flow_db
->sniffer
, MLX5_IB_NUM_SNIFFER_FTS
);
2289 fs_cleanup_anchor(dev
->flow_db
->egress
, MLX5_IB_NUM_EGRESS_FTS
);
2290 fs_cleanup_anchor(dev
->flow_db
->fdb
, MLX5_IB_NUM_FDB_FTS
);
2291 fs_cleanup_anchor(dev
->flow_db
->rdma_rx
, MLX5_IB_NUM_FLOW_FT
);
2292 fs_cleanup_anchor(dev
->flow_db
->rdma_tx
, MLX5_IB_NUM_FLOW_FT
);
2295 static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle
*attrs
,
2296 struct mlx5_ib_flow_matcher
*obj
)
2298 enum mlx5_ib_uapi_flow_table_type ft_type
=
2299 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX
;
2303 /* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2304 * users should switch to it. We leave this to not break userspace
2306 if (uverbs_attr_is_valid(attrs
, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE
) &&
2307 uverbs_attr_is_valid(attrs
, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS
))
2310 if (uverbs_attr_is_valid(attrs
, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE
)) {
2311 err
= uverbs_get_const(&ft_type
, attrs
,
2312 MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE
);
2316 err
= mlx5_ib_ft_type_to_namespace(ft_type
, &obj
->ns_type
);
2323 if (uverbs_attr_is_valid(attrs
, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS
)) {
2324 err
= uverbs_get_flags32(&flags
, attrs
,
2325 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS
,
2326 IB_FLOW_ATTR_FLAGS_EGRESS
);
2331 return mlx5_ib_ft_type_to_namespace(
2332 MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX
,
2336 obj
->ns_type
= MLX5_FLOW_NAMESPACE_BYPASS
;
2341 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE
)(
2342 struct uverbs_attr_bundle
*attrs
)
2344 struct ib_uobject
*uobj
= uverbs_attr_get_uobject(
2345 attrs
, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE
);
2346 struct mlx5_ib_dev
*dev
= mlx5_udata_to_mdev(&attrs
->driver_udata
);
2347 struct mlx5_ib_flow_matcher
*obj
;
2350 obj
= kzalloc(sizeof(struct mlx5_ib_flow_matcher
), GFP_KERNEL
);
2354 obj
->mask_len
= uverbs_attr_get_len(
2355 attrs
, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK
);
2356 err
= uverbs_copy_from(&obj
->matcher_mask
,
2358 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK
);
2362 obj
->flow_type
= uverbs_attr_get_enum_id(
2363 attrs
, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE
);
2365 if (obj
->flow_type
== MLX5_IB_FLOW_TYPE_NORMAL
) {
2366 err
= uverbs_copy_from(&obj
->priority
,
2368 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE
);
2373 err
= uverbs_copy_from(&obj
->match_criteria_enable
,
2375 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA
);
2379 err
= mlx5_ib_matcher_ns(attrs
, obj
);
2383 if (obj
->ns_type
== MLX5_FLOW_NAMESPACE_FDB_BYPASS
&&
2384 mlx5_eswitch_mode(dev
->mdev
) != MLX5_ESWITCH_OFFLOADS
) {
2390 obj
->mdev
= dev
->mdev
;
2391 atomic_set(&obj
->usecnt
, 0);
2399 static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE
)(
2400 struct uverbs_attr_bundle
*attrs
)
2402 struct ib_uobject
*uobj
= uverbs_attr_get_uobject(
2403 attrs
, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE
);
2404 struct mlx5_ib_dev
*dev
= mlx5_udata_to_mdev(&attrs
->driver_udata
);
2405 enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type
;
2406 enum mlx5_flow_namespace_type ns_type
;
2407 struct mlx5_ib_steering_anchor
*obj
;
2408 struct mlx5_ib_flow_prio
*ft_prio
;
2413 if (!capable(CAP_NET_RAW
))
2416 err
= uverbs_get_const(&ib_uapi_ft_type
, attrs
,
2417 MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE
);
2421 err
= mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type
, &ns_type
);
2425 err
= uverbs_copy_from(&priority
, attrs
,
2426 MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY
);
2430 obj
= kzalloc(sizeof(*obj
), GFP_KERNEL
);
2434 mutex_lock(&dev
->flow_db
->lock
);
2436 ft_prio
= _get_flow_table(dev
, priority
, ns_type
, 0);
2437 if (IS_ERR(ft_prio
)) {
2438 err
= PTR_ERR(ft_prio
);
2442 ft_prio
->refcount
++;
2444 if (!ft_prio
->anchor
.rule_goto_table_ref
) {
2445 err
= steering_anchor_create_res(dev
, ft_prio
, ns_type
);
2447 goto put_flow_table
;
2450 ft_prio
->anchor
.rule_goto_table_ref
++;
2452 ft_id
= mlx5_flow_table_id(ft_prio
->anchor
.ft
);
2454 err
= uverbs_copy_to(attrs
, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID
,
2455 &ft_id
, sizeof(ft_id
));
2459 mutex_unlock(&dev
->flow_db
->lock
);
2463 obj
->ft_prio
= ft_prio
;
2464 atomic_set(&obj
->usecnt
, 0);
2469 --ft_prio
->anchor
.rule_goto_table_ref
;
2470 mlx5_steering_anchor_destroy_res(ft_prio
);
2472 put_flow_table(dev
, ft_prio
, true);
2473 mutex_unlock(&dev
->flow_db
->lock
);
2480 static struct ib_flow_action
*
2481 mlx5_ib_create_modify_header(struct mlx5_ib_dev
*dev
,
2482 enum mlx5_ib_uapi_flow_table_type ft_type
,
2483 u8 num_actions
, void *in
)
2485 enum mlx5_flow_namespace_type
namespace;
2486 struct mlx5_ib_flow_action
*maction
;
2489 ret
= mlx5_ib_ft_type_to_namespace(ft_type
, &namespace);
2491 return ERR_PTR(-EINVAL
);
2493 maction
= kzalloc(sizeof(*maction
), GFP_KERNEL
);
2495 return ERR_PTR(-ENOMEM
);
2497 maction
->flow_action_raw
.modify_hdr
=
2498 mlx5_modify_header_alloc(dev
->mdev
, namespace, num_actions
, in
);
2500 if (IS_ERR(maction
->flow_action_raw
.modify_hdr
)) {
2501 ret
= PTR_ERR(maction
->flow_action_raw
.modify_hdr
);
2503 return ERR_PTR(ret
);
2505 maction
->flow_action_raw
.sub_type
=
2506 MLX5_IB_FLOW_ACTION_MODIFY_HEADER
;
2507 maction
->flow_action_raw
.dev
= dev
;
2509 return &maction
->ib_action
;
2512 static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev
*dev
)
2514 return MLX5_CAP_FLOWTABLE_NIC_RX(dev
->mdev
,
2515 max_modify_header_actions
) ||
2516 MLX5_CAP_FLOWTABLE_NIC_TX(dev
->mdev
,
2517 max_modify_header_actions
) ||
2518 MLX5_CAP_FLOWTABLE_RDMA_TX(dev
->mdev
,
2519 max_modify_header_actions
);
2522 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER
)(
2523 struct uverbs_attr_bundle
*attrs
)
2525 struct ib_uobject
*uobj
= uverbs_attr_get_uobject(
2526 attrs
, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE
);
2527 struct mlx5_ib_dev
*mdev
= mlx5_udata_to_mdev(&attrs
->driver_udata
);
2528 enum mlx5_ib_uapi_flow_table_type ft_type
;
2529 struct ib_flow_action
*action
;
2534 if (!mlx5_ib_modify_header_supported(mdev
))
2537 in
= uverbs_attr_get_alloced_ptr(attrs
,
2538 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM
);
2540 num_actions
= uverbs_attr_ptr_get_array_size(
2541 attrs
, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM
,
2542 MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto
));
2543 if (num_actions
< 0)
2546 ret
= uverbs_get_const(&ft_type
, attrs
,
2547 MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE
);
2550 action
= mlx5_ib_create_modify_header(mdev
, ft_type
, num_actions
, in
);
2552 return PTR_ERR(action
);
2554 uverbs_flow_action_fill_action(action
, uobj
, &mdev
->ib_dev
,
2555 IB_FLOW_ACTION_UNSPECIFIED
);
2560 static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev
*ibdev
,
2561 u8 packet_reformat_type
,
2564 switch (packet_reformat_type
) {
2565 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL
:
2566 if (ft_type
== MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX
)
2567 return MLX5_CAP_FLOWTABLE(ibdev
->mdev
,
2568 encap_general_header
);
2570 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL
:
2571 if (ft_type
== MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX
)
2572 return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev
->mdev
,
2573 reformat_l2_to_l3_tunnel
);
2575 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2
:
2576 if (ft_type
== MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX
)
2577 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev
->mdev
,
2578 reformat_l3_tunnel_to_l2
);
2580 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2
:
2581 if (ft_type
== MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX
)
2582 return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev
->mdev
, decap
);
2591 static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt
, u8
*prm_prt
)
2594 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL
:
2595 *prm_prt
= MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL
;
2597 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2
:
2598 *prm_prt
= MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2
;
2600 case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL
:
2601 *prm_prt
= MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL
;
2610 static int mlx5_ib_flow_action_create_packet_reformat_ctx(
2611 struct mlx5_ib_dev
*dev
,
2612 struct mlx5_ib_flow_action
*maction
,
2613 u8 ft_type
, u8 dv_prt
,
2614 void *in
, size_t len
)
2616 struct mlx5_pkt_reformat_params reformat_params
;
2617 enum mlx5_flow_namespace_type
namespace;
2621 ret
= mlx5_ib_ft_type_to_namespace(ft_type
, &namespace);
2625 ret
= mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt
, &prm_prt
);
2629 memset(&reformat_params
, 0, sizeof(reformat_params
));
2630 reformat_params
.type
= prm_prt
;
2631 reformat_params
.size
= len
;
2632 reformat_params
.data
= in
;
2633 maction
->flow_action_raw
.pkt_reformat
=
2634 mlx5_packet_reformat_alloc(dev
->mdev
, &reformat_params
,
2636 if (IS_ERR(maction
->flow_action_raw
.pkt_reformat
)) {
2637 ret
= PTR_ERR(maction
->flow_action_raw
.pkt_reformat
);
2641 maction
->flow_action_raw
.sub_type
=
2642 MLX5_IB_FLOW_ACTION_PACKET_REFORMAT
;
2643 maction
->flow_action_raw
.dev
= dev
;
2648 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT
)(
2649 struct uverbs_attr_bundle
*attrs
)
2651 struct ib_uobject
*uobj
= uverbs_attr_get_uobject(attrs
,
2652 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE
);
2653 struct mlx5_ib_dev
*mdev
= mlx5_udata_to_mdev(&attrs
->driver_udata
);
2654 enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt
;
2655 enum mlx5_ib_uapi_flow_table_type ft_type
;
2656 struct mlx5_ib_flow_action
*maction
;
2659 ret
= uverbs_get_const(&ft_type
, attrs
,
2660 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE
);
2664 ret
= uverbs_get_const(&dv_prt
, attrs
,
2665 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE
);
2669 if (!mlx5_ib_flow_action_packet_reformat_valid(mdev
, dv_prt
, ft_type
))
2672 maction
= kzalloc(sizeof(*maction
), GFP_KERNEL
);
2677 MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2
) {
2678 maction
->flow_action_raw
.sub_type
=
2679 MLX5_IB_FLOW_ACTION_DECAP
;
2680 maction
->flow_action_raw
.dev
= mdev
;
2685 in
= uverbs_attr_get_alloced_ptr(attrs
,
2686 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF
);
2692 len
= uverbs_attr_get_len(attrs
,
2693 MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF
);
2695 ret
= mlx5_ib_flow_action_create_packet_reformat_ctx(mdev
,
2696 maction
, ft_type
, dv_prt
, in
, len
);
2701 uverbs_flow_action_fill_action(&maction
->ib_action
, uobj
, &mdev
->ib_dev
,
2702 IB_FLOW_ACTION_UNSPECIFIED
);
2710 DECLARE_UVERBS_NAMED_METHOD(
2711 MLX5_IB_METHOD_CREATE_FLOW
,
2712 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE
,
2717 MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE
,
2718 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params
)),
2721 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER
,
2722 MLX5_IB_OBJECT_FLOW_MATCHER
,
2725 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP
,
2727 UVERBS_ACCESS_READ
),
2728 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX
,
2729 MLX5_IB_OBJECT_DEVX_OBJ
,
2730 UVERBS_ACCESS_READ
),
2731 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS
,
2732 UVERBS_OBJECT_FLOW_ACTION
,
2733 UVERBS_ACCESS_READ
, 1,
2734 MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS
,
2736 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG
,
2737 UVERBS_ATTR_TYPE(u32
),
2739 UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX
,
2740 MLX5_IB_OBJECT_DEVX_OBJ
,
2741 UVERBS_ACCESS_READ
, 1, 1,
2743 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET
,
2744 UVERBS_ATTR_MIN_SIZE(sizeof(u32
)),
2747 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS
,
2748 enum mlx5_ib_create_flow_flags
,
2751 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2752 MLX5_IB_METHOD_DESTROY_FLOW
,
2753 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE
,
2755 UVERBS_ACCESS_DESTROY
,
2758 ADD_UVERBS_METHODS(mlx5_ib_fs
,
2760 &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW
),
2761 &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW
));
2763 DECLARE_UVERBS_NAMED_METHOD(
2764 MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER
,
2765 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE
,
2766 UVERBS_OBJECT_FLOW_ACTION
,
2769 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM
,
2770 UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
2771 set_add_copy_action_in_auto
)),
2774 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE
,
2775 enum mlx5_ib_uapi_flow_table_type
,
2778 DECLARE_UVERBS_NAMED_METHOD(
2779 MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT
,
2780 UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE
,
2781 UVERBS_OBJECT_FLOW_ACTION
,
2784 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF
,
2785 UVERBS_ATTR_MIN_SIZE(1),
2788 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE
,
2789 enum mlx5_ib_uapi_flow_action_packet_reformat_type
,
2791 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE
,
2792 enum mlx5_ib_uapi_flow_table_type
,
2796 mlx5_ib_flow_actions
,
2797 UVERBS_OBJECT_FLOW_ACTION
,
2798 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER
),
2799 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT
));
2801 DECLARE_UVERBS_NAMED_METHOD(
2802 MLX5_IB_METHOD_FLOW_MATCHER_CREATE
,
2803 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE
,
2804 MLX5_IB_OBJECT_FLOW_MATCHER
,
2808 MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK
,
2809 UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params
)),
2811 UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE
,
2814 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA
,
2815 UVERBS_ATTR_TYPE(u8
),
2817 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS
,
2820 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE
,
2821 enum mlx5_ib_uapi_flow_table_type
,
2824 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2825 MLX5_IB_METHOD_FLOW_MATCHER_DESTROY
,
2826 UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE
,
2827 MLX5_IB_OBJECT_FLOW_MATCHER
,
2828 UVERBS_ACCESS_DESTROY
,
2831 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER
,
2832 UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup
),
2833 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE
),
2834 &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY
));
2836 DECLARE_UVERBS_NAMED_METHOD(
2837 MLX5_IB_METHOD_STEERING_ANCHOR_CREATE
,
2838 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE
,
2839 MLX5_IB_OBJECT_STEERING_ANCHOR
,
2842 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE
,
2843 enum mlx5_ib_uapi_flow_table_type
,
2845 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY
,
2846 UVERBS_ATTR_TYPE(u16
),
2848 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID
,
2849 UVERBS_ATTR_TYPE(u32
),
2852 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
2853 MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY
,
2854 UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE
,
2855 MLX5_IB_OBJECT_STEERING_ANCHOR
,
2856 UVERBS_ACCESS_DESTROY
,
2859 DECLARE_UVERBS_NAMED_OBJECT(
2860 MLX5_IB_OBJECT_STEERING_ANCHOR
,
2861 UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup
),
2862 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE
),
2863 &UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY
));
2865 const struct uapi_definition mlx5_ib_flow_defs
[] = {
2866 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2867 MLX5_IB_OBJECT_FLOW_MATCHER
),
2868 UAPI_DEF_CHAIN_OBJ_TREE(
2871 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION
,
2872 &mlx5_ib_flow_actions
),
2873 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2874 MLX5_IB_OBJECT_STEERING_ANCHOR
,
2875 UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed
)),
2879 static const struct ib_device_ops flow_ops
= {
2880 .create_flow
= mlx5_ib_create_flow
,
2881 .destroy_flow
= mlx5_ib_destroy_flow
,
2882 .destroy_flow_action
= mlx5_ib_destroy_flow_action
,
2885 int mlx5_ib_fs_init(struct mlx5_ib_dev
*dev
)
2887 dev
->flow_db
= kzalloc(sizeof(*dev
->flow_db
), GFP_KERNEL
);
2892 mutex_init(&dev
->flow_db
->lock
);
2894 ib_set_device_ops(&dev
->ib_dev
, &flow_ops
);