1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
4 #include <linux/skbuff.h>
5 #include <net/devlink.h>
6 #include <net/pkt_cls.h>
10 #include "conntrack.h"
11 #include "../nfpcore/nfp_cpp.h"
12 #include "../nfpcore/nfp_nsp.h"
13 #include "../nfp_app.h"
14 #include "../nfp_main.h"
15 #include "../nfp_net.h"
16 #include "../nfp_port.h"
18 #define NFP_FLOWER_SUPPORTED_TCPFLAGS \
19 (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
20 TCPHDR_PSH | TCPHDR_URG)
22 #define NFP_FLOWER_SUPPORTED_CTLFLAGS \
23 (FLOW_DIS_IS_FRAGMENT | \
26 #define NFP_FLOWER_WHITELIST_DISSECTOR \
27 (BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | \
28 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | \
29 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \
30 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \
31 BIT_ULL(FLOW_DISSECTOR_KEY_TCP) | \
32 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | \
33 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \
34 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | \
35 BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) | \
36 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
37 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
38 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
39 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
40 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
41 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
42 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) | \
43 BIT_ULL(FLOW_DISSECTOR_KEY_MPLS) | \
44 BIT_ULL(FLOW_DISSECTOR_KEY_CT) | \
45 BIT_ULL(FLOW_DISSECTOR_KEY_META) | \
46 BIT_ULL(FLOW_DISSECTOR_KEY_IP))
48 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR \
49 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
50 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
51 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
52 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
53 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
54 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
55 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP))
57 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R \
58 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
59 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS))
61 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_V6_R \
62 (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
63 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS))
65 #define NFP_FLOWER_MERGE_FIELDS \
66 (NFP_FLOWER_LAYER_PORT | \
67 NFP_FLOWER_LAYER_MAC | \
68 NFP_FLOWER_LAYER_TP | \
69 NFP_FLOWER_LAYER_IPV4 | \
70 NFP_FLOWER_LAYER_IPV6)
72 #define NFP_FLOWER_PRE_TUN_RULE_FIELDS \
73 (NFP_FLOWER_LAYER_EXT_META | \
74 NFP_FLOWER_LAYER_PORT | \
75 NFP_FLOWER_LAYER_MAC | \
76 NFP_FLOWER_LAYER_IPV4 | \
77 NFP_FLOWER_LAYER_IPV6)
79 struct nfp_flower_merge_check
{
83 struct nfp_flower_mac_mpls l2
;
84 struct nfp_flower_tp_ports l4
;
86 struct nfp_flower_ipv4 ipv4
;
87 struct nfp_flower_ipv6 ipv6
;
90 unsigned long vals
[8];
95 nfp_flower_xmit_flow(struct nfp_app
*app
, struct nfp_fl_payload
*nfp_flow
,
98 u32 meta_len
, key_len
, mask_len
, act_len
, tot_len
;
102 meta_len
= sizeof(struct nfp_fl_rule_metadata
);
103 key_len
= nfp_flow
->meta
.key_len
;
104 mask_len
= nfp_flow
->meta
.mask_len
;
105 act_len
= nfp_flow
->meta
.act_len
;
107 tot_len
= meta_len
+ key_len
+ mask_len
+ act_len
;
109 /* Convert to long words as firmware expects
110 * lengths in units of NFP_FL_LW_SIZ.
112 nfp_flow
->meta
.key_len
>>= NFP_FL_LW_SIZ
;
113 nfp_flow
->meta
.mask_len
>>= NFP_FL_LW_SIZ
;
114 nfp_flow
->meta
.act_len
>>= NFP_FL_LW_SIZ
;
116 skb
= nfp_flower_cmsg_alloc(app
, tot_len
, mtype
, GFP_KERNEL
);
120 msg
= nfp_flower_cmsg_get_data(skb
);
121 memcpy(msg
, &nfp_flow
->meta
, meta_len
);
122 memcpy(&msg
[meta_len
], nfp_flow
->unmasked_data
, key_len
);
123 memcpy(&msg
[meta_len
+ key_len
], nfp_flow
->mask_data
, mask_len
);
124 memcpy(&msg
[meta_len
+ key_len
+ mask_len
],
125 nfp_flow
->action_data
, act_len
);
127 /* Convert back to bytes as software expects
128 * lengths in units of bytes.
130 nfp_flow
->meta
.key_len
<<= NFP_FL_LW_SIZ
;
131 nfp_flow
->meta
.mask_len
<<= NFP_FL_LW_SIZ
;
132 nfp_flow
->meta
.act_len
<<= NFP_FL_LW_SIZ
;
134 nfp_ctrl_tx(app
->ctrl
, skb
);
139 static bool nfp_flower_check_higher_than_mac(struct flow_rule
*rule
)
141 return flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
) ||
142 flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
) ||
143 flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_PORTS
) ||
144 flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ICMP
);
147 static bool nfp_flower_check_higher_than_l3(struct flow_rule
*rule
)
149 return flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_PORTS
) ||
150 flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ICMP
);
154 nfp_flower_calc_opt_layer(struct flow_dissector_key_enc_opts
*enc_opts
,
155 u32
*key_layer_two
, int *key_size
, bool ipv6
,
156 struct netlink_ext_ack
*extack
)
158 if (enc_opts
->len
> NFP_FL_MAX_GENEVE_OPT_KEY
||
159 (ipv6
&& enc_opts
->len
> NFP_FL_MAX_GENEVE_OPT_KEY_V6
)) {
160 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: geneve options exceed maximum length");
164 if (enc_opts
->len
> 0) {
165 *key_layer_two
|= NFP_FLOWER_LAYER2_GENEVE_OP
;
166 *key_size
+= sizeof(struct nfp_flower_geneve_options
);
173 nfp_flower_calc_udp_tun_layer(struct flow_dissector_key_ports
*enc_ports
,
174 struct flow_dissector_key_enc_opts
*enc_op
,
175 u32
*key_layer_two
, u8
*key_layer
, int *key_size
,
176 struct nfp_flower_priv
*priv
,
177 enum nfp_flower_tun_type
*tun_type
, bool ipv6
,
178 struct netlink_ext_ack
*extack
)
182 switch (enc_ports
->dst
) {
183 case htons(IANA_VXLAN_UDP_PORT
):
184 *tun_type
= NFP_FL_TUNNEL_VXLAN
;
185 *key_layer
|= NFP_FLOWER_LAYER_VXLAN
;
188 *key_layer
|= NFP_FLOWER_LAYER_EXT_META
;
189 *key_size
+= sizeof(struct nfp_flower_ext_meta
);
190 *key_layer_two
|= NFP_FLOWER_LAYER2_TUN_IPV6
;
191 *key_size
+= sizeof(struct nfp_flower_ipv6_udp_tun
);
193 *key_size
+= sizeof(struct nfp_flower_ipv4_udp_tun
);
197 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: encap options not supported on vxlan tunnels");
201 case htons(GENEVE_UDP_PORT
):
202 if (!(priv
->flower_ext_feats
& NFP_FL_FEATS_GENEVE
)) {
203 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: loaded firmware does not support geneve offload");
206 *tun_type
= NFP_FL_TUNNEL_GENEVE
;
207 *key_layer
|= NFP_FLOWER_LAYER_EXT_META
;
208 *key_size
+= sizeof(struct nfp_flower_ext_meta
);
209 *key_layer_two
|= NFP_FLOWER_LAYER2_GENEVE
;
212 *key_layer_two
|= NFP_FLOWER_LAYER2_TUN_IPV6
;
213 *key_size
+= sizeof(struct nfp_flower_ipv6_udp_tun
);
215 *key_size
+= sizeof(struct nfp_flower_ipv4_udp_tun
);
220 if (!(priv
->flower_ext_feats
& NFP_FL_FEATS_GENEVE_OPT
)) {
221 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: loaded firmware does not support geneve option offload");
224 err
= nfp_flower_calc_opt_layer(enc_op
, key_layer_two
, key_size
,
230 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: tunnel type unknown");
238 nfp_flower_calculate_key_layers(struct nfp_app
*app
,
239 struct net_device
*netdev
,
240 struct nfp_fl_key_ls
*ret_key_ls
,
241 struct flow_rule
*rule
,
242 enum nfp_flower_tun_type
*tun_type
,
243 struct netlink_ext_ack
*extack
)
245 struct flow_dissector
*dissector
= rule
->match
.dissector
;
246 struct flow_match_basic basic
= { NULL
, NULL
};
247 struct nfp_flower_priv
*priv
= app
->priv
;
253 if (dissector
->used_keys
& ~NFP_FLOWER_WHITELIST_DISSECTOR
) {
254 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: match not supported");
258 /* If any tun dissector is used then the required set must be used. */
259 if (dissector
->used_keys
& NFP_FLOWER_WHITELIST_TUN_DISSECTOR
&&
260 (dissector
->used_keys
& NFP_FLOWER_WHITELIST_TUN_DISSECTOR_V6_R
)
261 != NFP_FLOWER_WHITELIST_TUN_DISSECTOR_V6_R
&&
262 (dissector
->used_keys
& NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R
)
263 != NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R
) {
264 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: tunnel match not supported");
269 key_layer
= NFP_FLOWER_LAYER_PORT
;
270 key_size
= sizeof(struct nfp_flower_meta_tci
) +
271 sizeof(struct nfp_flower_in_port
);
273 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ETH_ADDRS
) ||
274 flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_MPLS
)) {
275 key_layer
|= NFP_FLOWER_LAYER_MAC
;
276 key_size
+= sizeof(struct nfp_flower_mac_mpls
);
279 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_VLAN
)) {
280 struct flow_match_vlan vlan
;
282 flow_rule_match_vlan(rule
, &vlan
);
283 if (!(priv
->flower_ext_feats
& NFP_FL_FEATS_VLAN_PCP
) &&
284 vlan
.key
->vlan_priority
) {
285 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: loaded firmware does not support VLAN PCP offload");
288 if (priv
->flower_ext_feats
& NFP_FL_FEATS_VLAN_QINQ
&&
289 !(key_layer_two
& NFP_FLOWER_LAYER2_QINQ
)) {
290 key_layer
|= NFP_FLOWER_LAYER_EXT_META
;
291 key_size
+= sizeof(struct nfp_flower_ext_meta
);
292 key_size
+= sizeof(struct nfp_flower_vlan
);
293 key_layer_two
|= NFP_FLOWER_LAYER2_QINQ
;
297 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_CVLAN
)) {
298 struct flow_match_vlan cvlan
;
300 if (!(priv
->flower_ext_feats
& NFP_FL_FEATS_VLAN_QINQ
)) {
301 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: loaded firmware does not support VLAN QinQ offload");
305 flow_rule_match_vlan(rule
, &cvlan
);
306 if (!(key_layer_two
& NFP_FLOWER_LAYER2_QINQ
)) {
307 key_layer
|= NFP_FLOWER_LAYER_EXT_META
;
308 key_size
+= sizeof(struct nfp_flower_ext_meta
);
309 key_size
+= sizeof(struct nfp_flower_vlan
);
310 key_layer_two
|= NFP_FLOWER_LAYER2_QINQ
;
314 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ENC_CONTROL
)) {
315 struct flow_match_enc_opts enc_op
= { NULL
, NULL
};
316 struct flow_match_ipv4_addrs ipv4_addrs
;
317 struct flow_match_ipv6_addrs ipv6_addrs
;
318 struct flow_match_control enc_ctl
;
319 struct flow_match_ports enc_ports
;
320 bool ipv6_tun
= false;
322 flow_rule_match_enc_control(rule
, &enc_ctl
);
324 if (enc_ctl
.mask
->addr_type
!= 0xffff) {
325 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: wildcarded protocols on tunnels are not supported");
329 ipv6_tun
= enc_ctl
.key
->addr_type
==
330 FLOW_DISSECTOR_KEY_IPV6_ADDRS
;
332 !(priv
->flower_ext_feats
& NFP_FL_FEATS_IPV6_TUN
)) {
333 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: firmware does not support IPv6 tunnels");
338 enc_ctl
.key
->addr_type
!= FLOW_DISSECTOR_KEY_IPV4_ADDRS
) {
339 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: tunnel address type not IPv4 or IPv6");
344 flow_rule_match_enc_ipv6_addrs(rule
, &ipv6_addrs
);
345 if (memchr_inv(&ipv6_addrs
.mask
->dst
, 0xff,
346 sizeof(ipv6_addrs
.mask
->dst
))) {
347 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: only an exact match IPv6 destination address is supported");
351 flow_rule_match_enc_ipv4_addrs(rule
, &ipv4_addrs
);
352 if (ipv4_addrs
.mask
->dst
!= cpu_to_be32(~0)) {
353 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: only an exact match IPv4 destination address is supported");
358 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ENC_OPTS
))
359 flow_rule_match_enc_opts(rule
, &enc_op
);
361 if (!flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) {
362 /* Check if GRE, which has no enc_ports */
363 if (!netif_is_gretap(netdev
) && !netif_is_ip6gretap(netdev
)) {
364 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: an exact match on L4 destination port is required for non-GRE tunnels");
368 *tun_type
= NFP_FL_TUNNEL_GRE
;
369 key_layer
|= NFP_FLOWER_LAYER_EXT_META
;
370 key_size
+= sizeof(struct nfp_flower_ext_meta
);
371 key_layer_two
|= NFP_FLOWER_LAYER2_GRE
;
374 key_layer_two
|= NFP_FLOWER_LAYER2_TUN_IPV6
;
376 sizeof(struct nfp_flower_ipv6_gre_tun
);
379 sizeof(struct nfp_flower_ipv4_gre_tun
);
383 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: encap options not supported on GRE tunnels");
387 flow_rule_match_enc_ports(rule
, &enc_ports
);
388 if (enc_ports
.mask
->dst
!= cpu_to_be16(~0)) {
389 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: only an exact match L4 destination port is supported");
393 err
= nfp_flower_calc_udp_tun_layer(enc_ports
.key
,
403 /* Ensure the ingress netdev matches the expected
406 if (!nfp_fl_netdev_is_tunnel_type(netdev
, *tun_type
)) {
407 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: ingress netdev does not match the expected tunnel type");
413 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_BASIC
))
414 flow_rule_match_basic(rule
, &basic
);
416 if (basic
.mask
&& basic
.mask
->n_proto
) {
417 /* Ethernet type is present in the key. */
418 switch (basic
.key
->n_proto
) {
419 case cpu_to_be16(ETH_P_IP
):
420 key_layer
|= NFP_FLOWER_LAYER_IPV4
;
421 key_size
+= sizeof(struct nfp_flower_ipv4
);
424 case cpu_to_be16(ETH_P_IPV6
):
425 key_layer
|= NFP_FLOWER_LAYER_IPV6
;
426 key_size
+= sizeof(struct nfp_flower_ipv6
);
429 /* Currently we do not offload ARP
430 * because we rely on it to get to the host.
432 case cpu_to_be16(ETH_P_ARP
):
433 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: ARP not supported");
436 case cpu_to_be16(ETH_P_MPLS_UC
):
437 case cpu_to_be16(ETH_P_MPLS_MC
):
438 if (!(key_layer
& NFP_FLOWER_LAYER_MAC
)) {
439 key_layer
|= NFP_FLOWER_LAYER_MAC
;
440 key_size
+= sizeof(struct nfp_flower_mac_mpls
);
444 /* Will be included in layer 2. */
445 case cpu_to_be16(ETH_P_8021Q
):
449 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: match on given EtherType is not supported");
452 } else if (nfp_flower_check_higher_than_mac(rule
)) {
453 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: cannot match above L2 without specified EtherType");
457 if (basic
.mask
&& basic
.mask
->ip_proto
) {
458 switch (basic
.key
->ip_proto
) {
464 key_layer
|= NFP_FLOWER_LAYER_TP
;
465 key_size
+= sizeof(struct nfp_flower_tp_ports
);
470 if (!(key_layer
& NFP_FLOWER_LAYER_TP
) &&
471 nfp_flower_check_higher_than_l3(rule
)) {
472 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: cannot match on L4 information without specified IP protocol type");
476 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_TCP
)) {
477 struct flow_match_tcp tcp
;
480 flow_rule_match_tcp(rule
, &tcp
);
481 tcp_flags
= be16_to_cpu(tcp
.key
->flags
);
483 if (tcp_flags
& ~NFP_FLOWER_SUPPORTED_TCPFLAGS
) {
484 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: no match support for selected TCP flags");
488 /* We only support PSH and URG flags when either
489 * FIN, SYN or RST is present as well.
491 if ((tcp_flags
& (TCPHDR_PSH
| TCPHDR_URG
)) &&
492 !(tcp_flags
& (TCPHDR_FIN
| TCPHDR_SYN
| TCPHDR_RST
))) {
493 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: PSH and URG is only supported when used with FIN, SYN or RST");
497 /* We need to store TCP flags in the either the IPv4 or IPv6 key
498 * space, thus we need to ensure we include a IPv4/IPv6 key
499 * layer if we have not done so already.
502 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: match on TCP flags requires a match on L3 protocol");
506 if (!(key_layer
& NFP_FLOWER_LAYER_IPV4
) &&
507 !(key_layer
& NFP_FLOWER_LAYER_IPV6
)) {
508 switch (basic
.key
->n_proto
) {
509 case cpu_to_be16(ETH_P_IP
):
510 key_layer
|= NFP_FLOWER_LAYER_IPV4
;
511 key_size
+= sizeof(struct nfp_flower_ipv4
);
514 case cpu_to_be16(ETH_P_IPV6
):
515 key_layer
|= NFP_FLOWER_LAYER_IPV6
;
516 key_size
+= sizeof(struct nfp_flower_ipv6
);
520 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: match on TCP flags requires a match on IPv4/IPv6");
526 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_CONTROL
)) {
527 struct flow_match_control ctl
;
529 flow_rule_match_control(rule
, &ctl
);
530 if (ctl
.key
->flags
& ~NFP_FLOWER_SUPPORTED_CTLFLAGS
) {
531 NL_SET_ERR_MSG_MOD(extack
, "unsupported offload: match on unknown control flag");
536 ret_key_ls
->key_layer
= key_layer
;
537 ret_key_ls
->key_layer_two
= key_layer_two
;
538 ret_key_ls
->key_size
= key_size
;
543 struct nfp_fl_payload
*
544 nfp_flower_allocate_new(struct nfp_fl_key_ls
*key_layer
)
546 struct nfp_fl_payload
*flow_pay
;
548 flow_pay
= kmalloc(sizeof(*flow_pay
), GFP_KERNEL
);
552 flow_pay
->meta
.key_len
= key_layer
->key_size
;
553 flow_pay
->unmasked_data
= kmalloc(key_layer
->key_size
, GFP_KERNEL
);
554 if (!flow_pay
->unmasked_data
)
557 flow_pay
->meta
.mask_len
= key_layer
->key_size
;
558 flow_pay
->mask_data
= kmalloc(key_layer
->key_size
, GFP_KERNEL
);
559 if (!flow_pay
->mask_data
)
560 goto err_free_unmasked
;
562 flow_pay
->action_data
= kmalloc(NFP_FL_MAX_A_SIZ
, GFP_KERNEL
);
563 if (!flow_pay
->action_data
)
566 flow_pay
->nfp_tun_ipv4_addr
= 0;
567 flow_pay
->nfp_tun_ipv6
= NULL
;
568 flow_pay
->meta
.flags
= 0;
569 INIT_LIST_HEAD(&flow_pay
->linked_flows
);
570 flow_pay
->in_hw
= false;
571 flow_pay
->pre_tun_rule
.dev
= NULL
;
576 kfree(flow_pay
->mask_data
);
578 kfree(flow_pay
->unmasked_data
);
585 nfp_flower_update_merge_with_actions(struct nfp_fl_payload
*flow
,
586 struct nfp_flower_merge_check
*merge
,
587 u8
*last_act_id
, int *act_out
)
589 struct nfp_fl_set_ipv6_tc_hl_fl
*ipv6_tc_hl_fl
;
590 struct nfp_fl_set_ip4_ttl_tos
*ipv4_ttl_tos
;
591 struct nfp_fl_set_ip4_addrs
*ipv4_add
;
592 struct nfp_fl_set_ipv6_addr
*ipv6_add
;
593 struct nfp_fl_push_vlan
*push_vlan
;
594 struct nfp_fl_pre_tunnel
*pre_tun
;
595 struct nfp_fl_set_tport
*tport
;
596 struct nfp_fl_set_eth
*eth
;
597 struct nfp_fl_act_head
*a
;
598 unsigned int act_off
= 0;
599 bool ipv6_tun
= false;
604 while (act_off
< flow
->meta
.act_len
) {
605 a
= (struct nfp_fl_act_head
*)&flow
->action_data
[act_off
];
609 case NFP_FL_ACTION_OPCODE_OUTPUT
:
613 case NFP_FL_ACTION_OPCODE_PUSH_VLAN
:
614 push_vlan
= (struct nfp_fl_push_vlan
*)a
;
615 if (push_vlan
->vlan_tci
)
616 merge
->tci
= cpu_to_be16(0xffff);
618 case NFP_FL_ACTION_OPCODE_POP_VLAN
:
619 merge
->tci
= cpu_to_be16(0);
621 case NFP_FL_ACTION_OPCODE_SET_TUNNEL
:
622 /* New tunnel header means l2 to l4 can be matched. */
623 eth_broadcast_addr(&merge
->l2
.mac_dst
[0]);
624 eth_broadcast_addr(&merge
->l2
.mac_src
[0]);
625 memset(&merge
->l4
, 0xff,
626 sizeof(struct nfp_flower_tp_ports
));
628 memset(&merge
->ipv6
, 0xff,
629 sizeof(struct nfp_flower_ipv6
));
631 memset(&merge
->ipv4
, 0xff,
632 sizeof(struct nfp_flower_ipv4
));
634 case NFP_FL_ACTION_OPCODE_SET_ETHERNET
:
635 eth
= (struct nfp_fl_set_eth
*)a
;
636 for (i
= 0; i
< ETH_ALEN
; i
++)
637 merge
->l2
.mac_dst
[i
] |= eth
->eth_addr_mask
[i
];
638 for (i
= 0; i
< ETH_ALEN
; i
++)
639 merge
->l2
.mac_src
[i
] |=
640 eth
->eth_addr_mask
[ETH_ALEN
+ i
];
642 case NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS
:
643 ipv4_add
= (struct nfp_fl_set_ip4_addrs
*)a
;
644 merge
->ipv4
.ipv4_src
|= ipv4_add
->ipv4_src_mask
;
645 merge
->ipv4
.ipv4_dst
|= ipv4_add
->ipv4_dst_mask
;
647 case NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS
:
648 ipv4_ttl_tos
= (struct nfp_fl_set_ip4_ttl_tos
*)a
;
649 merge
->ipv4
.ip_ext
.ttl
|= ipv4_ttl_tos
->ipv4_ttl_mask
;
650 merge
->ipv4
.ip_ext
.tos
|= ipv4_ttl_tos
->ipv4_tos_mask
;
652 case NFP_FL_ACTION_OPCODE_SET_IPV6_SRC
:
653 ipv6_add
= (struct nfp_fl_set_ipv6_addr
*)a
;
654 for (i
= 0; i
< 4; i
++)
655 merge
->ipv6
.ipv6_src
.in6_u
.u6_addr32
[i
] |=
656 ipv6_add
->ipv6
[i
].mask
;
658 case NFP_FL_ACTION_OPCODE_SET_IPV6_DST
:
659 ipv6_add
= (struct nfp_fl_set_ipv6_addr
*)a
;
660 for (i
= 0; i
< 4; i
++)
661 merge
->ipv6
.ipv6_dst
.in6_u
.u6_addr32
[i
] |=
662 ipv6_add
->ipv6
[i
].mask
;
664 case NFP_FL_ACTION_OPCODE_SET_IPV6_TC_HL_FL
:
665 ipv6_tc_hl_fl
= (struct nfp_fl_set_ipv6_tc_hl_fl
*)a
;
666 merge
->ipv6
.ip_ext
.ttl
|=
667 ipv6_tc_hl_fl
->ipv6_hop_limit_mask
;
668 merge
->ipv6
.ip_ext
.tos
|= ipv6_tc_hl_fl
->ipv6_tc_mask
;
669 merge
->ipv6
.ipv6_flow_label_exthdr
|=
670 ipv6_tc_hl_fl
->ipv6_label_mask
;
672 case NFP_FL_ACTION_OPCODE_SET_UDP
:
673 case NFP_FL_ACTION_OPCODE_SET_TCP
:
674 tport
= (struct nfp_fl_set_tport
*)a
;
675 ports
= (u8
*)&merge
->l4
.port_src
;
676 for (i
= 0; i
< 4; i
++)
677 ports
[i
] |= tport
->tp_port_mask
[i
];
679 case NFP_FL_ACTION_OPCODE_PRE_TUNNEL
:
680 pre_tun
= (struct nfp_fl_pre_tunnel
*)a
;
681 ipv6_tun
= be16_to_cpu(pre_tun
->flags
) &
684 case NFP_FL_ACTION_OPCODE_PRE_LAG
:
685 case NFP_FL_ACTION_OPCODE_PUSH_GENEVE
:
691 act_off
+= a
->len_lw
<< NFP_FL_LW_SIZ
;
695 *last_act_id
= act_id
;
701 nfp_flower_populate_merge_match(struct nfp_fl_payload
*flow
,
702 struct nfp_flower_merge_check
*merge
,
705 struct nfp_flower_meta_tci
*meta_tci
;
706 u8
*mask
= flow
->mask_data
;
707 u8 key_layer
, match_size
;
709 memset(merge
, 0, sizeof(struct nfp_flower_merge_check
));
711 meta_tci
= (struct nfp_flower_meta_tci
*)mask
;
712 key_layer
= meta_tci
->nfp_flow_key_layer
;
714 if (key_layer
& ~NFP_FLOWER_MERGE_FIELDS
&& !extra_fields
)
717 merge
->tci
= meta_tci
->tci
;
718 mask
+= sizeof(struct nfp_flower_meta_tci
);
720 if (key_layer
& NFP_FLOWER_LAYER_EXT_META
)
721 mask
+= sizeof(struct nfp_flower_ext_meta
);
723 mask
+= sizeof(struct nfp_flower_in_port
);
725 if (key_layer
& NFP_FLOWER_LAYER_MAC
) {
726 match_size
= sizeof(struct nfp_flower_mac_mpls
);
727 memcpy(&merge
->l2
, mask
, match_size
);
731 if (key_layer
& NFP_FLOWER_LAYER_TP
) {
732 match_size
= sizeof(struct nfp_flower_tp_ports
);
733 memcpy(&merge
->l4
, mask
, match_size
);
737 if (key_layer
& NFP_FLOWER_LAYER_IPV4
) {
738 match_size
= sizeof(struct nfp_flower_ipv4
);
739 memcpy(&merge
->ipv4
, mask
, match_size
);
742 if (key_layer
& NFP_FLOWER_LAYER_IPV6
) {
743 match_size
= sizeof(struct nfp_flower_ipv6
);
744 memcpy(&merge
->ipv6
, mask
, match_size
);
751 nfp_flower_can_merge(struct nfp_fl_payload
*sub_flow1
,
752 struct nfp_fl_payload
*sub_flow2
)
754 /* Two flows can be merged if sub_flow2 only matches on bits that are
755 * either matched by sub_flow1 or set by a sub_flow1 action. This
756 * ensures that every packet that hits sub_flow1 and recirculates is
757 * guaranteed to hit sub_flow2.
759 struct nfp_flower_merge_check sub_flow1_merge
, sub_flow2_merge
;
760 int err
, act_out
= 0;
763 err
= nfp_flower_populate_merge_match(sub_flow1
, &sub_flow1_merge
,
768 err
= nfp_flower_populate_merge_match(sub_flow2
, &sub_flow2_merge
,
773 err
= nfp_flower_update_merge_with_actions(sub_flow1
, &sub_flow1_merge
,
774 &last_act_id
, &act_out
);
778 /* Must only be 1 output action and it must be the last in sequence. */
779 if (act_out
!= 1 || last_act_id
!= NFP_FL_ACTION_OPCODE_OUTPUT
)
782 /* Reject merge if sub_flow2 matches on something that is not matched
783 * on or set in an action by sub_flow1.
785 err
= bitmap_andnot(sub_flow2_merge
.vals
, sub_flow2_merge
.vals
,
786 sub_flow1_merge
.vals
,
787 sizeof(struct nfp_flower_merge_check
) * 8);
795 nfp_flower_copy_pre_actions(char *act_dst
, char *act_src
, int len
,
798 unsigned int act_off
= 0, act_len
;
799 struct nfp_fl_act_head
*a
;
802 while (act_off
< len
) {
803 a
= (struct nfp_fl_act_head
*)&act_src
[act_off
];
804 act_len
= a
->len_lw
<< NFP_FL_LW_SIZ
;
808 case NFP_FL_ACTION_OPCODE_PRE_TUNNEL
:
812 case NFP_FL_ACTION_OPCODE_PRE_LAG
:
813 memcpy(act_dst
+ act_off
, act_src
+ act_off
, act_len
);
826 nfp_fl_verify_post_tun_acts(char *acts
, int len
, struct nfp_fl_push_vlan
**vlan
)
828 struct nfp_fl_act_head
*a
;
829 unsigned int act_off
= 0;
831 while (act_off
< len
) {
832 a
= (struct nfp_fl_act_head
*)&acts
[act_off
];
834 if (a
->jump_id
== NFP_FL_ACTION_OPCODE_PUSH_VLAN
&& !act_off
)
835 *vlan
= (struct nfp_fl_push_vlan
*)a
;
836 else if (a
->jump_id
!= NFP_FL_ACTION_OPCODE_OUTPUT
)
839 act_off
+= a
->len_lw
<< NFP_FL_LW_SIZ
;
842 /* Ensure any VLAN push also has an egress action. */
843 if (*vlan
&& act_off
<= sizeof(struct nfp_fl_push_vlan
))
850 nfp_fl_push_vlan_after_tun(char *acts
, int len
, struct nfp_fl_push_vlan
*vlan
)
852 struct nfp_fl_set_tun
*tun
;
853 struct nfp_fl_act_head
*a
;
854 unsigned int act_off
= 0;
856 while (act_off
< len
) {
857 a
= (struct nfp_fl_act_head
*)&acts
[act_off
];
859 if (a
->jump_id
== NFP_FL_ACTION_OPCODE_SET_TUNNEL
) {
860 tun
= (struct nfp_fl_set_tun
*)a
;
861 tun
->outer_vlan_tpid
= vlan
->vlan_tpid
;
862 tun
->outer_vlan_tci
= vlan
->vlan_tci
;
867 act_off
+= a
->len_lw
<< NFP_FL_LW_SIZ
;
870 /* Return error if no tunnel action is found. */
875 nfp_flower_merge_action(struct nfp_fl_payload
*sub_flow1
,
876 struct nfp_fl_payload
*sub_flow2
,
877 struct nfp_fl_payload
*merge_flow
)
879 unsigned int sub1_act_len
, sub2_act_len
, pre_off1
, pre_off2
;
880 struct nfp_fl_push_vlan
*post_tun_push_vlan
= NULL
;
881 bool tunnel_act
= false;
885 /* The last action of sub_flow1 must be output - do not merge this. */
886 sub1_act_len
= sub_flow1
->meta
.act_len
- sizeof(struct nfp_fl_output
);
887 sub2_act_len
= sub_flow2
->meta
.act_len
;
892 if (sub1_act_len
+ sub2_act_len
> NFP_FL_MAX_A_SIZ
)
895 /* A shortcut can only be applied if there is a single action. */
897 merge_flow
->meta
.shortcut
= cpu_to_be32(NFP_FL_SC_ACT_NULL
);
899 merge_flow
->meta
.shortcut
= sub_flow2
->meta
.shortcut
;
901 merge_flow
->meta
.act_len
= sub1_act_len
+ sub2_act_len
;
902 merge_act
= merge_flow
->action_data
;
904 /* Copy any pre-actions to the start of merge flow action list. */
905 pre_off1
= nfp_flower_copy_pre_actions(merge_act
,
906 sub_flow1
->action_data
,
907 sub1_act_len
, &tunnel_act
);
908 merge_act
+= pre_off1
;
909 sub1_act_len
-= pre_off1
;
910 pre_off2
= nfp_flower_copy_pre_actions(merge_act
,
911 sub_flow2
->action_data
,
913 merge_act
+= pre_off2
;
914 sub2_act_len
-= pre_off2
;
916 /* FW does a tunnel push when egressing, therefore, if sub_flow 1 pushes
917 * a tunnel, there are restrictions on what sub_flow 2 actions lead to a
921 char *post_tun_acts
= &sub_flow2
->action_data
[pre_off2
];
923 err
= nfp_fl_verify_post_tun_acts(post_tun_acts
, sub2_act_len
,
924 &post_tun_push_vlan
);
928 if (post_tun_push_vlan
) {
929 pre_off2
+= sizeof(*post_tun_push_vlan
);
930 sub2_act_len
-= sizeof(*post_tun_push_vlan
);
934 /* Copy remaining actions from sub_flows 1 and 2. */
935 memcpy(merge_act
, sub_flow1
->action_data
+ pre_off1
, sub1_act_len
);
937 if (post_tun_push_vlan
) {
938 /* Update tunnel action in merge to include VLAN push. */
939 err
= nfp_fl_push_vlan_after_tun(merge_act
, sub1_act_len
,
944 merge_flow
->meta
.act_len
-= sizeof(*post_tun_push_vlan
);
947 merge_act
+= sub1_act_len
;
948 memcpy(merge_act
, sub_flow2
->action_data
+ pre_off2
, sub2_act_len
);
953 /* Flow link code should only be accessed under RTNL. */
954 static void nfp_flower_unlink_flow(struct nfp_fl_payload_link
*link
)
956 list_del(&link
->merge_flow
.list
);
957 list_del(&link
->sub_flow
.list
);
961 static void nfp_flower_unlink_flows(struct nfp_fl_payload
*merge_flow
,
962 struct nfp_fl_payload
*sub_flow
)
964 struct nfp_fl_payload_link
*link
;
966 list_for_each_entry(link
, &merge_flow
->linked_flows
, merge_flow
.list
)
967 if (link
->sub_flow
.flow
== sub_flow
) {
968 nfp_flower_unlink_flow(link
);
973 static int nfp_flower_link_flows(struct nfp_fl_payload
*merge_flow
,
974 struct nfp_fl_payload
*sub_flow
)
976 struct nfp_fl_payload_link
*link
;
978 link
= kmalloc(sizeof(*link
), GFP_KERNEL
);
982 link
->merge_flow
.flow
= merge_flow
;
983 list_add_tail(&link
->merge_flow
.list
, &merge_flow
->linked_flows
);
984 link
->sub_flow
.flow
= sub_flow
;
985 list_add_tail(&link
->sub_flow
.list
, &sub_flow
->linked_flows
);
991 * nfp_flower_merge_offloaded_flows() - Merge 2 existing flows to single flow.
992 * @app: Pointer to the APP handle
993 * @sub_flow1: Initial flow matched to produce merge hint
994 * @sub_flow2: Post recirculation flow matched in merge hint
996 * Combines 2 flows (if valid) to a single flow, removing the initial from hw
997 * and offloading the new, merged flow.
999 * Return: negative value on error, 0 in success.
1001 int nfp_flower_merge_offloaded_flows(struct nfp_app
*app
,
1002 struct nfp_fl_payload
*sub_flow1
,
1003 struct nfp_fl_payload
*sub_flow2
)
1005 struct nfp_flower_priv
*priv
= app
->priv
;
1006 struct nfp_fl_payload
*merge_flow
;
1007 struct nfp_fl_key_ls merge_key_ls
;
1008 struct nfp_merge_info
*merge_info
;
1012 if (sub_flow1
== sub_flow2
||
1013 nfp_flower_is_merge_flow(sub_flow1
) ||
1014 nfp_flower_is_merge_flow(sub_flow2
))
1017 /* Check if the two flows are already merged */
1018 parent_ctx
= (u64
)(be32_to_cpu(sub_flow1
->meta
.host_ctx_id
)) << 32;
1019 parent_ctx
|= (u64
)(be32_to_cpu(sub_flow2
->meta
.host_ctx_id
));
1020 if (rhashtable_lookup_fast(&priv
->merge_table
,
1021 &parent_ctx
, merge_table_params
)) {
1022 nfp_flower_cmsg_warn(app
, "The two flows are already merged.\n");
1026 err
= nfp_flower_can_merge(sub_flow1
, sub_flow2
);
1030 merge_key_ls
.key_size
= sub_flow1
->meta
.key_len
;
1032 merge_flow
= nfp_flower_allocate_new(&merge_key_ls
);
1036 merge_flow
->tc_flower_cookie
= (unsigned long)merge_flow
;
1037 merge_flow
->ingress_dev
= sub_flow1
->ingress_dev
;
1039 memcpy(merge_flow
->unmasked_data
, sub_flow1
->unmasked_data
,
1040 sub_flow1
->meta
.key_len
);
1041 memcpy(merge_flow
->mask_data
, sub_flow1
->mask_data
,
1042 sub_flow1
->meta
.mask_len
);
1044 err
= nfp_flower_merge_action(sub_flow1
, sub_flow2
, merge_flow
);
1046 goto err_destroy_merge_flow
;
1048 err
= nfp_flower_link_flows(merge_flow
, sub_flow1
);
1050 goto err_destroy_merge_flow
;
1052 err
= nfp_flower_link_flows(merge_flow
, sub_flow2
);
1054 goto err_unlink_sub_flow1
;
1056 err
= nfp_compile_flow_metadata(app
, merge_flow
->tc_flower_cookie
, merge_flow
,
1057 merge_flow
->ingress_dev
, NULL
);
1059 goto err_unlink_sub_flow2
;
1061 err
= rhashtable_insert_fast(&priv
->flow_table
, &merge_flow
->fl_node
,
1062 nfp_flower_table_params
);
1064 goto err_release_metadata
;
1066 merge_info
= kmalloc(sizeof(*merge_info
), GFP_KERNEL
);
1069 goto err_remove_rhash
;
1071 merge_info
->parent_ctx
= parent_ctx
;
1072 err
= rhashtable_insert_fast(&priv
->merge_table
, &merge_info
->ht_node
,
1073 merge_table_params
);
1075 goto err_destroy_merge_info
;
1077 err
= nfp_flower_xmit_flow(app
, merge_flow
,
1078 NFP_FLOWER_CMSG_TYPE_FLOW_MOD
);
1080 goto err_remove_merge_info
;
1082 merge_flow
->in_hw
= true;
1083 sub_flow1
->in_hw
= false;
1087 err_remove_merge_info
:
1088 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->merge_table
,
1089 &merge_info
->ht_node
,
1090 merge_table_params
));
1091 err_destroy_merge_info
:
1094 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->flow_table
,
1095 &merge_flow
->fl_node
,
1096 nfp_flower_table_params
));
1097 err_release_metadata
:
1098 nfp_modify_flow_metadata(app
, merge_flow
);
1099 err_unlink_sub_flow2
:
1100 nfp_flower_unlink_flows(merge_flow
, sub_flow2
);
1101 err_unlink_sub_flow1
:
1102 nfp_flower_unlink_flows(merge_flow
, sub_flow1
);
1103 err_destroy_merge_flow
:
1104 kfree(merge_flow
->action_data
);
1105 kfree(merge_flow
->mask_data
);
1106 kfree(merge_flow
->unmasked_data
);
1112 * nfp_flower_validate_pre_tun_rule()
1113 * @app: Pointer to the APP handle
1114 * @flow: Pointer to NFP flow representation of rule
1115 * @key_ls: Pointer to NFP key layers structure
1116 * @extack: Netlink extended ACK report
1118 * Verifies the flow as a pre-tunnel rule.
1120 * Return: negative value on error, 0 if verified.
1123 nfp_flower_validate_pre_tun_rule(struct nfp_app
*app
,
1124 struct nfp_fl_payload
*flow
,
1125 struct nfp_fl_key_ls
*key_ls
,
1126 struct netlink_ext_ack
*extack
)
1128 struct nfp_flower_priv
*priv
= app
->priv
;
1129 struct nfp_flower_meta_tci
*meta_tci
;
1130 struct nfp_flower_mac_mpls
*mac
;
1131 u8
*ext
= flow
->unmasked_data
;
1132 struct nfp_fl_act_head
*act
;
1133 u8
*mask
= flow
->mask_data
;
1138 meta_tci
= (struct nfp_flower_meta_tci
*)flow
->unmasked_data
;
1139 key_layer
= key_ls
->key_layer
;
1140 if (!(priv
->flower_ext_feats
& NFP_FL_FEATS_VLAN_QINQ
)) {
1141 if (meta_tci
->tci
& cpu_to_be16(NFP_FLOWER_MASK_VLAN_PRESENT
)) {
1142 u16 vlan_tci
= be16_to_cpu(meta_tci
->tci
);
1144 vlan_tci
&= ~NFP_FLOWER_MASK_VLAN_PRESENT
;
1145 flow
->pre_tun_rule
.vlan_tci
= cpu_to_be16(vlan_tci
);
1148 flow
->pre_tun_rule
.vlan_tci
= cpu_to_be16(0xffff);
1152 if (key_layer
& ~NFP_FLOWER_PRE_TUN_RULE_FIELDS
) {
1153 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: too many match fields");
1155 } else if (key_ls
->key_layer_two
& ~NFP_FLOWER_LAYER2_QINQ
) {
1156 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: non-vlan in extended match fields");
1160 if (!(key_layer
& NFP_FLOWER_LAYER_MAC
)) {
1161 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: MAC fields match required");
1165 if (!(key_layer
& NFP_FLOWER_LAYER_IPV4
) &&
1166 !(key_layer
& NFP_FLOWER_LAYER_IPV6
)) {
1167 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: match on ipv4/ipv6 eth_type must be present");
1171 if (key_layer
& NFP_FLOWER_LAYER_IPV6
)
1172 flow
->pre_tun_rule
.is_ipv6
= true;
1174 flow
->pre_tun_rule
.is_ipv6
= false;
1176 /* Skip fields known to exist. */
1177 mask
+= sizeof(struct nfp_flower_meta_tci
);
1178 ext
+= sizeof(struct nfp_flower_meta_tci
);
1179 if (key_ls
->key_layer_two
) {
1180 mask
+= sizeof(struct nfp_flower_ext_meta
);
1181 ext
+= sizeof(struct nfp_flower_ext_meta
);
1183 mask
+= sizeof(struct nfp_flower_in_port
);
1184 ext
+= sizeof(struct nfp_flower_in_port
);
1186 /* Ensure destination MAC address is fully matched. */
1187 mac
= (struct nfp_flower_mac_mpls
*)mask
;
1188 if (!is_broadcast_ether_addr(&mac
->mac_dst
[0])) {
1189 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: dest MAC field must not be masked");
1193 /* Ensure source MAC address is fully matched. This is only needed
1194 * for firmware with the DECAP_V2 feature enabled. Don't do this
1195 * for firmware without this feature to keep old behaviour.
1197 if (priv
->flower_ext_feats
& NFP_FL_FEATS_DECAP_V2
) {
1198 mac
= (struct nfp_flower_mac_mpls
*)mask
;
1199 if (!is_broadcast_ether_addr(&mac
->mac_src
[0])) {
1200 NL_SET_ERR_MSG_MOD(extack
,
1201 "unsupported pre-tunnel rule: source MAC field must not be masked");
1206 if (mac
->mpls_lse
) {
1207 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: MPLS not supported");
1211 /* Ensure destination MAC address matches pre_tun_dev. */
1212 mac
= (struct nfp_flower_mac_mpls
*)ext
;
1213 if (memcmp(&mac
->mac_dst
[0], flow
->pre_tun_rule
.dev
->dev_addr
, 6)) {
1214 NL_SET_ERR_MSG_MOD(extack
,
1215 "unsupported pre-tunnel rule: dest MAC must match output dev MAC");
1219 /* Save mac addresses in pre_tun_rule entry for later use */
1220 memcpy(&flow
->pre_tun_rule
.loc_mac
, &mac
->mac_dst
[0], ETH_ALEN
);
1221 memcpy(&flow
->pre_tun_rule
.rem_mac
, &mac
->mac_src
[0], ETH_ALEN
);
1223 mask
+= sizeof(struct nfp_flower_mac_mpls
);
1224 ext
+= sizeof(struct nfp_flower_mac_mpls
);
1225 if (key_layer
& NFP_FLOWER_LAYER_IPV4
||
1226 key_layer
& NFP_FLOWER_LAYER_IPV6
) {
1227 /* Flags and proto fields have same offset in IPv4 and IPv6. */
1228 int ip_flags
= offsetof(struct nfp_flower_ipv4
, ip_ext
.flags
);
1229 int ip_proto
= offsetof(struct nfp_flower_ipv4
, ip_ext
.proto
);
1233 size
= key_layer
& NFP_FLOWER_LAYER_IPV4
?
1234 sizeof(struct nfp_flower_ipv4
) :
1235 sizeof(struct nfp_flower_ipv6
);
1238 /* Ensure proto and flags are the only IP layer fields. */
1239 for (i
= 0; i
< size
; i
++)
1240 if (mask
[i
] && i
!= ip_flags
&& i
!= ip_proto
) {
1241 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: only flags and proto can be matched in ip header");
1248 if ((priv
->flower_ext_feats
& NFP_FL_FEATS_VLAN_QINQ
)) {
1249 if (key_ls
->key_layer_two
& NFP_FLOWER_LAYER2_QINQ
) {
1250 struct nfp_flower_vlan
*vlan_tags
;
1254 vlan_tags
= (struct nfp_flower_vlan
*)ext
;
1256 vlan_tci
= be16_to_cpu(vlan_tags
->outer_tci
);
1257 vlan_tpid
= be16_to_cpu(vlan_tags
->outer_tpid
);
1259 vlan_tci
&= ~NFP_FLOWER_MASK_VLAN_PRESENT
;
1260 flow
->pre_tun_rule
.vlan_tci
= cpu_to_be16(vlan_tci
);
1261 flow
->pre_tun_rule
.vlan_tpid
= cpu_to_be16(vlan_tpid
);
1264 flow
->pre_tun_rule
.vlan_tci
= cpu_to_be16(0xffff);
1265 flow
->pre_tun_rule
.vlan_tpid
= cpu_to_be16(0xffff);
1269 /* Action must be a single egress or pop_vlan and egress. */
1271 act
= (struct nfp_fl_act_head
*)&flow
->action_data
[act_offset
];
1273 if (act
->jump_id
!= NFP_FL_ACTION_OPCODE_POP_VLAN
) {
1274 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: match on VLAN must have VLAN pop as first action");
1278 act_offset
+= act
->len_lw
<< NFP_FL_LW_SIZ
;
1279 act
= (struct nfp_fl_act_head
*)&flow
->action_data
[act_offset
];
1282 if (act
->jump_id
!= NFP_FL_ACTION_OPCODE_OUTPUT
) {
1283 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: non egress action detected where egress was expected");
1287 act_offset
+= act
->len_lw
<< NFP_FL_LW_SIZ
;
1289 /* Ensure there are no more actions after egress. */
1290 if (act_offset
!= flow
->meta
.act_len
) {
1291 NL_SET_ERR_MSG_MOD(extack
, "unsupported pre-tunnel rule: egress is not the last action");
1298 static bool offload_pre_check(struct flow_cls_offload
*flow
)
1300 struct flow_rule
*rule
= flow_cls_offload_flow_rule(flow
);
1301 struct flow_dissector
*dissector
= rule
->match
.dissector
;
1302 struct flow_match_ct ct
;
1304 if (dissector
->used_keys
& BIT_ULL(FLOW_DISSECTOR_KEY_CT
)) {
1305 flow_rule_match_ct(rule
, &ct
);
1306 /* Allow special case where CT match is all 0 */
1307 if (memchr_inv(ct
.key
, 0, sizeof(*ct
.key
)))
1311 if (flow
->common
.chain_index
)
1318 * nfp_flower_add_offload() - Adds a new flow to hardware.
1319 * @app: Pointer to the APP handle
1320 * @netdev: netdev structure.
1321 * @flow: TC flower classifier offload structure.
1323 * Adds a new flow to the repeated hash structure and action payload.
1325 * Return: negative value on error, 0 if configured successfully.
1328 nfp_flower_add_offload(struct nfp_app
*app
, struct net_device
*netdev
,
1329 struct flow_cls_offload
*flow
)
1331 struct flow_rule
*rule
= flow_cls_offload_flow_rule(flow
);
1332 enum nfp_flower_tun_type tun_type
= NFP_FL_TUNNEL_NONE
;
1333 struct nfp_flower_priv
*priv
= app
->priv
;
1334 struct netlink_ext_ack
*extack
= NULL
;
1335 struct nfp_fl_payload
*flow_pay
;
1336 struct nfp_fl_key_ls
*key_layer
;
1337 struct nfp_port
*port
= NULL
;
1340 extack
= flow
->common
.extack
;
1341 if (nfp_netdev_is_nfp_repr(netdev
))
1342 port
= nfp_port_from_netdev(netdev
);
1344 if (is_pre_ct_flow(flow
))
1345 return nfp_fl_ct_handle_pre_ct(priv
, netdev
, flow
, extack
, NULL
);
1347 if (is_post_ct_flow(flow
))
1348 return nfp_fl_ct_handle_post_ct(priv
, netdev
, flow
, extack
);
1350 if (!offload_pre_check(flow
))
1353 key_layer
= kmalloc(sizeof(*key_layer
), GFP_KERNEL
);
1357 err
= nfp_flower_calculate_key_layers(app
, netdev
, key_layer
, rule
,
1360 goto err_free_key_ls
;
1362 flow_pay
= nfp_flower_allocate_new(key_layer
);
1365 goto err_free_key_ls
;
1368 err
= nfp_flower_compile_flow_match(app
, rule
, key_layer
, netdev
,
1369 flow_pay
, tun_type
, extack
);
1371 goto err_destroy_flow
;
1373 err
= nfp_flower_compile_action(app
, rule
, netdev
, flow_pay
, extack
);
1375 goto err_destroy_flow
;
1377 if (flow_pay
->pre_tun_rule
.dev
) {
1378 err
= nfp_flower_validate_pre_tun_rule(app
, flow_pay
, key_layer
, extack
);
1380 goto err_destroy_flow
;
1383 err
= nfp_compile_flow_metadata(app
, flow
->cookie
, flow_pay
, netdev
, extack
);
1385 goto err_destroy_flow
;
1387 flow_pay
->tc_flower_cookie
= flow
->cookie
;
1388 err
= rhashtable_insert_fast(&priv
->flow_table
, &flow_pay
->fl_node
,
1389 nfp_flower_table_params
);
1391 NL_SET_ERR_MSG_MOD(extack
, "invalid entry: cannot insert flow into tables for offloads");
1392 goto err_release_metadata
;
1395 if (flow_pay
->pre_tun_rule
.dev
) {
1396 if (priv
->flower_ext_feats
& NFP_FL_FEATS_DECAP_V2
) {
1397 struct nfp_predt_entry
*predt
;
1399 predt
= kzalloc(sizeof(*predt
), GFP_KERNEL
);
1402 goto err_remove_rhash
;
1404 predt
->flow_pay
= flow_pay
;
1405 INIT_LIST_HEAD(&predt
->nn_list
);
1406 spin_lock_bh(&priv
->predt_lock
);
1407 list_add(&predt
->list_head
, &priv
->predt_list
);
1408 flow_pay
->pre_tun_rule
.predt
= predt
;
1409 nfp_tun_link_and_update_nn_entries(app
, predt
);
1410 spin_unlock_bh(&priv
->predt_lock
);
1412 err
= nfp_flower_xmit_pre_tun_flow(app
, flow_pay
);
1415 err
= nfp_flower_xmit_flow(app
, flow_pay
,
1416 NFP_FLOWER_CMSG_TYPE_FLOW_ADD
);
1420 goto err_remove_rhash
;
1423 port
->tc_offload_cnt
++;
1425 flow_pay
->in_hw
= true;
1427 /* Deallocate flow payload when flower rule has been destroyed. */
1433 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->flow_table
,
1435 nfp_flower_table_params
));
1436 err_release_metadata
:
1437 nfp_modify_flow_metadata(app
, flow_pay
);
1439 if (flow_pay
->nfp_tun_ipv6
)
1440 nfp_tunnel_put_ipv6_off(app
, flow_pay
->nfp_tun_ipv6
);
1441 kfree(flow_pay
->action_data
);
1442 kfree(flow_pay
->mask_data
);
1443 kfree(flow_pay
->unmasked_data
);
1451 nfp_flower_remove_merge_flow(struct nfp_app
*app
,
1452 struct nfp_fl_payload
*del_sub_flow
,
1453 struct nfp_fl_payload
*merge_flow
)
1455 struct nfp_flower_priv
*priv
= app
->priv
;
1456 struct nfp_fl_payload_link
*link
, *temp
;
1457 struct nfp_merge_info
*merge_info
;
1458 struct nfp_fl_payload
*origin
;
1463 link
= list_first_entry(&merge_flow
->linked_flows
,
1464 struct nfp_fl_payload_link
, merge_flow
.list
);
1465 origin
= link
->sub_flow
.flow
;
1467 /* Re-add rule the merge had overwritten if it has not been deleted. */
1468 if (origin
!= del_sub_flow
)
1471 err
= nfp_modify_flow_metadata(app
, merge_flow
);
1473 nfp_flower_cmsg_warn(app
, "Metadata fail for merge flow delete.\n");
1474 goto err_free_links
;
1478 err
= nfp_flower_xmit_flow(app
, merge_flow
,
1479 NFP_FLOWER_CMSG_TYPE_FLOW_DEL
);
1481 nfp_flower_cmsg_warn(app
, "Failed to delete merged flow.\n");
1482 goto err_free_links
;
1485 __nfp_modify_flow_metadata(priv
, origin
);
1486 err
= nfp_flower_xmit_flow(app
, origin
,
1487 NFP_FLOWER_CMSG_TYPE_FLOW_MOD
);
1489 nfp_flower_cmsg_warn(app
, "Failed to revert merge flow.\n");
1490 origin
->in_hw
= true;
1494 /* Clean any links connected with the merged flow. */
1495 list_for_each_entry_safe(link
, temp
, &merge_flow
->linked_flows
,
1497 u32 ctx_id
= be32_to_cpu(link
->sub_flow
.flow
->meta
.host_ctx_id
);
1499 parent_ctx
= (parent_ctx
<< 32) | (u64
)(ctx_id
);
1500 nfp_flower_unlink_flow(link
);
1503 merge_info
= rhashtable_lookup_fast(&priv
->merge_table
,
1505 merge_table_params
);
1507 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->merge_table
,
1508 &merge_info
->ht_node
,
1509 merge_table_params
));
1513 kfree(merge_flow
->action_data
);
1514 kfree(merge_flow
->mask_data
);
1515 kfree(merge_flow
->unmasked_data
);
1516 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->flow_table
,
1517 &merge_flow
->fl_node
,
1518 nfp_flower_table_params
));
1519 kfree_rcu(merge_flow
, rcu
);
1523 nfp_flower_del_linked_merge_flows(struct nfp_app
*app
,
1524 struct nfp_fl_payload
*sub_flow
)
1526 struct nfp_fl_payload_link
*link
, *temp
;
1528 /* Remove any merge flow formed from the deleted sub_flow. */
1529 list_for_each_entry_safe(link
, temp
, &sub_flow
->linked_flows
,
1531 nfp_flower_remove_merge_flow(app
, sub_flow
,
1532 link
->merge_flow
.flow
);
1536 * nfp_flower_del_offload() - Removes a flow from hardware.
1537 * @app: Pointer to the APP handle
1538 * @netdev: netdev structure.
1539 * @flow: TC flower classifier offload structure
1541 * Removes a flow from the repeated hash structure and clears the
1542 * action payload. Any flows merged from this are also deleted.
1544 * Return: negative value on error, 0 if removed successfully.
1547 nfp_flower_del_offload(struct nfp_app
*app
, struct net_device
*netdev
,
1548 struct flow_cls_offload
*flow
)
1550 struct nfp_flower_priv
*priv
= app
->priv
;
1551 struct nfp_fl_ct_map_entry
*ct_map_ent
;
1552 struct netlink_ext_ack
*extack
= NULL
;
1553 struct nfp_fl_payload
*nfp_flow
;
1554 struct nfp_port
*port
= NULL
;
1557 extack
= flow
->common
.extack
;
1558 if (nfp_netdev_is_nfp_repr(netdev
))
1559 port
= nfp_port_from_netdev(netdev
);
1561 /* Check ct_map_table */
1562 ct_map_ent
= rhashtable_lookup_fast(&priv
->ct_map_table
, &flow
->cookie
,
1565 err
= nfp_fl_ct_del_flow(ct_map_ent
);
1569 nfp_flow
= nfp_flower_search_fl_table(app
, flow
->cookie
, netdev
);
1571 NL_SET_ERR_MSG_MOD(extack
, "invalid entry: cannot remove flow that does not exist");
1575 err
= nfp_modify_flow_metadata(app
, nfp_flow
);
1577 goto err_free_merge_flow
;
1579 if (nfp_flow
->nfp_tun_ipv4_addr
)
1580 nfp_tunnel_del_ipv4_off(app
, nfp_flow
->nfp_tun_ipv4_addr
);
1582 if (nfp_flow
->nfp_tun_ipv6
)
1583 nfp_tunnel_put_ipv6_off(app
, nfp_flow
->nfp_tun_ipv6
);
1585 if (!nfp_flow
->in_hw
) {
1587 goto err_free_merge_flow
;
1590 if (nfp_flow
->pre_tun_rule
.dev
) {
1591 if (priv
->flower_ext_feats
& NFP_FL_FEATS_DECAP_V2
) {
1592 struct nfp_predt_entry
*predt
;
1594 predt
= nfp_flow
->pre_tun_rule
.predt
;
1596 spin_lock_bh(&priv
->predt_lock
);
1597 nfp_tun_unlink_and_update_nn_entries(app
, predt
);
1598 list_del(&predt
->list_head
);
1599 spin_unlock_bh(&priv
->predt_lock
);
1603 err
= nfp_flower_xmit_pre_tun_del_flow(app
, nfp_flow
);
1606 err
= nfp_flower_xmit_flow(app
, nfp_flow
,
1607 NFP_FLOWER_CMSG_TYPE_FLOW_DEL
);
1609 /* Fall through on error. */
1611 err_free_merge_flow
:
1612 nfp_flower_del_linked_merge_flows(app
, nfp_flow
);
1614 port
->tc_offload_cnt
--;
1615 kfree(nfp_flow
->action_data
);
1616 kfree(nfp_flow
->mask_data
);
1617 kfree(nfp_flow
->unmasked_data
);
1618 WARN_ON_ONCE(rhashtable_remove_fast(&priv
->flow_table
,
1620 nfp_flower_table_params
));
1621 kfree_rcu(nfp_flow
, rcu
);
1626 __nfp_flower_update_merge_stats(struct nfp_app
*app
,
1627 struct nfp_fl_payload
*merge_flow
)
1629 struct nfp_flower_priv
*priv
= app
->priv
;
1630 struct nfp_fl_payload_link
*link
;
1631 struct nfp_fl_payload
*sub_flow
;
1632 u64 pkts
, bytes
, used
;
1635 ctx_id
= be32_to_cpu(merge_flow
->meta
.host_ctx_id
);
1636 pkts
= priv
->stats
[ctx_id
].pkts
;
1637 /* Do not cycle subflows if no stats to distribute. */
1640 bytes
= priv
->stats
[ctx_id
].bytes
;
1641 used
= priv
->stats
[ctx_id
].used
;
1643 /* Reset stats for the merge flow. */
1644 priv
->stats
[ctx_id
].pkts
= 0;
1645 priv
->stats
[ctx_id
].bytes
= 0;
1647 /* The merge flow has received stats updates from firmware.
1648 * Distribute these stats to all subflows that form the merge.
1649 * The stats will collected from TC via the subflows.
1651 list_for_each_entry(link
, &merge_flow
->linked_flows
, merge_flow
.list
) {
1652 sub_flow
= link
->sub_flow
.flow
;
1653 ctx_id
= be32_to_cpu(sub_flow
->meta
.host_ctx_id
);
1654 priv
->stats
[ctx_id
].pkts
+= pkts
;
1655 priv
->stats
[ctx_id
].bytes
+= bytes
;
1656 priv
->stats
[ctx_id
].used
= max_t(u64
, used
,
1657 priv
->stats
[ctx_id
].used
);
1662 nfp_flower_update_merge_stats(struct nfp_app
*app
,
1663 struct nfp_fl_payload
*sub_flow
)
1665 struct nfp_fl_payload_link
*link
;
1667 /* Get merge flows that the subflow forms to distribute their stats. */
1668 list_for_each_entry(link
, &sub_flow
->linked_flows
, sub_flow
.list
)
1669 __nfp_flower_update_merge_stats(app
, link
->merge_flow
.flow
);
1673 * nfp_flower_get_stats() - Populates flow stats obtained from hardware.
1674 * @app: Pointer to the APP handle
1675 * @netdev: Netdev structure.
1676 * @flow: TC flower classifier offload structure
1678 * Populates a flow statistics structure which which corresponds to a
1681 * Return: negative value on error, 0 if stats populated successfully.
1684 nfp_flower_get_stats(struct nfp_app
*app
, struct net_device
*netdev
,
1685 struct flow_cls_offload
*flow
)
1687 struct nfp_flower_priv
*priv
= app
->priv
;
1688 struct nfp_fl_ct_map_entry
*ct_map_ent
;
1689 struct netlink_ext_ack
*extack
= NULL
;
1690 struct nfp_fl_payload
*nfp_flow
;
1693 /* Check ct_map table first */
1694 ct_map_ent
= rhashtable_lookup_fast(&priv
->ct_map_table
, &flow
->cookie
,
1697 return nfp_fl_ct_stats(flow
, ct_map_ent
);
1699 extack
= flow
->common
.extack
;
1700 nfp_flow
= nfp_flower_search_fl_table(app
, flow
->cookie
, netdev
);
1702 NL_SET_ERR_MSG_MOD(extack
, "invalid entry: cannot dump stats for flow that does not exist");
1706 ctx_id
= be32_to_cpu(nfp_flow
->meta
.host_ctx_id
);
1708 spin_lock_bh(&priv
->stats_lock
);
1709 /* If request is for a sub_flow, update stats from merged flows. */
1710 if (!list_empty(&nfp_flow
->linked_flows
))
1711 nfp_flower_update_merge_stats(app
, nfp_flow
);
1713 flow_stats_update(&flow
->stats
, priv
->stats
[ctx_id
].bytes
,
1714 priv
->stats
[ctx_id
].pkts
, 0, priv
->stats
[ctx_id
].used
,
1715 FLOW_ACTION_HW_STATS_DELAYED
);
1717 priv
->stats
[ctx_id
].pkts
= 0;
1718 priv
->stats
[ctx_id
].bytes
= 0;
1719 spin_unlock_bh(&priv
->stats_lock
);
1725 nfp_flower_repr_offload(struct nfp_app
*app
, struct net_device
*netdev
,
1726 struct flow_cls_offload
*flower
)
1728 struct nfp_flower_priv
*priv
= app
->priv
;
1731 if (!eth_proto_is_802_3(flower
->common
.protocol
))
1734 mutex_lock(&priv
->nfp_fl_lock
);
1735 switch (flower
->command
) {
1736 case FLOW_CLS_REPLACE
:
1737 ret
= nfp_flower_add_offload(app
, netdev
, flower
);
1739 case FLOW_CLS_DESTROY
:
1740 ret
= nfp_flower_del_offload(app
, netdev
, flower
);
1742 case FLOW_CLS_STATS
:
1743 ret
= nfp_flower_get_stats(app
, netdev
, flower
);
1749 mutex_unlock(&priv
->nfp_fl_lock
);
1754 static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type
,
1755 void *type_data
, void *cb_priv
)
1757 struct flow_cls_common_offload
*common
= type_data
;
1758 struct nfp_repr
*repr
= cb_priv
;
1760 if (!tc_can_offload_extack(repr
->netdev
, common
->extack
))
1764 case TC_SETUP_CLSFLOWER
:
1765 return nfp_flower_repr_offload(repr
->app
, repr
->netdev
,
1767 case TC_SETUP_CLSMATCHALL
:
1768 return nfp_flower_setup_qos_offload(repr
->app
, repr
->netdev
,
1775 static LIST_HEAD(nfp_block_cb_list
);
1777 static int nfp_flower_setup_tc_block(struct net_device
*netdev
,
1778 struct flow_block_offload
*f
)
1780 struct nfp_repr
*repr
= netdev_priv(netdev
);
1781 struct nfp_flower_repr_priv
*repr_priv
;
1782 struct flow_block_cb
*block_cb
;
1784 if (f
->binder_type
!= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS
)
1787 repr_priv
= repr
->app_priv
;
1788 repr_priv
->block_shared
= f
->block_shared
;
1789 f
->driver_block_list
= &nfp_block_cb_list
;
1790 f
->unlocked_driver_cb
= true;
1792 switch (f
->command
) {
1793 case FLOW_BLOCK_BIND
:
1794 if (flow_block_cb_is_busy(nfp_flower_setup_tc_block_cb
, repr
,
1795 &nfp_block_cb_list
))
1798 block_cb
= flow_block_cb_alloc(nfp_flower_setup_tc_block_cb
,
1800 if (IS_ERR(block_cb
))
1801 return PTR_ERR(block_cb
);
1803 flow_block_cb_add(block_cb
, f
);
1804 list_add_tail(&block_cb
->driver_list
, &nfp_block_cb_list
);
1806 case FLOW_BLOCK_UNBIND
:
1807 block_cb
= flow_block_cb_lookup(f
->block
,
1808 nfp_flower_setup_tc_block_cb
,
1813 flow_block_cb_remove(block_cb
, f
);
1814 list_del(&block_cb
->driver_list
);
1821 int nfp_flower_setup_tc(struct nfp_app
*app
, struct net_device
*netdev
,
1822 enum tc_setup_type type
, void *type_data
)
1825 case TC_SETUP_BLOCK
:
1826 return nfp_flower_setup_tc_block(netdev
, type_data
);
1832 struct nfp_flower_indr_block_cb_priv
{
1833 struct net_device
*netdev
;
1834 struct nfp_app
*app
;
1835 struct list_head list
;
1838 static struct nfp_flower_indr_block_cb_priv
*
1839 nfp_flower_indr_block_cb_priv_lookup(struct nfp_app
*app
,
1840 struct net_device
*netdev
)
1842 struct nfp_flower_indr_block_cb_priv
*cb_priv
;
1843 struct nfp_flower_priv
*priv
= app
->priv
;
1845 list_for_each_entry(cb_priv
, &priv
->indr_block_cb_priv
, list
)
1846 if (cb_priv
->netdev
== netdev
)
1852 static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type
,
1853 void *type_data
, void *cb_priv
)
1855 struct nfp_flower_indr_block_cb_priv
*priv
= cb_priv
;
1858 case TC_SETUP_CLSFLOWER
:
1859 return nfp_flower_repr_offload(priv
->app
, priv
->netdev
,
1866 void nfp_flower_setup_indr_tc_release(void *cb_priv
)
1868 struct nfp_flower_indr_block_cb_priv
*priv
= cb_priv
;
1870 list_del(&priv
->list
);
1875 nfp_flower_setup_indr_tc_block(struct net_device
*netdev
, struct Qdisc
*sch
, struct nfp_app
*app
,
1876 struct flow_block_offload
*f
, void *data
,
1877 void (*cleanup
)(struct flow_block_cb
*block_cb
))
1879 struct nfp_flower_indr_block_cb_priv
*cb_priv
;
1880 struct nfp_flower_priv
*priv
= app
->priv
;
1881 struct flow_block_cb
*block_cb
;
1883 if ((f
->binder_type
!= FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS
&&
1884 !nfp_flower_internal_port_can_offload(app
, netdev
)) ||
1885 (f
->binder_type
!= FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS
&&
1886 nfp_flower_internal_port_can_offload(app
, netdev
)))
1889 f
->unlocked_driver_cb
= true;
1891 switch (f
->command
) {
1892 case FLOW_BLOCK_BIND
:
1893 cb_priv
= nfp_flower_indr_block_cb_priv_lookup(app
, netdev
);
1895 flow_block_cb_is_busy(nfp_flower_setup_indr_block_cb
,
1897 &nfp_block_cb_list
))
1900 cb_priv
= kmalloc(sizeof(*cb_priv
), GFP_KERNEL
);
1904 cb_priv
->netdev
= netdev
;
1906 list_add(&cb_priv
->list
, &priv
->indr_block_cb_priv
);
1908 block_cb
= flow_indr_block_cb_alloc(nfp_flower_setup_indr_block_cb
,
1910 nfp_flower_setup_indr_tc_release
,
1911 f
, netdev
, sch
, data
, app
, cleanup
);
1912 if (IS_ERR(block_cb
)) {
1913 list_del(&cb_priv
->list
);
1915 return PTR_ERR(block_cb
);
1918 flow_block_cb_add(block_cb
, f
);
1919 list_add_tail(&block_cb
->driver_list
, &nfp_block_cb_list
);
1921 case FLOW_BLOCK_UNBIND
:
1922 cb_priv
= nfp_flower_indr_block_cb_priv_lookup(app
, netdev
);
1926 block_cb
= flow_block_cb_lookup(f
->block
,
1927 nfp_flower_setup_indr_block_cb
,
1932 flow_indr_block_cb_remove(block_cb
, f
);
1933 list_del(&block_cb
->driver_list
);
1942 nfp_setup_tc_no_dev(struct nfp_app
*app
, enum tc_setup_type type
, void *data
)
1949 return nfp_setup_tc_act_offload(app
, data
);
1956 nfp_flower_indr_setup_tc_cb(struct net_device
*netdev
, struct Qdisc
*sch
, void *cb_priv
,
1957 enum tc_setup_type type
, void *type_data
,
1959 void (*cleanup
)(struct flow_block_cb
*block_cb
))
1962 return nfp_setup_tc_no_dev(cb_priv
, type
, data
);
1964 if (!nfp_fl_is_netdev_to_offload(netdev
))
1968 case TC_SETUP_BLOCK
:
1969 return nfp_flower_setup_indr_tc_block(netdev
, sch
, cb_priv
,
1970 type_data
, data
, cleanup
);