1 /* Broadcom NetXtreme-C/E network driver.
3 * Copyright (c) 2017 Broadcom Limited
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
10 #include <linux/netdevice.h>
11 #include <linux/inetdevice.h>
12 #include <linux/if_vlan.h>
13 #include <net/flow_dissector.h>
14 #include <net/pkt_cls.h>
15 #include <net/tc_act/tc_gact.h>
16 #include <net/tc_act/tc_skbedit.h>
17 #include <net/tc_act/tc_mirred.h>
18 #include <net/tc_act/tc_vlan.h>
19 #include <net/tc_act/tc_tunnel_key.h>
23 #include "bnxt_sriov.h"
27 #define BNXT_FID_INVALID 0xffff
28 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT))
30 #define is_vlan_pcp_wildcarded(vlan_tci_mask) \
31 ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == 0x0000)
32 #define is_vlan_pcp_exactmatch(vlan_tci_mask) \
33 ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == VLAN_PRIO_MASK)
34 #define is_vlan_pcp_zero(vlan_tci) \
35 ((ntohs(vlan_tci) & VLAN_PRIO_MASK) == 0x0000)
36 #define is_vid_exactmatch(vlan_tci_mask) \
37 ((ntohs(vlan_tci_mask) & VLAN_VID_MASK) == VLAN_VID_MASK)
39 /* Return the dst fid of the func for flow forwarding
40 * For PFs: src_fid is the fid of the PF
41 * For VF-reps: src_fid the fid of the VF
43 static u16
bnxt_flow_get_dst_fid(struct bnxt
*pf_bp
, struct net_device
*dev
)
47 /* check if dev belongs to the same switch */
48 if (!switchdev_port_same_parent_id(pf_bp
->dev
, dev
)) {
49 netdev_info(pf_bp
->dev
, "dev(ifindex=%d) not on same switch",
51 return BNXT_FID_INVALID
;
54 /* Is dev a VF-rep? */
55 if (bnxt_dev_is_vf_rep(dev
))
56 return bnxt_vf_rep_get_fid(dev
);
58 bp
= netdev_priv(dev
);
62 static int bnxt_tc_parse_redir(struct bnxt
*bp
,
63 struct bnxt_tc_actions
*actions
,
64 const struct tc_action
*tc_act
)
66 struct net_device
*dev
= tcf_mirred_dev(tc_act
);
69 netdev_info(bp
->dev
, "no dev in mirred action");
73 actions
->flags
|= BNXT_TC_ACTION_FLAG_FWD
;
74 actions
->dst_dev
= dev
;
78 static int bnxt_tc_parse_vlan(struct bnxt
*bp
,
79 struct bnxt_tc_actions
*actions
,
80 const struct tc_action
*tc_act
)
82 switch (tcf_vlan_action(tc_act
)) {
83 case TCA_VLAN_ACT_POP
:
84 actions
->flags
|= BNXT_TC_ACTION_FLAG_POP_VLAN
;
86 case TCA_VLAN_ACT_PUSH
:
87 actions
->flags
|= BNXT_TC_ACTION_FLAG_PUSH_VLAN
;
88 actions
->push_vlan_tci
= htons(tcf_vlan_push_vid(tc_act
));
89 actions
->push_vlan_tpid
= tcf_vlan_push_proto(tc_act
);
97 static int bnxt_tc_parse_tunnel_set(struct bnxt
*bp
,
98 struct bnxt_tc_actions
*actions
,
99 const struct tc_action
*tc_act
)
101 struct ip_tunnel_info
*tun_info
= tcf_tunnel_info(tc_act
);
102 struct ip_tunnel_key
*tun_key
= &tun_info
->key
;
104 if (ip_tunnel_info_af(tun_info
) != AF_INET
) {
105 netdev_info(bp
->dev
, "only IPv4 tunnel-encap is supported");
109 actions
->tun_encap_key
= *tun_key
;
110 actions
->flags
|= BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
;
114 static int bnxt_tc_parse_actions(struct bnxt
*bp
,
115 struct bnxt_tc_actions
*actions
,
116 struct tcf_exts
*tc_exts
)
118 const struct tc_action
*tc_act
;
121 if (!tcf_exts_has_actions(tc_exts
)) {
122 netdev_info(bp
->dev
, "no actions");
126 tcf_exts_for_each_action(i
, tc_act
, tc_exts
) {
128 if (is_tcf_gact_shot(tc_act
)) {
129 actions
->flags
|= BNXT_TC_ACTION_FLAG_DROP
;
130 return 0; /* don't bother with other actions */
133 /* Redirect action */
134 if (is_tcf_mirred_egress_redirect(tc_act
)) {
135 rc
= bnxt_tc_parse_redir(bp
, actions
, tc_act
);
142 if (is_tcf_vlan(tc_act
)) {
143 rc
= bnxt_tc_parse_vlan(bp
, actions
, tc_act
);
150 if (is_tcf_tunnel_set(tc_act
)) {
151 rc
= bnxt_tc_parse_tunnel_set(bp
, actions
, tc_act
);
158 if (is_tcf_tunnel_release(tc_act
)) {
159 actions
->flags
|= BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
;
164 if (actions
->flags
& BNXT_TC_ACTION_FLAG_FWD
) {
165 if (actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
) {
166 /* dst_fid is PF's fid */
167 actions
->dst_fid
= bp
->pf
.fw_fid
;
169 /* find the FID from dst_dev */
171 bnxt_flow_get_dst_fid(bp
, actions
->dst_dev
);
172 if (actions
->dst_fid
== BNXT_FID_INVALID
)
180 #define GET_KEY(flow_cmd, key_type) \
181 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
183 #define GET_MASK(flow_cmd, key_type) \
184 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
187 static int bnxt_tc_parse_flow(struct bnxt
*bp
,
188 struct tc_cls_flower_offload
*tc_flow_cmd
,
189 struct bnxt_tc_flow
*flow
)
191 struct flow_dissector
*dissector
= tc_flow_cmd
->dissector
;
193 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
194 if ((dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_CONTROL
)) == 0 ||
195 (dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_BASIC
)) == 0) {
196 netdev_info(bp
->dev
, "cannot form TC key: used_keys = 0x%x",
197 dissector
->used_keys
);
201 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_BASIC
)) {
202 struct flow_dissector_key_basic
*key
=
203 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
204 struct flow_dissector_key_basic
*mask
=
205 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
207 flow
->l2_key
.ether_type
= key
->n_proto
;
208 flow
->l2_mask
.ether_type
= mask
->n_proto
;
210 if (key
->n_proto
== htons(ETH_P_IP
) ||
211 key
->n_proto
== htons(ETH_P_IPV6
)) {
212 flow
->l4_key
.ip_proto
= key
->ip_proto
;
213 flow
->l4_mask
.ip_proto
= mask
->ip_proto
;
217 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
218 struct flow_dissector_key_eth_addrs
*key
=
219 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
220 struct flow_dissector_key_eth_addrs
*mask
=
221 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
223 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ETH_ADDRS
;
224 ether_addr_copy(flow
->l2_key
.dmac
, key
->dst
);
225 ether_addr_copy(flow
->l2_mask
.dmac
, mask
->dst
);
226 ether_addr_copy(flow
->l2_key
.smac
, key
->src
);
227 ether_addr_copy(flow
->l2_mask
.smac
, mask
->src
);
230 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_VLAN
)) {
231 struct flow_dissector_key_vlan
*key
=
232 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
233 struct flow_dissector_key_vlan
*mask
=
234 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
236 flow
->l2_key
.inner_vlan_tci
=
237 cpu_to_be16(VLAN_TCI(key
->vlan_id
, key
->vlan_priority
));
238 flow
->l2_mask
.inner_vlan_tci
=
239 cpu_to_be16((VLAN_TCI(mask
->vlan_id
, mask
->vlan_priority
)));
240 flow
->l2_key
.inner_vlan_tpid
= htons(ETH_P_8021Q
);
241 flow
->l2_mask
.inner_vlan_tpid
= htons(0xffff);
242 flow
->l2_key
.num_vlans
= 1;
245 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
)) {
246 struct flow_dissector_key_ipv4_addrs
*key
=
247 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
248 struct flow_dissector_key_ipv4_addrs
*mask
=
249 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
251 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
;
252 flow
->l3_key
.ipv4
.daddr
.s_addr
= key
->dst
;
253 flow
->l3_mask
.ipv4
.daddr
.s_addr
= mask
->dst
;
254 flow
->l3_key
.ipv4
.saddr
.s_addr
= key
->src
;
255 flow
->l3_mask
.ipv4
.saddr
.s_addr
= mask
->src
;
256 } else if (dissector_uses_key(dissector
,
257 FLOW_DISSECTOR_KEY_IPV6_ADDRS
)) {
258 struct flow_dissector_key_ipv6_addrs
*key
=
259 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
260 struct flow_dissector_key_ipv6_addrs
*mask
=
261 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
263 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
;
264 flow
->l3_key
.ipv6
.daddr
= key
->dst
;
265 flow
->l3_mask
.ipv6
.daddr
= mask
->dst
;
266 flow
->l3_key
.ipv6
.saddr
= key
->src
;
267 flow
->l3_mask
.ipv6
.saddr
= mask
->src
;
270 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_PORTS
)) {
271 struct flow_dissector_key_ports
*key
=
272 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
273 struct flow_dissector_key_ports
*mask
=
274 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
276 flow
->flags
|= BNXT_TC_FLOW_FLAGS_PORTS
;
277 flow
->l4_key
.ports
.dport
= key
->dst
;
278 flow
->l4_mask
.ports
.dport
= mask
->dst
;
279 flow
->l4_key
.ports
.sport
= key
->src
;
280 flow
->l4_mask
.ports
.sport
= mask
->src
;
283 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ICMP
)) {
284 struct flow_dissector_key_icmp
*key
=
285 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
286 struct flow_dissector_key_icmp
*mask
=
287 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
289 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ICMP
;
290 flow
->l4_key
.icmp
.type
= key
->type
;
291 flow
->l4_key
.icmp
.code
= key
->code
;
292 flow
->l4_mask
.icmp
.type
= mask
->type
;
293 flow
->l4_mask
.icmp
.code
= mask
->code
;
296 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
)) {
297 struct flow_dissector_key_ipv4_addrs
*key
=
298 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
);
299 struct flow_dissector_key_ipv4_addrs
*mask
=
300 GET_MASK(tc_flow_cmd
,
301 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
);
303 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS
;
304 flow
->tun_key
.u
.ipv4
.dst
= key
->dst
;
305 flow
->tun_mask
.u
.ipv4
.dst
= mask
->dst
;
306 flow
->tun_key
.u
.ipv4
.src
= key
->src
;
307 flow
->tun_mask
.u
.ipv4
.src
= mask
->src
;
308 } else if (dissector_uses_key(dissector
,
309 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
)) {
313 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_KEYID
)) {
314 struct flow_dissector_key_keyid
*key
=
315 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_KEYID
);
316 struct flow_dissector_key_keyid
*mask
=
317 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_KEYID
);
319 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_ID
;
320 flow
->tun_key
.tun_id
= key32_to_tunnel_id(key
->keyid
);
321 flow
->tun_mask
.tun_id
= key32_to_tunnel_id(mask
->keyid
);
324 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) {
325 struct flow_dissector_key_ports
*key
=
326 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_PORTS
);
327 struct flow_dissector_key_ports
*mask
=
328 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_PORTS
);
330 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_PORTS
;
331 flow
->tun_key
.tp_dst
= key
->dst
;
332 flow
->tun_mask
.tp_dst
= mask
->dst
;
333 flow
->tun_key
.tp_src
= key
->src
;
334 flow
->tun_mask
.tp_src
= mask
->src
;
337 return bnxt_tc_parse_actions(bp
, &flow
->actions
, tc_flow_cmd
->exts
);
340 static int bnxt_hwrm_cfa_flow_free(struct bnxt
*bp
,
341 struct bnxt_tc_flow_node
*flow_node
)
343 struct hwrm_cfa_flow_free_input req
= { 0 };
346 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_FREE
, -1, -1);
347 if (bp
->fw_cap
& BNXT_FW_CAP_OVS_64BIT_HANDLE
)
348 req
.ext_flow_handle
= flow_node
->ext_flow_handle
;
350 req
.flow_handle
= flow_node
->flow_handle
;
352 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
354 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
361 static int ipv6_mask_len(struct in6_addr
*mask
)
365 for (i
= 0; i
< 4; i
++)
366 mask_len
+= inet_mask_len(mask
->s6_addr32
[i
]);
371 static bool is_wildcard(void *mask
, int len
)
376 for (i
= 0; i
< len
; i
++) {
383 static bool is_exactmatch(void *mask
, int len
)
388 for (i
= 0; i
< len
; i
++)
395 static bool is_vlan_tci_allowed(__be16 vlan_tci_mask
,
398 /* VLAN priority must be either exactly zero or fully wildcarded and
399 * VLAN id must be exact match.
401 if (is_vid_exactmatch(vlan_tci_mask
) &&
402 ((is_vlan_pcp_exactmatch(vlan_tci_mask
) &&
403 is_vlan_pcp_zero(vlan_tci
)) ||
404 is_vlan_pcp_wildcarded(vlan_tci_mask
)))
410 static bool bits_set(void *key
, int len
)
415 for (i
= 0; i
< len
; i
++)
422 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
423 __le16 ref_flow_handle
,
424 __le32 tunnel_handle
,
425 struct bnxt_tc_flow_node
*flow_node
)
427 struct bnxt_tc_actions
*actions
= &flow
->actions
;
428 struct bnxt_tc_l3_key
*l3_mask
= &flow
->l3_mask
;
429 struct bnxt_tc_l3_key
*l3_key
= &flow
->l3_key
;
430 struct hwrm_cfa_flow_alloc_input req
= { 0 };
431 struct hwrm_cfa_flow_alloc_output
*resp
;
432 u16 flow_flags
= 0, action_flags
= 0;
435 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_ALLOC
, -1, -1);
437 req
.src_fid
= cpu_to_le16(flow
->src_fid
);
438 req
.ref_flow_handle
= ref_flow_handle
;
440 if (actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
||
441 actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
) {
442 req
.tunnel_handle
= tunnel_handle
;
443 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL
;
444 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL
;
447 req
.ethertype
= flow
->l2_key
.ether_type
;
448 req
.ip_proto
= flow
->l4_key
.ip_proto
;
450 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ETH_ADDRS
) {
451 memcpy(req
.dmac
, flow
->l2_key
.dmac
, ETH_ALEN
);
452 memcpy(req
.smac
, flow
->l2_key
.smac
, ETH_ALEN
);
455 if (flow
->l2_key
.num_vlans
> 0) {
456 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE
;
457 /* FW expects the inner_vlan_tci value to be set
458 * in outer_vlan_tci when num_vlans is 1 (which is
459 * always the case in TC.)
461 req
.outer_vlan_tci
= flow
->l2_key
.inner_vlan_tci
;
464 /* If all IP and L4 fields are wildcarded then this is an L2 flow */
465 if (is_wildcard(l3_mask
, sizeof(*l3_mask
)) &&
466 is_wildcard(&flow
->l4_mask
, sizeof(flow
->l4_mask
))) {
467 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2
;
469 flow_flags
|= flow
->l2_key
.ether_type
== htons(ETH_P_IP
) ?
470 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4
:
471 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
;
473 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
) {
474 req
.ip_dst
[0] = l3_key
->ipv4
.daddr
.s_addr
;
475 req
.ip_dst_mask_len
=
476 inet_mask_len(l3_mask
->ipv4
.daddr
.s_addr
);
477 req
.ip_src
[0] = l3_key
->ipv4
.saddr
.s_addr
;
478 req
.ip_src_mask_len
=
479 inet_mask_len(l3_mask
->ipv4
.saddr
.s_addr
);
480 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
) {
481 memcpy(req
.ip_dst
, l3_key
->ipv6
.daddr
.s6_addr32
,
483 req
.ip_dst_mask_len
=
484 ipv6_mask_len(&l3_mask
->ipv6
.daddr
);
485 memcpy(req
.ip_src
, l3_key
->ipv6
.saddr
.s6_addr32
,
487 req
.ip_src_mask_len
=
488 ipv6_mask_len(&l3_mask
->ipv6
.saddr
);
492 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) {
493 req
.l4_src_port
= flow
->l4_key
.ports
.sport
;
494 req
.l4_src_port_mask
= flow
->l4_mask
.ports
.sport
;
495 req
.l4_dst_port
= flow
->l4_key
.ports
.dport
;
496 req
.l4_dst_port_mask
= flow
->l4_mask
.ports
.dport
;
497 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ICMP
) {
498 /* l4 ports serve as type/code when ip_proto is ICMP */
499 req
.l4_src_port
= htons(flow
->l4_key
.icmp
.type
);
500 req
.l4_src_port_mask
= htons(flow
->l4_mask
.icmp
.type
);
501 req
.l4_dst_port
= htons(flow
->l4_key
.icmp
.code
);
502 req
.l4_dst_port_mask
= htons(flow
->l4_mask
.icmp
.code
);
504 req
.flags
= cpu_to_le16(flow_flags
);
506 if (actions
->flags
& BNXT_TC_ACTION_FLAG_DROP
) {
507 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP
;
509 if (actions
->flags
& BNXT_TC_ACTION_FLAG_FWD
) {
510 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD
;
511 req
.dst_fid
= cpu_to_le16(actions
->dst_fid
);
513 if (actions
->flags
& BNXT_TC_ACTION_FLAG_PUSH_VLAN
) {
515 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
516 req
.l2_rewrite_vlan_tpid
= actions
->push_vlan_tpid
;
517 req
.l2_rewrite_vlan_tci
= actions
->push_vlan_tci
;
518 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
519 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
521 if (actions
->flags
& BNXT_TC_ACTION_FLAG_POP_VLAN
) {
523 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
524 /* Rewrite config with tpid = 0 implies vlan pop */
525 req
.l2_rewrite_vlan_tpid
= 0;
526 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
527 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
530 req
.action_flags
= cpu_to_le16(action_flags
);
532 mutex_lock(&bp
->hwrm_cmd_lock
);
533 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
535 resp
= bnxt_get_hwrm_resp_addr(bp
, &req
);
536 /* CFA_FLOW_ALLOC response interpretation:
539 * flow handle flow handle
540 * =========== ===========
541 * flow_handle flow handle flow context id
542 * ext_flow_handle INVALID flow handle
543 * flow_id INVALID flow counter id
545 flow_node
->flow_handle
= resp
->flow_handle
;
546 if (bp
->fw_cap
& BNXT_FW_CAP_OVS_64BIT_HANDLE
) {
547 flow_node
->ext_flow_handle
= resp
->ext_flow_handle
;
548 flow_node
->flow_id
= resp
->flow_id
;
551 mutex_unlock(&bp
->hwrm_cmd_lock
);
553 if (rc
== HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR
)
560 static int hwrm_cfa_decap_filter_alloc(struct bnxt
*bp
,
561 struct bnxt_tc_flow
*flow
,
562 struct bnxt_tc_l2_key
*l2_info
,
563 __le32 ref_decap_handle
,
564 __le32
*decap_filter_handle
)
566 struct hwrm_cfa_decap_filter_alloc_input req
= { 0 };
567 struct hwrm_cfa_decap_filter_alloc_output
*resp
;
568 struct ip_tunnel_key
*tun_key
= &flow
->tun_key
;
572 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_DECAP_FILTER_ALLOC
, -1, -1);
574 req
.flags
= cpu_to_le32(CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL
);
575 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
|
576 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL
;
577 req
.tunnel_type
= CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN
;
578 req
.ip_protocol
= CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP
;
580 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_ID
) {
581 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID
;
582 /* tunnel_id is wrongly defined in hsi defn. as __le32 */
583 req
.tunnel_id
= tunnel_id_to_key32(tun_key
->tun_id
);
586 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS
) {
587 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR
;
588 ether_addr_copy(req
.dst_macaddr
, l2_info
->dmac
);
590 if (l2_info
->num_vlans
) {
591 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID
;
592 req
.t_ivlan_vid
= l2_info
->inner_vlan_tci
;
595 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE
;
596 req
.ethertype
= htons(ETH_P_IP
);
598 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS
) {
599 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR
|
600 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR
|
601 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE
;
602 req
.ip_addr_type
= CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4
;
603 req
.dst_ipaddr
[0] = tun_key
->u
.ipv4
.dst
;
604 req
.src_ipaddr
[0] = tun_key
->u
.ipv4
.src
;
607 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_PORTS
) {
608 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT
;
609 req
.dst_port
= tun_key
->tp_dst
;
612 /* Eventhough the decap_handle returned by hwrm_cfa_decap_filter_alloc
613 * is defined as __le32, l2_ctxt_ref_id is defined in HSI as __le16.
615 req
.l2_ctxt_ref_id
= (__force __le16
)ref_decap_handle
;
616 req
.enables
= cpu_to_le32(enables
);
618 mutex_lock(&bp
->hwrm_cmd_lock
);
619 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
621 resp
= bnxt_get_hwrm_resp_addr(bp
, &req
);
622 *decap_filter_handle
= resp
->decap_filter_id
;
624 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
626 mutex_unlock(&bp
->hwrm_cmd_lock
);
633 static int hwrm_cfa_decap_filter_free(struct bnxt
*bp
,
634 __le32 decap_filter_handle
)
636 struct hwrm_cfa_decap_filter_free_input req
= { 0 };
639 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_DECAP_FILTER_FREE
, -1, -1);
640 req
.decap_filter_id
= decap_filter_handle
;
642 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
644 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
651 static int hwrm_cfa_encap_record_alloc(struct bnxt
*bp
,
652 struct ip_tunnel_key
*encap_key
,
653 struct bnxt_tc_l2_key
*l2_info
,
654 __le32
*encap_record_handle
)
656 struct hwrm_cfa_encap_record_alloc_input req
= { 0 };
657 struct hwrm_cfa_encap_record_alloc_output
*resp
;
658 struct hwrm_cfa_encap_data_vxlan
*encap
=
659 (struct hwrm_cfa_encap_data_vxlan
*)&req
.encap_data
;
660 struct hwrm_vxlan_ipv4_hdr
*encap_ipv4
=
661 (struct hwrm_vxlan_ipv4_hdr
*)encap
->l3
;
664 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_ENCAP_RECORD_ALLOC
, -1, -1);
666 req
.encap_type
= CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN
;
668 ether_addr_copy(encap
->dst_mac_addr
, l2_info
->dmac
);
669 ether_addr_copy(encap
->src_mac_addr
, l2_info
->smac
);
670 if (l2_info
->num_vlans
) {
671 encap
->num_vlan_tags
= l2_info
->num_vlans
;
672 encap
->ovlan_tci
= l2_info
->inner_vlan_tci
;
673 encap
->ovlan_tpid
= l2_info
->inner_vlan_tpid
;
676 encap_ipv4
->ver_hlen
= 4 << VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT
;
677 encap_ipv4
->ver_hlen
|= 5 << VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT
;
678 encap_ipv4
->ttl
= encap_key
->ttl
;
680 encap_ipv4
->dest_ip_addr
= encap_key
->u
.ipv4
.dst
;
681 encap_ipv4
->src_ip_addr
= encap_key
->u
.ipv4
.src
;
682 encap_ipv4
->protocol
= IPPROTO_UDP
;
684 encap
->dst_port
= encap_key
->tp_dst
;
685 encap
->vni
= tunnel_id_to_key32(encap_key
->tun_id
);
687 mutex_lock(&bp
->hwrm_cmd_lock
);
688 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
690 resp
= bnxt_get_hwrm_resp_addr(bp
, &req
);
691 *encap_record_handle
= resp
->encap_record_id
;
693 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
695 mutex_unlock(&bp
->hwrm_cmd_lock
);
702 static int hwrm_cfa_encap_record_free(struct bnxt
*bp
,
703 __le32 encap_record_handle
)
705 struct hwrm_cfa_encap_record_free_input req
= { 0 };
708 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_ENCAP_RECORD_FREE
, -1, -1);
709 req
.encap_record_id
= encap_record_handle
;
711 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
713 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
720 static int bnxt_tc_put_l2_node(struct bnxt
*bp
,
721 struct bnxt_tc_flow_node
*flow_node
)
723 struct bnxt_tc_l2_node
*l2_node
= flow_node
->l2_node
;
724 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
727 /* remove flow_node from the L2 shared flow list */
728 list_del(&flow_node
->l2_list_node
);
729 if (--l2_node
->refcount
== 0) {
730 rc
= rhashtable_remove_fast(&tc_info
->l2_table
, &l2_node
->node
,
731 tc_info
->l2_ht_params
);
734 "Error: %s: rhashtable_remove_fast: %d",
736 kfree_rcu(l2_node
, rcu
);
741 static struct bnxt_tc_l2_node
*
742 bnxt_tc_get_l2_node(struct bnxt
*bp
, struct rhashtable
*l2_table
,
743 struct rhashtable_params ht_params
,
744 struct bnxt_tc_l2_key
*l2_key
)
746 struct bnxt_tc_l2_node
*l2_node
;
749 l2_node
= rhashtable_lookup_fast(l2_table
, l2_key
, ht_params
);
751 l2_node
= kzalloc(sizeof(*l2_node
), GFP_KERNEL
);
757 l2_node
->key
= *l2_key
;
758 rc
= rhashtable_insert_fast(l2_table
, &l2_node
->node
,
761 kfree_rcu(l2_node
, rcu
);
763 "Error: %s: rhashtable_insert_fast: %d",
767 INIT_LIST_HEAD(&l2_node
->common_l2_flows
);
772 /* Get the ref_flow_handle for a flow by checking if there are any other
773 * flows that share the same L2 key as this flow.
776 bnxt_tc_get_ref_flow_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
777 struct bnxt_tc_flow_node
*flow_node
,
778 __le16
*ref_flow_handle
)
780 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
781 struct bnxt_tc_flow_node
*ref_flow_node
;
782 struct bnxt_tc_l2_node
*l2_node
;
784 l2_node
= bnxt_tc_get_l2_node(bp
, &tc_info
->l2_table
,
785 tc_info
->l2_ht_params
,
790 /* If any other flow is using this l2_node, use it's flow_handle
791 * as the ref_flow_handle
793 if (l2_node
->refcount
> 0) {
794 ref_flow_node
= list_first_entry(&l2_node
->common_l2_flows
,
795 struct bnxt_tc_flow_node
,
797 *ref_flow_handle
= ref_flow_node
->flow_handle
;
799 *ref_flow_handle
= cpu_to_le16(0xffff);
802 /* Insert the l2_node into the flow_node so that subsequent flows
803 * with a matching l2 key can use the flow_handle of this flow
804 * as their ref_flow_handle
806 flow_node
->l2_node
= l2_node
;
807 list_add(&flow_node
->l2_list_node
, &l2_node
->common_l2_flows
);
812 /* After the flow parsing is done, this routine is used for checking
813 * if there are any aspects of the flow that prevent it from being
816 static bool bnxt_tc_can_offload(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
)
818 /* If L4 ports are specified then ip_proto must be TCP or UDP */
819 if ((flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) &&
820 (flow
->l4_key
.ip_proto
!= IPPROTO_TCP
&&
821 flow
->l4_key
.ip_proto
!= IPPROTO_UDP
)) {
822 netdev_info(bp
->dev
, "Cannot offload non-TCP/UDP (%d) ports",
823 flow
->l4_key
.ip_proto
);
827 /* Currently source/dest MAC cannot be partial wildcard */
828 if (bits_set(&flow
->l2_key
.smac
, sizeof(flow
->l2_key
.smac
)) &&
829 !is_exactmatch(flow
->l2_mask
.smac
, sizeof(flow
->l2_mask
.smac
))) {
830 netdev_info(bp
->dev
, "Wildcard match unsupported for Source MAC\n");
833 if (bits_set(&flow
->l2_key
.dmac
, sizeof(flow
->l2_key
.dmac
)) &&
834 !is_exactmatch(&flow
->l2_mask
.dmac
, sizeof(flow
->l2_mask
.dmac
))) {
835 netdev_info(bp
->dev
, "Wildcard match unsupported for Dest MAC\n");
839 /* Currently VLAN fields cannot be partial wildcard */
840 if (bits_set(&flow
->l2_key
.inner_vlan_tci
,
841 sizeof(flow
->l2_key
.inner_vlan_tci
)) &&
842 !is_vlan_tci_allowed(flow
->l2_mask
.inner_vlan_tci
,
843 flow
->l2_key
.inner_vlan_tci
)) {
844 netdev_info(bp
->dev
, "Unsupported VLAN TCI\n");
847 if (bits_set(&flow
->l2_key
.inner_vlan_tpid
,
848 sizeof(flow
->l2_key
.inner_vlan_tpid
)) &&
849 !is_exactmatch(&flow
->l2_mask
.inner_vlan_tpid
,
850 sizeof(flow
->l2_mask
.inner_vlan_tpid
))) {
851 netdev_info(bp
->dev
, "Wildcard match unsupported for VLAN TPID\n");
855 /* Currently Ethertype must be set */
856 if (!is_exactmatch(&flow
->l2_mask
.ether_type
,
857 sizeof(flow
->l2_mask
.ether_type
))) {
858 netdev_info(bp
->dev
, "Wildcard match unsupported for Ethertype\n");
865 /* Returns the final refcount of the node on success
866 * or a -ve error code on failure
868 static int bnxt_tc_put_tunnel_node(struct bnxt
*bp
,
869 struct rhashtable
*tunnel_table
,
870 struct rhashtable_params
*ht_params
,
871 struct bnxt_tc_tunnel_node
*tunnel_node
)
875 if (--tunnel_node
->refcount
== 0) {
876 rc
= rhashtable_remove_fast(tunnel_table
, &tunnel_node
->node
,
879 netdev_err(bp
->dev
, "rhashtable_remove_fast rc=%d", rc
);
882 kfree_rcu(tunnel_node
, rcu
);
885 return tunnel_node
->refcount
;
889 /* Get (or add) either encap or decap tunnel node from/to the supplied
892 static struct bnxt_tc_tunnel_node
*
893 bnxt_tc_get_tunnel_node(struct bnxt
*bp
, struct rhashtable
*tunnel_table
,
894 struct rhashtable_params
*ht_params
,
895 struct ip_tunnel_key
*tun_key
)
897 struct bnxt_tc_tunnel_node
*tunnel_node
;
900 tunnel_node
= rhashtable_lookup_fast(tunnel_table
, tun_key
, *ht_params
);
902 tunnel_node
= kzalloc(sizeof(*tunnel_node
), GFP_KERNEL
);
908 tunnel_node
->key
= *tun_key
;
909 tunnel_node
->tunnel_handle
= INVALID_TUNNEL_HANDLE
;
910 rc
= rhashtable_insert_fast(tunnel_table
, &tunnel_node
->node
,
913 kfree_rcu(tunnel_node
, rcu
);
917 tunnel_node
->refcount
++;
920 netdev_info(bp
->dev
, "error rc=%d", rc
);
924 static int bnxt_tc_get_ref_decap_handle(struct bnxt
*bp
,
925 struct bnxt_tc_flow
*flow
,
926 struct bnxt_tc_l2_key
*l2_key
,
927 struct bnxt_tc_flow_node
*flow_node
,
928 __le32
*ref_decap_handle
)
930 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
931 struct bnxt_tc_flow_node
*ref_flow_node
;
932 struct bnxt_tc_l2_node
*decap_l2_node
;
934 decap_l2_node
= bnxt_tc_get_l2_node(bp
, &tc_info
->decap_l2_table
,
935 tc_info
->decap_l2_ht_params
,
940 /* If any other flow is using this decap_l2_node, use it's decap_handle
941 * as the ref_decap_handle
943 if (decap_l2_node
->refcount
> 0) {
945 list_first_entry(&decap_l2_node
->common_l2_flows
,
946 struct bnxt_tc_flow_node
,
948 *ref_decap_handle
= ref_flow_node
->decap_node
->tunnel_handle
;
950 *ref_decap_handle
= INVALID_TUNNEL_HANDLE
;
953 /* Insert the l2_node into the flow_node so that subsequent flows
954 * with a matching decap l2 key can use the decap_filter_handle of
955 * this flow as their ref_decap_handle
957 flow_node
->decap_l2_node
= decap_l2_node
;
958 list_add(&flow_node
->decap_l2_list_node
,
959 &decap_l2_node
->common_l2_flows
);
960 decap_l2_node
->refcount
++;
964 static void bnxt_tc_put_decap_l2_node(struct bnxt
*bp
,
965 struct bnxt_tc_flow_node
*flow_node
)
967 struct bnxt_tc_l2_node
*decap_l2_node
= flow_node
->decap_l2_node
;
968 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
971 /* remove flow_node from the decap L2 sharing flow list */
972 list_del(&flow_node
->decap_l2_list_node
);
973 if (--decap_l2_node
->refcount
== 0) {
974 rc
= rhashtable_remove_fast(&tc_info
->decap_l2_table
,
975 &decap_l2_node
->node
,
976 tc_info
->decap_l2_ht_params
);
978 netdev_err(bp
->dev
, "rhashtable_remove_fast rc=%d", rc
);
979 kfree_rcu(decap_l2_node
, rcu
);
983 static void bnxt_tc_put_decap_handle(struct bnxt
*bp
,
984 struct bnxt_tc_flow_node
*flow_node
)
986 __le32 decap_handle
= flow_node
->decap_node
->tunnel_handle
;
987 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
990 if (flow_node
->decap_l2_node
)
991 bnxt_tc_put_decap_l2_node(bp
, flow_node
);
993 rc
= bnxt_tc_put_tunnel_node(bp
, &tc_info
->decap_table
,
994 &tc_info
->decap_ht_params
,
995 flow_node
->decap_node
);
996 if (!rc
&& decap_handle
!= INVALID_TUNNEL_HANDLE
)
997 hwrm_cfa_decap_filter_free(bp
, decap_handle
);
1000 static int bnxt_tc_resolve_tunnel_hdrs(struct bnxt
*bp
,
1001 struct ip_tunnel_key
*tun_key
,
1002 struct bnxt_tc_l2_key
*l2_info
)
1005 struct net_device
*real_dst_dev
= bp
->dev
;
1006 struct flowi4 flow
= { {0} };
1007 struct net_device
*dst_dev
;
1008 struct neighbour
*nbr
;
1012 flow
.flowi4_proto
= IPPROTO_UDP
;
1013 flow
.fl4_dport
= tun_key
->tp_dst
;
1014 flow
.daddr
= tun_key
->u
.ipv4
.dst
;
1016 rt
= ip_route_output_key(dev_net(real_dst_dev
), &flow
);
1018 netdev_info(bp
->dev
, "no route to %pI4b", &flow
.daddr
);
1022 /* The route must either point to the real_dst_dev or a dst_dev that
1023 * uses the real_dst_dev.
1025 dst_dev
= rt
->dst
.dev
;
1026 if (is_vlan_dev(dst_dev
)) {
1027 #if IS_ENABLED(CONFIG_VLAN_8021Q)
1028 struct vlan_dev_priv
*vlan
= vlan_dev_priv(dst_dev
);
1030 if (vlan
->real_dev
!= real_dst_dev
) {
1031 netdev_info(bp
->dev
,
1032 "dst_dev(%s) doesn't use PF-if(%s)",
1033 netdev_name(dst_dev
),
1034 netdev_name(real_dst_dev
));
1038 l2_info
->inner_vlan_tci
= htons(vlan
->vlan_id
);
1039 l2_info
->inner_vlan_tpid
= vlan
->vlan_proto
;
1040 l2_info
->num_vlans
= 1;
1042 } else if (dst_dev
!= real_dst_dev
) {
1043 netdev_info(bp
->dev
,
1044 "dst_dev(%s) for %pI4b is not PF-if(%s)",
1045 netdev_name(dst_dev
), &flow
.daddr
,
1046 netdev_name(real_dst_dev
));
1051 nbr
= dst_neigh_lookup(&rt
->dst
, &flow
.daddr
);
1053 netdev_info(bp
->dev
, "can't lookup neighbor for %pI4b",
1059 tun_key
->u
.ipv4
.src
= flow
.saddr
;
1060 tun_key
->ttl
= ip4_dst_hoplimit(&rt
->dst
);
1061 neigh_ha_snapshot(l2_info
->dmac
, nbr
, dst_dev
);
1062 ether_addr_copy(l2_info
->smac
, dst_dev
->dev_addr
);
1075 static int bnxt_tc_get_decap_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
1076 struct bnxt_tc_flow_node
*flow_node
,
1077 __le32
*decap_filter_handle
)
1079 struct ip_tunnel_key
*decap_key
= &flow
->tun_key
;
1080 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1081 struct bnxt_tc_l2_key l2_info
= { {0} };
1082 struct bnxt_tc_tunnel_node
*decap_node
;
1083 struct ip_tunnel_key tun_key
= { 0 };
1084 struct bnxt_tc_l2_key
*decap_l2_info
;
1085 __le32 ref_decap_handle
;
1088 /* Check if there's another flow using the same tunnel decap.
1089 * If not, add this tunnel to the table and resolve the other
1090 * tunnel header fileds. Ignore src_port in the tunnel_key,
1091 * since it is not required for decap filters.
1093 decap_key
->tp_src
= 0;
1094 decap_node
= bnxt_tc_get_tunnel_node(bp
, &tc_info
->decap_table
,
1095 &tc_info
->decap_ht_params
,
1100 flow_node
->decap_node
= decap_node
;
1102 if (decap_node
->tunnel_handle
!= INVALID_TUNNEL_HANDLE
)
1105 /* Resolve the L2 fields for tunnel decap
1106 * Resolve the route for remote vtep (saddr) of the decap key
1107 * Find it's next-hop mac addrs
1109 tun_key
.u
.ipv4
.dst
= flow
->tun_key
.u
.ipv4
.src
;
1110 tun_key
.tp_dst
= flow
->tun_key
.tp_dst
;
1111 rc
= bnxt_tc_resolve_tunnel_hdrs(bp
, &tun_key
, &l2_info
);
1115 decap_l2_info
= &decap_node
->l2_info
;
1116 /* decap smac is wildcarded */
1117 ether_addr_copy(decap_l2_info
->dmac
, l2_info
.smac
);
1118 if (l2_info
.num_vlans
) {
1119 decap_l2_info
->num_vlans
= l2_info
.num_vlans
;
1120 decap_l2_info
->inner_vlan_tpid
= l2_info
.inner_vlan_tpid
;
1121 decap_l2_info
->inner_vlan_tci
= l2_info
.inner_vlan_tci
;
1123 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS
;
1125 /* For getting a decap_filter_handle we first need to check if
1126 * there are any other decap flows that share the same tunnel L2
1127 * key and if so, pass that flow's decap_filter_handle as the
1128 * ref_decap_handle for this flow.
1130 rc
= bnxt_tc_get_ref_decap_handle(bp
, flow
, decap_l2_info
, flow_node
,
1135 /* Issue the hwrm cmd to allocate a decap filter handle */
1136 rc
= hwrm_cfa_decap_filter_alloc(bp
, flow
, decap_l2_info
,
1138 &decap_node
->tunnel_handle
);
1143 *decap_filter_handle
= decap_node
->tunnel_handle
;
1147 bnxt_tc_put_decap_l2_node(bp
, flow_node
);
1149 bnxt_tc_put_tunnel_node(bp
, &tc_info
->decap_table
,
1150 &tc_info
->decap_ht_params
,
1151 flow_node
->decap_node
);
1155 static void bnxt_tc_put_encap_handle(struct bnxt
*bp
,
1156 struct bnxt_tc_tunnel_node
*encap_node
)
1158 __le32 encap_handle
= encap_node
->tunnel_handle
;
1159 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1162 rc
= bnxt_tc_put_tunnel_node(bp
, &tc_info
->encap_table
,
1163 &tc_info
->encap_ht_params
, encap_node
);
1164 if (!rc
&& encap_handle
!= INVALID_TUNNEL_HANDLE
)
1165 hwrm_cfa_encap_record_free(bp
, encap_handle
);
1168 /* Lookup the tunnel encap table and check if there's an encap_handle
1170 * If not, query L2 info via a route lookup and issue an encap_record_alloc
1173 static int bnxt_tc_get_encap_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
1174 struct bnxt_tc_flow_node
*flow_node
,
1175 __le32
*encap_handle
)
1177 struct ip_tunnel_key
*encap_key
= &flow
->actions
.tun_encap_key
;
1178 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1179 struct bnxt_tc_tunnel_node
*encap_node
;
1182 /* Check if there's another flow using the same tunnel encap.
1183 * If not, add this tunnel to the table and resolve the other
1184 * tunnel header fileds
1186 encap_node
= bnxt_tc_get_tunnel_node(bp
, &tc_info
->encap_table
,
1187 &tc_info
->encap_ht_params
,
1192 flow_node
->encap_node
= encap_node
;
1194 if (encap_node
->tunnel_handle
!= INVALID_TUNNEL_HANDLE
)
1197 rc
= bnxt_tc_resolve_tunnel_hdrs(bp
, encap_key
, &encap_node
->l2_info
);
1201 /* Allocate a new tunnel encap record */
1202 rc
= hwrm_cfa_encap_record_alloc(bp
, encap_key
, &encap_node
->l2_info
,
1203 &encap_node
->tunnel_handle
);
1208 *encap_handle
= encap_node
->tunnel_handle
;
1212 bnxt_tc_put_tunnel_node(bp
, &tc_info
->encap_table
,
1213 &tc_info
->encap_ht_params
, encap_node
);
1217 static void bnxt_tc_put_tunnel_handle(struct bnxt
*bp
,
1218 struct bnxt_tc_flow
*flow
,
1219 struct bnxt_tc_flow_node
*flow_node
)
1221 if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
)
1222 bnxt_tc_put_decap_handle(bp
, flow_node
);
1223 else if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
)
1224 bnxt_tc_put_encap_handle(bp
, flow_node
->encap_node
);
1227 static int bnxt_tc_get_tunnel_handle(struct bnxt
*bp
,
1228 struct bnxt_tc_flow
*flow
,
1229 struct bnxt_tc_flow_node
*flow_node
,
1230 __le32
*tunnel_handle
)
1232 if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
)
1233 return bnxt_tc_get_decap_handle(bp
, flow
, flow_node
,
1235 else if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
)
1236 return bnxt_tc_get_encap_handle(bp
, flow
, flow_node
,
1241 static int __bnxt_tc_del_flow(struct bnxt
*bp
,
1242 struct bnxt_tc_flow_node
*flow_node
)
1244 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1247 /* send HWRM cmd to free the flow-id */
1248 bnxt_hwrm_cfa_flow_free(bp
, flow_node
);
1250 mutex_lock(&tc_info
->lock
);
1252 /* release references to any tunnel encap/decap nodes */
1253 bnxt_tc_put_tunnel_handle(bp
, &flow_node
->flow
, flow_node
);
1255 /* release reference to l2 node */
1256 bnxt_tc_put_l2_node(bp
, flow_node
);
1258 mutex_unlock(&tc_info
->lock
);
1260 rc
= rhashtable_remove_fast(&tc_info
->flow_table
, &flow_node
->node
,
1261 tc_info
->flow_ht_params
);
1263 netdev_err(bp
->dev
, "Error: %s: rhashtable_remove_fast rc=%d",
1266 kfree_rcu(flow_node
, rcu
);
1270 static void bnxt_tc_set_flow_dir(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
1273 flow
->dir
= (bp
->pf
.fw_fid
== src_fid
) ? BNXT_DIR_RX
: BNXT_DIR_TX
;
1276 static void bnxt_tc_set_src_fid(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
1279 if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
)
1280 flow
->src_fid
= bp
->pf
.fw_fid
;
1282 flow
->src_fid
= src_fid
;
1285 /* Add a new flow or replace an existing flow.
1287 * There are essentially two critical sections here.
1288 * 1. while adding a new flow
1290 * b) issue HWRM cmd and get flow_handle
1291 * c) link l2-key with flow
1292 * 2. while deleting a flow
1293 * a) unlinking l2-key from flow
1294 * A lock is needed to protect these two critical sections.
1296 * The hash-tables are already protected by the rhashtable API.
1298 static int bnxt_tc_add_flow(struct bnxt
*bp
, u16 src_fid
,
1299 struct tc_cls_flower_offload
*tc_flow_cmd
)
1301 struct bnxt_tc_flow_node
*new_node
, *old_node
;
1302 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1303 struct bnxt_tc_flow
*flow
;
1304 __le32 tunnel_handle
= 0;
1305 __le16 ref_flow_handle
;
1308 /* allocate memory for the new flow and it's node */
1309 new_node
= kzalloc(sizeof(*new_node
), GFP_KERNEL
);
1314 new_node
->cookie
= tc_flow_cmd
->cookie
;
1315 flow
= &new_node
->flow
;
1317 rc
= bnxt_tc_parse_flow(bp
, tc_flow_cmd
, flow
);
1321 bnxt_tc_set_src_fid(bp
, flow
, src_fid
);
1323 if (bp
->fw_cap
& BNXT_FW_CAP_OVS_64BIT_HANDLE
)
1324 bnxt_tc_set_flow_dir(bp
, flow
, src_fid
);
1326 if (!bnxt_tc_can_offload(bp
, flow
)) {
1331 /* If a flow exists with the same cookie, delete it */
1332 old_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1333 &tc_flow_cmd
->cookie
,
1334 tc_info
->flow_ht_params
);
1336 __bnxt_tc_del_flow(bp
, old_node
);
1338 /* Check if the L2 part of the flow has been offloaded already.
1339 * If so, bump up it's refcnt and get it's reference handle.
1341 mutex_lock(&tc_info
->lock
);
1342 rc
= bnxt_tc_get_ref_flow_handle(bp
, flow
, new_node
, &ref_flow_handle
);
1346 /* If the flow involves tunnel encap/decap, get tunnel_handle */
1347 rc
= bnxt_tc_get_tunnel_handle(bp
, flow
, new_node
, &tunnel_handle
);
1351 /* send HWRM cmd to alloc the flow */
1352 rc
= bnxt_hwrm_cfa_flow_alloc(bp
, flow
, ref_flow_handle
,
1353 tunnel_handle
, new_node
);
1357 flow
->lastused
= jiffies
;
1358 spin_lock_init(&flow
->stats_lock
);
1359 /* add new flow to flow-table */
1360 rc
= rhashtable_insert_fast(&tc_info
->flow_table
, &new_node
->node
,
1361 tc_info
->flow_ht_params
);
1363 goto hwrm_flow_free
;
1365 mutex_unlock(&tc_info
->lock
);
1369 bnxt_hwrm_cfa_flow_free(bp
, new_node
);
1371 bnxt_tc_put_tunnel_handle(bp
, flow
, new_node
);
1373 bnxt_tc_put_l2_node(bp
, new_node
);
1375 mutex_unlock(&tc_info
->lock
);
1377 kfree_rcu(new_node
, rcu
);
1379 netdev_err(bp
->dev
, "Error: %s: cookie=0x%lx error=%d",
1380 __func__
, tc_flow_cmd
->cookie
, rc
);
1384 static int bnxt_tc_del_flow(struct bnxt
*bp
,
1385 struct tc_cls_flower_offload
*tc_flow_cmd
)
1387 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1388 struct bnxt_tc_flow_node
*flow_node
;
1390 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1391 &tc_flow_cmd
->cookie
,
1392 tc_info
->flow_ht_params
);
1396 return __bnxt_tc_del_flow(bp
, flow_node
);
1399 static int bnxt_tc_get_flow_stats(struct bnxt
*bp
,
1400 struct tc_cls_flower_offload
*tc_flow_cmd
)
1402 struct bnxt_tc_flow_stats stats
, *curr_stats
, *prev_stats
;
1403 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1404 struct bnxt_tc_flow_node
*flow_node
;
1405 struct bnxt_tc_flow
*flow
;
1406 unsigned long lastused
;
1408 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1409 &tc_flow_cmd
->cookie
,
1410 tc_info
->flow_ht_params
);
1414 flow
= &flow_node
->flow
;
1415 curr_stats
= &flow
->stats
;
1416 prev_stats
= &flow
->prev_stats
;
1418 spin_lock(&flow
->stats_lock
);
1419 stats
.packets
= curr_stats
->packets
- prev_stats
->packets
;
1420 stats
.bytes
= curr_stats
->bytes
- prev_stats
->bytes
;
1421 *prev_stats
= *curr_stats
;
1422 lastused
= flow
->lastused
;
1423 spin_unlock(&flow
->stats_lock
);
1425 tcf_exts_stats_update(tc_flow_cmd
->exts
, stats
.bytes
, stats
.packets
,
1430 static void bnxt_fill_cfa_stats_req(struct bnxt
*bp
,
1431 struct bnxt_tc_flow_node
*flow_node
,
1432 __le16
*flow_handle
, __le32
*flow_id
)
1436 if (bp
->fw_cap
& BNXT_FW_CAP_OVS_64BIT_HANDLE
) {
1437 *flow_id
= flow_node
->flow_id
;
1439 /* If flow_id is used to fetch flow stats then:
1440 * 1. lower 12 bits of flow_handle must be set to all 1s.
1441 * 2. 15th bit of flow_handle must specify the flow
1442 * direction (TX/RX).
1444 if (flow_node
->flow
.dir
== BNXT_DIR_RX
)
1445 handle
= CFA_FLOW_INFO_REQ_FLOW_HANDLE_DIR_RX
|
1446 CFA_FLOW_INFO_REQ_FLOW_HANDLE_MAX_MASK
;
1448 handle
= CFA_FLOW_INFO_REQ_FLOW_HANDLE_MAX_MASK
;
1450 *flow_handle
= cpu_to_le16(handle
);
1452 *flow_handle
= flow_node
->flow_handle
;
1457 bnxt_hwrm_cfa_flow_stats_get(struct bnxt
*bp
, int num_flows
,
1458 struct bnxt_tc_stats_batch stats_batch
[])
1460 struct hwrm_cfa_flow_stats_input req
= { 0 };
1461 struct hwrm_cfa_flow_stats_output
*resp
;
1462 __le16
*req_flow_handles
= &req
.flow_handle_0
;
1463 __le32
*req_flow_ids
= &req
.flow_id_0
;
1466 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_STATS
, -1, -1);
1467 req
.num_flows
= cpu_to_le16(num_flows
);
1468 for (i
= 0; i
< num_flows
; i
++) {
1469 struct bnxt_tc_flow_node
*flow_node
= stats_batch
[i
].flow_node
;
1471 bnxt_fill_cfa_stats_req(bp
, flow_node
,
1472 &req_flow_handles
[i
], &req_flow_ids
[i
]);
1475 mutex_lock(&bp
->hwrm_cmd_lock
);
1476 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
1478 __le64
*resp_packets
;
1481 resp
= bnxt_get_hwrm_resp_addr(bp
, &req
);
1482 resp_packets
= &resp
->packet_0
;
1483 resp_bytes
= &resp
->byte_0
;
1485 for (i
= 0; i
< num_flows
; i
++) {
1486 stats_batch
[i
].hw_stats
.packets
=
1487 le64_to_cpu(resp_packets
[i
]);
1488 stats_batch
[i
].hw_stats
.bytes
=
1489 le64_to_cpu(resp_bytes
[i
]);
1492 netdev_info(bp
->dev
, "error rc=%d", rc
);
1494 mutex_unlock(&bp
->hwrm_cmd_lock
);
1501 /* Add val to accum while handling a possible wraparound
1502 * of val. Eventhough val is of type u64, its actual width
1503 * is denoted by mask and will wrap-around beyond that width.
1505 static void accumulate_val(u64
*accum
, u64 val
, u64 mask
)
1507 #define low_bits(x, mask) ((x) & (mask))
1508 #define high_bits(x, mask) ((x) & ~(mask))
1509 bool wrapped
= val
< low_bits(*accum
, mask
);
1511 *accum
= high_bits(*accum
, mask
) + val
;
1513 *accum
+= (mask
+ 1);
1516 /* The HW counters' width is much less than 64bits.
1517 * Handle possible wrap-around while updating the stat counters
1519 static void bnxt_flow_stats_accum(struct bnxt_tc_info
*tc_info
,
1520 struct bnxt_tc_flow_stats
*acc_stats
,
1521 struct bnxt_tc_flow_stats
*hw_stats
)
1523 accumulate_val(&acc_stats
->bytes
, hw_stats
->bytes
, tc_info
->bytes_mask
);
1524 accumulate_val(&acc_stats
->packets
, hw_stats
->packets
,
1525 tc_info
->packets_mask
);
1529 bnxt_tc_flow_stats_batch_update(struct bnxt
*bp
, int num_flows
,
1530 struct bnxt_tc_stats_batch stats_batch
[])
1532 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1535 rc
= bnxt_hwrm_cfa_flow_stats_get(bp
, num_flows
, stats_batch
);
1539 for (i
= 0; i
< num_flows
; i
++) {
1540 struct bnxt_tc_flow_node
*flow_node
= stats_batch
[i
].flow_node
;
1541 struct bnxt_tc_flow
*flow
= &flow_node
->flow
;
1543 spin_lock(&flow
->stats_lock
);
1544 bnxt_flow_stats_accum(tc_info
, &flow
->stats
,
1545 &stats_batch
[i
].hw_stats
);
1546 if (flow
->stats
.packets
!= flow
->prev_stats
.packets
)
1547 flow
->lastused
= jiffies
;
1548 spin_unlock(&flow
->stats_lock
);
1555 bnxt_tc_flow_stats_batch_prep(struct bnxt
*bp
,
1556 struct bnxt_tc_stats_batch stats_batch
[],
1559 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1560 struct rhashtable_iter
*iter
= &tc_info
->iter
;
1564 rhashtable_walk_start(iter
);
1567 for (i
= 0; i
< BNXT_FLOW_STATS_BATCH_MAX
; i
++) {
1568 flow_node
= rhashtable_walk_next(iter
);
1569 if (IS_ERR(flow_node
)) {
1571 if (PTR_ERR(flow_node
) == -EAGAIN
) {
1574 rc
= PTR_ERR(flow_node
);
1583 stats_batch
[i
].flow_node
= flow_node
;
1586 rhashtable_walk_stop(iter
);
1591 void bnxt_tc_flow_stats_work(struct bnxt
*bp
)
1593 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1596 num_flows
= atomic_read(&tc_info
->flow_table
.nelems
);
1600 rhashtable_walk_enter(&tc_info
->flow_table
, &tc_info
->iter
);
1603 rc
= bnxt_tc_flow_stats_batch_prep(bp
, tc_info
->stats_batch
,
1614 bnxt_tc_flow_stats_batch_update(bp
, num_flows
,
1615 tc_info
->stats_batch
);
1618 rhashtable_walk_exit(&tc_info
->iter
);
1621 int bnxt_tc_setup_flower(struct bnxt
*bp
, u16 src_fid
,
1622 struct tc_cls_flower_offload
*cls_flower
)
1624 switch (cls_flower
->command
) {
1625 case TC_CLSFLOWER_REPLACE
:
1626 return bnxt_tc_add_flow(bp
, src_fid
, cls_flower
);
1627 case TC_CLSFLOWER_DESTROY
:
1628 return bnxt_tc_del_flow(bp
, cls_flower
);
1629 case TC_CLSFLOWER_STATS
:
1630 return bnxt_tc_get_flow_stats(bp
, cls_flower
);
1636 static const struct rhashtable_params bnxt_tc_flow_ht_params
= {
1637 .head_offset
= offsetof(struct bnxt_tc_flow_node
, node
),
1638 .key_offset
= offsetof(struct bnxt_tc_flow_node
, cookie
),
1639 .key_len
= sizeof(((struct bnxt_tc_flow_node
*)0)->cookie
),
1640 .automatic_shrinking
= true
1643 static const struct rhashtable_params bnxt_tc_l2_ht_params
= {
1644 .head_offset
= offsetof(struct bnxt_tc_l2_node
, node
),
1645 .key_offset
= offsetof(struct bnxt_tc_l2_node
, key
),
1646 .key_len
= BNXT_TC_L2_KEY_LEN
,
1647 .automatic_shrinking
= true
1650 static const struct rhashtable_params bnxt_tc_decap_l2_ht_params
= {
1651 .head_offset
= offsetof(struct bnxt_tc_l2_node
, node
),
1652 .key_offset
= offsetof(struct bnxt_tc_l2_node
, key
),
1653 .key_len
= BNXT_TC_L2_KEY_LEN
,
1654 .automatic_shrinking
= true
1657 static const struct rhashtable_params bnxt_tc_tunnel_ht_params
= {
1658 .head_offset
= offsetof(struct bnxt_tc_tunnel_node
, node
),
1659 .key_offset
= offsetof(struct bnxt_tc_tunnel_node
, key
),
1660 .key_len
= sizeof(struct ip_tunnel_key
),
1661 .automatic_shrinking
= true
1664 /* convert counter width in bits to a mask */
1665 #define mask(width) ((u64)~0 >> (64 - (width)))
1667 int bnxt_init_tc(struct bnxt
*bp
)
1669 struct bnxt_tc_info
*tc_info
;
1672 if (bp
->hwrm_spec_code
< 0x10803) {
1673 netdev_warn(bp
->dev
,
1674 "Firmware does not support TC flower offload.\n");
1678 tc_info
= kzalloc(sizeof(*tc_info
), GFP_KERNEL
);
1681 mutex_init(&tc_info
->lock
);
1683 /* Counter widths are programmed by FW */
1684 tc_info
->bytes_mask
= mask(36);
1685 tc_info
->packets_mask
= mask(28);
1687 tc_info
->flow_ht_params
= bnxt_tc_flow_ht_params
;
1688 rc
= rhashtable_init(&tc_info
->flow_table
, &tc_info
->flow_ht_params
);
1692 tc_info
->l2_ht_params
= bnxt_tc_l2_ht_params
;
1693 rc
= rhashtable_init(&tc_info
->l2_table
, &tc_info
->l2_ht_params
);
1695 goto destroy_flow_table
;
1697 tc_info
->decap_l2_ht_params
= bnxt_tc_decap_l2_ht_params
;
1698 rc
= rhashtable_init(&tc_info
->decap_l2_table
,
1699 &tc_info
->decap_l2_ht_params
);
1701 goto destroy_l2_table
;
1703 tc_info
->decap_ht_params
= bnxt_tc_tunnel_ht_params
;
1704 rc
= rhashtable_init(&tc_info
->decap_table
,
1705 &tc_info
->decap_ht_params
);
1707 goto destroy_decap_l2_table
;
1709 tc_info
->encap_ht_params
= bnxt_tc_tunnel_ht_params
;
1710 rc
= rhashtable_init(&tc_info
->encap_table
,
1711 &tc_info
->encap_ht_params
);
1713 goto destroy_decap_table
;
1715 tc_info
->enabled
= true;
1716 bp
->dev
->hw_features
|= NETIF_F_HW_TC
;
1717 bp
->dev
->features
|= NETIF_F_HW_TC
;
1718 bp
->tc_info
= tc_info
;
1721 destroy_decap_table
:
1722 rhashtable_destroy(&tc_info
->decap_table
);
1723 destroy_decap_l2_table
:
1724 rhashtable_destroy(&tc_info
->decap_l2_table
);
1726 rhashtable_destroy(&tc_info
->l2_table
);
1728 rhashtable_destroy(&tc_info
->flow_table
);
1734 void bnxt_shutdown_tc(struct bnxt
*bp
)
1736 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1738 if (!bnxt_tc_flower_enabled(bp
))
1741 rhashtable_destroy(&tc_info
->flow_table
);
1742 rhashtable_destroy(&tc_info
->l2_table
);
1743 rhashtable_destroy(&tc_info
->decap_l2_table
);
1744 rhashtable_destroy(&tc_info
->decap_table
);
1745 rhashtable_destroy(&tc_info
->encap_table
);