]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'x86-fpu-2020-06-01' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
[thirdparty/linux.git] / drivers / net / ethernet / mellanox / mlx5 / core / en / tc_ct.c
CommitLineData
4c3844d9
PB		 1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2019 Mellanox Technologies. */
3
4#include <net/netfilter/nf_conntrack.h>
5#include <net/netfilter/nf_conntrack_core.h>
6#include <net/netfilter/nf_conntrack_zones.h>
7#include <net/netfilter/nf_conntrack_labels.h>
8#include <net/netfilter/nf_conntrack_helper.h>
9#include <net/netfilter/nf_conntrack_acct.h>
10#include <uapi/linux/tc_act/tc_pedit.h>
11#include <net/tc_act/tc_ct.h>
12#include <net/flow_offload.h>
ac991b48 13#include <net/netfilter/nf_flow_table.h>
4c3844d9 14#include <linux/workqueue.h>
70840b66 15#include <linux/xarray.h>
4c3844d9 16
49964352 17#include "esw/chains.h"
4c3844d9
PB		 18#include "en/tc_ct.h"
19#include "en.h"
20#include "en_tc.h"
21#include "en_rep.h"
4c3844d9
PB		 22
23#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen * 8)
24#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0)
25#define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
26#define MLX5_CT_STATE_TRK_BIT BIT(2)
27
28#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen * 8)
29#define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0)
30#define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX
31
32#define ct_dbg(fmt, args...)\
33 netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args)
34
35struct mlx5_tc_ct_priv {
36 struct mlx5_eswitch *esw;
37 const struct net_device *netdev;
38 struct idr fte_ids;
70840b66 39 struct xarray tuple_ids;
ac991b48 40 struct rhashtable zone_ht;
4c3844d9
PB		 41 struct mlx5_flow_table *ct;
42 struct mlx5_flow_table *ct_nat;
43 struct mlx5_flow_table *post_ct;
44 struct mutex control_lock; /* guards parallel adds/dels */
45};
46
47struct mlx5_ct_flow {
48 struct mlx5_esw_flow_attr pre_ct_attr;
49 struct mlx5_esw_flow_attr post_ct_attr;
50 struct mlx5_flow_handle *pre_ct_rule;
51 struct mlx5_flow_handle *post_ct_rule;
ac991b48 52 struct mlx5_ct_ft *ft;
4c3844d9
PB		 53 u32 fte_id;
54 u32 chain_mapping;
55};
56
ac991b48
PB		 57struct mlx5_ct_zone_rule {
58 struct mlx5_flow_handle *rule;
59 struct mlx5_esw_flow_attr attr;
5c6b9460 60 int tupleid;
ac991b48
PB		 61 bool nat;
62};
63
64struct mlx5_ct_ft {
65 struct rhash_head node;
66 u16 zone;
67 refcount_t refcount;
68 struct nf_flowtable *nf_ft;
69 struct mlx5_tc_ct_priv *ct_priv;
70 struct rhashtable ct_entries_ht;
ac991b48
PB		 71};
72
73struct mlx5_ct_entry {
ac991b48
PB		 74 u16 zone;
75 struct rhash_head node;
76 struct flow_rule *flow_rule;
77 struct mlx5_fc *counter;
78 unsigned long lastuse;
79 unsigned long cookie;
5c6b9460 80 unsigned long restore_cookie;
ac991b48
PB		 81 struct mlx5_ct_zone_rule zone_rules[2];
82};
83
84static const struct rhashtable_params cts_ht_params = {
85 .head_offset = offsetof(struct mlx5_ct_entry, node),
86 .key_offset = offsetof(struct mlx5_ct_entry, cookie),
87 .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie),
88 .automatic_shrinking = true,
89 .min_size = 16 * 1024,
90};
91
92static const struct rhashtable_params zone_params = {
93 .head_offset = offsetof(struct mlx5_ct_ft, node),
94 .key_offset = offsetof(struct mlx5_ct_ft, zone),
95 .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone),
96 .automatic_shrinking = true,
97};
98
4c3844d9
PB		 99static struct mlx5_tc_ct_priv *
100mlx5_tc_ct_get_ct_priv(struct mlx5e_priv *priv)
101{
102 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
103 struct mlx5_rep_uplink_priv *uplink_priv;
104 struct mlx5e_rep_priv *uplink_rpriv;
105
106 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
107 uplink_priv = &uplink_rpriv->uplink_priv;
108 return uplink_priv->ct_priv;
109}
110
ac991b48
PB		 111static int
112mlx5_tc_ct_set_tuple_match(struct mlx5_flow_spec *spec,
113 struct flow_rule *rule)
114{
115 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
116 outer_headers);
117 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
118 outer_headers);
119 u16 addr_type = 0;
120 u8 ip_proto = 0;
121
122 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
123 struct flow_match_basic match;
124
125 flow_rule_match_basic(rule, &match);
126
127 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
128 ntohs(match.mask->n_proto));
129 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
130 ntohs(match.key->n_proto));
131 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
132 match.mask->ip_proto);
133 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
134 match.key->ip_proto);
135
136 ip_proto = match.key->ip_proto;
137 }
138
139 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
140 struct flow_match_control match;
141
142 flow_rule_match_control(rule, &match);
143 addr_type = match.key->addr_type;
144 }
145
146 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
147 struct flow_match_ipv4_addrs match;
148
149 flow_rule_match_ipv4_addrs(rule, &match);
150 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
151 src_ipv4_src_ipv6.ipv4_layout.ipv4),
152 &match.mask->src, sizeof(match.mask->src));
153 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
154 src_ipv4_src_ipv6.ipv4_layout.ipv4),
155 &match.key->src, sizeof(match.key->src));
156 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
157 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
158 &match.mask->dst, sizeof(match.mask->dst));
159 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
160 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
161 &match.key->dst, sizeof(match.key->dst));
162 }
163
164 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
165 struct flow_match_ipv6_addrs match;
166
167 flow_rule_match_ipv6_addrs(rule, &match);
168 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
169 src_ipv4_src_ipv6.ipv6_layout.ipv6),
170 &match.mask->src, sizeof(match.mask->src));
171 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
172 src_ipv4_src_ipv6.ipv6_layout.ipv6),
173 &match.key->src, sizeof(match.key->src));
174
175 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
176 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
177 &match.mask->dst, sizeof(match.mask->dst));
178 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
179 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
180 &match.key->dst, sizeof(match.key->dst));
181 }
182
183 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
184 struct flow_match_ports match;
185
186 flow_rule_match_ports(rule, &match);
187 switch (ip_proto) {
188 case IPPROTO_TCP:
189 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
190 tcp_sport, ntohs(match.mask->src));
191 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
192 tcp_sport, ntohs(match.key->src));
193
194 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
195 tcp_dport, ntohs(match.mask->dst));
196 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
197 tcp_dport, ntohs(match.key->dst));
198 break;
199
200 case IPPROTO_UDP:
201 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
202 udp_sport, ntohs(match.mask->src));
203 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
204 udp_sport, ntohs(match.key->src));
205
206 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
207 udp_dport, ntohs(match.mask->dst));
208 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
209 udp_dport, ntohs(match.key->dst));
210 break;
211 default:
212 break;
213 }
214 }
215
216 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
217 struct flow_match_tcp match;
218
219 flow_rule_match_tcp(rule, &match);
220 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
221 ntohs(match.mask->flags));
222 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
223 ntohs(match.key->flags));
224 }
225
226 return 0;
227}
228
229static void
230mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv,
231 struct mlx5_ct_entry *entry,
232 bool nat)
233{
234 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
235 struct mlx5_esw_flow_attr *attr = &zone_rule->attr;
236 struct mlx5_eswitch *esw = ct_priv->esw;
237
238 ct_dbg("Deleting ct entry rule in zone %d", entry->zone);
239
240 mlx5_eswitch_del_offloaded_rule(esw, zone_rule->rule, attr);
241 mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr);
70840b66 242 xa_erase(&ct_priv->tuple_ids, zone_rule->tupleid);
ac991b48
PB		 243}
244
245static void
246mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv,
247 struct mlx5_ct_entry *entry)
248{
249 mlx5_tc_ct_entry_del_rule(ct_priv, entry, true);
250 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
251
252 mlx5_fc_destroy(ct_priv->esw->dev, entry->counter);
253}
254
255static struct flow_action_entry *
256mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule)
257{
258 struct flow_action *flow_action = &flow_rule->action;
259 struct flow_action_entry *act;
260 int i;
261
262 flow_action_for_each(i, act, flow_action) {
263 if (act->id == FLOW_ACTION_CT_METADATA)
264 return act;
265 }
266
267 return NULL;
268}
269
270static int
271mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
272 struct mlx5e_tc_mod_hdr_acts *mod_acts,
273 u8 ct_state,
274 u32 mark,
5c6b9460
PB		 275 u32 label,
276 u32 tupleid)
ac991b48
PB		 277{
278 struct mlx5_eswitch *esw = ct_priv->esw;
279 int err;
280
281 err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
282 CTSTATE_TO_REG, ct_state);
283 if (err)
284 return err;
285
286 err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
287 MARK_TO_REG, mark);
288 if (err)
289 return err;
290
291 err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
292 LABELS_TO_REG, label);
293 if (err)
294 return err;
295
5c6b9460
PB		 296 err = mlx5e_tc_match_to_reg_set(esw->dev, mod_acts,
297 TUPLEID_TO_REG, tupleid);
298 if (err)
299 return err;
300
ac991b48
PB		 301 return 0;
302}
303
304static int
305mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
306 char *modact)
307{
308 u32 offset = act->mangle.offset, field;
309
310 switch (act->mangle.htype) {
311 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
312 MLX5_SET(set_action_in, modact, length, 0);
313 if (offset == offsetof(struct iphdr, saddr))
314 field = MLX5_ACTION_IN_FIELD_OUT_SIPV4;
315 else if (offset == offsetof(struct iphdr, daddr))
316 field = MLX5_ACTION_IN_FIELD_OUT_DIPV4;
317 else
318 return -EOPNOTSUPP;
319 break;
320
321 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
322 MLX5_SET(set_action_in, modact, length, 0);
323 if (offset == offsetof(struct ipv6hdr, saddr))
324 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
325 else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
326 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
327 else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
328 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
329 else if (offset == offsetof(struct ipv6hdr, saddr) + 12)
330 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
331 else if (offset == offsetof(struct ipv6hdr, daddr))
332 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
333 else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
334 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
335 else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
336 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
337 else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
338 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
339 else
340 return -EOPNOTSUPP;
341 break;
342
343 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
344 MLX5_SET(set_action_in, modact, length, 16);
345 if (offset == offsetof(struct tcphdr, source))
346 field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT;
347 else if (offset == offsetof(struct tcphdr, dest))
348 field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT;
349 else
350 return -EOPNOTSUPP;
351 break;
352
353 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
354 MLX5_SET(set_action_in, modact, length, 16);
355 if (offset == offsetof(struct udphdr, source))
356 field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT;
357 else if (offset == offsetof(struct udphdr, dest))
358 field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT;
359 else
360 return -EOPNOTSUPP;
361 break;
362
363 default:
364 return -EOPNOTSUPP;
365 }
366
367 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
368 MLX5_SET(set_action_in, modact, offset, 0);
369 MLX5_SET(set_action_in, modact, field, field);
370 MLX5_SET(set_action_in, modact, data, act->mangle.val);
371
372 return 0;
373}
374
375static int
376mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv,
377 struct flow_rule *flow_rule,
378 struct mlx5e_tc_mod_hdr_acts *mod_acts)
379{
380 struct flow_action *flow_action = &flow_rule->action;
381 struct mlx5_core_dev *mdev = ct_priv->esw->dev;
382 struct flow_action_entry *act;
383 size_t action_size;
384 char *modact;
385 int err, i;
386
387 action_size = MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto);
388
389 flow_action_for_each(i, act, flow_action) {
390 switch (act->id) {
391 case FLOW_ACTION_MANGLE: {
392 err = alloc_mod_hdr_actions(mdev,
393 MLX5_FLOW_NAMESPACE_FDB,
394 mod_acts);
395 if (err)
396 return err;
397
398 modact = mod_acts->actions +
399 mod_acts->num_actions * action_size;
400
401 err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact);
402 if (err)
403 return err;
404
405 mod_acts->num_actions++;
406 }
407 break;
408
409 case FLOW_ACTION_CT_METADATA:
410 /* Handled earlier */
411 continue;
412 default:
413 return -EOPNOTSUPP;
414 }
415 }
416
417 return 0;
418}
419
420static int
421mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
422 struct mlx5_esw_flow_attr *attr,
423 struct flow_rule *flow_rule,
5c6b9460 424 u32 tupleid,
ac991b48
PB		 425 bool nat)
426{
427 struct mlx5e_tc_mod_hdr_acts mod_acts = {};
428 struct mlx5_eswitch *esw = ct_priv->esw;
429 struct mlx5_modify_hdr *mod_hdr;
430 struct flow_action_entry *meta;
431 int err;
432
433 meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
434 if (!meta)
435 return -EOPNOTSUPP;
436
437 if (meta->ct_metadata.labels[1] ||
438 meta->ct_metadata.labels[2] ||
439 meta->ct_metadata.labels[3]) {
440 ct_dbg("Failed to offload ct entry due to unsupported label");
441 return -EOPNOTSUPP;
442 }
443
444 if (nat) {
445 err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule,
446 &mod_acts);
447 if (err)
448 goto err_mapping;
449 }
450
451 err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
452 (MLX5_CT_STATE_ESTABLISHED_BIT |
453 MLX5_CT_STATE_TRK_BIT),
454 meta->ct_metadata.mark,
5c6b9460
PB		 455 meta->ct_metadata.labels[0],
456 tupleid);
ac991b48
PB		 457 if (err)
458 goto err_mapping;
459
460 mod_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB,
461 mod_acts.num_actions,
462 mod_acts.actions);
463 if (IS_ERR(mod_hdr)) {
464 err = PTR_ERR(mod_hdr);
465 goto err_mapping;
466 }
467 attr->modify_hdr = mod_hdr;
468
469 dealloc_mod_hdr_actions(&mod_acts);
470 return 0;
471
472err_mapping:
473 dealloc_mod_hdr_actions(&mod_acts);
474 return err;
475}
476
477static int
478mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
479 struct flow_rule *flow_rule,
480 struct mlx5_ct_entry *entry,
481 bool nat)
482{
483 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
484 struct mlx5_esw_flow_attr *attr = &zone_rule->attr;
485 struct mlx5_eswitch *esw = ct_priv->esw;
aded104d 486 struct mlx5_flow_spec *spec = NULL;
70840b66 487 u32 tupleid;
ac991b48
PB		 488 int err;
489
490 zone_rule->nat = nat;
491
aded104d
SM		 492 spec = kzalloc(sizeof(*spec), GFP_KERNEL);
493 if (!spec)
494 return -ENOMEM;
495
5c6b9460 496 /* Get tuple unique id */
70840b66
PB		 497 err = xa_alloc(&ct_priv->tuple_ids, &tupleid, zone_rule,
498 XA_LIMIT(1, TUPLE_ID_MAX), GFP_KERNEL);
ac991b48 499 if (err) {
5c6b9460
PB		 500 netdev_warn(ct_priv->netdev,
501 "Failed to allocate tuple id, err: %d\n", err);
70840b66 502 goto err_xa_alloc;
ac991b48 503 }
5c6b9460
PB		 504 zone_rule->tupleid = tupleid;
505
506 err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule,
507 tupleid, nat);
508 if (err) {
509 ct_dbg("Failed to create ct entry mod hdr");
510 goto err_mod_hdr;
511 }
ac991b48
PB		 512
513 attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
514 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
515 MLX5_FLOW_CONTEXT_ACTION_COUNT;
516 attr->dest_chain = 0;
517 attr->dest_ft = ct_priv->post_ct;
518 attr->fdb = nat ? ct_priv->ct_nat : ct_priv->ct;
519 attr->outer_match_level = MLX5_MATCH_L4;
520 attr->counter = entry->counter;
521 attr->flags |= MLX5_ESW_ATTR_FLAG_NO_IN_PORT;
522
aded104d
SM		 523 mlx5_tc_ct_set_tuple_match(spec, flow_rule);
524 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
ac991b48
PB		 525 entry->zone & MLX5_CT_ZONE_MASK,
526 MLX5_CT_ZONE_MASK);
527
aded104d 528 zone_rule->rule = mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
ac991b48
PB		 529 if (IS_ERR(zone_rule->rule)) {
530 err = PTR_ERR(zone_rule->rule);
531 ct_dbg("Failed to add ct entry rule, nat: %d", nat);
532 goto err_rule;
533 }
534
aded104d 535 kfree(spec);
ac991b48
PB		 536 ct_dbg("Offloaded ct entry rule in zone %d", entry->zone);
537
538 return 0;
539
540err_rule:
541 mlx5_modify_header_dealloc(esw->dev, attr->modify_hdr);
5c6b9460 542err_mod_hdr:
70840b66
PB		 543 xa_erase(&ct_priv->tuple_ids, zone_rule->tupleid);
544err_xa_alloc:
aded104d 545 kfree(spec);
ac991b48
PB		 546 return err;
547}
548
549static int
550mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
551 struct flow_rule *flow_rule,
552 struct mlx5_ct_entry *entry)
553{
554 struct mlx5_eswitch *esw = ct_priv->esw;
555 int err;
556
557 entry->counter = mlx5_fc_create(esw->dev, true);
558 if (IS_ERR(entry->counter)) {
559 err = PTR_ERR(entry->counter);
560 ct_dbg("Failed to create counter for ct entry");
561 return err;
562 }
563
564 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false);
565 if (err)
566 goto err_orig;
567
568 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true);
569 if (err)
570 goto err_nat;
571
572 return 0;
573
574err_nat:
575 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
576err_orig:
577 mlx5_fc_destroy(esw->dev, entry->counter);
578 return err;
579}
580
581static int
582mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
583 struct flow_cls_offload *flow)
584{
585 struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow);
586 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
587 struct flow_action_entry *meta_action;
588 unsigned long cookie = flow->cookie;
589 struct mlx5_ct_entry *entry;
590 int err;
591
592 meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
593 if (!meta_action)
594 return -EOPNOTSUPP;
595
596 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
597 cts_ht_params);
598 if (entry)
599 return 0;
600
601 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
602 if (!entry)
603 return -ENOMEM;
604
605 entry->zone = ft->zone;
606 entry->flow_rule = flow_rule;
607 entry->cookie = flow->cookie;
5c6b9460 608 entry->restore_cookie = meta_action->ct_metadata.cookie;
ac991b48
PB		 609
610 err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry);
611 if (err)
612 goto err_rules;
613
614 err = rhashtable_insert_fast(&ft->ct_entries_ht, &entry->node,
615 cts_ht_params);
616 if (err)
617 goto err_insert;
618
ac991b48
PB		 619 return 0;
620
621err_insert:
622 mlx5_tc_ct_entry_del_rules(ct_priv, entry);
623err_rules:
624 kfree(entry);
625 netdev_warn(ct_priv->netdev,
626 "Failed to offload ct entry, err: %d\n", err);
627 return err;
628}
629
630static int
631mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft,
632 struct flow_cls_offload *flow)
633{
634 unsigned long cookie = flow->cookie;
635 struct mlx5_ct_entry *entry;
636
637 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
638 cts_ht_params);
639 if (!entry)
640 return -ENOENT;
641
642 mlx5_tc_ct_entry_del_rules(ft->ct_priv, entry);
643 WARN_ON(rhashtable_remove_fast(&ft->ct_entries_ht,
644 &entry->node,
645 cts_ht_params));
ac991b48
PB		 646 kfree(entry);
647
648 return 0;
649}
650
651static int
652mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft,
653 struct flow_cls_offload *f)
654{
655 unsigned long cookie = f->cookie;
656 struct mlx5_ct_entry *entry;
657 u64 lastuse, packets, bytes;
658
659 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie,
660 cts_ht_params);
661 if (!entry)
662 return -ENOENT;
663
664 mlx5_fc_query_cached(entry->counter, &bytes, &packets, &lastuse);
93a129eb
JP		 665 flow_stats_update(&f->stats, bytes, packets, lastuse,
666 FLOW_ACTION_HW_STATS_DELAYED);
ac991b48
PB		 667
668 return 0;
669}
670
671static int
672mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data,
673 void *cb_priv)
674{
675 struct flow_cls_offload *f = type_data;
676 struct mlx5_ct_ft *ft = cb_priv;
677
678 if (type != TC_SETUP_CLSFLOWER)
679 return -EOPNOTSUPP;
680
681 switch (f->command) {
682 case FLOW_CLS_REPLACE:
683 return mlx5_tc_ct_block_flow_offload_add(ft, f);
684 case FLOW_CLS_DESTROY:
685 return mlx5_tc_ct_block_flow_offload_del(ft, f);
686 case FLOW_CLS_STATS:
687 return mlx5_tc_ct_block_flow_offload_stats(ft, f);
688 default:
689 break;
690 };
691
692 return -EOPNOTSUPP;
693}
694
4c3844d9
PB		 695int
696mlx5_tc_ct_parse_match(struct mlx5e_priv *priv,
697 struct mlx5_flow_spec *spec,
698 struct flow_cls_offload *f,
699 struct netlink_ext_ack *extack)
700{
701 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
d37bd5e8 702 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
4c3844d9 703 struct flow_dissector_key_ct *mask, *key;
35e725e1 704 bool trk, est, untrk, unest, new;
4c3844d9
PB		 705 u32 ctstate = 0, ctstate_mask = 0;
706 u16 ct_state_on, ct_state_off;
707 u16 ct_state, ct_state_mask;
708 struct flow_match_ct match;
709
d37bd5e8 710 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT))
4c3844d9
PB		 711 return 0;
712
713 if (!ct_priv) {
714 NL_SET_ERR_MSG_MOD(extack,
715 "offload of ct matching isn't available");
716 return -EOPNOTSUPP;
717 }
718
d37bd5e8 719 flow_rule_match_ct(rule, &match);
4c3844d9
PB		 720
721 key = match.key;
722 mask = match.mask;
723
724 ct_state = key->ct_state;
725 ct_state_mask = mask->ct_state;
726
727 if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
728 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
729 TCA_FLOWER_KEY_CT_FLAGS_NEW)) {
730 NL_SET_ERR_MSG_MOD(extack,
731 "only ct_state trk, est and new are supported for offload");
732 return -EOPNOTSUPP;
733 }
734
735 if (mask->ct_labels[1] || mask->ct_labels[2] || mask->ct_labels[3]) {
736 NL_SET_ERR_MSG_MOD(extack,
737 "only lower 32bits of ct_labels are supported for offload");
738 return -EOPNOTSUPP;
739 }
740
741 ct_state_on = ct_state & ct_state_mask;
742 ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask;
743 trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
744 new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW;
745 est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
746 untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
4c3844d9
PB		 747 unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
748
749 ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
750 ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
751 ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
752 ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
753
754 if (new) {
755 NL_SET_ERR_MSG_MOD(extack,
756 "matching on ct_state +new isn't supported");
757 return -EOPNOTSUPP;
758 }
759
760 if (mask->ct_zone)
761 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
762 key->ct_zone, MLX5_CT_ZONE_MASK);
763 if (ctstate_mask)
764 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
765 ctstate, ctstate_mask);
766 if (mask->ct_mark)
767 mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG,
768 key->ct_mark, mask->ct_mark);
769 if (mask->ct_labels[0])
770 mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG,
771 key->ct_labels[0],
772 mask->ct_labels[0]);
773
774 return 0;
775}
776
777int
778mlx5_tc_ct_parse_action(struct mlx5e_priv *priv,
779 struct mlx5_esw_flow_attr *attr,
780 const struct flow_action_entry *act,
781 struct netlink_ext_ack *extack)
782{
783 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
784
785 if (!ct_priv) {
786 NL_SET_ERR_MSG_MOD(extack,
787 "offload of ct action isn't available");
788 return -EOPNOTSUPP;
789 }
790
791 attr->ct_attr.zone = act->ct.zone;
792 attr->ct_attr.ct_action = act->ct.action;
ac991b48 793 attr->ct_attr.nf_ft = act->ct.flow_table;
4c3844d9
PB		 794
795 return 0;
796}
797
ac991b48
PB		 798static struct mlx5_ct_ft *
799mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone,
800 struct nf_flowtable *nf_ft)
801{
802 struct mlx5_ct_ft *ft;
803 int err;
804
805 ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params);
806 if (ft) {
807 refcount_inc(&ft->refcount);
808 return ft;
809 }
810
811 ft = kzalloc(sizeof(*ft), GFP_KERNEL);
812 if (!ft)
813 return ERR_PTR(-ENOMEM);
814
815 ft->zone = zone;
816 ft->nf_ft = nf_ft;
817 ft->ct_priv = ct_priv;
ac991b48
PB		 818 refcount_set(&ft->refcount, 1);
819
820 err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params);
821 if (err)
822 goto err_init;
823
824 err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node,
825 zone_params);
826 if (err)
827 goto err_insert;
828
829 err = nf_flow_table_offload_add_cb(ft->nf_ft,
830 mlx5_tc_ct_block_flow_offload, ft);
831 if (err)
832 goto err_add_cb;
833
834 return ft;
835
836err_add_cb:
837 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
838err_insert:
839 rhashtable_destroy(&ft->ct_entries_ht);
840err_init:
841 kfree(ft);
842 return ERR_PTR(err);
843}
844
845static void
9808dd0a 846mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg)
ac991b48 847{
9808dd0a
PB		 848 struct mlx5_tc_ct_priv *ct_priv = arg;
849 struct mlx5_ct_entry *entry = ptr;
ac991b48 850
9808dd0a 851 mlx5_tc_ct_entry_del_rules(ct_priv, entry);
ac991b48
PB		 852}
853
854static void
855mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
856{
857 if (!refcount_dec_and_test(&ft->refcount))
858 return;
859
860 nf_flow_table_offload_del_cb(ft->nf_ft,
861 mlx5_tc_ct_block_flow_offload, ft);
ac991b48 862 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
9808dd0a
PB		 863 rhashtable_free_and_destroy(&ft->ct_entries_ht,
864 mlx5_tc_ct_flush_ft_entry,
865 ct_priv);
ac991b48
PB		 866 kfree(ft);
867}
868
4c3844d9
PB		 869/* We translate the tc filter with CT action to the following HW model:
870 *
871 * +-------------------+ +--------------------+ +--------------+
872 * + pre_ct (tc chain) +----->+ CT (nat or no nat) +--->+ post_ct +----->
873 * + original match + | + tuple + zone match + | + fte_id match + |
874 * +-------------------+ | +--------------------+ | +--------------+ |
875 * v v v
876 * set chain miss mapping set mark original
877 * set fte_id set label filter
878 * set zone set established actions
879 * set tunnel_id do nat (if needed)
880 * do decap
881 */
882static int
883__mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv,
884 struct mlx5e_tc_flow *flow,
885 struct mlx5_flow_spec *orig_spec,
886 struct mlx5_esw_flow_attr *attr,
887 struct mlx5_flow_handle **flow_rule)
888{
889 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
890 bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
891 struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
aded104d 892 struct mlx5_flow_spec *post_ct_spec = NULL;
4c3844d9 893 struct mlx5_eswitch *esw = ct_priv->esw;
4c3844d9
PB		 894 struct mlx5_esw_flow_attr *pre_ct_attr;
895 struct mlx5_modify_hdr *mod_hdr;
896 struct mlx5_flow_handle *rule;
897 struct mlx5_ct_flow *ct_flow;
898 int chain_mapping = 0, err;
ac991b48 899 struct mlx5_ct_ft *ft;
4c3844d9
PB		 900 u32 fte_id = 1;
901
aded104d 902 post_ct_spec = kzalloc(sizeof(*post_ct_spec), GFP_KERNEL);
4c3844d9 903 ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
aded104d
SM		 904 if (!post_ct_spec || !ct_flow) {
905 kfree(post_ct_spec);
906 kfree(ct_flow);
4c3844d9 907 return -ENOMEM;
aded104d 908 }
4c3844d9 909
ac991b48
PB		 910 /* Register for CT established events */
911 ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
912 attr->ct_attr.nf_ft);
913 if (IS_ERR(ft)) {
914 err = PTR_ERR(ft);
915 ct_dbg("Failed to register to ft callback");
916 goto err_ft;
917 }
918 ct_flow->ft = ft;
919
4c3844d9
PB		 920 err = idr_alloc_u32(&ct_priv->fte_ids, ct_flow, &fte_id,
921 MLX5_FTE_ID_MAX, GFP_KERNEL);
922 if (err) {
923 netdev_warn(priv->netdev,
924 "Failed to allocate fte id, err: %d\n", err);
925 goto err_idr;
926 }
927 ct_flow->fte_id = fte_id;
928
929 /* Base esw attributes of both rules on original rule attribute */
930 pre_ct_attr = &ct_flow->pre_ct_attr;
931 memcpy(pre_ct_attr, attr, sizeof(*attr));
932 memcpy(&ct_flow->post_ct_attr, attr, sizeof(*attr));
933
934 /* Modify the original rule's action to fwd and modify, leave decap */
935 pre_ct_attr->action = attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP;
936 pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
937 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
938
939 /* Write chain miss tag for miss in ct table as we
940 * don't go though all prios of this chain as normal tc rules
941 * miss.
942 */
943 err = mlx5_esw_chains_get_chain_mapping(esw, attr->chain,
944 &chain_mapping);
945 if (err) {
946 ct_dbg("Failed to get chain register mapping for chain");
947 goto err_get_chain;
948 }
949 ct_flow->chain_mapping = chain_mapping;
950
951 err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
952 CHAIN_TO_REG, chain_mapping);
953 if (err) {
954 ct_dbg("Failed to set chain register mapping");
955 goto err_mapping;
956 }
957
958 err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts, ZONE_TO_REG,
959 attr->ct_attr.zone &
960 MLX5_CT_ZONE_MASK);
961 if (err) {
962 ct_dbg("Failed to set zone register mapping");
963 goto err_mapping;
964 }
965
966 err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
967 FTEID_TO_REG, fte_id);
968 if (err) {
969 ct_dbg("Failed to set fte_id register mapping");
970 goto err_mapping;
971 }
972
973 /* If original flow is decap, we do it before going into ct table
974 * so add a rewrite for the tunnel match_id.
975 */
976 if ((pre_ct_attr->action & MLX5_FLOW_CONTEXT_ACTION_DECAP) &&
977 attr->chain == 0) {
978 u32 tun_id = mlx5e_tc_get_flow_tun_id(flow);
979
980 err = mlx5e_tc_match_to_reg_set(esw->dev, &pre_mod_acts,
981 TUNNEL_TO_REG,
982 tun_id);
983 if (err) {
984 ct_dbg("Failed to set tunnel register mapping");
985 goto err_mapping;
986 }
987 }
988
989 mod_hdr = mlx5_modify_header_alloc(esw->dev,
990 MLX5_FLOW_NAMESPACE_FDB,
991 pre_mod_acts.num_actions,
992 pre_mod_acts.actions);
993 if (IS_ERR(mod_hdr)) {
994 err = PTR_ERR(mod_hdr);
995 ct_dbg("Failed to create pre ct mod hdr");
996 goto err_mapping;
997 }
998 pre_ct_attr->modify_hdr = mod_hdr;
999
1000 /* Post ct rule matches on fte_id and executes original rule's
1001 * tc rule action
1002 */
aded104d 1003 mlx5e_tc_match_to_reg_match(post_ct_spec, FTEID_TO_REG,
4c3844d9
PB		 1004 fte_id, MLX5_FTE_ID_MASK);
1005
1006 /* Put post_ct rule on post_ct fdb */
1007 ct_flow->post_ct_attr.chain = 0;
1008 ct_flow->post_ct_attr.prio = 0;
1009 ct_flow->post_ct_attr.fdb = ct_priv->post_ct;
1010
1011 ct_flow->post_ct_attr.inner_match_level = MLX5_MATCH_NONE;
1012 ct_flow->post_ct_attr.outer_match_level = MLX5_MATCH_NONE;
1013 ct_flow->post_ct_attr.action &= ~(MLX5_FLOW_CONTEXT_ACTION_DECAP);
aded104d 1014 rule = mlx5_eswitch_add_offloaded_rule(esw, post_ct_spec,
4c3844d9
PB		 1015 &ct_flow->post_ct_attr);
1016 ct_flow->post_ct_rule = rule;
1017 if (IS_ERR(ct_flow->post_ct_rule)) {
1018 err = PTR_ERR(ct_flow->post_ct_rule);
1019 ct_dbg("Failed to add post ct rule");
1020 goto err_insert_post_ct;
1021 }
1022
1023 /* Change original rule point to ct table */
1024 pre_ct_attr->dest_chain = 0;
1025 pre_ct_attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct;
1026 ct_flow->pre_ct_rule = mlx5_eswitch_add_offloaded_rule(esw,
1027 orig_spec,
1028 pre_ct_attr);
1029 if (IS_ERR(ct_flow->pre_ct_rule)) {
1030 err = PTR_ERR(ct_flow->pre_ct_rule);
1031 ct_dbg("Failed to add pre ct rule");
1032 goto err_insert_orig;
1033 }
1034
1035 attr->ct_attr.ct_flow = ct_flow;
1036 *flow_rule = ct_flow->post_ct_rule;
1037 dealloc_mod_hdr_actions(&pre_mod_acts);
aded104d 1038 kfree(post_ct_spec);
4c3844d9
PB		 1039
1040 return 0;
1041
1042err_insert_orig:
1043 mlx5_eswitch_del_offloaded_rule(ct_priv->esw, ct_flow->post_ct_rule,
1044 &ct_flow->post_ct_attr);
1045err_insert_post_ct:
1046 mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr);
1047err_mapping:
1048 dealloc_mod_hdr_actions(&pre_mod_acts);
1049 mlx5_esw_chains_put_chain_mapping(esw, ct_flow->chain_mapping);
1050err_get_chain:
1051 idr_remove(&ct_priv->fte_ids, fte_id);
1052err_idr:
ac991b48
PB		 1053 mlx5_tc_ct_del_ft_cb(ct_priv, ft);
1054err_ft:
aded104d 1055 kfree(post_ct_spec);
4c3844d9
PB		 1056 kfree(ct_flow);
1057 netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
1058 return err;
1059}
1060
1ef3018f
PB		 1061static int
1062__mlx5_tc_ct_flow_offload_clear(struct mlx5e_priv *priv,
1063 struct mlx5e_tc_flow *flow,
1064 struct mlx5_flow_spec *orig_spec,
1065 struct mlx5_esw_flow_attr *attr,
1066 struct mlx5e_tc_mod_hdr_acts *mod_acts,
1067 struct mlx5_flow_handle **flow_rule)
1068{
1069 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1070 struct mlx5_eswitch *esw = ct_priv->esw;
1071 struct mlx5_esw_flow_attr *pre_ct_attr;
1072 struct mlx5_modify_hdr *mod_hdr;
1073 struct mlx5_flow_handle *rule;
1074 struct mlx5_ct_flow *ct_flow;
1075 int err;
1076
1077 ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL);
1078 if (!ct_flow)
1079 return -ENOMEM;
1080
1081 /* Base esw attributes on original rule attribute */
1082 pre_ct_attr = &ct_flow->pre_ct_attr;
1083 memcpy(pre_ct_attr, attr, sizeof(*attr));
1084
1085 err = mlx5_tc_ct_entry_set_registers(ct_priv, mod_acts, 0, 0, 0, 0);
1086 if (err) {
1087 ct_dbg("Failed to set register for ct clear");
1088 goto err_set_registers;
1089 }
1090
1091 mod_hdr = mlx5_modify_header_alloc(esw->dev,
1092 MLX5_FLOW_NAMESPACE_FDB,
1093 mod_acts->num_actions,
1094 mod_acts->actions);
1095 if (IS_ERR(mod_hdr)) {
1096 err = PTR_ERR(mod_hdr);
1097 ct_dbg("Failed to add create ct clear mod hdr");
1098 goto err_set_registers;
1099 }
1100
1101 dealloc_mod_hdr_actions(mod_acts);
1102 pre_ct_attr->modify_hdr = mod_hdr;
1103 pre_ct_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1104
1105 rule = mlx5_eswitch_add_offloaded_rule(esw, orig_spec, pre_ct_attr);
1106 if (IS_ERR(rule)) {
1107 err = PTR_ERR(rule);
1108 ct_dbg("Failed to add ct clear rule");
1109 goto err_insert;
1110 }
1111
1112 attr->ct_attr.ct_flow = ct_flow;
1113 ct_flow->pre_ct_rule = rule;
1114 *flow_rule = rule;
1115
1116 return 0;
1117
1118err_insert:
1119 mlx5_modify_header_dealloc(priv->mdev, mod_hdr);
1120err_set_registers:
1121 netdev_warn(priv->netdev,
1122 "Failed to offload ct clear flow, err %d\n", err);
1123 return err;
1124}
1125
4c3844d9
PB		 1126struct mlx5_flow_handle *
1127mlx5_tc_ct_flow_offload(struct mlx5e_priv *priv,
1128 struct mlx5e_tc_flow *flow,
1129 struct mlx5_flow_spec *spec,
1ef3018f
PB		 1130 struct mlx5_esw_flow_attr *attr,
1131 struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts)
4c3844d9 1132{
1ef3018f 1133 bool clear_action = attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR;
4c3844d9
PB		 1134 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1135 struct mlx5_flow_handle *rule;
1136 int err;
1137
1138 if (!ct_priv)
1139 return ERR_PTR(-EOPNOTSUPP);
1140
1141 mutex_lock(&ct_priv->control_lock);
1ef3018f
PB		 1142 if (clear_action)
1143 err = __mlx5_tc_ct_flow_offload_clear(priv, flow, spec, attr,
1144 mod_hdr_acts, &rule);
1145 else
1146 err = __mlx5_tc_ct_flow_offload(priv, flow, spec, attr,
1147 &rule);
4c3844d9
PB		 1148 mutex_unlock(&ct_priv->control_lock);
1149 if (err)
1150 return ERR_PTR(err);
1151
1152 return rule;
1153}
1154
1155static void
1156__mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
1157 struct mlx5_ct_flow *ct_flow)
1158{
1159 struct mlx5_esw_flow_attr *pre_ct_attr = &ct_flow->pre_ct_attr;
1160 struct mlx5_eswitch *esw = ct_priv->esw;
1161
1162 mlx5_eswitch_del_offloaded_rule(esw, ct_flow->pre_ct_rule,
1163 pre_ct_attr);
1164 mlx5_modify_header_dealloc(esw->dev, pre_ct_attr->modify_hdr);
1ef3018f
PB		 1165
1166 if (ct_flow->post_ct_rule) {
1167 mlx5_eswitch_del_offloaded_rule(esw, ct_flow->post_ct_rule,
1168 &ct_flow->post_ct_attr);
1169 mlx5_esw_chains_put_chain_mapping(esw, ct_flow->chain_mapping);
1170 idr_remove(&ct_priv->fte_ids, ct_flow->fte_id);
1171 mlx5_tc_ct_del_ft_cb(ct_priv, ct_flow->ft);
1172 }
1173
4c3844d9
PB		 1174 kfree(ct_flow);
1175}
1176
1177void
1178mlx5_tc_ct_delete_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow,
1179 struct mlx5_esw_flow_attr *attr)
1180{
1181 struct mlx5_tc_ct_priv *ct_priv = mlx5_tc_ct_get_ct_priv(priv);
1182 struct mlx5_ct_flow *ct_flow = attr->ct_attr.ct_flow;
1183
1184 /* We are called on error to clean up stuff from parsing
1185 * but we don't have anything for now
1186 */
1187 if (!ct_flow)
1188 return;
1189
1190 mutex_lock(&ct_priv->control_lock);
1191 __mlx5_tc_ct_delete_flow(ct_priv, ct_flow);
1192 mutex_unlock(&ct_priv->control_lock);
1193}
1194
1195static int
1196mlx5_tc_ct_init_check_support(struct mlx5_eswitch *esw,
1197 const char **err_msg)
1198{
1199#if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
1200 /* cannot restore chain ID on HW miss */
1201
1202 *err_msg = "tc skb extension missing";
1203 return -EOPNOTSUPP;
1204#endif
1205
1206 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) {
1207 *err_msg = "firmware level support is missing";
1208 return -EOPNOTSUPP;
1209 }
1210
1211 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) {
1212 /* vlan workaround should be avoided for multi chain rules.
1213 * This is just a sanity check as pop vlan action should
1214 * be supported by any FW that supports ignore_flow_level
1215 */
1216
1217 *err_msg = "firmware vlan actions support is missing";
1218 return -EOPNOTSUPP;
1219 }
1220
1221 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev,
1222 fdb_modify_header_fwd_to_table)) {
1223 /* CT always writes to registers which are mod header actions.
1224 * Therefore, mod header and goto is required
1225 */
1226
1227 *err_msg = "firmware fwd and modify support is missing";
1228 return -EOPNOTSUPP;
1229 }
1230
1231 if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1232 *err_msg = "register loopback isn't supported";
1233 return -EOPNOTSUPP;
1234 }
1235
1236 return 0;
1237}
1238
1239static void
1240mlx5_tc_ct_init_err(struct mlx5e_rep_priv *rpriv, const char *msg, int err)
1241{
1242 if (msg)
1243 netdev_warn(rpriv->netdev,
1244 "tc ct offload not supported, %s, err: %d\n",
1245 msg, err);
1246 else
1247 netdev_warn(rpriv->netdev,
1248 "tc ct offload not supported, err: %d\n",
1249 err);
1250}
1251
1252int
1253mlx5_tc_ct_init(struct mlx5_rep_uplink_priv *uplink_priv)
1254{
1255 struct mlx5_tc_ct_priv *ct_priv;
1256 struct mlx5e_rep_priv *rpriv;
1257 struct mlx5_eswitch *esw;
1258 struct mlx5e_priv *priv;
1259 const char *msg;
1260 int err;
1261
1262 rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv);
1263 priv = netdev_priv(rpriv->netdev);
1264 esw = priv->mdev->priv.eswitch;
1265
1266 err = mlx5_tc_ct_init_check_support(esw, &msg);
1267 if (err) {
1268 mlx5_tc_ct_init_err(rpriv, msg, err);
1269 goto err_support;
1270 }
1271
1272 ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
1273 if (!ct_priv) {
1274 mlx5_tc_ct_init_err(rpriv, NULL, -ENOMEM);
1275 goto err_alloc;
1276 }
1277
1278 ct_priv->esw = esw;
1279 ct_priv->netdev = rpriv->netdev;
1280 ct_priv->ct = mlx5_esw_chains_create_global_table(esw);
1281 if (IS_ERR(ct_priv->ct)) {
1282 err = PTR_ERR(ct_priv->ct);
1283 mlx5_tc_ct_init_err(rpriv, "failed to create ct table", err);
1284 goto err_ct_tbl;
1285 }
1286
1287 ct_priv->ct_nat = mlx5_esw_chains_create_global_table(esw);
1288 if (IS_ERR(ct_priv->ct_nat)) {
1289 err = PTR_ERR(ct_priv->ct_nat);
1290 mlx5_tc_ct_init_err(rpriv, "failed to create ct nat table",
1291 err);
1292 goto err_ct_nat_tbl;
1293 }
1294
1295 ct_priv->post_ct = mlx5_esw_chains_create_global_table(esw);
1296 if (IS_ERR(ct_priv->post_ct)) {
1297 err = PTR_ERR(ct_priv->post_ct);
1298 mlx5_tc_ct_init_err(rpriv, "failed to create post ct table",
1299 err);
1300 goto err_post_ct_tbl;
1301 }
1302
1303 idr_init(&ct_priv->fte_ids);
70840b66 1304 xa_init_flags(&ct_priv->tuple_ids, XA_FLAGS_ALLOC1);
4c3844d9 1305 mutex_init(&ct_priv->control_lock);
ac991b48 1306 rhashtable_init(&ct_priv->zone_ht, &zone_params);
4c3844d9
PB		 1307
1308 /* Done, set ct_priv to know it initializted */
1309 uplink_priv->ct_priv = ct_priv;
1310
1311 return 0;
1312
1313err_post_ct_tbl:
1314 mlx5_esw_chains_destroy_global_table(esw, ct_priv->ct_nat);
1315err_ct_nat_tbl:
1316 mlx5_esw_chains_destroy_global_table(esw, ct_priv->ct);
1317err_ct_tbl:
1318 kfree(ct_priv);
1319err_alloc:
1320err_support:
1321
1322 return 0;
1323}
1324
1325void
1326mlx5_tc_ct_clean(struct mlx5_rep_uplink_priv *uplink_priv)
1327{
1328 struct mlx5_tc_ct_priv *ct_priv = uplink_priv->ct_priv;
1329
1330 if (!ct_priv)
1331 return;
1332
1333 mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->post_ct);
1334 mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct_nat);
1335 mlx5_esw_chains_destroy_global_table(ct_priv->esw, ct_priv->ct);
1336
ac991b48 1337 rhashtable_destroy(&ct_priv->zone_ht);
4c3844d9 1338 mutex_destroy(&ct_priv->control_lock);
70840b66 1339 xa_destroy(&ct_priv->tuple_ids);
4c3844d9
PB		 1340 idr_destroy(&ct_priv->fte_ids);
1341 kfree(ct_priv);
1342
1343 uplink_priv->ct_priv = NULL;
1344}
5c6b9460
PB		 1345
1346bool
1347mlx5e_tc_ct_restore_flow(struct mlx5_rep_uplink_priv *uplink_priv,
1348 struct sk_buff *skb, u32 tupleid)
1349{
1350 struct mlx5_tc_ct_priv *ct_priv = uplink_priv->ct_priv;
1351 struct mlx5_ct_zone_rule *zone_rule;
1352 struct mlx5_ct_entry *entry;
1353
1354 if (!ct_priv || !tupleid)
1355 return true;
1356
70840b66 1357 zone_rule = xa_load(&ct_priv->tuple_ids, tupleid);
5c6b9460
PB		 1358 if (!zone_rule)
1359 return false;
1360
1361 entry = container_of(zone_rule, struct mlx5_ct_entry,
1362 zone_rules[zone_rule->nat]);
1363 tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie);
1364
1365 return true;
1366}
A mirror of Linus' kernel repository