1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2008, Intel Corporation.
5 * Author: Alexander Duyck <alexander.h.duyck@intel.com>
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/skbuff.h>
12 #include <linux/rtnetlink.h>
13 #include <net/netlink.h>
14 #include <net/pkt_sched.h>
17 #include <net/dsfield.h>
18 #include <net/pkt_cls.h>
20 #include <linux/tc_act/tc_skbedit.h>
21 #include <net/tc_act/tc_skbedit.h>
23 static unsigned int skbedit_net_id
;
24 static struct tc_action_ops act_skbedit_ops
;
26 static u16
tcf_skbedit_hash(struct tcf_skbedit_params
*params
,
29 u16 queue_mapping
= params
->queue_mapping
;
31 if (params
->flags
& SKBEDIT_F_TXQ_SKBHASH
) {
32 u32 hash
= skb_get_hash(skb
);
34 queue_mapping
+= hash
% params
->mapping_mod
;
37 return netdev_cap_txqueue(skb
->dev
, queue_mapping
);
40 static int tcf_skbedit_act(struct sk_buff
*skb
, const struct tc_action
*a
,
41 struct tcf_result
*res
)
43 struct tcf_skbedit
*d
= to_skbedit(a
);
44 struct tcf_skbedit_params
*params
;
47 tcf_lastuse_update(&d
->tcf_tm
);
48 bstats_update(this_cpu_ptr(d
->common
.cpu_bstats
), skb
);
50 params
= rcu_dereference_bh(d
->params
);
51 action
= READ_ONCE(d
->tcf_action
);
53 if (params
->flags
& SKBEDIT_F_PRIORITY
)
54 skb
->priority
= params
->priority
;
55 if (params
->flags
& SKBEDIT_F_INHERITDSFIELD
) {
56 int wlen
= skb_network_offset(skb
);
58 switch (skb_protocol(skb
, true)) {
60 wlen
+= sizeof(struct iphdr
);
61 if (!pskb_may_pull(skb
, wlen
))
63 skb
->priority
= ipv4_get_dsfield(ip_hdr(skb
)) >> 2;
66 case htons(ETH_P_IPV6
):
67 wlen
+= sizeof(struct ipv6hdr
);
68 if (!pskb_may_pull(skb
, wlen
))
70 skb
->priority
= ipv6_get_dsfield(ipv6_hdr(skb
)) >> 2;
74 if (params
->flags
& SKBEDIT_F_QUEUE_MAPPING
&&
75 skb
->dev
->real_num_tx_queues
> params
->queue_mapping
) {
76 #ifdef CONFIG_NET_EGRESS
77 netdev_xmit_skip_txqueue(true);
79 skb_set_queue_mapping(skb
, tcf_skbedit_hash(params
, skb
));
81 if (params
->flags
& SKBEDIT_F_MARK
) {
82 skb
->mark
&= ~params
->mask
;
83 skb
->mark
|= params
->mark
& params
->mask
;
85 if (params
->flags
& SKBEDIT_F_PTYPE
)
86 skb
->pkt_type
= params
->ptype
;
90 qstats_drop_inc(this_cpu_ptr(d
->common
.cpu_qstats
));
94 static void tcf_skbedit_stats_update(struct tc_action
*a
, u64 bytes
,
95 u64 packets
, u64 drops
,
98 struct tcf_skbedit
*d
= to_skbedit(a
);
99 struct tcf_t
*tm
= &d
->tcf_tm
;
101 tcf_action_update_stats(a
, bytes
, packets
, drops
, hw
);
102 tm
->lastuse
= max_t(u64
, tm
->lastuse
, lastuse
);
105 static const struct nla_policy skbedit_policy
[TCA_SKBEDIT_MAX
+ 1] = {
106 [TCA_SKBEDIT_PARMS
] = { .len
= sizeof(struct tc_skbedit
) },
107 [TCA_SKBEDIT_PRIORITY
] = { .len
= sizeof(u32
) },
108 [TCA_SKBEDIT_QUEUE_MAPPING
] = { .len
= sizeof(u16
) },
109 [TCA_SKBEDIT_MARK
] = { .len
= sizeof(u32
) },
110 [TCA_SKBEDIT_PTYPE
] = { .len
= sizeof(u16
) },
111 [TCA_SKBEDIT_MASK
] = { .len
= sizeof(u32
) },
112 [TCA_SKBEDIT_FLAGS
] = { .len
= sizeof(u64
) },
113 [TCA_SKBEDIT_QUEUE_MAPPING_MAX
] = { .len
= sizeof(u16
) },
116 static int tcf_skbedit_init(struct net
*net
, struct nlattr
*nla
,
117 struct nlattr
*est
, struct tc_action
**a
,
118 struct tcf_proto
*tp
, u32 act_flags
,
119 struct netlink_ext_ack
*extack
)
121 struct tc_action_net
*tn
= net_generic(net
, skbedit_net_id
);
122 bool bind
= act_flags
& TCA_ACT_FLAGS_BIND
;
123 struct tcf_skbedit_params
*params_new
;
124 struct nlattr
*tb
[TCA_SKBEDIT_MAX
+ 1];
125 struct tcf_chain
*goto_ch
= NULL
;
126 struct tc_skbedit
*parm
;
127 struct tcf_skbedit
*d
;
128 u32 flags
= 0, *priority
= NULL
, *mark
= NULL
, *mask
= NULL
;
129 u16
*queue_mapping
= NULL
, *ptype
= NULL
;
138 err
= nla_parse_nested_deprecated(tb
, TCA_SKBEDIT_MAX
, nla
,
139 skbedit_policy
, NULL
);
143 if (tb
[TCA_SKBEDIT_PARMS
] == NULL
)
146 if (tb
[TCA_SKBEDIT_PRIORITY
] != NULL
) {
147 flags
|= SKBEDIT_F_PRIORITY
;
148 priority
= nla_data(tb
[TCA_SKBEDIT_PRIORITY
]);
151 if (tb
[TCA_SKBEDIT_QUEUE_MAPPING
] != NULL
) {
152 flags
|= SKBEDIT_F_QUEUE_MAPPING
;
153 queue_mapping
= nla_data(tb
[TCA_SKBEDIT_QUEUE_MAPPING
]);
156 if (tb
[TCA_SKBEDIT_PTYPE
] != NULL
) {
157 ptype
= nla_data(tb
[TCA_SKBEDIT_PTYPE
]);
158 if (!skb_pkt_type_ok(*ptype
))
160 flags
|= SKBEDIT_F_PTYPE
;
163 if (tb
[TCA_SKBEDIT_MARK
] != NULL
) {
164 flags
|= SKBEDIT_F_MARK
;
165 mark
= nla_data(tb
[TCA_SKBEDIT_MARK
]);
168 if (tb
[TCA_SKBEDIT_MASK
] != NULL
) {
169 flags
|= SKBEDIT_F_MASK
;
170 mask
= nla_data(tb
[TCA_SKBEDIT_MASK
]);
173 if (tb
[TCA_SKBEDIT_FLAGS
] != NULL
) {
174 u64
*pure_flags
= nla_data(tb
[TCA_SKBEDIT_FLAGS
]);
176 if (*pure_flags
& SKBEDIT_F_TXQ_SKBHASH
) {
177 u16
*queue_mapping_max
;
179 if (!tb
[TCA_SKBEDIT_QUEUE_MAPPING
] ||
180 !tb
[TCA_SKBEDIT_QUEUE_MAPPING_MAX
]) {
181 NL_SET_ERR_MSG_MOD(extack
, "Missing required range of queue_mapping.");
186 nla_data(tb
[TCA_SKBEDIT_QUEUE_MAPPING_MAX
]);
187 if (*queue_mapping_max
< *queue_mapping
) {
188 NL_SET_ERR_MSG_MOD(extack
, "The range of queue_mapping is invalid, max < min.");
192 mapping_mod
= *queue_mapping_max
- *queue_mapping
+ 1;
193 flags
|= SKBEDIT_F_TXQ_SKBHASH
;
195 if (*pure_flags
& SKBEDIT_F_INHERITDSFIELD
)
196 flags
|= SKBEDIT_F_INHERITDSFIELD
;
199 parm
= nla_data(tb
[TCA_SKBEDIT_PARMS
]);
201 err
= tcf_idr_check_alloc(tn
, &index
, a
, bind
);
210 tcf_idr_release(*a
, bind
);
212 tcf_idr_cleanup(tn
, index
);
217 ret
= tcf_idr_create(tn
, index
, est
, a
,
218 &act_skbedit_ops
, bind
, true, act_flags
);
220 tcf_idr_cleanup(tn
, index
);
228 if (!(act_flags
& TCA_ACT_FLAGS_REPLACE
)) {
229 tcf_idr_release(*a
, bind
);
233 err
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
237 params_new
= kzalloc(sizeof(*params_new
), GFP_KERNEL
);
238 if (unlikely(!params_new
)) {
243 params_new
->flags
= flags
;
244 if (flags
& SKBEDIT_F_PRIORITY
)
245 params_new
->priority
= *priority
;
246 if (flags
& SKBEDIT_F_QUEUE_MAPPING
) {
247 params_new
->queue_mapping
= *queue_mapping
;
248 params_new
->mapping_mod
= mapping_mod
;
250 if (flags
& SKBEDIT_F_MARK
)
251 params_new
->mark
= *mark
;
252 if (flags
& SKBEDIT_F_PTYPE
)
253 params_new
->ptype
= *ptype
;
254 /* default behaviour is to use all the bits */
255 params_new
->mask
= 0xffffffff;
256 if (flags
& SKBEDIT_F_MASK
)
257 params_new
->mask
= *mask
;
259 spin_lock_bh(&d
->tcf_lock
);
260 goto_ch
= tcf_action_set_ctrlact(*a
, parm
->action
, goto_ch
);
261 params_new
= rcu_replace_pointer(d
->params
, params_new
,
262 lockdep_is_held(&d
->tcf_lock
));
263 spin_unlock_bh(&d
->tcf_lock
);
265 kfree_rcu(params_new
, rcu
);
267 tcf_chain_put_by_act(goto_ch
);
272 tcf_chain_put_by_act(goto_ch
);
274 tcf_idr_release(*a
, bind
);
278 static int tcf_skbedit_dump(struct sk_buff
*skb
, struct tc_action
*a
,
281 unsigned char *b
= skb_tail_pointer(skb
);
282 struct tcf_skbedit
*d
= to_skbedit(a
);
283 struct tcf_skbedit_params
*params
;
284 struct tc_skbedit opt
= {
285 .index
= d
->tcf_index
,
286 .refcnt
= refcount_read(&d
->tcf_refcnt
) - ref
,
287 .bindcnt
= atomic_read(&d
->tcf_bindcnt
) - bind
,
292 spin_lock_bh(&d
->tcf_lock
);
293 params
= rcu_dereference_protected(d
->params
,
294 lockdep_is_held(&d
->tcf_lock
));
295 opt
.action
= d
->tcf_action
;
297 if (nla_put(skb
, TCA_SKBEDIT_PARMS
, sizeof(opt
), &opt
))
298 goto nla_put_failure
;
299 if ((params
->flags
& SKBEDIT_F_PRIORITY
) &&
300 nla_put_u32(skb
, TCA_SKBEDIT_PRIORITY
, params
->priority
))
301 goto nla_put_failure
;
302 if ((params
->flags
& SKBEDIT_F_QUEUE_MAPPING
) &&
303 nla_put_u16(skb
, TCA_SKBEDIT_QUEUE_MAPPING
, params
->queue_mapping
))
304 goto nla_put_failure
;
305 if ((params
->flags
& SKBEDIT_F_MARK
) &&
306 nla_put_u32(skb
, TCA_SKBEDIT_MARK
, params
->mark
))
307 goto nla_put_failure
;
308 if ((params
->flags
& SKBEDIT_F_PTYPE
) &&
309 nla_put_u16(skb
, TCA_SKBEDIT_PTYPE
, params
->ptype
))
310 goto nla_put_failure
;
311 if ((params
->flags
& SKBEDIT_F_MASK
) &&
312 nla_put_u32(skb
, TCA_SKBEDIT_MASK
, params
->mask
))
313 goto nla_put_failure
;
314 if (params
->flags
& SKBEDIT_F_INHERITDSFIELD
)
315 pure_flags
|= SKBEDIT_F_INHERITDSFIELD
;
316 if (params
->flags
& SKBEDIT_F_TXQ_SKBHASH
) {
317 if (nla_put_u16(skb
, TCA_SKBEDIT_QUEUE_MAPPING_MAX
,
318 params
->queue_mapping
+ params
->mapping_mod
- 1))
319 goto nla_put_failure
;
321 pure_flags
|= SKBEDIT_F_TXQ_SKBHASH
;
323 if (pure_flags
!= 0 &&
324 nla_put(skb
, TCA_SKBEDIT_FLAGS
, sizeof(pure_flags
), &pure_flags
))
325 goto nla_put_failure
;
327 tcf_tm_dump(&t
, &d
->tcf_tm
);
328 if (nla_put_64bit(skb
, TCA_SKBEDIT_TM
, sizeof(t
), &t
, TCA_SKBEDIT_PAD
))
329 goto nla_put_failure
;
330 spin_unlock_bh(&d
->tcf_lock
);
335 spin_unlock_bh(&d
->tcf_lock
);
340 static void tcf_skbedit_cleanup(struct tc_action
*a
)
342 struct tcf_skbedit
*d
= to_skbedit(a
);
343 struct tcf_skbedit_params
*params
;
345 params
= rcu_dereference_protected(d
->params
, 1);
347 kfree_rcu(params
, rcu
);
350 static int tcf_skbedit_walker(struct net
*net
, struct sk_buff
*skb
,
351 struct netlink_callback
*cb
, int type
,
352 const struct tc_action_ops
*ops
,
353 struct netlink_ext_ack
*extack
)
355 struct tc_action_net
*tn
= net_generic(net
, skbedit_net_id
);
357 return tcf_generic_walker(tn
, skb
, cb
, type
, ops
, extack
);
360 static int tcf_skbedit_search(struct net
*net
, struct tc_action
**a
, u32 index
)
362 struct tc_action_net
*tn
= net_generic(net
, skbedit_net_id
);
364 return tcf_idr_search(tn
, a
, index
);
367 static size_t tcf_skbedit_get_fill_size(const struct tc_action
*act
)
369 return nla_total_size(sizeof(struct tc_skbedit
))
370 + nla_total_size(sizeof(u32
)) /* TCA_SKBEDIT_PRIORITY */
371 + nla_total_size(sizeof(u16
)) /* TCA_SKBEDIT_QUEUE_MAPPING */
372 + nla_total_size(sizeof(u16
)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */
373 + nla_total_size(sizeof(u32
)) /* TCA_SKBEDIT_MARK */
374 + nla_total_size(sizeof(u16
)) /* TCA_SKBEDIT_PTYPE */
375 + nla_total_size(sizeof(u32
)) /* TCA_SKBEDIT_MASK */
376 + nla_total_size_64bit(sizeof(u64
)); /* TCA_SKBEDIT_FLAGS */
379 static int tcf_skbedit_offload_act_setup(struct tc_action
*act
, void *entry_data
,
380 u32
*index_inc
, bool bind
,
381 struct netlink_ext_ack
*extack
)
384 struct flow_action_entry
*entry
= entry_data
;
386 if (is_tcf_skbedit_mark(act
)) {
387 entry
->id
= FLOW_ACTION_MARK
;
388 entry
->mark
= tcf_skbedit_mark(act
);
389 } else if (is_tcf_skbedit_ptype(act
)) {
390 entry
->id
= FLOW_ACTION_PTYPE
;
391 entry
->ptype
= tcf_skbedit_ptype(act
);
392 } else if (is_tcf_skbedit_priority(act
)) {
393 entry
->id
= FLOW_ACTION_PRIORITY
;
394 entry
->priority
= tcf_skbedit_priority(act
);
395 } else if (is_tcf_skbedit_queue_mapping(act
)) {
396 NL_SET_ERR_MSG_MOD(extack
, "Offload not supported when \"queue_mapping\" option is used");
398 } else if (is_tcf_skbedit_inheritdsfield(act
)) {
399 NL_SET_ERR_MSG_MOD(extack
, "Offload not supported when \"inheritdsfield\" option is used");
402 NL_SET_ERR_MSG_MOD(extack
, "Unsupported skbedit option offload");
407 struct flow_offload_action
*fl_action
= entry_data
;
409 if (is_tcf_skbedit_mark(act
))
410 fl_action
->id
= FLOW_ACTION_MARK
;
411 else if (is_tcf_skbedit_ptype(act
))
412 fl_action
->id
= FLOW_ACTION_PTYPE
;
413 else if (is_tcf_skbedit_priority(act
))
414 fl_action
->id
= FLOW_ACTION_PRIORITY
;
422 static struct tc_action_ops act_skbedit_ops
= {
424 .id
= TCA_ID_SKBEDIT
,
425 .owner
= THIS_MODULE
,
426 .act
= tcf_skbedit_act
,
427 .stats_update
= tcf_skbedit_stats_update
,
428 .dump
= tcf_skbedit_dump
,
429 .init
= tcf_skbedit_init
,
430 .cleanup
= tcf_skbedit_cleanup
,
431 .walk
= tcf_skbedit_walker
,
432 .get_fill_size
= tcf_skbedit_get_fill_size
,
433 .lookup
= tcf_skbedit_search
,
434 .offload_act_setup
= tcf_skbedit_offload_act_setup
,
435 .size
= sizeof(struct tcf_skbedit
),
438 static __net_init
int skbedit_init_net(struct net
*net
)
440 struct tc_action_net
*tn
= net_generic(net
, skbedit_net_id
);
442 return tc_action_net_init(net
, tn
, &act_skbedit_ops
);
445 static void __net_exit
skbedit_exit_net(struct list_head
*net_list
)
447 tc_action_net_exit(net_list
, skbedit_net_id
);
450 static struct pernet_operations skbedit_net_ops
= {
451 .init
= skbedit_init_net
,
452 .exit_batch
= skbedit_exit_net
,
453 .id
= &skbedit_net_id
,
454 .size
= sizeof(struct tc_action_net
),
457 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>");
458 MODULE_DESCRIPTION("SKB Editing");
459 MODULE_LICENSE("GPL");
461 static int __init
skbedit_init_module(void)
463 return tcf_register_action(&act_skbedit_ops
, &skbedit_net_ops
);
466 static void __exit
skbedit_cleanup_module(void)
468 tcf_unregister_action(&act_skbedit_ops
, &skbedit_net_ops
);
471 module_init(skbedit_init_module
);
472 module_exit(skbedit_cleanup_module
);