2 * net/sched/act_api.c Packet action API.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Author: Jamal Hadi Salim
14 #include <linux/types.h>
15 #include <linux/kernel.h>
16 #include <linux/string.h>
17 #include <linux/errno.h>
18 #include <linux/slab.h>
19 #include <linux/skbuff.h>
20 #include <linux/init.h>
21 #include <linux/kmod.h>
22 #include <linux/err.h>
23 #include <linux/module.h>
24 #include <linux/rhashtable.h>
25 #include <linux/list.h>
26 #include <net/net_namespace.h>
28 #include <net/sch_generic.h>
29 #include <net/pkt_cls.h>
30 #include <net/act_api.h>
31 #include <net/netlink.h>
33 static int tcf_action_goto_chain_init(struct tc_action
*a
, struct tcf_proto
*tp
)
35 u32 chain_index
= a
->tcfa_action
& TC_ACT_EXT_VAL_MASK
;
39 a
->goto_chain
= tcf_chain_get(tp
->chain
->block
, chain_index
, true);
45 static void tcf_action_goto_chain_fini(struct tc_action
*a
)
47 tcf_chain_put(a
->goto_chain
);
50 static void tcf_action_goto_chain_exec(const struct tc_action
*a
,
51 struct tcf_result
*res
)
53 const struct tcf_chain
*chain
= a
->goto_chain
;
55 res
->goto_tp
= rcu_dereference_bh(chain
->filter_chain
);
58 static void tcf_free_cookie_rcu(struct rcu_head
*p
)
60 struct tc_cookie
*cookie
= container_of(p
, struct tc_cookie
, rcu
);
66 static void tcf_set_action_cookie(struct tc_cookie __rcu
**old_cookie
,
67 struct tc_cookie
*new_cookie
)
69 struct tc_cookie
*old
;
71 old
= xchg(old_cookie
, new_cookie
);
73 call_rcu(&old
->rcu
, tcf_free_cookie_rcu
);
76 /* XXX: For standalone actions, we don't need a RCU grace period either, because
77 * actions are always connected to filters and filters are already destroyed in
78 * RCU callbacks, so after a RCU grace period actions are already disconnected
79 * from filters. Readers later can not find us.
81 static void free_tcf(struct tc_action
*p
)
83 free_percpu(p
->cpu_bstats
);
84 free_percpu(p
->cpu_qstats
);
86 tcf_set_action_cookie(&p
->act_cookie
, NULL
);
88 tcf_action_goto_chain_fini(p
);
93 static void tcf_action_cleanup(struct tc_action
*p
)
98 gen_kill_estimator(&p
->tcfa_rate_est
);
102 static int __tcf_action_put(struct tc_action
*p
, bool bind
)
104 struct tcf_idrinfo
*idrinfo
= p
->idrinfo
;
106 if (refcount_dec_and_lock(&p
->tcfa_refcnt
, &idrinfo
->lock
)) {
108 atomic_dec(&p
->tcfa_bindcnt
);
109 idr_remove(&idrinfo
->action_idr
, p
->tcfa_index
);
110 spin_unlock(&idrinfo
->lock
);
112 tcf_action_cleanup(p
);
117 atomic_dec(&p
->tcfa_bindcnt
);
122 int __tcf_idr_release(struct tc_action
*p
, bool bind
, bool strict
)
126 /* Release with strict==1 and bind==0 is only called through act API
127 * interface (classifiers always bind). Only case when action with
128 * positive reference count and zero bind count can exist is when it was
129 * also created with act API (unbinding last classifier will destroy the
130 * action if it was created by classifier). So only case when bind count
131 * can be changed after initial check is when unbound action is
132 * destroyed by act API while classifier binds to action with same id
133 * concurrently. This result either creation of new action(same behavior
134 * as before), or reusing existing action if concurrent process
135 * increments reference count before action is deleted. Both scenarios
139 if (!bind
&& strict
&& atomic_read(&p
->tcfa_bindcnt
) > 0)
142 if (__tcf_action_put(p
, bind
))
148 EXPORT_SYMBOL(__tcf_idr_release
);
150 static size_t tcf_action_shared_attrs_size(const struct tc_action
*act
)
155 cookie_len
= nla_total_size(act
->act_cookie
->len
);
157 return nla_total_size(0) /* action number nested */
158 + nla_total_size(IFNAMSIZ
) /* TCA_ACT_KIND */
159 + cookie_len
/* TCA_ACT_COOKIE */
160 + nla_total_size(0) /* TCA_ACT_STATS nested */
161 /* TCA_STATS_BASIC */
162 + nla_total_size_64bit(sizeof(struct gnet_stats_basic
))
163 /* TCA_STATS_QUEUE */
164 + nla_total_size_64bit(sizeof(struct gnet_stats_queue
))
165 + nla_total_size(0) /* TCA_OPTIONS nested */
166 + nla_total_size(sizeof(struct tcf_t
)); /* TCA_GACT_TM */
169 static size_t tcf_action_full_attrs_size(size_t sz
)
171 return NLMSG_HDRLEN
/* struct nlmsghdr */
172 + sizeof(struct tcamsg
)
173 + nla_total_size(0) /* TCA_ACT_TAB nested */
177 static size_t tcf_action_fill_size(const struct tc_action
*act
)
179 size_t sz
= tcf_action_shared_attrs_size(act
);
181 if (act
->ops
->get_fill_size
)
182 return act
->ops
->get_fill_size(act
) + sz
;
186 static int tcf_dump_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
187 struct netlink_callback
*cb
)
189 int err
= 0, index
= -1, s_i
= 0, n_i
= 0;
190 u32 act_flags
= cb
->args
[2];
191 unsigned long jiffy_since
= cb
->args
[3];
193 struct idr
*idr
= &idrinfo
->action_idr
;
195 unsigned long id
= 1;
197 spin_lock(&idrinfo
->lock
);
201 idr_for_each_entry_ul(idr
, p
, id
) {
207 time_after(jiffy_since
,
208 (unsigned long)p
->tcfa_tm
.lastuse
))
211 nest
= nla_nest_start(skb
, n_i
);
214 goto nla_put_failure
;
216 err
= tcf_action_dump_1(skb
, p
, 0, 0);
219 nlmsg_trim(skb
, nest
);
222 nla_nest_end(skb
, nest
);
224 if (!(act_flags
& TCA_FLAG_LARGE_DUMP_ON
) &&
225 n_i
>= TCA_ACT_MAX_PRIO
)
230 cb
->args
[0] = index
+ 1;
232 spin_unlock(&idrinfo
->lock
);
234 if (act_flags
& TCA_FLAG_LARGE_DUMP_ON
)
240 nla_nest_cancel(skb
, nest
);
244 static int tcf_del_walker(struct tcf_idrinfo
*idrinfo
, struct sk_buff
*skb
,
245 const struct tc_action_ops
*ops
)
250 struct idr
*idr
= &idrinfo
->action_idr
;
252 unsigned long id
= 1;
254 nest
= nla_nest_start(skb
, 0);
256 goto nla_put_failure
;
257 if (nla_put_string(skb
, TCA_KIND
, ops
->kind
))
258 goto nla_put_failure
;
260 idr_for_each_entry_ul(idr
, p
, id
) {
261 ret
= __tcf_idr_release(p
, false, true);
262 if (ret
== ACT_P_DELETED
) {
263 module_put(ops
->owner
);
265 } else if (ret
< 0) {
266 goto nla_put_failure
;
269 if (nla_put_u32(skb
, TCA_FCNT
, n_i
))
270 goto nla_put_failure
;
271 nla_nest_end(skb
, nest
);
275 nla_nest_cancel(skb
, nest
);
279 int tcf_generic_walker(struct tc_action_net
*tn
, struct sk_buff
*skb
,
280 struct netlink_callback
*cb
, int type
,
281 const struct tc_action_ops
*ops
,
282 struct netlink_ext_ack
*extack
)
284 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
286 if (type
== RTM_DELACTION
) {
287 return tcf_del_walker(idrinfo
, skb
, ops
);
288 } else if (type
== RTM_GETACTION
) {
289 return tcf_dump_walker(idrinfo
, skb
, cb
);
291 WARN(1, "tcf_generic_walker: unknown command %d\n", type
);
292 NL_SET_ERR_MSG(extack
, "tcf_generic_walker: unknown command");
296 EXPORT_SYMBOL(tcf_generic_walker
);
298 static bool __tcf_idr_check(struct tc_action_net
*tn
, u32 index
,
299 struct tc_action
**a
, int bind
)
301 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
304 spin_lock(&idrinfo
->lock
);
305 p
= idr_find(&idrinfo
->action_idr
, index
);
309 refcount_inc(&p
->tcfa_refcnt
);
311 atomic_inc(&p
->tcfa_bindcnt
);
313 spin_unlock(&idrinfo
->lock
);
322 int tcf_idr_search(struct tc_action_net
*tn
, struct tc_action
**a
, u32 index
)
324 return __tcf_idr_check(tn
, index
, a
, 0);
326 EXPORT_SYMBOL(tcf_idr_search
);
328 bool tcf_idr_check(struct tc_action_net
*tn
, u32 index
, struct tc_action
**a
,
331 return __tcf_idr_check(tn
, index
, a
, bind
);
333 EXPORT_SYMBOL(tcf_idr_check
);
335 int tcf_idr_delete_index(struct tc_action_net
*tn
, u32 index
)
337 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
341 spin_lock(&idrinfo
->lock
);
342 p
= idr_find(&idrinfo
->action_idr
, index
);
344 spin_unlock(&idrinfo
->lock
);
348 if (!atomic_read(&p
->tcfa_bindcnt
)) {
349 if (refcount_dec_and_test(&p
->tcfa_refcnt
)) {
350 struct module
*owner
= p
->ops
->owner
;
352 WARN_ON(p
!= idr_remove(&idrinfo
->action_idr
,
354 spin_unlock(&idrinfo
->lock
);
356 tcf_action_cleanup(p
);
365 spin_unlock(&idrinfo
->lock
);
368 EXPORT_SYMBOL(tcf_idr_delete_index
);
370 int tcf_idr_create(struct tc_action_net
*tn
, u32 index
, struct nlattr
*est
,
371 struct tc_action
**a
, const struct tc_action_ops
*ops
,
372 int bind
, bool cpustats
)
374 struct tc_action
*p
= kzalloc(ops
->size
, GFP_KERNEL
);
375 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
380 refcount_set(&p
->tcfa_refcnt
, 1);
382 atomic_set(&p
->tcfa_bindcnt
, 1);
385 p
->cpu_bstats
= netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu
);
388 p
->cpu_qstats
= alloc_percpu(struct gnet_stats_queue
);
392 spin_lock_init(&p
->tcfa_lock
);
393 p
->tcfa_index
= index
;
394 p
->tcfa_tm
.install
= jiffies
;
395 p
->tcfa_tm
.lastuse
= jiffies
;
396 p
->tcfa_tm
.firstuse
= 0;
398 err
= gen_new_estimator(&p
->tcfa_bstats
, p
->cpu_bstats
,
400 &p
->tcfa_lock
, NULL
, est
);
405 p
->idrinfo
= idrinfo
;
407 INIT_LIST_HEAD(&p
->list
);
411 free_percpu(p
->cpu_qstats
);
413 free_percpu(p
->cpu_bstats
);
418 EXPORT_SYMBOL(tcf_idr_create
);
420 void tcf_idr_insert(struct tc_action_net
*tn
, struct tc_action
*a
)
422 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
424 spin_lock(&idrinfo
->lock
);
425 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
426 WARN_ON(!IS_ERR(idr_replace(&idrinfo
->action_idr
, a
, a
->tcfa_index
)));
427 spin_unlock(&idrinfo
->lock
);
429 EXPORT_SYMBOL(tcf_idr_insert
);
431 /* Cleanup idr index that was allocated but not initialized. */
433 void tcf_idr_cleanup(struct tc_action_net
*tn
, u32 index
)
435 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
437 spin_lock(&idrinfo
->lock
);
438 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
439 WARN_ON(!IS_ERR(idr_remove(&idrinfo
->action_idr
, index
)));
440 spin_unlock(&idrinfo
->lock
);
442 EXPORT_SYMBOL(tcf_idr_cleanup
);
444 /* Check if action with specified index exists. If actions is found, increments
445 * its reference and bind counters, and return 1. Otherwise insert temporary
446 * error pointer (to prevent concurrent users from inserting actions with same
447 * index) and return 0.
450 int tcf_idr_check_alloc(struct tc_action_net
*tn
, u32
*index
,
451 struct tc_action
**a
, int bind
)
453 struct tcf_idrinfo
*idrinfo
= tn
->idrinfo
;
458 spin_lock(&idrinfo
->lock
);
460 p
= idr_find(&idrinfo
->action_idr
, *index
);
462 /* This means that another process allocated
463 * index but did not assign the pointer yet.
465 spin_unlock(&idrinfo
->lock
);
470 refcount_inc(&p
->tcfa_refcnt
);
472 atomic_inc(&p
->tcfa_bindcnt
);
477 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
480 idr_replace(&idrinfo
->action_idr
,
481 ERR_PTR(-EBUSY
), *index
);
486 ret
= idr_alloc_u32(&idrinfo
->action_idr
, NULL
, index
,
487 UINT_MAX
, GFP_ATOMIC
);
489 idr_replace(&idrinfo
->action_idr
, ERR_PTR(-EBUSY
),
492 spin_unlock(&idrinfo
->lock
);
495 EXPORT_SYMBOL(tcf_idr_check_alloc
);
497 void tcf_idrinfo_destroy(const struct tc_action_ops
*ops
,
498 struct tcf_idrinfo
*idrinfo
)
500 struct idr
*idr
= &idrinfo
->action_idr
;
503 unsigned long id
= 1;
505 idr_for_each_entry_ul(idr
, p
, id
) {
506 ret
= __tcf_idr_release(p
, false, true);
507 if (ret
== ACT_P_DELETED
)
508 module_put(ops
->owner
);
512 idr_destroy(&idrinfo
->action_idr
);
514 EXPORT_SYMBOL(tcf_idrinfo_destroy
);
516 static LIST_HEAD(act_base
);
517 static DEFINE_RWLOCK(act_mod_lock
);
519 int tcf_register_action(struct tc_action_ops
*act
,
520 struct pernet_operations
*ops
)
522 struct tc_action_ops
*a
;
525 if (!act
->act
|| !act
->dump
|| !act
->init
|| !act
->walk
|| !act
->lookup
)
528 /* We have to register pernet ops before making the action ops visible,
529 * otherwise tcf_action_init_1() could get a partially initialized
532 ret
= register_pernet_subsys(ops
);
536 write_lock(&act_mod_lock
);
537 list_for_each_entry(a
, &act_base
, head
) {
538 if (act
->type
== a
->type
|| (strcmp(act
->kind
, a
->kind
) == 0)) {
539 write_unlock(&act_mod_lock
);
540 unregister_pernet_subsys(ops
);
544 list_add_tail(&act
->head
, &act_base
);
545 write_unlock(&act_mod_lock
);
549 EXPORT_SYMBOL(tcf_register_action
);
551 int tcf_unregister_action(struct tc_action_ops
*act
,
552 struct pernet_operations
*ops
)
554 struct tc_action_ops
*a
;
557 write_lock(&act_mod_lock
);
558 list_for_each_entry(a
, &act_base
, head
) {
560 list_del(&act
->head
);
565 write_unlock(&act_mod_lock
);
567 unregister_pernet_subsys(ops
);
570 EXPORT_SYMBOL(tcf_unregister_action
);
573 static struct tc_action_ops
*tc_lookup_action_n(char *kind
)
575 struct tc_action_ops
*a
, *res
= NULL
;
578 read_lock(&act_mod_lock
);
579 list_for_each_entry(a
, &act_base
, head
) {
580 if (strcmp(kind
, a
->kind
) == 0) {
581 if (try_module_get(a
->owner
))
586 read_unlock(&act_mod_lock
);
591 /* lookup by nlattr */
592 static struct tc_action_ops
*tc_lookup_action(struct nlattr
*kind
)
594 struct tc_action_ops
*a
, *res
= NULL
;
597 read_lock(&act_mod_lock
);
598 list_for_each_entry(a
, &act_base
, head
) {
599 if (nla_strcmp(kind
, a
->kind
) == 0) {
600 if (try_module_get(a
->owner
))
605 read_unlock(&act_mod_lock
);
610 /*TCA_ACT_MAX_PRIO is 32, there count upto 32 */
611 #define TCA_ACT_MAX_PRIO_MASK 0x1FF
612 int tcf_action_exec(struct sk_buff
*skb
, struct tc_action
**actions
,
613 int nr_actions
, struct tcf_result
*res
)
616 u32 jmp_ttl
= TCA_ACT_MAX_PRIO
; /*matches actions per filter */
620 if (skb_skip_tc_classify(skb
))
624 for (i
= 0; i
< nr_actions
; i
++) {
625 const struct tc_action
*a
= actions
[i
];
627 if (jmp_prgcnt
> 0) {
632 ret
= a
->ops
->act(skb
, a
, res
);
633 if (ret
== TC_ACT_REPEAT
)
634 goto repeat
; /* we need a ttl - JHS */
636 if (TC_ACT_EXT_CMP(ret
, TC_ACT_JUMP
)) {
637 jmp_prgcnt
= ret
& TCA_ACT_MAX_PRIO_MASK
;
638 if (!jmp_prgcnt
|| (jmp_prgcnt
> nr_actions
)) {
639 /* faulty opcode, stop pipeline */
644 goto restart_act_graph
;
645 else /* faulty graph, stop pipeline */
648 } else if (TC_ACT_EXT_CMP(ret
, TC_ACT_GOTO_CHAIN
)) {
649 tcf_action_goto_chain_exec(a
, res
);
652 if (ret
!= TC_ACT_PIPE
)
658 EXPORT_SYMBOL(tcf_action_exec
);
660 int tcf_action_destroy(struct list_head
*actions
, int bind
)
662 const struct tc_action_ops
*ops
;
663 struct tc_action
*a
, *tmp
;
666 list_for_each_entry_safe(a
, tmp
, actions
, list
) {
668 ret
= __tcf_idr_release(a
, bind
, true);
669 if (ret
== ACT_P_DELETED
)
670 module_put(ops
->owner
);
677 static int tcf_action_put(struct tc_action
*p
)
679 return __tcf_action_put(p
, false);
682 static void tcf_action_put_lst(struct list_head
*actions
)
684 struct tc_action
*a
, *tmp
;
686 list_for_each_entry_safe(a
, tmp
, actions
, list
) {
687 const struct tc_action_ops
*ops
= a
->ops
;
689 if (tcf_action_put(a
))
690 module_put(ops
->owner
);
695 tcf_action_dump_old(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
697 return a
->ops
->dump(skb
, a
, bind
, ref
);
701 tcf_action_dump_1(struct sk_buff
*skb
, struct tc_action
*a
, int bind
, int ref
)
704 unsigned char *b
= skb_tail_pointer(skb
);
706 struct tc_cookie
*cookie
;
708 if (nla_put_string(skb
, TCA_KIND
, a
->ops
->kind
))
709 goto nla_put_failure
;
710 if (tcf_action_copy_stats(skb
, a
, 0))
711 goto nla_put_failure
;
714 cookie
= rcu_dereference(a
->act_cookie
);
716 if (nla_put(skb
, TCA_ACT_COOKIE
, cookie
->len
, cookie
->data
)) {
718 goto nla_put_failure
;
723 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
725 goto nla_put_failure
;
726 err
= tcf_action_dump_old(skb
, a
, bind
, ref
);
728 nla_nest_end(skb
, nest
);
736 EXPORT_SYMBOL(tcf_action_dump_1
);
738 int tcf_action_dump(struct sk_buff
*skb
, struct list_head
*actions
,
745 list_for_each_entry(a
, actions
, list
) {
746 nest
= nla_nest_start(skb
, a
->order
);
748 goto nla_put_failure
;
749 err
= tcf_action_dump_1(skb
, a
, bind
, ref
);
752 nla_nest_end(skb
, nest
);
760 nla_nest_cancel(skb
, nest
);
764 static struct tc_cookie
*nla_memdup_cookie(struct nlattr
**tb
)
766 struct tc_cookie
*c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
770 c
->data
= nla_memdup(tb
[TCA_ACT_COOKIE
], GFP_KERNEL
);
775 c
->len
= nla_len(tb
[TCA_ACT_COOKIE
]);
780 struct tc_action
*tcf_action_init_1(struct net
*net
, struct tcf_proto
*tp
,
781 struct nlattr
*nla
, struct nlattr
*est
,
782 char *name
, int ovr
, int bind
,
784 struct netlink_ext_ack
*extack
)
787 struct tc_action_ops
*a_o
;
788 struct tc_cookie
*cookie
= NULL
;
789 char act_name
[IFNAMSIZ
];
790 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
795 err
= nla_parse_nested(tb
, TCA_ACT_MAX
, nla
, NULL
, extack
);
799 kind
= tb
[TCA_ACT_KIND
];
801 NL_SET_ERR_MSG(extack
, "TC action kind must be specified");
804 if (nla_strlcpy(act_name
, kind
, IFNAMSIZ
) >= IFNAMSIZ
) {
805 NL_SET_ERR_MSG(extack
, "TC action name too long");
808 if (tb
[TCA_ACT_COOKIE
]) {
809 int cklen
= nla_len(tb
[TCA_ACT_COOKIE
]);
811 if (cklen
> TC_COOKIE_MAX_SIZE
) {
812 NL_SET_ERR_MSG(extack
, "TC cookie size above the maximum");
816 cookie
= nla_memdup_cookie(tb
);
818 NL_SET_ERR_MSG(extack
, "No memory to generate TC cookie");
824 if (strlcpy(act_name
, name
, IFNAMSIZ
) >= IFNAMSIZ
) {
825 NL_SET_ERR_MSG(extack
, "TC action name too long");
831 a_o
= tc_lookup_action_n(act_name
);
833 #ifdef CONFIG_MODULES
836 request_module("act_%s", act_name
);
840 a_o
= tc_lookup_action_n(act_name
);
842 /* We dropped the RTNL semaphore in order to
843 * perform the module load. So, even if we
844 * succeeded in loading the module we have to
845 * tell the caller to replay the request. We
846 * indicate this using -EAGAIN.
853 NL_SET_ERR_MSG(extack
, "Failed to load TC action module");
858 /* backward compatibility for policer */
860 err
= a_o
->init(net
, tb
[TCA_ACT_OPTIONS
], est
, &a
, ovr
, bind
,
863 err
= a_o
->init(net
, nla
, est
, &a
, ovr
, bind
, rtnl_held
,
868 if (!name
&& tb
[TCA_ACT_COOKIE
])
869 tcf_set_action_cookie(&a
->act_cookie
, cookie
);
871 /* module count goes up only when brand new policy is created
872 * if it exists and is only bound to in a_o->init() then
873 * ACT_P_CREATED is not returned (a zero is).
875 if (err
!= ACT_P_CREATED
)
876 module_put(a_o
->owner
);
878 if (TC_ACT_EXT_CMP(a
->tcfa_action
, TC_ACT_GOTO_CHAIN
)) {
879 err
= tcf_action_goto_chain_init(a
, tp
);
883 list_add_tail(&a
->list
, &actions
);
884 tcf_action_destroy(&actions
, bind
);
885 NL_SET_ERR_MSG(extack
, "Failed to init TC action chain");
893 module_put(a_o
->owner
);
902 int tcf_action_init(struct net
*net
, struct tcf_proto
*tp
, struct nlattr
*nla
,
903 struct nlattr
*est
, char *name
, int ovr
, int bind
,
904 struct list_head
*actions
, size_t *attr_size
,
905 bool rtnl_held
, struct netlink_ext_ack
*extack
)
907 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
908 struct tc_action
*act
;
913 err
= nla_parse_nested(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
, extack
);
917 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
918 act
= tcf_action_init_1(net
, tp
, tb
[i
], est
, name
, ovr
, bind
,
925 sz
+= tcf_action_fill_size(act
);
926 list_add_tail(&act
->list
, actions
);
929 *attr_size
= tcf_action_full_attrs_size(sz
);
933 tcf_action_destroy(actions
, bind
);
937 int tcf_action_copy_stats(struct sk_buff
*skb
, struct tc_action
*p
,
946 /* compat_mode being true specifies a call that is supposed
947 * to add additional backward compatibility statistic TLVs.
950 if (p
->type
== TCA_OLD_COMPAT
)
951 err
= gnet_stats_start_copy_compat(skb
, 0,
959 err
= gnet_stats_start_copy(skb
, TCA_ACT_STATS
,
960 &p
->tcfa_lock
, &d
, TCA_ACT_PAD
);
965 if (gnet_stats_copy_basic(NULL
, &d
, p
->cpu_bstats
, &p
->tcfa_bstats
) < 0 ||
966 gnet_stats_copy_rate_est(&d
, &p
->tcfa_rate_est
) < 0 ||
967 gnet_stats_copy_queue(&d
, p
->cpu_qstats
,
969 p
->tcfa_qstats
.qlen
) < 0)
972 if (gnet_stats_finish_copy(&d
) < 0)
981 static int tca_get_fill(struct sk_buff
*skb
, struct list_head
*actions
,
982 u32 portid
, u32 seq
, u16 flags
, int event
, int bind
,
986 struct nlmsghdr
*nlh
;
987 unsigned char *b
= skb_tail_pointer(skb
);
990 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*t
), flags
);
994 t
->tca_family
= AF_UNSPEC
;
998 nest
= nla_nest_start(skb
, TCA_ACT_TAB
);
1000 goto out_nlmsg_trim
;
1002 if (tcf_action_dump(skb
, actions
, bind
, ref
) < 0)
1003 goto out_nlmsg_trim
;
1005 nla_nest_end(skb
, nest
);
1007 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1016 tcf_get_notify(struct net
*net
, u32 portid
, struct nlmsghdr
*n
,
1017 struct list_head
*actions
, int event
,
1018 struct netlink_ext_ack
*extack
)
1020 struct sk_buff
*skb
;
1022 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1025 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, event
,
1027 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1032 return rtnl_unicast(skb
, net
, portid
);
1035 static struct tc_action
*tcf_action_get_1(struct net
*net
, struct nlattr
*nla
,
1036 struct nlmsghdr
*n
, u32 portid
,
1037 struct netlink_ext_ack
*extack
)
1039 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1040 const struct tc_action_ops
*ops
;
1041 struct tc_action
*a
;
1045 err
= nla_parse_nested(tb
, TCA_ACT_MAX
, nla
, NULL
, extack
);
1050 if (tb
[TCA_ACT_INDEX
] == NULL
||
1051 nla_len(tb
[TCA_ACT_INDEX
]) < sizeof(index
)) {
1052 NL_SET_ERR_MSG(extack
, "Invalid TC action index value");
1055 index
= nla_get_u32(tb
[TCA_ACT_INDEX
]);
1058 ops
= tc_lookup_action(tb
[TCA_ACT_KIND
]);
1059 if (!ops
) { /* could happen in batch of actions */
1060 NL_SET_ERR_MSG(extack
, "Specified TC action not found");
1064 if (ops
->lookup(net
, &a
, index
, extack
) == 0)
1067 module_put(ops
->owner
);
1071 module_put(ops
->owner
);
1073 return ERR_PTR(err
);
1076 static int tca_action_flush(struct net
*net
, struct nlattr
*nla
,
1077 struct nlmsghdr
*n
, u32 portid
,
1078 struct netlink_ext_ack
*extack
)
1080 struct sk_buff
*skb
;
1082 struct nlmsghdr
*nlh
;
1084 struct netlink_callback dcb
;
1085 struct nlattr
*nest
;
1086 struct nlattr
*tb
[TCA_ACT_MAX
+ 1];
1087 const struct tc_action_ops
*ops
;
1088 struct nlattr
*kind
;
1091 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1095 b
= skb_tail_pointer(skb
);
1097 err
= nla_parse_nested(tb
, TCA_ACT_MAX
, nla
, NULL
, extack
);
1102 kind
= tb
[TCA_ACT_KIND
];
1103 ops
= tc_lookup_action(kind
);
1104 if (!ops
) { /*some idjot trying to flush unknown action */
1105 NL_SET_ERR_MSG(extack
, "Cannot flush unknown TC action");
1109 nlh
= nlmsg_put(skb
, portid
, n
->nlmsg_seq
, RTM_DELACTION
,
1112 NL_SET_ERR_MSG(extack
, "Failed to create TC action flush notification");
1113 goto out_module_put
;
1115 t
= nlmsg_data(nlh
);
1116 t
->tca_family
= AF_UNSPEC
;
1120 nest
= nla_nest_start(skb
, TCA_ACT_TAB
);
1122 NL_SET_ERR_MSG(extack
, "Failed to add new netlink message");
1123 goto out_module_put
;
1126 err
= ops
->walk(net
, skb
, &dcb
, RTM_DELACTION
, ops
, extack
);
1128 nla_nest_cancel(skb
, nest
);
1129 goto out_module_put
;
1132 nla_nest_end(skb
, nest
);
1134 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1135 nlh
->nlmsg_flags
|= NLM_F_ROOT
;
1136 module_put(ops
->owner
);
1137 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1138 n
->nlmsg_flags
& NLM_F_ECHO
);
1142 NL_SET_ERR_MSG(extack
, "Failed to send TC action flush notification");
1147 module_put(ops
->owner
);
1153 static int tcf_action_delete(struct net
*net
, struct list_head
*actions
,
1154 struct netlink_ext_ack
*extack
)
1156 struct tc_action
*a
, *tmp
;
1160 list_for_each_entry_safe(a
, tmp
, actions
, list
) {
1161 const struct tc_action_ops
*ops
= a
->ops
;
1163 /* Actions can be deleted concurrently so we must save their
1164 * type and id to search again after reference is released.
1166 act_index
= a
->tcfa_index
;
1169 if (tcf_action_put(a
)) {
1170 /* last reference, action was deleted concurrently */
1171 module_put(ops
->owner
);
1173 /* now do the delete */
1174 ret
= ops
->delete(net
, act_index
);
1183 tcf_del_notify(struct net
*net
, struct nlmsghdr
*n
, struct list_head
*actions
,
1184 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1187 struct sk_buff
*skb
;
1189 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1194 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, 0, RTM_DELACTION
,
1196 NL_SET_ERR_MSG(extack
, "Failed to fill netlink TC action attributes");
1201 /* now do the delete */
1202 ret
= tcf_action_delete(net
, actions
, extack
);
1204 NL_SET_ERR_MSG(extack
, "Failed to delete TC action");
1209 ret
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1210 n
->nlmsg_flags
& NLM_F_ECHO
);
1217 tca_action_gd(struct net
*net
, struct nlattr
*nla
, struct nlmsghdr
*n
,
1218 u32 portid
, int event
, struct netlink_ext_ack
*extack
)
1221 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1222 struct tc_action
*act
;
1223 size_t attr_size
= 0;
1226 ret
= nla_parse_nested(tb
, TCA_ACT_MAX_PRIO
, nla
, NULL
, extack
);
1230 if (event
== RTM_DELACTION
&& n
->nlmsg_flags
& NLM_F_ROOT
) {
1232 return tca_action_flush(net
, tb
[1], n
, portid
, extack
);
1234 NL_SET_ERR_MSG(extack
, "Invalid netlink attributes while flushing TC action");
1238 for (i
= 1; i
<= TCA_ACT_MAX_PRIO
&& tb
[i
]; i
++) {
1239 act
= tcf_action_get_1(net
, tb
[i
], n
, portid
, extack
);
1245 attr_size
+= tcf_action_fill_size(act
);
1246 list_add_tail(&act
->list
, &actions
);
1249 attr_size
= tcf_action_full_attrs_size(attr_size
);
1251 if (event
== RTM_GETACTION
)
1252 ret
= tcf_get_notify(net
, portid
, n
, &actions
, event
, extack
);
1254 ret
= tcf_del_notify(net
, n
, &actions
, portid
, attr_size
, extack
);
1260 tcf_action_put_lst(&actions
);
1265 tcf_add_notify(struct net
*net
, struct nlmsghdr
*n
, struct list_head
*actions
,
1266 u32 portid
, size_t attr_size
, struct netlink_ext_ack
*extack
)
1268 struct sk_buff
*skb
;
1271 skb
= alloc_skb(attr_size
<= NLMSG_GOODSIZE
? NLMSG_GOODSIZE
: attr_size
,
1276 if (tca_get_fill(skb
, actions
, portid
, n
->nlmsg_seq
, n
->nlmsg_flags
,
1277 RTM_NEWACTION
, 0, 0) <= 0) {
1278 NL_SET_ERR_MSG(extack
, "Failed to fill netlink attributes while adding TC action");
1283 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1284 n
->nlmsg_flags
& NLM_F_ECHO
);
1290 static int tcf_action_add(struct net
*net
, struct nlattr
*nla
,
1291 struct nlmsghdr
*n
, u32 portid
, int ovr
,
1292 struct netlink_ext_ack
*extack
)
1294 size_t attr_size
= 0;
1298 ret
= tcf_action_init(net
, NULL
, nla
, NULL
, NULL
, ovr
, 0, &actions
,
1299 &attr_size
, true, extack
);
1302 ret
= tcf_add_notify(net
, n
, &actions
, portid
, attr_size
, extack
);
1304 tcf_action_put_lst(&actions
);
1309 static u32 tcaa_root_flags_allowed
= TCA_FLAG_LARGE_DUMP_ON
;
1310 static const struct nla_policy tcaa_policy
[TCA_ROOT_MAX
+ 1] = {
1311 [TCA_ROOT_FLAGS
] = { .type
= NLA_BITFIELD32
,
1312 .validation_data
= &tcaa_root_flags_allowed
},
1313 [TCA_ROOT_TIME_DELTA
] = { .type
= NLA_U32
},
1316 static int tc_ctl_action(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1317 struct netlink_ext_ack
*extack
)
1319 struct net
*net
= sock_net(skb
->sk
);
1320 struct nlattr
*tca
[TCA_ROOT_MAX
+ 1];
1321 u32 portid
= skb
? NETLINK_CB(skb
).portid
: 0;
1322 int ret
= 0, ovr
= 0;
1324 if ((n
->nlmsg_type
!= RTM_GETACTION
) &&
1325 !netlink_capable(skb
, CAP_NET_ADMIN
))
1328 ret
= nlmsg_parse(n
, sizeof(struct tcamsg
), tca
, TCA_ROOT_MAX
, NULL
,
1333 if (tca
[TCA_ACT_TAB
] == NULL
) {
1334 NL_SET_ERR_MSG(extack
, "Netlink action attributes missing");
1338 /* n->nlmsg_flags & NLM_F_CREATE */
1339 switch (n
->nlmsg_type
) {
1341 /* we are going to assume all other flags
1342 * imply create only if it doesn't exist
1343 * Note that CREATE | EXCL implies that
1344 * but since we want avoid ambiguity (eg when flags
1345 * is zero) then just set this
1347 if (n
->nlmsg_flags
& NLM_F_REPLACE
)
1350 ret
= tcf_action_add(net
, tca
[TCA_ACT_TAB
], n
, portid
, ovr
,
1356 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1357 portid
, RTM_DELACTION
, extack
);
1360 ret
= tca_action_gd(net
, tca
[TCA_ACT_TAB
], n
,
1361 portid
, RTM_GETACTION
, extack
);
1370 static struct nlattr
*find_dump_kind(struct nlattr
**nla
)
1372 struct nlattr
*tb1
, *tb2
[TCA_ACT_MAX
+ 1];
1373 struct nlattr
*tb
[TCA_ACT_MAX_PRIO
+ 1];
1374 struct nlattr
*kind
;
1376 tb1
= nla
[TCA_ACT_TAB
];
1380 if (nla_parse(tb
, TCA_ACT_MAX_PRIO
, nla_data(tb1
),
1381 NLMSG_ALIGN(nla_len(tb1
)), NULL
, NULL
) < 0)
1386 if (nla_parse_nested(tb2
, TCA_ACT_MAX
, tb
[1], NULL
, NULL
) < 0)
1388 kind
= tb2
[TCA_ACT_KIND
];
1393 static int tc_dump_action(struct sk_buff
*skb
, struct netlink_callback
*cb
)
1395 struct net
*net
= sock_net(skb
->sk
);
1396 struct nlmsghdr
*nlh
;
1397 unsigned char *b
= skb_tail_pointer(skb
);
1398 struct nlattr
*nest
;
1399 struct tc_action_ops
*a_o
;
1401 struct tcamsg
*t
= (struct tcamsg
*) nlmsg_data(cb
->nlh
);
1402 struct nlattr
*tb
[TCA_ROOT_MAX
+ 1];
1403 struct nlattr
*count_attr
= NULL
;
1404 unsigned long jiffy_since
= 0;
1405 struct nlattr
*kind
= NULL
;
1406 struct nla_bitfield32 bf
;
1407 u32 msecs_since
= 0;
1410 ret
= nlmsg_parse(cb
->nlh
, sizeof(struct tcamsg
), tb
, TCA_ROOT_MAX
,
1415 kind
= find_dump_kind(tb
);
1417 pr_info("tc_dump_action: action bad kind\n");
1421 a_o
= tc_lookup_action(kind
);
1426 if (tb
[TCA_ROOT_FLAGS
]) {
1427 bf
= nla_get_bitfield32(tb
[TCA_ROOT_FLAGS
]);
1428 cb
->args
[2] = bf
.value
;
1431 if (tb
[TCA_ROOT_TIME_DELTA
]) {
1432 msecs_since
= nla_get_u32(tb
[TCA_ROOT_TIME_DELTA
]);
1435 nlh
= nlmsg_put(skb
, NETLINK_CB(cb
->skb
).portid
, cb
->nlh
->nlmsg_seq
,
1436 cb
->nlh
->nlmsg_type
, sizeof(*t
), 0);
1438 goto out_module_put
;
1441 jiffy_since
= jiffies
- msecs_to_jiffies(msecs_since
);
1443 t
= nlmsg_data(nlh
);
1444 t
->tca_family
= AF_UNSPEC
;
1447 cb
->args
[3] = jiffy_since
;
1448 count_attr
= nla_reserve(skb
, TCA_ROOT_COUNT
, sizeof(u32
));
1450 goto out_module_put
;
1452 nest
= nla_nest_start(skb
, TCA_ACT_TAB
);
1454 goto out_module_put
;
1456 ret
= a_o
->walk(net
, skb
, cb
, RTM_GETACTION
, a_o
, NULL
);
1458 goto out_module_put
;
1461 nla_nest_end(skb
, nest
);
1463 act_count
= cb
->args
[1];
1464 memcpy(nla_data(count_attr
), &act_count
, sizeof(u32
));
1469 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1470 if (NETLINK_CB(cb
->skb
).portid
&& ret
)
1471 nlh
->nlmsg_flags
|= NLM_F_MULTI
;
1472 module_put(a_o
->owner
);
1476 module_put(a_o
->owner
);
1481 struct tcf_action_net
{
1482 struct rhashtable egdev_ht
;
1485 static unsigned int tcf_action_net_id
;
1487 struct tcf_action_egdev_cb
{
1488 struct list_head list
;
1493 struct tcf_action_egdev
{
1494 struct rhash_head ht_node
;
1495 const struct net_device
*dev
;
1496 unsigned int refcnt
;
1497 struct list_head cb_list
;
1500 static const struct rhashtable_params tcf_action_egdev_ht_params
= {
1501 .key_offset
= offsetof(struct tcf_action_egdev
, dev
),
1502 .head_offset
= offsetof(struct tcf_action_egdev
, ht_node
),
1503 .key_len
= sizeof(const struct net_device
*),
1506 static struct tcf_action_egdev
*
1507 tcf_action_egdev_lookup(const struct net_device
*dev
)
1509 struct net
*net
= dev_net(dev
);
1510 struct tcf_action_net
*tan
= net_generic(net
, tcf_action_net_id
);
1512 return rhashtable_lookup_fast(&tan
->egdev_ht
, &dev
,
1513 tcf_action_egdev_ht_params
);
1516 static struct tcf_action_egdev
*
1517 tcf_action_egdev_get(const struct net_device
*dev
)
1519 struct tcf_action_egdev
*egdev
;
1520 struct tcf_action_net
*tan
;
1522 egdev
= tcf_action_egdev_lookup(dev
);
1526 egdev
= kzalloc(sizeof(*egdev
), GFP_KERNEL
);
1529 INIT_LIST_HEAD(&egdev
->cb_list
);
1531 tan
= net_generic(dev_net(dev
), tcf_action_net_id
);
1532 rhashtable_insert_fast(&tan
->egdev_ht
, &egdev
->ht_node
,
1533 tcf_action_egdev_ht_params
);
1540 static void tcf_action_egdev_put(struct tcf_action_egdev
*egdev
)
1542 struct tcf_action_net
*tan
;
1544 if (--egdev
->refcnt
)
1546 tan
= net_generic(dev_net(egdev
->dev
), tcf_action_net_id
);
1547 rhashtable_remove_fast(&tan
->egdev_ht
, &egdev
->ht_node
,
1548 tcf_action_egdev_ht_params
);
1552 static struct tcf_action_egdev_cb
*
1553 tcf_action_egdev_cb_lookup(struct tcf_action_egdev
*egdev
,
1554 tc_setup_cb_t
*cb
, void *cb_priv
)
1556 struct tcf_action_egdev_cb
*egdev_cb
;
1558 list_for_each_entry(egdev_cb
, &egdev
->cb_list
, list
)
1559 if (egdev_cb
->cb
== cb
&& egdev_cb
->cb_priv
== cb_priv
)
1564 static int tcf_action_egdev_cb_call(struct tcf_action_egdev
*egdev
,
1565 enum tc_setup_type type
,
1566 void *type_data
, bool err_stop
)
1568 struct tcf_action_egdev_cb
*egdev_cb
;
1572 list_for_each_entry(egdev_cb
, &egdev
->cb_list
, list
) {
1573 err
= egdev_cb
->cb(type
, type_data
, egdev_cb
->cb_priv
);
1584 static int tcf_action_egdev_cb_add(struct tcf_action_egdev
*egdev
,
1585 tc_setup_cb_t
*cb
, void *cb_priv
)
1587 struct tcf_action_egdev_cb
*egdev_cb
;
1589 egdev_cb
= tcf_action_egdev_cb_lookup(egdev
, cb
, cb_priv
);
1590 if (WARN_ON(egdev_cb
))
1592 egdev_cb
= kzalloc(sizeof(*egdev_cb
), GFP_KERNEL
);
1596 egdev_cb
->cb_priv
= cb_priv
;
1597 list_add(&egdev_cb
->list
, &egdev
->cb_list
);
1601 static void tcf_action_egdev_cb_del(struct tcf_action_egdev
*egdev
,
1602 tc_setup_cb_t
*cb
, void *cb_priv
)
1604 struct tcf_action_egdev_cb
*egdev_cb
;
1606 egdev_cb
= tcf_action_egdev_cb_lookup(egdev
, cb
, cb_priv
);
1607 if (WARN_ON(!egdev_cb
))
1609 list_del(&egdev_cb
->list
);
1613 static int __tc_setup_cb_egdev_register(const struct net_device
*dev
,
1614 tc_setup_cb_t
*cb
, void *cb_priv
)
1616 struct tcf_action_egdev
*egdev
= tcf_action_egdev_get(dev
);
1621 err
= tcf_action_egdev_cb_add(egdev
, cb
, cb_priv
);
1627 tcf_action_egdev_put(egdev
);
1630 int tc_setup_cb_egdev_register(const struct net_device
*dev
,
1631 tc_setup_cb_t
*cb
, void *cb_priv
)
1636 err
= __tc_setup_cb_egdev_register(dev
, cb
, cb_priv
);
1640 EXPORT_SYMBOL_GPL(tc_setup_cb_egdev_register
);
1642 static void __tc_setup_cb_egdev_unregister(const struct net_device
*dev
,
1643 tc_setup_cb_t
*cb
, void *cb_priv
)
1645 struct tcf_action_egdev
*egdev
= tcf_action_egdev_lookup(dev
);
1647 if (WARN_ON(!egdev
))
1649 tcf_action_egdev_cb_del(egdev
, cb
, cb_priv
);
1650 tcf_action_egdev_put(egdev
);
1652 void tc_setup_cb_egdev_unregister(const struct net_device
*dev
,
1653 tc_setup_cb_t
*cb
, void *cb_priv
)
1656 __tc_setup_cb_egdev_unregister(dev
, cb
, cb_priv
);
1659 EXPORT_SYMBOL_GPL(tc_setup_cb_egdev_unregister
);
1661 int tc_setup_cb_egdev_call(const struct net_device
*dev
,
1662 enum tc_setup_type type
, void *type_data
,
1665 struct tcf_action_egdev
*egdev
= tcf_action_egdev_lookup(dev
);
1669 return tcf_action_egdev_cb_call(egdev
, type
, type_data
, err_stop
);
1671 EXPORT_SYMBOL_GPL(tc_setup_cb_egdev_call
);
1673 static __net_init
int tcf_action_net_init(struct net
*net
)
1675 struct tcf_action_net
*tan
= net_generic(net
, tcf_action_net_id
);
1677 return rhashtable_init(&tan
->egdev_ht
, &tcf_action_egdev_ht_params
);
1680 static void __net_exit
tcf_action_net_exit(struct net
*net
)
1682 struct tcf_action_net
*tan
= net_generic(net
, tcf_action_net_id
);
1684 rhashtable_destroy(&tan
->egdev_ht
);
1687 static struct pernet_operations tcf_action_net_ops
= {
1688 .init
= tcf_action_net_init
,
1689 .exit
= tcf_action_net_exit
,
1690 .id
= &tcf_action_net_id
,
1691 .size
= sizeof(struct tcf_action_net
),
1694 static int __init
tc_action_init(void)
1698 err
= register_pernet_subsys(&tcf_action_net_ops
);
1702 rtnl_register(PF_UNSPEC
, RTM_NEWACTION
, tc_ctl_action
, NULL
, 0);
1703 rtnl_register(PF_UNSPEC
, RTM_DELACTION
, tc_ctl_action
, NULL
, 0);
1704 rtnl_register(PF_UNSPEC
, RTM_GETACTION
, tc_ctl_action
, tc_dump_action
,
1710 subsys_initcall(tc_action_init
);