2 * net/sched/cls_api.c Packet classifier API.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
17 #include <linux/module.h>
18 #include <linux/types.h>
19 #include <linux/kernel.h>
20 #include <linux/string.h>
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <linux/skbuff.h>
24 #include <linux/init.h>
25 #include <linux/kmod.h>
26 #include <linux/slab.h>
27 #include <linux/idr.h>
28 #include <net/net_namespace.h>
30 #include <net/netlink.h>
31 #include <net/pkt_sched.h>
32 #include <net/pkt_cls.h>
34 /* The list of all installed classifier types */
35 static LIST_HEAD(tcf_proto_base
);
37 /* Protects list of registered TC modules. It is pure SMP lock. */
38 static DEFINE_RWLOCK(cls_mod_lock
);
40 /* Find classifier type by string name */
42 static const struct tcf_proto_ops
*__tcf_proto_lookup_ops(const char *kind
)
44 const struct tcf_proto_ops
*t
, *res
= NULL
;
47 read_lock(&cls_mod_lock
);
48 list_for_each_entry(t
, &tcf_proto_base
, head
) {
49 if (strcmp(kind
, t
->kind
) == 0) {
50 if (try_module_get(t
->owner
))
55 read_unlock(&cls_mod_lock
);
60 static const struct tcf_proto_ops
*
61 tcf_proto_lookup_ops(const char *kind
, struct netlink_ext_ack
*extack
)
63 const struct tcf_proto_ops
*ops
;
65 ops
= __tcf_proto_lookup_ops(kind
);
70 request_module("cls_%s", kind
);
72 ops
= __tcf_proto_lookup_ops(kind
);
73 /* We dropped the RTNL semaphore in order to perform
74 * the module load. So, even if we succeeded in loading
75 * the module we have to replay the request. We indicate
79 module_put(ops
->owner
);
80 return ERR_PTR(-EAGAIN
);
83 NL_SET_ERR_MSG(extack
, "TC classifier not found");
84 return ERR_PTR(-ENOENT
);
87 /* Register(unregister) new classifier type */
89 int register_tcf_proto_ops(struct tcf_proto_ops
*ops
)
91 struct tcf_proto_ops
*t
;
94 write_lock(&cls_mod_lock
);
95 list_for_each_entry(t
, &tcf_proto_base
, head
)
96 if (!strcmp(ops
->kind
, t
->kind
))
99 list_add_tail(&ops
->head
, &tcf_proto_base
);
102 write_unlock(&cls_mod_lock
);
105 EXPORT_SYMBOL(register_tcf_proto_ops
);
107 static struct workqueue_struct
*tc_filter_wq
;
109 int unregister_tcf_proto_ops(struct tcf_proto_ops
*ops
)
111 struct tcf_proto_ops
*t
;
114 /* Wait for outstanding call_rcu()s, if any, from a
115 * tcf_proto_ops's destroy() handler.
118 flush_workqueue(tc_filter_wq
);
120 write_lock(&cls_mod_lock
);
121 list_for_each_entry(t
, &tcf_proto_base
, head
) {
128 write_unlock(&cls_mod_lock
);
131 EXPORT_SYMBOL(unregister_tcf_proto_ops
);
133 bool tcf_queue_work(struct rcu_work
*rwork
, work_func_t func
)
135 INIT_RCU_WORK(rwork
, func
);
136 return queue_rcu_work(tc_filter_wq
, rwork
);
138 EXPORT_SYMBOL(tcf_queue_work
);
140 /* Select new prio value from the range, managed by kernel. */
142 static inline u32
tcf_auto_prio(struct tcf_proto
*tp
)
144 u32 first
= TC_H_MAKE(0xC0000000U
, 0U);
147 first
= tp
->prio
- 1;
149 return TC_H_MAJ(first
);
152 static struct tcf_proto
*tcf_proto_create(const char *kind
, u32 protocol
,
153 u32 prio
, struct tcf_chain
*chain
,
154 struct netlink_ext_ack
*extack
)
156 struct tcf_proto
*tp
;
159 tp
= kzalloc(sizeof(*tp
), GFP_KERNEL
);
161 return ERR_PTR(-ENOBUFS
);
163 tp
->ops
= tcf_proto_lookup_ops(kind
, extack
);
164 if (IS_ERR(tp
->ops
)) {
165 err
= PTR_ERR(tp
->ops
);
168 tp
->classify
= tp
->ops
->classify
;
169 tp
->protocol
= protocol
;
173 err
= tp
->ops
->init(tp
);
175 module_put(tp
->ops
->owner
);
185 static void tcf_proto_destroy(struct tcf_proto
*tp
,
186 struct netlink_ext_ack
*extack
)
188 tp
->ops
->destroy(tp
, extack
);
189 module_put(tp
->ops
->owner
);
193 struct tcf_filter_chain_list_item
{
194 struct list_head list
;
195 tcf_chain_head_change_t
*chain_head_change
;
196 void *chain_head_change_priv
;
199 static struct tcf_chain
*tcf_chain_create(struct tcf_block
*block
,
202 struct tcf_chain
*chain
;
204 chain
= kzalloc(sizeof(*chain
), GFP_KERNEL
);
207 list_add_tail(&chain
->list
, &block
->chain_list
);
208 chain
->block
= block
;
209 chain
->index
= chain_index
;
212 block
->chain0
.chain
= chain
;
216 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item
*item
,
217 struct tcf_proto
*tp_head
)
219 if (item
->chain_head_change
)
220 item
->chain_head_change(tp_head
, item
->chain_head_change_priv
);
223 static void tcf_chain0_head_change(struct tcf_chain
*chain
,
224 struct tcf_proto
*tp_head
)
226 struct tcf_filter_chain_list_item
*item
;
227 struct tcf_block
*block
= chain
->block
;
231 list_for_each_entry(item
, &block
->chain0
.filter_chain_list
, list
)
232 tcf_chain_head_change_item(item
, tp_head
);
235 static void tcf_chain_destroy(struct tcf_chain
*chain
)
237 struct tcf_block
*block
= chain
->block
;
239 list_del(&chain
->list
);
241 block
->chain0
.chain
= NULL
;
243 if (list_empty(&block
->chain_list
) && !refcount_read(&block
->refcnt
))
247 static void tcf_chain_hold(struct tcf_chain
*chain
)
252 static bool tcf_chain_held_by_acts_only(struct tcf_chain
*chain
)
254 /* In case all the references are action references, this
255 * chain should not be shown to the user.
257 return chain
->refcnt
== chain
->action_refcnt
;
260 static struct tcf_chain
*tcf_chain_lookup(struct tcf_block
*block
,
263 struct tcf_chain
*chain
;
265 list_for_each_entry(chain
, &block
->chain_list
, list
) {
266 if (chain
->index
== chain_index
)
272 static int tc_chain_notify(struct tcf_chain
*chain
, struct sk_buff
*oskb
,
273 u32 seq
, u16 flags
, int event
, bool unicast
);
275 static struct tcf_chain
*__tcf_chain_get(struct tcf_block
*block
,
276 u32 chain_index
, bool create
,
279 struct tcf_chain
*chain
= tcf_chain_lookup(block
, chain_index
);
282 tcf_chain_hold(chain
);
286 chain
= tcf_chain_create(block
, chain_index
);
292 ++chain
->action_refcnt
;
294 /* Send notification only in case we got the first
295 * non-action reference. Until then, the chain acts only as
296 * a placeholder for actions pointing to it and user ought
297 * not know about them.
299 if (chain
->refcnt
- chain
->action_refcnt
== 1 && !by_act
)
300 tc_chain_notify(chain
, NULL
, 0, NLM_F_CREATE
| NLM_F_EXCL
,
301 RTM_NEWCHAIN
, false);
306 static struct tcf_chain
*tcf_chain_get(struct tcf_block
*block
, u32 chain_index
,
309 return __tcf_chain_get(block
, chain_index
, create
, false);
312 struct tcf_chain
*tcf_chain_get_by_act(struct tcf_block
*block
, u32 chain_index
)
314 return __tcf_chain_get(block
, chain_index
, true, true);
316 EXPORT_SYMBOL(tcf_chain_get_by_act
);
318 static void tc_chain_tmplt_del(struct tcf_chain
*chain
);
320 static void __tcf_chain_put(struct tcf_chain
*chain
, bool by_act
)
323 chain
->action_refcnt
--;
326 /* The last dropped non-action reference will trigger notification. */
327 if (chain
->refcnt
- chain
->action_refcnt
== 0 && !by_act
)
328 tc_chain_notify(chain
, NULL
, 0, 0, RTM_DELCHAIN
, false);
330 if (chain
->refcnt
== 0) {
331 tc_chain_tmplt_del(chain
);
332 tcf_chain_destroy(chain
);
336 static void tcf_chain_put(struct tcf_chain
*chain
)
338 __tcf_chain_put(chain
, false);
341 void tcf_chain_put_by_act(struct tcf_chain
*chain
)
343 __tcf_chain_put(chain
, true);
345 EXPORT_SYMBOL(tcf_chain_put_by_act
);
347 static void tcf_chain_put_explicitly_created(struct tcf_chain
*chain
)
349 if (chain
->explicitly_created
)
350 tcf_chain_put(chain
);
353 static void tcf_chain_flush(struct tcf_chain
*chain
)
355 struct tcf_proto
*tp
= rtnl_dereference(chain
->filter_chain
);
357 tcf_chain0_head_change(chain
, NULL
);
359 RCU_INIT_POINTER(chain
->filter_chain
, tp
->next
);
360 tcf_proto_destroy(tp
, NULL
);
361 tp
= rtnl_dereference(chain
->filter_chain
);
362 tcf_chain_put(chain
);
366 static bool tcf_block_offload_in_use(struct tcf_block
*block
)
368 return block
->offloadcnt
;
371 static int tcf_block_offload_cmd(struct tcf_block
*block
,
372 struct net_device
*dev
,
373 struct tcf_block_ext_info
*ei
,
374 enum tc_block_command command
,
375 struct netlink_ext_ack
*extack
)
377 struct tc_block_offload bo
= {};
379 bo
.command
= command
;
380 bo
.binder_type
= ei
->binder_type
;
383 return dev
->netdev_ops
->ndo_setup_tc(dev
, TC_SETUP_BLOCK
, &bo
);
386 static int tcf_block_offload_bind(struct tcf_block
*block
, struct Qdisc
*q
,
387 struct tcf_block_ext_info
*ei
,
388 struct netlink_ext_ack
*extack
)
390 struct net_device
*dev
= q
->dev_queue
->dev
;
393 if (!dev
->netdev_ops
->ndo_setup_tc
)
394 goto no_offload_dev_inc
;
396 /* If tc offload feature is disabled and the block we try to bind
397 * to already has some offloaded filters, forbid to bind.
399 if (!tc_can_offload(dev
) && tcf_block_offload_in_use(block
)) {
400 NL_SET_ERR_MSG(extack
, "Bind to offloaded block failed as dev has offload disabled");
404 err
= tcf_block_offload_cmd(block
, dev
, ei
, TC_BLOCK_BIND
, extack
);
405 if (err
== -EOPNOTSUPP
)
406 goto no_offload_dev_inc
;
410 if (tcf_block_offload_in_use(block
))
412 block
->nooffloaddevcnt
++;
416 static void tcf_block_offload_unbind(struct tcf_block
*block
, struct Qdisc
*q
,
417 struct tcf_block_ext_info
*ei
)
419 struct net_device
*dev
= q
->dev_queue
->dev
;
422 if (!dev
->netdev_ops
->ndo_setup_tc
)
423 goto no_offload_dev_dec
;
424 err
= tcf_block_offload_cmd(block
, dev
, ei
, TC_BLOCK_UNBIND
, NULL
);
425 if (err
== -EOPNOTSUPP
)
426 goto no_offload_dev_dec
;
430 WARN_ON(block
->nooffloaddevcnt
-- == 0);
434 tcf_chain0_head_change_cb_add(struct tcf_block
*block
,
435 struct tcf_block_ext_info
*ei
,
436 struct netlink_ext_ack
*extack
)
438 struct tcf_chain
*chain0
= block
->chain0
.chain
;
439 struct tcf_filter_chain_list_item
*item
;
441 item
= kmalloc(sizeof(*item
), GFP_KERNEL
);
443 NL_SET_ERR_MSG(extack
, "Memory allocation for head change callback item failed");
446 item
->chain_head_change
= ei
->chain_head_change
;
447 item
->chain_head_change_priv
= ei
->chain_head_change_priv
;
448 if (chain0
&& chain0
->filter_chain
)
449 tcf_chain_head_change_item(item
, chain0
->filter_chain
);
450 list_add(&item
->list
, &block
->chain0
.filter_chain_list
);
455 tcf_chain0_head_change_cb_del(struct tcf_block
*block
,
456 struct tcf_block_ext_info
*ei
)
458 struct tcf_chain
*chain0
= block
->chain0
.chain
;
459 struct tcf_filter_chain_list_item
*item
;
461 list_for_each_entry(item
, &block
->chain0
.filter_chain_list
, list
) {
462 if ((!ei
->chain_head_change
&& !ei
->chain_head_change_priv
) ||
463 (item
->chain_head_change
== ei
->chain_head_change
&&
464 item
->chain_head_change_priv
== ei
->chain_head_change_priv
)) {
466 tcf_chain_head_change_item(item
, NULL
);
467 list_del(&item
->list
);
476 spinlock_t idr_lock
; /* Protects idr */
480 static unsigned int tcf_net_id
;
482 static int tcf_block_insert(struct tcf_block
*block
, struct net
*net
,
483 struct netlink_ext_ack
*extack
)
485 struct tcf_net
*tn
= net_generic(net
, tcf_net_id
);
488 idr_preload(GFP_KERNEL
);
489 spin_lock(&tn
->idr_lock
);
490 err
= idr_alloc_u32(&tn
->idr
, block
, &block
->index
, block
->index
,
492 spin_unlock(&tn
->idr_lock
);
498 static void tcf_block_remove(struct tcf_block
*block
, struct net
*net
)
500 struct tcf_net
*tn
= net_generic(net
, tcf_net_id
);
502 spin_lock(&tn
->idr_lock
);
503 idr_remove(&tn
->idr
, block
->index
);
504 spin_unlock(&tn
->idr_lock
);
507 static struct tcf_block
*tcf_block_create(struct net
*net
, struct Qdisc
*q
,
509 struct netlink_ext_ack
*extack
)
511 struct tcf_block
*block
;
513 block
= kzalloc(sizeof(*block
), GFP_KERNEL
);
515 NL_SET_ERR_MSG(extack
, "Memory allocation for block failed");
516 return ERR_PTR(-ENOMEM
);
518 INIT_LIST_HEAD(&block
->chain_list
);
519 INIT_LIST_HEAD(&block
->cb_list
);
520 INIT_LIST_HEAD(&block
->owner_list
);
521 INIT_LIST_HEAD(&block
->chain0
.filter_chain_list
);
523 refcount_set(&block
->refcnt
, 1);
525 block
->index
= block_index
;
527 /* Don't store q pointer for blocks which are shared */
528 if (!tcf_block_shared(block
))
533 static struct tcf_block
*tcf_block_lookup(struct net
*net
, u32 block_index
)
535 struct tcf_net
*tn
= net_generic(net
, tcf_net_id
);
537 return idr_find(&tn
->idr
, block_index
);
540 static void tcf_block_flush_all_chains(struct tcf_block
*block
)
542 struct tcf_chain
*chain
;
544 /* Hold a refcnt for all chains, so that they don't disappear
545 * while we are iterating.
547 list_for_each_entry(chain
, &block
->chain_list
, list
)
548 tcf_chain_hold(chain
);
550 list_for_each_entry(chain
, &block
->chain_list
, list
)
551 tcf_chain_flush(chain
);
554 static void tcf_block_put_all_chains(struct tcf_block
*block
)
556 struct tcf_chain
*chain
, *tmp
;
558 /* At this point, all the chains should have refcnt >= 1. */
559 list_for_each_entry_safe(chain
, tmp
, &block
->chain_list
, list
) {
560 tcf_chain_put_explicitly_created(chain
);
561 tcf_chain_put(chain
);
566 * Set q, parent, cl when appropriate.
569 static struct tcf_block
*tcf_block_find(struct net
*net
, struct Qdisc
**q
,
570 u32
*parent
, unsigned long *cl
,
571 int ifindex
, u32 block_index
,
572 struct netlink_ext_ack
*extack
)
574 struct tcf_block
*block
;
577 if (ifindex
== TCM_IFINDEX_MAGIC_BLOCK
) {
578 block
= tcf_block_lookup(net
, block_index
);
580 NL_SET_ERR_MSG(extack
, "Block of given index was not found");
581 return ERR_PTR(-EINVAL
);
584 const struct Qdisc_class_ops
*cops
;
585 struct net_device
*dev
;
590 dev
= dev_get_by_index_rcu(net
, ifindex
);
593 return ERR_PTR(-ENODEV
);
599 *parent
= (*q
)->handle
;
601 *q
= qdisc_lookup_rcu(dev
, TC_H_MAJ(*parent
));
603 NL_SET_ERR_MSG(extack
, "Parent Qdisc doesn't exists");
609 *q
= qdisc_refcount_inc_nz(*q
);
611 NL_SET_ERR_MSG(extack
, "Parent Qdisc doesn't exists");
616 /* Is it classful? */
617 cops
= (*q
)->ops
->cl_ops
;
619 NL_SET_ERR_MSG(extack
, "Qdisc not classful");
624 if (!cops
->tcf_block
) {
625 NL_SET_ERR_MSG(extack
, "Class doesn't support blocks");
630 /* At this point we know that qdisc is not noop_qdisc,
631 * which means that qdisc holds a reference to net_device
632 * and we hold a reference to qdisc, so it is safe to release
637 /* Do we search for filter, attached to class? */
638 if (TC_H_MIN(*parent
)) {
639 *cl
= cops
->find(*q
, *parent
);
641 NL_SET_ERR_MSG(extack
, "Specified class doesn't exist");
647 /* And the last stroke */
648 block
= cops
->tcf_block(*q
, *cl
, extack
);
653 if (tcf_block_shared(block
)) {
654 NL_SET_ERR_MSG(extack
, "This filter block is shared. Please use the block index to manipulate the filters");
670 static void tcf_block_release(struct Qdisc
*q
, struct tcf_block
*block
)
676 struct tcf_block_owner_item
{
677 struct list_head list
;
679 enum tcf_block_binder_type binder_type
;
683 tcf_block_owner_netif_keep_dst(struct tcf_block
*block
,
685 enum tcf_block_binder_type binder_type
)
687 if (block
->keep_dst
&&
688 binder_type
!= TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS
&&
689 binder_type
!= TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS
)
690 netif_keep_dst(qdisc_dev(q
));
693 void tcf_block_netif_keep_dst(struct tcf_block
*block
)
695 struct tcf_block_owner_item
*item
;
697 block
->keep_dst
= true;
698 list_for_each_entry(item
, &block
->owner_list
, list
)
699 tcf_block_owner_netif_keep_dst(block
, item
->q
,
702 EXPORT_SYMBOL(tcf_block_netif_keep_dst
);
704 static int tcf_block_owner_add(struct tcf_block
*block
,
706 enum tcf_block_binder_type binder_type
)
708 struct tcf_block_owner_item
*item
;
710 item
= kmalloc(sizeof(*item
), GFP_KERNEL
);
714 item
->binder_type
= binder_type
;
715 list_add(&item
->list
, &block
->owner_list
);
719 static void tcf_block_owner_del(struct tcf_block
*block
,
721 enum tcf_block_binder_type binder_type
)
723 struct tcf_block_owner_item
*item
;
725 list_for_each_entry(item
, &block
->owner_list
, list
) {
726 if (item
->q
== q
&& item
->binder_type
== binder_type
) {
727 list_del(&item
->list
);
735 int tcf_block_get_ext(struct tcf_block
**p_block
, struct Qdisc
*q
,
736 struct tcf_block_ext_info
*ei
,
737 struct netlink_ext_ack
*extack
)
739 struct net
*net
= qdisc_net(q
);
740 struct tcf_block
*block
= NULL
;
741 bool created
= false;
744 if (ei
->block_index
) {
745 /* block_index not 0 means the shared block is requested */
746 block
= tcf_block_lookup(net
, ei
->block_index
);
748 refcount_inc(&block
->refcnt
);
752 block
= tcf_block_create(net
, q
, ei
->block_index
, extack
);
754 return PTR_ERR(block
);
756 if (tcf_block_shared(block
)) {
757 err
= tcf_block_insert(block
, net
, extack
);
759 goto err_block_insert
;
763 err
= tcf_block_owner_add(block
, q
, ei
->binder_type
);
765 goto err_block_owner_add
;
767 tcf_block_owner_netif_keep_dst(block
, q
, ei
->binder_type
);
769 err
= tcf_chain0_head_change_cb_add(block
, ei
, extack
);
771 goto err_chain0_head_change_cb_add
;
773 err
= tcf_block_offload_bind(block
, q
, ei
, extack
);
775 goto err_block_offload_bind
;
780 err_block_offload_bind
:
781 tcf_chain0_head_change_cb_del(block
, ei
);
782 err_chain0_head_change_cb_add
:
783 tcf_block_owner_del(block
, q
, ei
->binder_type
);
786 if (tcf_block_shared(block
))
787 tcf_block_remove(block
, net
);
791 refcount_dec(&block
->refcnt
);
795 EXPORT_SYMBOL(tcf_block_get_ext
);
797 static void tcf_chain_head_change_dflt(struct tcf_proto
*tp_head
, void *priv
)
799 struct tcf_proto __rcu
**p_filter_chain
= priv
;
801 rcu_assign_pointer(*p_filter_chain
, tp_head
);
804 int tcf_block_get(struct tcf_block
**p_block
,
805 struct tcf_proto __rcu
**p_filter_chain
, struct Qdisc
*q
,
806 struct netlink_ext_ack
*extack
)
808 struct tcf_block_ext_info ei
= {
809 .chain_head_change
= tcf_chain_head_change_dflt
,
810 .chain_head_change_priv
= p_filter_chain
,
813 WARN_ON(!p_filter_chain
);
814 return tcf_block_get_ext(p_block
, q
, &ei
, extack
);
816 EXPORT_SYMBOL(tcf_block_get
);
818 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
819 * actions should be all removed after flushing.
821 void tcf_block_put_ext(struct tcf_block
*block
, struct Qdisc
*q
,
822 struct tcf_block_ext_info
*ei
)
826 tcf_chain0_head_change_cb_del(block
, ei
);
827 tcf_block_owner_del(block
, q
, ei
->binder_type
);
829 if (refcount_dec_and_test(&block
->refcnt
)) {
830 /* Flushing/putting all chains will cause the block to be
831 * deallocated when last chain is freed. However, if chain_list
832 * is empty, block has to be manually deallocated. After block
833 * reference counter reached 0, it is no longer possible to
834 * increment it or add new chains to block.
836 bool free_block
= list_empty(&block
->chain_list
);
838 if (tcf_block_shared(block
))
839 tcf_block_remove(block
, block
->net
);
841 tcf_block_flush_all_chains(block
);
842 tcf_block_offload_unbind(block
, q
, ei
);
847 tcf_block_put_all_chains(block
);
849 tcf_block_offload_unbind(block
, q
, ei
);
852 EXPORT_SYMBOL(tcf_block_put_ext
);
854 void tcf_block_put(struct tcf_block
*block
)
856 struct tcf_block_ext_info ei
= {0, };
860 tcf_block_put_ext(block
, block
->q
, &ei
);
863 EXPORT_SYMBOL(tcf_block_put
);
865 struct tcf_block_cb
{
866 struct list_head list
;
873 void *tcf_block_cb_priv(struct tcf_block_cb
*block_cb
)
875 return block_cb
->cb_priv
;
877 EXPORT_SYMBOL(tcf_block_cb_priv
);
879 struct tcf_block_cb
*tcf_block_cb_lookup(struct tcf_block
*block
,
880 tc_setup_cb_t
*cb
, void *cb_ident
)
881 { struct tcf_block_cb
*block_cb
;
883 list_for_each_entry(block_cb
, &block
->cb_list
, list
)
884 if (block_cb
->cb
== cb
&& block_cb
->cb_ident
== cb_ident
)
888 EXPORT_SYMBOL(tcf_block_cb_lookup
);
890 void tcf_block_cb_incref(struct tcf_block_cb
*block_cb
)
894 EXPORT_SYMBOL(tcf_block_cb_incref
);
896 unsigned int tcf_block_cb_decref(struct tcf_block_cb
*block_cb
)
898 return --block_cb
->refcnt
;
900 EXPORT_SYMBOL(tcf_block_cb_decref
);
903 tcf_block_playback_offloads(struct tcf_block
*block
, tc_setup_cb_t
*cb
,
904 void *cb_priv
, bool add
, bool offload_in_use
,
905 struct netlink_ext_ack
*extack
)
907 struct tcf_chain
*chain
;
908 struct tcf_proto
*tp
;
911 list_for_each_entry(chain
, &block
->chain_list
, list
) {
912 for (tp
= rtnl_dereference(chain
->filter_chain
); tp
;
913 tp
= rtnl_dereference(tp
->next
)) {
914 if (tp
->ops
->reoffload
) {
915 err
= tp
->ops
->reoffload(tp
, add
, cb
, cb_priv
,
918 goto err_playback_remove
;
919 } else if (add
&& offload_in_use
) {
921 NL_SET_ERR_MSG(extack
, "Filter HW offload failed - classifier without re-offloading support");
922 goto err_playback_remove
;
930 tcf_block_playback_offloads(block
, cb
, cb_priv
, false, offload_in_use
,
935 struct tcf_block_cb
*__tcf_block_cb_register(struct tcf_block
*block
,
936 tc_setup_cb_t
*cb
, void *cb_ident
,
938 struct netlink_ext_ack
*extack
)
940 struct tcf_block_cb
*block_cb
;
943 /* Replay any already present rules */
944 err
= tcf_block_playback_offloads(block
, cb
, cb_priv
, true,
945 tcf_block_offload_in_use(block
),
950 block_cb
= kzalloc(sizeof(*block_cb
), GFP_KERNEL
);
952 return ERR_PTR(-ENOMEM
);
954 block_cb
->cb_ident
= cb_ident
;
955 block_cb
->cb_priv
= cb_priv
;
956 list_add(&block_cb
->list
, &block
->cb_list
);
959 EXPORT_SYMBOL(__tcf_block_cb_register
);
961 int tcf_block_cb_register(struct tcf_block
*block
,
962 tc_setup_cb_t
*cb
, void *cb_ident
,
963 void *cb_priv
, struct netlink_ext_ack
*extack
)
965 struct tcf_block_cb
*block_cb
;
967 block_cb
= __tcf_block_cb_register(block
, cb
, cb_ident
, cb_priv
,
969 return PTR_ERR_OR_ZERO(block_cb
);
971 EXPORT_SYMBOL(tcf_block_cb_register
);
973 void __tcf_block_cb_unregister(struct tcf_block
*block
,
974 struct tcf_block_cb
*block_cb
)
976 tcf_block_playback_offloads(block
, block_cb
->cb
, block_cb
->cb_priv
,
977 false, tcf_block_offload_in_use(block
),
979 list_del(&block_cb
->list
);
982 EXPORT_SYMBOL(__tcf_block_cb_unregister
);
984 void tcf_block_cb_unregister(struct tcf_block
*block
,
985 tc_setup_cb_t
*cb
, void *cb_ident
)
987 struct tcf_block_cb
*block_cb
;
989 block_cb
= tcf_block_cb_lookup(block
, cb
, cb_ident
);
992 __tcf_block_cb_unregister(block
, block_cb
);
994 EXPORT_SYMBOL(tcf_block_cb_unregister
);
996 static int tcf_block_cb_call(struct tcf_block
*block
, enum tc_setup_type type
,
997 void *type_data
, bool err_stop
)
999 struct tcf_block_cb
*block_cb
;
1003 /* Make sure all netdevs sharing this block are offload-capable. */
1004 if (block
->nooffloaddevcnt
&& err_stop
)
1007 list_for_each_entry(block_cb
, &block
->cb_list
, list
) {
1008 err
= block_cb
->cb(type
, type_data
, block_cb
->cb_priv
);
1019 /* Main classifier routine: scans classifier chain attached
1020 * to this qdisc, (optionally) tests for protocol and asks
1021 * specific classifiers.
1023 int tcf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
1024 struct tcf_result
*res
, bool compat_mode
)
1026 __be16 protocol
= tc_skb_protocol(skb
);
1027 #ifdef CONFIG_NET_CLS_ACT
1028 const int max_reclassify_loop
= 4;
1029 const struct tcf_proto
*orig_tp
= tp
;
1030 const struct tcf_proto
*first_tp
;
1035 for (; tp
; tp
= rcu_dereference_bh(tp
->next
)) {
1038 if (tp
->protocol
!= protocol
&&
1039 tp
->protocol
!= htons(ETH_P_ALL
))
1042 err
= tp
->classify(skb
, tp
, res
);
1043 #ifdef CONFIG_NET_CLS_ACT
1044 if (unlikely(err
== TC_ACT_RECLASSIFY
&& !compat_mode
)) {
1047 } else if (unlikely(TC_ACT_EXT_CMP(err
, TC_ACT_GOTO_CHAIN
))) {
1048 first_tp
= res
->goto_tp
;
1056 return TC_ACT_UNSPEC
; /* signal: continue lookup */
1057 #ifdef CONFIG_NET_CLS_ACT
1059 if (unlikely(limit
++ >= max_reclassify_loop
)) {
1060 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1061 tp
->chain
->block
->index
,
1063 ntohs(tp
->protocol
));
1068 protocol
= tc_skb_protocol(skb
);
1072 EXPORT_SYMBOL(tcf_classify
);
1074 struct tcf_chain_info
{
1075 struct tcf_proto __rcu
**pprev
;
1076 struct tcf_proto __rcu
*next
;
1079 static struct tcf_proto
*tcf_chain_tp_prev(struct tcf_chain_info
*chain_info
)
1081 return rtnl_dereference(*chain_info
->pprev
);
1084 static void tcf_chain_tp_insert(struct tcf_chain
*chain
,
1085 struct tcf_chain_info
*chain_info
,
1086 struct tcf_proto
*tp
)
1088 if (*chain_info
->pprev
== chain
->filter_chain
)
1089 tcf_chain0_head_change(chain
, tp
);
1090 RCU_INIT_POINTER(tp
->next
, tcf_chain_tp_prev(chain_info
));
1091 rcu_assign_pointer(*chain_info
->pprev
, tp
);
1092 tcf_chain_hold(chain
);
1095 static void tcf_chain_tp_remove(struct tcf_chain
*chain
,
1096 struct tcf_chain_info
*chain_info
,
1097 struct tcf_proto
*tp
)
1099 struct tcf_proto
*next
= rtnl_dereference(chain_info
->next
);
1101 if (tp
== chain
->filter_chain
)
1102 tcf_chain0_head_change(chain
, next
);
1103 RCU_INIT_POINTER(*chain_info
->pprev
, next
);
1104 tcf_chain_put(chain
);
1107 static struct tcf_proto
*tcf_chain_tp_find(struct tcf_chain
*chain
,
1108 struct tcf_chain_info
*chain_info
,
1109 u32 protocol
, u32 prio
,
1112 struct tcf_proto
**pprev
;
1113 struct tcf_proto
*tp
;
1115 /* Check the chain for existence of proto-tcf with this priority */
1116 for (pprev
= &chain
->filter_chain
;
1117 (tp
= rtnl_dereference(*pprev
)); pprev
= &tp
->next
) {
1118 if (tp
->prio
>= prio
) {
1119 if (tp
->prio
== prio
) {
1120 if (prio_allocate
||
1121 (tp
->protocol
!= protocol
&& protocol
))
1122 return ERR_PTR(-EINVAL
);
1129 chain_info
->pprev
= pprev
;
1130 chain_info
->next
= tp
? tp
->next
: NULL
;
1134 static int tcf_fill_node(struct net
*net
, struct sk_buff
*skb
,
1135 struct tcf_proto
*tp
, struct tcf_block
*block
,
1136 struct Qdisc
*q
, u32 parent
, void *fh
,
1137 u32 portid
, u32 seq
, u16 flags
, int event
)
1140 struct nlmsghdr
*nlh
;
1141 unsigned char *b
= skb_tail_pointer(skb
);
1143 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*tcm
), flags
);
1145 goto out_nlmsg_trim
;
1146 tcm
= nlmsg_data(nlh
);
1147 tcm
->tcm_family
= AF_UNSPEC
;
1151 tcm
->tcm_ifindex
= qdisc_dev(q
)->ifindex
;
1152 tcm
->tcm_parent
= parent
;
1154 tcm
->tcm_ifindex
= TCM_IFINDEX_MAGIC_BLOCK
;
1155 tcm
->tcm_block_index
= block
->index
;
1157 tcm
->tcm_info
= TC_H_MAKE(tp
->prio
, tp
->protocol
);
1158 if (nla_put_string(skb
, TCA_KIND
, tp
->ops
->kind
))
1159 goto nla_put_failure
;
1160 if (nla_put_u32(skb
, TCA_CHAIN
, tp
->chain
->index
))
1161 goto nla_put_failure
;
1163 tcm
->tcm_handle
= 0;
1165 if (tp
->ops
->dump
&& tp
->ops
->dump(net
, tp
, fh
, skb
, tcm
) < 0)
1166 goto nla_put_failure
;
1168 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1177 static int tfilter_notify(struct net
*net
, struct sk_buff
*oskb
,
1178 struct nlmsghdr
*n
, struct tcf_proto
*tp
,
1179 struct tcf_block
*block
, struct Qdisc
*q
,
1180 u32 parent
, void *fh
, int event
, bool unicast
)
1182 struct sk_buff
*skb
;
1183 u32 portid
= oskb
? NETLINK_CB(oskb
).portid
: 0;
1185 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1189 if (tcf_fill_node(net
, skb
, tp
, block
, q
, parent
, fh
, portid
,
1190 n
->nlmsg_seq
, n
->nlmsg_flags
, event
) <= 0) {
1196 return netlink_unicast(net
->rtnl
, skb
, portid
, MSG_DONTWAIT
);
1198 return rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1199 n
->nlmsg_flags
& NLM_F_ECHO
);
1202 static int tfilter_del_notify(struct net
*net
, struct sk_buff
*oskb
,
1203 struct nlmsghdr
*n
, struct tcf_proto
*tp
,
1204 struct tcf_block
*block
, struct Qdisc
*q
,
1205 u32 parent
, void *fh
, bool unicast
, bool *last
,
1206 struct netlink_ext_ack
*extack
)
1208 struct sk_buff
*skb
;
1209 u32 portid
= oskb
? NETLINK_CB(oskb
).portid
: 0;
1212 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1216 if (tcf_fill_node(net
, skb
, tp
, block
, q
, parent
, fh
, portid
,
1217 n
->nlmsg_seq
, n
->nlmsg_flags
, RTM_DELTFILTER
) <= 0) {
1218 NL_SET_ERR_MSG(extack
, "Failed to build del event notification");
1223 err
= tp
->ops
->delete(tp
, fh
, last
, extack
);
1230 return netlink_unicast(net
->rtnl
, skb
, portid
, MSG_DONTWAIT
);
1232 err
= rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
,
1233 n
->nlmsg_flags
& NLM_F_ECHO
);
1235 NL_SET_ERR_MSG(extack
, "Failed to send filter delete notification");
1239 static void tfilter_notify_chain(struct net
*net
, struct sk_buff
*oskb
,
1240 struct tcf_block
*block
, struct Qdisc
*q
,
1241 u32 parent
, struct nlmsghdr
*n
,
1242 struct tcf_chain
*chain
, int event
)
1244 struct tcf_proto
*tp
;
1246 for (tp
= rtnl_dereference(chain
->filter_chain
);
1247 tp
; tp
= rtnl_dereference(tp
->next
))
1248 tfilter_notify(net
, oskb
, n
, tp
, block
,
1249 q
, parent
, NULL
, event
, false);
1252 static int tc_new_tfilter(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1253 struct netlink_ext_ack
*extack
)
1255 struct net
*net
= sock_net(skb
->sk
);
1256 struct nlattr
*tca
[TCA_MAX
+ 1];
1263 struct Qdisc
*q
= NULL
;
1264 struct tcf_chain_info chain_info
;
1265 struct tcf_chain
*chain
= NULL
;
1266 struct tcf_block
*block
;
1267 struct tcf_proto
*tp
;
1273 if (!netlink_ns_capable(skb
, net
->user_ns
, CAP_NET_ADMIN
))
1279 err
= nlmsg_parse(n
, sizeof(*t
), tca
, TCA_MAX
, NULL
, extack
);
1284 protocol
= TC_H_MIN(t
->tcm_info
);
1285 prio
= TC_H_MAJ(t
->tcm_info
);
1286 prio_allocate
= false;
1287 parent
= t
->tcm_parent
;
1291 /* If no priority is provided by the user,
1294 if (n
->nlmsg_flags
& NLM_F_CREATE
) {
1295 prio
= TC_H_MAKE(0x80000000U
, 0U);
1296 prio_allocate
= true;
1298 NL_SET_ERR_MSG(extack
, "Invalid filter command with priority of zero");
1303 /* Find head of filter chain. */
1305 block
= tcf_block_find(net
, &q
, &parent
, &cl
,
1306 t
->tcm_ifindex
, t
->tcm_block_index
, extack
);
1307 if (IS_ERR(block
)) {
1308 err
= PTR_ERR(block
);
1312 chain_index
= tca
[TCA_CHAIN
] ? nla_get_u32(tca
[TCA_CHAIN
]) : 0;
1313 if (chain_index
> TC_ACT_EXT_VAL_MASK
) {
1314 NL_SET_ERR_MSG(extack
, "Specified chain index exceeds upper limit");
1318 chain
= tcf_chain_get(block
, chain_index
, true);
1320 NL_SET_ERR_MSG(extack
, "Cannot create specified filter chain");
1325 tp
= tcf_chain_tp_find(chain
, &chain_info
, protocol
,
1326 prio
, prio_allocate
);
1328 NL_SET_ERR_MSG(extack
, "Filter with specified priority/protocol not found");
1334 /* Proto-tcf does not exist, create new one */
1336 if (tca
[TCA_KIND
] == NULL
|| !protocol
) {
1337 NL_SET_ERR_MSG(extack
, "Filter kind and protocol must be specified");
1342 if (!(n
->nlmsg_flags
& NLM_F_CREATE
)) {
1343 NL_SET_ERR_MSG(extack
, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1349 prio
= tcf_auto_prio(tcf_chain_tp_prev(&chain_info
));
1351 tp
= tcf_proto_create(nla_data(tca
[TCA_KIND
]),
1352 protocol
, prio
, chain
, extack
);
1358 } else if (tca
[TCA_KIND
] && nla_strcmp(tca
[TCA_KIND
], tp
->ops
->kind
)) {
1359 NL_SET_ERR_MSG(extack
, "Specified filter kind does not match existing one");
1364 fh
= tp
->ops
->get(tp
, t
->tcm_handle
);
1367 if (!(n
->nlmsg_flags
& NLM_F_CREATE
)) {
1368 NL_SET_ERR_MSG(extack
, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1372 } else if (n
->nlmsg_flags
& NLM_F_EXCL
) {
1373 NL_SET_ERR_MSG(extack
, "Filter already exists");
1378 if (chain
->tmplt_ops
&& chain
->tmplt_ops
!= tp
->ops
) {
1379 NL_SET_ERR_MSG(extack
, "Chain template is set to a different filter kind");
1384 err
= tp
->ops
->change(net
, skb
, tp
, cl
, t
->tcm_handle
, tca
, &fh
,
1385 n
->nlmsg_flags
& NLM_F_CREATE
? TCA_ACT_NOREPLACE
: TCA_ACT_REPLACE
,
1389 tcf_chain_tp_insert(chain
, &chain_info
, tp
);
1390 tfilter_notify(net
, skb
, n
, tp
, block
, q
, parent
, fh
,
1391 RTM_NEWTFILTER
, false);
1394 tcf_proto_destroy(tp
, NULL
);
1399 tcf_chain_put(chain
);
1400 tcf_block_release(q
, block
);
1402 /* Replay the request. */
1407 static int tc_del_tfilter(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1408 struct netlink_ext_ack
*extack
)
1410 struct net
*net
= sock_net(skb
->sk
);
1411 struct nlattr
*tca
[TCA_MAX
+ 1];
1417 struct Qdisc
*q
= NULL
;
1418 struct tcf_chain_info chain_info
;
1419 struct tcf_chain
*chain
= NULL
;
1420 struct tcf_block
*block
;
1421 struct tcf_proto
*tp
= NULL
;
1422 unsigned long cl
= 0;
1426 if (!netlink_ns_capable(skb
, net
->user_ns
, CAP_NET_ADMIN
))
1429 err
= nlmsg_parse(n
, sizeof(*t
), tca
, TCA_MAX
, NULL
, extack
);
1434 protocol
= TC_H_MIN(t
->tcm_info
);
1435 prio
= TC_H_MAJ(t
->tcm_info
);
1436 parent
= t
->tcm_parent
;
1438 if (prio
== 0 && (protocol
|| t
->tcm_handle
|| tca
[TCA_KIND
])) {
1439 NL_SET_ERR_MSG(extack
, "Cannot flush filters with protocol, handle or kind set");
1443 /* Find head of filter chain. */
1445 block
= tcf_block_find(net
, &q
, &parent
, &cl
,
1446 t
->tcm_ifindex
, t
->tcm_block_index
, extack
);
1447 if (IS_ERR(block
)) {
1448 err
= PTR_ERR(block
);
1452 chain_index
= tca
[TCA_CHAIN
] ? nla_get_u32(tca
[TCA_CHAIN
]) : 0;
1453 if (chain_index
> TC_ACT_EXT_VAL_MASK
) {
1454 NL_SET_ERR_MSG(extack
, "Specified chain index exceeds upper limit");
1458 chain
= tcf_chain_get(block
, chain_index
, false);
1460 /* User requested flush on non-existent chain. Nothing to do,
1461 * so just return success.
1467 NL_SET_ERR_MSG(extack
, "Cannot find specified filter chain");
1473 tfilter_notify_chain(net
, skb
, block
, q
, parent
, n
,
1474 chain
, RTM_DELTFILTER
);
1475 tcf_chain_flush(chain
);
1480 tp
= tcf_chain_tp_find(chain
, &chain_info
, protocol
,
1482 if (!tp
|| IS_ERR(tp
)) {
1483 NL_SET_ERR_MSG(extack
, "Filter with specified priority/protocol not found");
1484 err
= tp
? PTR_ERR(tp
) : -ENOENT
;
1486 } else if (tca
[TCA_KIND
] && nla_strcmp(tca
[TCA_KIND
], tp
->ops
->kind
)) {
1487 NL_SET_ERR_MSG(extack
, "Specified filter kind does not match existing one");
1492 fh
= tp
->ops
->get(tp
, t
->tcm_handle
);
1495 if (t
->tcm_handle
== 0) {
1496 tcf_chain_tp_remove(chain
, &chain_info
, tp
);
1497 tfilter_notify(net
, skb
, n
, tp
, block
, q
, parent
, fh
,
1498 RTM_DELTFILTER
, false);
1499 tcf_proto_destroy(tp
, extack
);
1502 NL_SET_ERR_MSG(extack
, "Specified filter handle not found");
1508 err
= tfilter_del_notify(net
, skb
, n
, tp
, block
,
1509 q
, parent
, fh
, false, &last
,
1514 tcf_chain_tp_remove(chain
, &chain_info
, tp
);
1515 tcf_proto_destroy(tp
, extack
);
1521 tcf_chain_put(chain
);
1522 tcf_block_release(q
, block
);
1526 static int tc_get_tfilter(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1527 struct netlink_ext_ack
*extack
)
1529 struct net
*net
= sock_net(skb
->sk
);
1530 struct nlattr
*tca
[TCA_MAX
+ 1];
1536 struct Qdisc
*q
= NULL
;
1537 struct tcf_chain_info chain_info
;
1538 struct tcf_chain
*chain
= NULL
;
1539 struct tcf_block
*block
;
1540 struct tcf_proto
*tp
= NULL
;
1541 unsigned long cl
= 0;
1545 err
= nlmsg_parse(n
, sizeof(*t
), tca
, TCA_MAX
, NULL
, extack
);
1550 protocol
= TC_H_MIN(t
->tcm_info
);
1551 prio
= TC_H_MAJ(t
->tcm_info
);
1552 parent
= t
->tcm_parent
;
1555 NL_SET_ERR_MSG(extack
, "Invalid filter command with priority of zero");
1559 /* Find head of filter chain. */
1561 block
= tcf_block_find(net
, &q
, &parent
, &cl
,
1562 t
->tcm_ifindex
, t
->tcm_block_index
, extack
);
1563 if (IS_ERR(block
)) {
1564 err
= PTR_ERR(block
);
1568 chain_index
= tca
[TCA_CHAIN
] ? nla_get_u32(tca
[TCA_CHAIN
]) : 0;
1569 if (chain_index
> TC_ACT_EXT_VAL_MASK
) {
1570 NL_SET_ERR_MSG(extack
, "Specified chain index exceeds upper limit");
1574 chain
= tcf_chain_get(block
, chain_index
, false);
1576 NL_SET_ERR_MSG(extack
, "Cannot find specified filter chain");
1581 tp
= tcf_chain_tp_find(chain
, &chain_info
, protocol
,
1583 if (!tp
|| IS_ERR(tp
)) {
1584 NL_SET_ERR_MSG(extack
, "Filter with specified priority/protocol not found");
1585 err
= tp
? PTR_ERR(tp
) : -ENOENT
;
1587 } else if (tca
[TCA_KIND
] && nla_strcmp(tca
[TCA_KIND
], tp
->ops
->kind
)) {
1588 NL_SET_ERR_MSG(extack
, "Specified filter kind does not match existing one");
1593 fh
= tp
->ops
->get(tp
, t
->tcm_handle
);
1596 NL_SET_ERR_MSG(extack
, "Specified filter handle not found");
1599 err
= tfilter_notify(net
, skb
, n
, tp
, block
, q
, parent
,
1600 fh
, RTM_NEWTFILTER
, true);
1602 NL_SET_ERR_MSG(extack
, "Failed to send filter notify message");
1607 tcf_chain_put(chain
);
1608 tcf_block_release(q
, block
);
1612 struct tcf_dump_args
{
1613 struct tcf_walker w
;
1614 struct sk_buff
*skb
;
1615 struct netlink_callback
*cb
;
1616 struct tcf_block
*block
;
1621 static int tcf_node_dump(struct tcf_proto
*tp
, void *n
, struct tcf_walker
*arg
)
1623 struct tcf_dump_args
*a
= (void *)arg
;
1624 struct net
*net
= sock_net(a
->skb
->sk
);
1626 return tcf_fill_node(net
, a
->skb
, tp
, a
->block
, a
->q
, a
->parent
,
1627 n
, NETLINK_CB(a
->cb
->skb
).portid
,
1628 a
->cb
->nlh
->nlmsg_seq
, NLM_F_MULTI
,
1632 static bool tcf_chain_dump(struct tcf_chain
*chain
, struct Qdisc
*q
, u32 parent
,
1633 struct sk_buff
*skb
, struct netlink_callback
*cb
,
1634 long index_start
, long *p_index
)
1636 struct net
*net
= sock_net(skb
->sk
);
1637 struct tcf_block
*block
= chain
->block
;
1638 struct tcmsg
*tcm
= nlmsg_data(cb
->nlh
);
1639 struct tcf_dump_args arg
;
1640 struct tcf_proto
*tp
;
1642 for (tp
= rtnl_dereference(chain
->filter_chain
);
1643 tp
; tp
= rtnl_dereference(tp
->next
), (*p_index
)++) {
1644 if (*p_index
< index_start
)
1646 if (TC_H_MAJ(tcm
->tcm_info
) &&
1647 TC_H_MAJ(tcm
->tcm_info
) != tp
->prio
)
1649 if (TC_H_MIN(tcm
->tcm_info
) &&
1650 TC_H_MIN(tcm
->tcm_info
) != tp
->protocol
)
1652 if (*p_index
> index_start
)
1653 memset(&cb
->args
[1], 0,
1654 sizeof(cb
->args
) - sizeof(cb
->args
[0]));
1655 if (cb
->args
[1] == 0) {
1656 if (tcf_fill_node(net
, skb
, tp
, block
, q
, parent
, NULL
,
1657 NETLINK_CB(cb
->skb
).portid
,
1658 cb
->nlh
->nlmsg_seq
, NLM_F_MULTI
,
1659 RTM_NEWTFILTER
) <= 0)
1666 arg
.w
.fn
= tcf_node_dump
;
1671 arg
.parent
= parent
;
1673 arg
.w
.skip
= cb
->args
[1] - 1;
1675 arg
.w
.cookie
= cb
->args
[2];
1676 tp
->ops
->walk(tp
, &arg
.w
);
1677 cb
->args
[2] = arg
.w
.cookie
;
1678 cb
->args
[1] = arg
.w
.count
+ 1;
1685 /* called with RTNL */
1686 static int tc_dump_tfilter(struct sk_buff
*skb
, struct netlink_callback
*cb
)
1688 struct net
*net
= sock_net(skb
->sk
);
1689 struct nlattr
*tca
[TCA_MAX
+ 1];
1690 struct Qdisc
*q
= NULL
;
1691 struct tcf_block
*block
;
1692 struct tcf_chain
*chain
;
1693 struct tcmsg
*tcm
= nlmsg_data(cb
->nlh
);
1699 if (nlmsg_len(cb
->nlh
) < sizeof(*tcm
))
1702 err
= nlmsg_parse(cb
->nlh
, sizeof(*tcm
), tca
, TCA_MAX
, NULL
, NULL
);
1706 if (tcm
->tcm_ifindex
== TCM_IFINDEX_MAGIC_BLOCK
) {
1707 block
= tcf_block_lookup(net
, tcm
->tcm_block_index
);
1710 /* If we work with block index, q is NULL and parent value
1711 * will never be used in the following code. The check
1712 * in tcf_fill_node prevents it. However, compiler does not
1713 * see that far, so set parent to zero to silence the warning
1714 * about parent being uninitialized.
1718 const struct Qdisc_class_ops
*cops
;
1719 struct net_device
*dev
;
1720 unsigned long cl
= 0;
1722 dev
= __dev_get_by_index(net
, tcm
->tcm_ifindex
);
1726 parent
= tcm
->tcm_parent
;
1731 q
= qdisc_lookup(dev
, TC_H_MAJ(tcm
->tcm_parent
));
1735 cops
= q
->ops
->cl_ops
;
1738 if (!cops
->tcf_block
)
1740 if (TC_H_MIN(tcm
->tcm_parent
)) {
1741 cl
= cops
->find(q
, tcm
->tcm_parent
);
1745 block
= cops
->tcf_block(q
, cl
, NULL
);
1748 if (tcf_block_shared(block
))
1752 index_start
= cb
->args
[0];
1755 list_for_each_entry(chain
, &block
->chain_list
, list
) {
1756 if (tca
[TCA_CHAIN
] &&
1757 nla_get_u32(tca
[TCA_CHAIN
]) != chain
->index
)
1759 if (!tcf_chain_dump(chain
, q
, parent
, skb
, cb
,
1760 index_start
, &index
)) {
1766 cb
->args
[0] = index
;
1769 /* If we did no progress, the error (EMSGSIZE) is real */
1770 if (skb
->len
== 0 && err
)
1775 static int tc_chain_fill_node(struct tcf_chain
*chain
, struct net
*net
,
1776 struct sk_buff
*skb
, struct tcf_block
*block
,
1777 u32 portid
, u32 seq
, u16 flags
, int event
)
1779 unsigned char *b
= skb_tail_pointer(skb
);
1780 const struct tcf_proto_ops
*ops
;
1781 struct nlmsghdr
*nlh
;
1785 ops
= chain
->tmplt_ops
;
1786 priv
= chain
->tmplt_priv
;
1788 nlh
= nlmsg_put(skb
, portid
, seq
, event
, sizeof(*tcm
), flags
);
1790 goto out_nlmsg_trim
;
1791 tcm
= nlmsg_data(nlh
);
1792 tcm
->tcm_family
= AF_UNSPEC
;
1795 tcm
->tcm_handle
= 0;
1797 tcm
->tcm_ifindex
= qdisc_dev(block
->q
)->ifindex
;
1798 tcm
->tcm_parent
= block
->q
->handle
;
1800 tcm
->tcm_ifindex
= TCM_IFINDEX_MAGIC_BLOCK
;
1801 tcm
->tcm_block_index
= block
->index
;
1804 if (nla_put_u32(skb
, TCA_CHAIN
, chain
->index
))
1805 goto nla_put_failure
;
1808 if (nla_put_string(skb
, TCA_KIND
, ops
->kind
))
1809 goto nla_put_failure
;
1810 if (ops
->tmplt_dump(skb
, net
, priv
) < 0)
1811 goto nla_put_failure
;
1814 nlh
->nlmsg_len
= skb_tail_pointer(skb
) - b
;
1823 static int tc_chain_notify(struct tcf_chain
*chain
, struct sk_buff
*oskb
,
1824 u32 seq
, u16 flags
, int event
, bool unicast
)
1826 u32 portid
= oskb
? NETLINK_CB(oskb
).portid
: 0;
1827 struct tcf_block
*block
= chain
->block
;
1828 struct net
*net
= block
->net
;
1829 struct sk_buff
*skb
;
1831 skb
= alloc_skb(NLMSG_GOODSIZE
, GFP_KERNEL
);
1835 if (tc_chain_fill_node(chain
, net
, skb
, block
, portid
,
1836 seq
, flags
, event
) <= 0) {
1842 return netlink_unicast(net
->rtnl
, skb
, portid
, MSG_DONTWAIT
);
1844 return rtnetlink_send(skb
, net
, portid
, RTNLGRP_TC
, flags
& NLM_F_ECHO
);
1847 static int tc_chain_tmplt_add(struct tcf_chain
*chain
, struct net
*net
,
1848 struct nlattr
**tca
,
1849 struct netlink_ext_ack
*extack
)
1851 const struct tcf_proto_ops
*ops
;
1854 /* If kind is not set, user did not specify template. */
1858 ops
= tcf_proto_lookup_ops(nla_data(tca
[TCA_KIND
]), extack
);
1860 return PTR_ERR(ops
);
1861 if (!ops
->tmplt_create
|| !ops
->tmplt_destroy
|| !ops
->tmplt_dump
) {
1862 NL_SET_ERR_MSG(extack
, "Chain templates are not supported with specified classifier");
1866 tmplt_priv
= ops
->tmplt_create(net
, chain
, tca
, extack
);
1867 if (IS_ERR(tmplt_priv
)) {
1868 module_put(ops
->owner
);
1869 return PTR_ERR(tmplt_priv
);
1871 chain
->tmplt_ops
= ops
;
1872 chain
->tmplt_priv
= tmplt_priv
;
1876 static void tc_chain_tmplt_del(struct tcf_chain
*chain
)
1878 const struct tcf_proto_ops
*ops
= chain
->tmplt_ops
;
1880 /* If template ops are set, no work to do for us. */
1884 ops
->tmplt_destroy(chain
->tmplt_priv
);
1885 module_put(ops
->owner
);
1888 /* Add/delete/get a chain */
1890 static int tc_ctl_chain(struct sk_buff
*skb
, struct nlmsghdr
*n
,
1891 struct netlink_ext_ack
*extack
)
1893 struct net
*net
= sock_net(skb
->sk
);
1894 struct nlattr
*tca
[TCA_MAX
+ 1];
1898 struct Qdisc
*q
= NULL
;
1899 struct tcf_chain
*chain
= NULL
;
1900 struct tcf_block
*block
;
1904 if (n
->nlmsg_type
!= RTM_GETCHAIN
&&
1905 !netlink_ns_capable(skb
, net
->user_ns
, CAP_NET_ADMIN
))
1909 err
= nlmsg_parse(n
, sizeof(*t
), tca
, TCA_MAX
, NULL
, extack
);
1914 parent
= t
->tcm_parent
;
1917 block
= tcf_block_find(net
, &q
, &parent
, &cl
,
1918 t
->tcm_ifindex
, t
->tcm_block_index
, extack
);
1920 return PTR_ERR(block
);
1922 chain_index
= tca
[TCA_CHAIN
] ? nla_get_u32(tca
[TCA_CHAIN
]) : 0;
1923 if (chain_index
> TC_ACT_EXT_VAL_MASK
) {
1924 NL_SET_ERR_MSG(extack
, "Specified chain index exceeds upper limit");
1928 chain
= tcf_chain_lookup(block
, chain_index
);
1929 if (n
->nlmsg_type
== RTM_NEWCHAIN
) {
1931 if (tcf_chain_held_by_acts_only(chain
)) {
1932 /* The chain exists only because there is
1933 * some action referencing it.
1935 tcf_chain_hold(chain
);
1937 NL_SET_ERR_MSG(extack
, "Filter chain already exists");
1942 if (!(n
->nlmsg_flags
& NLM_F_CREATE
)) {
1943 NL_SET_ERR_MSG(extack
, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
1947 chain
= tcf_chain_create(block
, chain_index
);
1949 NL_SET_ERR_MSG(extack
, "Failed to create filter chain");
1955 if (!chain
|| tcf_chain_held_by_acts_only(chain
)) {
1956 NL_SET_ERR_MSG(extack
, "Cannot find specified filter chain");
1960 tcf_chain_hold(chain
);
1963 switch (n
->nlmsg_type
) {
1965 err
= tc_chain_tmplt_add(chain
, net
, tca
, extack
);
1968 /* In case the chain was successfully added, take a reference
1969 * to the chain. This ensures that an empty chain
1970 * does not disappear at the end of this function.
1972 tcf_chain_hold(chain
);
1973 chain
->explicitly_created
= true;
1974 tc_chain_notify(chain
, NULL
, 0, NLM_F_CREATE
| NLM_F_EXCL
,
1975 RTM_NEWCHAIN
, false);
1978 tfilter_notify_chain(net
, skb
, block
, q
, parent
, n
,
1979 chain
, RTM_DELTFILTER
);
1980 /* Flush the chain first as the user requested chain removal. */
1981 tcf_chain_flush(chain
);
1982 /* In case the chain was successfully deleted, put a reference
1983 * to the chain previously taken during addition.
1985 tcf_chain_put_explicitly_created(chain
);
1986 chain
->explicitly_created
= false;
1989 err
= tc_chain_notify(chain
, skb
, n
->nlmsg_seq
,
1990 n
->nlmsg_seq
, n
->nlmsg_type
, true);
1992 NL_SET_ERR_MSG(extack
, "Failed to send chain notify message");
1996 NL_SET_ERR_MSG(extack
, "Unsupported message type");
2001 tcf_chain_put(chain
);
2003 tcf_block_release(q
, block
);
2005 /* Replay the request. */
2010 /* called with RTNL */
2011 static int tc_dump_chain(struct sk_buff
*skb
, struct netlink_callback
*cb
)
2013 struct net
*net
= sock_net(skb
->sk
);
2014 struct nlattr
*tca
[TCA_MAX
+ 1];
2015 struct Qdisc
*q
= NULL
;
2016 struct tcf_block
*block
;
2017 struct tcf_chain
*chain
;
2018 struct tcmsg
*tcm
= nlmsg_data(cb
->nlh
);
2024 if (nlmsg_len(cb
->nlh
) < sizeof(*tcm
))
2027 err
= nlmsg_parse(cb
->nlh
, sizeof(*tcm
), tca
, TCA_MAX
, NULL
, NULL
);
2031 if (tcm
->tcm_ifindex
== TCM_IFINDEX_MAGIC_BLOCK
) {
2032 block
= tcf_block_lookup(net
, tcm
->tcm_block_index
);
2035 /* If we work with block index, q is NULL and parent value
2036 * will never be used in the following code. The check
2037 * in tcf_fill_node prevents it. However, compiler does not
2038 * see that far, so set parent to zero to silence the warning
2039 * about parent being uninitialized.
2043 const struct Qdisc_class_ops
*cops
;
2044 struct net_device
*dev
;
2045 unsigned long cl
= 0;
2047 dev
= __dev_get_by_index(net
, tcm
->tcm_ifindex
);
2051 parent
= tcm
->tcm_parent
;
2056 q
= qdisc_lookup(dev
, TC_H_MAJ(tcm
->tcm_parent
));
2060 cops
= q
->ops
->cl_ops
;
2063 if (!cops
->tcf_block
)
2065 if (TC_H_MIN(tcm
->tcm_parent
)) {
2066 cl
= cops
->find(q
, tcm
->tcm_parent
);
2070 block
= cops
->tcf_block(q
, cl
, NULL
);
2073 if (tcf_block_shared(block
))
2077 index_start
= cb
->args
[0];
2080 list_for_each_entry(chain
, &block
->chain_list
, list
) {
2081 if ((tca
[TCA_CHAIN
] &&
2082 nla_get_u32(tca
[TCA_CHAIN
]) != chain
->index
))
2084 if (index
< index_start
) {
2088 if (tcf_chain_held_by_acts_only(chain
))
2090 err
= tc_chain_fill_node(chain
, net
, skb
, block
,
2091 NETLINK_CB(cb
->skb
).portid
,
2092 cb
->nlh
->nlmsg_seq
, NLM_F_MULTI
,
2099 cb
->args
[0] = index
;
2102 /* If we did no progress, the error (EMSGSIZE) is real */
2103 if (skb
->len
== 0 && err
)
2108 void tcf_exts_destroy(struct tcf_exts
*exts
)
2110 #ifdef CONFIG_NET_CLS_ACT
2111 tcf_action_destroy(exts
->actions
, TCA_ACT_UNBIND
);
2112 kfree(exts
->actions
);
2113 exts
->nr_actions
= 0;
2116 EXPORT_SYMBOL(tcf_exts_destroy
);
2118 int tcf_exts_validate(struct net
*net
, struct tcf_proto
*tp
, struct nlattr
**tb
,
2119 struct nlattr
*rate_tlv
, struct tcf_exts
*exts
, bool ovr
,
2120 struct netlink_ext_ack
*extack
)
2122 #ifdef CONFIG_NET_CLS_ACT
2124 struct tc_action
*act
;
2125 size_t attr_size
= 0;
2127 if (exts
->police
&& tb
[exts
->police
]) {
2128 act
= tcf_action_init_1(net
, tp
, tb
[exts
->police
],
2129 rate_tlv
, "police", ovr
,
2130 TCA_ACT_BIND
, true, extack
);
2132 return PTR_ERR(act
);
2134 act
->type
= exts
->type
= TCA_OLD_COMPAT
;
2135 exts
->actions
[0] = act
;
2136 exts
->nr_actions
= 1;
2137 } else if (exts
->action
&& tb
[exts
->action
]) {
2140 err
= tcf_action_init(net
, tp
, tb
[exts
->action
],
2141 rate_tlv
, NULL
, ovr
, TCA_ACT_BIND
,
2142 exts
->actions
, &attr_size
, true,
2146 exts
->nr_actions
= err
;
2151 if ((exts
->action
&& tb
[exts
->action
]) ||
2152 (exts
->police
&& tb
[exts
->police
])) {
2153 NL_SET_ERR_MSG(extack
, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
2160 EXPORT_SYMBOL(tcf_exts_validate
);
2162 void tcf_exts_change(struct tcf_exts
*dst
, struct tcf_exts
*src
)
2164 #ifdef CONFIG_NET_CLS_ACT
2165 struct tcf_exts old
= *dst
;
2168 tcf_exts_destroy(&old
);
2171 EXPORT_SYMBOL(tcf_exts_change
);
2173 #ifdef CONFIG_NET_CLS_ACT
2174 static struct tc_action
*tcf_exts_first_act(struct tcf_exts
*exts
)
2176 if (exts
->nr_actions
== 0)
2179 return exts
->actions
[0];
2183 int tcf_exts_dump(struct sk_buff
*skb
, struct tcf_exts
*exts
)
2185 #ifdef CONFIG_NET_CLS_ACT
2186 struct nlattr
*nest
;
2188 if (exts
->action
&& tcf_exts_has_actions(exts
)) {
2190 * again for backward compatible mode - we want
2191 * to work with both old and new modes of entering
2192 * tc data even if iproute2 was newer - jhs
2194 if (exts
->type
!= TCA_OLD_COMPAT
) {
2195 nest
= nla_nest_start(skb
, exts
->action
);
2197 goto nla_put_failure
;
2199 if (tcf_action_dump(skb
, exts
->actions
, 0, 0) < 0)
2200 goto nla_put_failure
;
2201 nla_nest_end(skb
, nest
);
2202 } else if (exts
->police
) {
2203 struct tc_action
*act
= tcf_exts_first_act(exts
);
2204 nest
= nla_nest_start(skb
, exts
->police
);
2205 if (nest
== NULL
|| !act
)
2206 goto nla_put_failure
;
2207 if (tcf_action_dump_old(skb
, act
, 0, 0) < 0)
2208 goto nla_put_failure
;
2209 nla_nest_end(skb
, nest
);
2215 nla_nest_cancel(skb
, nest
);
2221 EXPORT_SYMBOL(tcf_exts_dump
);
2224 int tcf_exts_dump_stats(struct sk_buff
*skb
, struct tcf_exts
*exts
)
2226 #ifdef CONFIG_NET_CLS_ACT
2227 struct tc_action
*a
= tcf_exts_first_act(exts
);
2228 if (a
!= NULL
&& tcf_action_copy_stats(skb
, a
, 1) < 0)
2233 EXPORT_SYMBOL(tcf_exts_dump_stats
);
2235 static int tc_exts_setup_cb_egdev_call(struct tcf_exts
*exts
,
2236 enum tc_setup_type type
,
2237 void *type_data
, bool err_stop
)
2240 #ifdef CONFIG_NET_CLS_ACT
2241 const struct tc_action
*a
;
2242 struct net_device
*dev
;
2245 if (!tcf_exts_has_actions(exts
))
2248 for (i
= 0; i
< exts
->nr_actions
; i
++) {
2249 a
= exts
->actions
[i
];
2250 if (!a
->ops
->get_dev
)
2252 dev
= a
->ops
->get_dev(a
);
2255 ret
= tc_setup_cb_egdev_call(dev
, type
, type_data
, err_stop
);
2256 a
->ops
->put_dev(dev
);
2265 int tc_setup_cb_call(struct tcf_block
*block
, struct tcf_exts
*exts
,
2266 enum tc_setup_type type
, void *type_data
, bool err_stop
)
2271 ret
= tcf_block_cb_call(block
, type
, type_data
, err_stop
);
2276 if (!exts
|| ok_count
)
2278 ret
= tc_exts_setup_cb_egdev_call(exts
, type
, type_data
, err_stop
);
2285 EXPORT_SYMBOL(tc_setup_cb_call
);
2287 static __net_init
int tcf_net_init(struct net
*net
)
2289 struct tcf_net
*tn
= net_generic(net
, tcf_net_id
);
2291 spin_lock_init(&tn
->idr_lock
);
2296 static void __net_exit
tcf_net_exit(struct net
*net
)
2298 struct tcf_net
*tn
= net_generic(net
, tcf_net_id
);
2300 idr_destroy(&tn
->idr
);
2303 static struct pernet_operations tcf_net_ops
= {
2304 .init
= tcf_net_init
,
2305 .exit
= tcf_net_exit
,
2307 .size
= sizeof(struct tcf_net
),
2310 static int __init
tc_filter_init(void)
2314 tc_filter_wq
= alloc_ordered_workqueue("tc_filter_workqueue", 0);
2318 err
= register_pernet_subsys(&tcf_net_ops
);
2320 goto err_register_pernet_subsys
;
2322 rtnl_register(PF_UNSPEC
, RTM_NEWTFILTER
, tc_new_tfilter
, NULL
, 0);
2323 rtnl_register(PF_UNSPEC
, RTM_DELTFILTER
, tc_del_tfilter
, NULL
, 0);
2324 rtnl_register(PF_UNSPEC
, RTM_GETTFILTER
, tc_get_tfilter
,
2325 tc_dump_tfilter
, 0);
2326 rtnl_register(PF_UNSPEC
, RTM_NEWCHAIN
, tc_ctl_chain
, NULL
, 0);
2327 rtnl_register(PF_UNSPEC
, RTM_DELCHAIN
, tc_ctl_chain
, NULL
, 0);
2328 rtnl_register(PF_UNSPEC
, RTM_GETCHAIN
, tc_ctl_chain
,
2333 err_register_pernet_subsys
:
2334 destroy_workqueue(tc_filter_wq
);
2338 subsys_initcall(tc_filter_init
);