net: sched: extend Qdisc with rcu

[ Upstream commit 3a7d0d07a386716b459b00783b11a8211cefcc0f ]

Currently, Qdisc API functions assume that users have rtnl lock taken. To
implement rtnl unlocked classifiers update interface, Qdisc API must be
extended with functions that do not require rtnl lock.

Extend Qdisc structure with rcu. Implement special version of put function
qdisc_put_unlocked() that is called without rtnl lock taken. This function
only takes rtnl lock if Qdisc reference counter reached zero and is
intended to be used as optimization.

Signed-off-by: Vlad Buslov <vladbu@mellanox.com>
Acked-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
[Lee: Sent to Stable]
Link: https://syzkaller.appspot.com/bug?id=d7e411c5472dd5da33d8cc921ccadc747743a568
Reported-by: syzbot+5f229e48cccc804062c0@syzkaller.appspotmail.com
Signed-off-by: Lee Jones <lee.jones@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h
index 9cdd76348d9ab8849ea712b95d2d02a3ded0283d..bb9cb84114c151059ccac74ca07a113ea22c347a 100644 (file)
--- a/include/linux/rtnetlink.h
+++ b/include/linux/rtnetlink.h
@@ -85,6 +85,11 @@ static inline struct netdev_queue *dev_ingress_queue(struct net_device *dev)
        return rtnl_dereference(dev->ingress_queue);
 }
 
+static inline struct netdev_queue *dev_ingress_queue_rcu(struct net_device *dev)
+{
+       return rcu_dereference(dev->ingress_queue);
+}
+
 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev);
 
 #ifdef CONFIG_NET_INGRESS

diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index edca90ef3bdc4e4aacf83c691e09ef214e58a1b5..1a6ac924266dbd97f1e1be6acdff5500cbb01c6a 100644 (file)
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -103,6 +103,7 @@ int qdisc_set_default(const char *id);
 void qdisc_hash_add(struct Qdisc *q, bool invisible);
 void qdisc_hash_del(struct Qdisc *q);
 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle);
+struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle);
 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
                                        struct nlattr *tab,
                                        struct netlink_ext_ack *extack);

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 70dfbd0437530d122a09d326e360b3e601a9d8c8..04a5133d875e8f2b383dfd0cace759a0c19550de 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -108,6 +108,7 @@ struct Qdisc {
 
        spinlock_t              busylock ____cacheline_aligned_in_smp;
        spinlock_t              seqlock;
+       struct rcu_head         rcu;
 };
 
 static inline void qdisc_refcount_inc(struct Qdisc *qdisc)
@@ -560,6 +561,7 @@ struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
                              struct Qdisc *qdisc);
 void qdisc_reset(struct Qdisc *qdisc);
 void qdisc_put(struct Qdisc *qdisc);
+void qdisc_put_unlocked(struct Qdisc *qdisc);
 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n,
                               unsigned int len);
 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,

diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 60f40a396a039f20c6f455c55aa6711206ed764a..af035431bec60624e91427dfe94de83c7cc56cc1 100644 (file)
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -315,6 +315,24 @@ out:
        return q;
 }
 
+struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
+{
+       struct netdev_queue *nq;
+       struct Qdisc *q;
+
+       if (!handle)
+               return NULL;
+       q = qdisc_match_from_root(dev->qdisc, handle);
+       if (q)
+               goto out;
+
+       nq = dev_ingress_queue_rcu(dev);
+       if (nq)
+               q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
+out:
+       return q;
+}
+
 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
 {
        unsigned long cl;

diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 6fee2731b4c22f936b59532f5ac1f92a7de8ca18..80498c22d001bbd10a3ebb5f800356d726dd132d 100644 (file)
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -958,6 +958,13 @@ void qdisc_free(struct Qdisc *qdisc)
        kfree((char *) qdisc - qdisc->padded);
 }
 
+void qdisc_free_cb(struct rcu_head *head)
+{
+       struct Qdisc *q = container_of(head, struct Qdisc, rcu);
+
+       qdisc_free(q);
+}
+
 static void qdisc_destroy(struct Qdisc *qdisc)
 {
        const struct Qdisc_ops *ops;
@@ -991,7 +998,7 @@ static void qdisc_destroy(struct Qdisc *qdisc)
                kfree_skb_list(skb);
        }
 
-       qdisc_free(qdisc);
+       call_rcu(&qdisc->rcu, qdisc_free_cb);
 }
 
 void qdisc_put(struct Qdisc *qdisc)
@@ -1004,6 +1011,22 @@ void qdisc_put(struct Qdisc *qdisc)
 }
 EXPORT_SYMBOL(qdisc_put);
 
+/* Version of qdisc_put() that is called with rtnl mutex unlocked.
+ * Intended to be used as optimization, this function only takes rtnl lock if
+ * qdisc reference counter reached zero.
+ */
+
+void qdisc_put_unlocked(struct Qdisc *qdisc)
+{
+       if (qdisc->flags & TCQ_F_BUILTIN ||
+           !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
+               return;
+
+       qdisc_destroy(qdisc);
+       rtnl_unlock();
+}
+EXPORT_SYMBOL(qdisc_put_unlocked);
+
 /* Attach toplevel qdisc to device queue. */
 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
                              struct Qdisc *qdisc)