2 * net/sched/sch_mq.c Classful multiqueue dummy scheduler
4 * Copyright (c) 2009 Patrick McHardy <kaber@trash.net>
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * version 2 as published by the Free Software Foundation.
11 #include <linux/types.h>
12 #include <linux/slab.h>
13 #include <linux/kernel.h>
14 #include <linux/export.h>
15 #include <linux/string.h>
16 #include <linux/errno.h>
17 #include <linux/skbuff.h>
18 #include <net/netlink.h>
19 #include <net/pkt_cls.h>
20 #include <net/pkt_sched.h>
21 #include <net/sch_generic.h>
24 struct Qdisc
**qdiscs
;
27 static int mq_offload(struct Qdisc
*sch
, enum tc_mq_command cmd
)
29 struct net_device
*dev
= qdisc_dev(sch
);
30 struct tc_mq_qopt_offload opt
= {
32 .handle
= sch
->handle
,
35 if (!tc_can_offload(dev
) || !dev
->netdev_ops
->ndo_setup_tc
)
38 return dev
->netdev_ops
->ndo_setup_tc(dev
, TC_SETUP_QDISC_MQ
, &opt
);
41 static int mq_offload_stats(struct Qdisc
*sch
)
43 struct tc_mq_qopt_offload opt
= {
44 .command
= TC_MQ_STATS
,
45 .handle
= sch
->handle
,
47 .bstats
= &sch
->bstats
,
48 .qstats
= &sch
->qstats
,
52 return qdisc_offload_dump_helper(sch
, TC_SETUP_QDISC_MQ
, &opt
);
55 static void mq_destroy(struct Qdisc
*sch
)
57 struct net_device
*dev
= qdisc_dev(sch
);
58 struct mq_sched
*priv
= qdisc_priv(sch
);
61 mq_offload(sch
, TC_MQ_DESTROY
);
65 for (ntx
= 0; ntx
< dev
->num_tx_queues
&& priv
->qdiscs
[ntx
]; ntx
++)
66 qdisc_put(priv
->qdiscs
[ntx
]);
70 static int mq_init(struct Qdisc
*sch
, struct nlattr
*opt
,
71 struct netlink_ext_ack
*extack
)
73 struct net_device
*dev
= qdisc_dev(sch
);
74 struct mq_sched
*priv
= qdisc_priv(sch
);
75 struct netdev_queue
*dev_queue
;
79 if (sch
->parent
!= TC_H_ROOT
)
82 if (!netif_is_multiqueue(dev
))
85 /* pre-allocate qdiscs, attachment can't fail */
86 priv
->qdiscs
= kcalloc(dev
->num_tx_queues
, sizeof(priv
->qdiscs
[0]),
91 for (ntx
= 0; ntx
< dev
->num_tx_queues
; ntx
++) {
92 dev_queue
= netdev_get_tx_queue(dev
, ntx
);
93 qdisc
= qdisc_create_dflt(dev_queue
, get_default_qdisc_ops(dev
, ntx
),
94 TC_H_MAKE(TC_H_MAJ(sch
->handle
),
99 priv
->qdiscs
[ntx
] = qdisc
;
100 qdisc
->flags
|= TCQ_F_ONETXQUEUE
| TCQ_F_NOPARENT
;
103 sch
->flags
|= TCQ_F_MQROOT
;
105 mq_offload(sch
, TC_MQ_CREATE
);
109 static void mq_attach(struct Qdisc
*sch
)
111 struct net_device
*dev
= qdisc_dev(sch
);
112 struct mq_sched
*priv
= qdisc_priv(sch
);
113 struct Qdisc
*qdisc
, *old
;
116 for (ntx
= 0; ntx
< dev
->num_tx_queues
; ntx
++) {
117 qdisc
= priv
->qdiscs
[ntx
];
118 old
= dev_graft_qdisc(qdisc
->dev_queue
, qdisc
);
121 #ifdef CONFIG_NET_SCHED
122 if (ntx
< dev
->real_num_tx_queues
)
123 qdisc_hash_add(qdisc
, false);
131 static int mq_dump(struct Qdisc
*sch
, struct sk_buff
*skb
)
133 struct net_device
*dev
= qdisc_dev(sch
);
139 memset(&sch
->bstats
, 0, sizeof(sch
->bstats
));
140 memset(&sch
->qstats
, 0, sizeof(sch
->qstats
));
142 /* MQ supports lockless qdiscs. However, statistics accounting needs
143 * to account for all, none, or a mix of locked and unlocked child
144 * qdiscs. Percpu stats are added to counters in-band and locking
145 * qdisc totals are added at end.
147 for (ntx
= 0; ntx
< dev
->num_tx_queues
; ntx
++) {
148 qdisc
= netdev_get_tx_queue(dev
, ntx
)->qdisc_sleeping
;
149 spin_lock_bh(qdisc_lock(qdisc
));
151 if (qdisc_is_percpu_stats(qdisc
)) {
152 qlen
= qdisc_qlen_sum(qdisc
);
153 __gnet_stats_copy_basic(NULL
, &sch
->bstats
,
156 __gnet_stats_copy_queue(&sch
->qstats
,
158 &qdisc
->qstats
, qlen
);
160 sch
->q
.qlen
+= qdisc
->q
.qlen
;
161 sch
->bstats
.bytes
+= qdisc
->bstats
.bytes
;
162 sch
->bstats
.packets
+= qdisc
->bstats
.packets
;
163 sch
->qstats
.qlen
+= qdisc
->qstats
.qlen
;
164 sch
->qstats
.backlog
+= qdisc
->qstats
.backlog
;
165 sch
->qstats
.drops
+= qdisc
->qstats
.drops
;
166 sch
->qstats
.requeues
+= qdisc
->qstats
.requeues
;
167 sch
->qstats
.overlimits
+= qdisc
->qstats
.overlimits
;
170 spin_unlock_bh(qdisc_lock(qdisc
));
173 return mq_offload_stats(sch
);
176 static struct netdev_queue
*mq_queue_get(struct Qdisc
*sch
, unsigned long cl
)
178 struct net_device
*dev
= qdisc_dev(sch
);
179 unsigned long ntx
= cl
- 1;
181 if (ntx
>= dev
->num_tx_queues
)
183 return netdev_get_tx_queue(dev
, ntx
);
186 static struct netdev_queue
*mq_select_queue(struct Qdisc
*sch
,
189 return mq_queue_get(sch
, TC_H_MIN(tcm
->tcm_parent
));
192 static int mq_graft(struct Qdisc
*sch
, unsigned long cl
, struct Qdisc
*new,
193 struct Qdisc
**old
, struct netlink_ext_ack
*extack
)
195 struct netdev_queue
*dev_queue
= mq_queue_get(sch
, cl
);
196 struct tc_mq_qopt_offload graft_offload
;
197 struct net_device
*dev
= qdisc_dev(sch
);
199 if (dev
->flags
& IFF_UP
)
202 *old
= dev_graft_qdisc(dev_queue
, new);
204 new->flags
|= TCQ_F_ONETXQUEUE
| TCQ_F_NOPARENT
;
205 if (dev
->flags
& IFF_UP
)
208 graft_offload
.handle
= sch
->handle
;
209 graft_offload
.graft_params
.queue
= cl
- 1;
210 graft_offload
.graft_params
.child_handle
= new ? new->handle
: 0;
211 graft_offload
.command
= TC_MQ_GRAFT
;
213 qdisc_offload_graft_helper(qdisc_dev(sch
), sch
, new, *old
,
214 TC_SETUP_QDISC_MQ
, &graft_offload
, extack
);
218 static struct Qdisc
*mq_leaf(struct Qdisc
*sch
, unsigned long cl
)
220 struct netdev_queue
*dev_queue
= mq_queue_get(sch
, cl
);
222 return dev_queue
->qdisc_sleeping
;
225 static unsigned long mq_find(struct Qdisc
*sch
, u32 classid
)
227 unsigned int ntx
= TC_H_MIN(classid
);
229 if (!mq_queue_get(sch
, ntx
))
234 static int mq_dump_class(struct Qdisc
*sch
, unsigned long cl
,
235 struct sk_buff
*skb
, struct tcmsg
*tcm
)
237 struct netdev_queue
*dev_queue
= mq_queue_get(sch
, cl
);
239 tcm
->tcm_parent
= TC_H_ROOT
;
240 tcm
->tcm_handle
|= TC_H_MIN(cl
);
241 tcm
->tcm_info
= dev_queue
->qdisc_sleeping
->handle
;
245 static int mq_dump_class_stats(struct Qdisc
*sch
, unsigned long cl
,
248 struct netdev_queue
*dev_queue
= mq_queue_get(sch
, cl
);
250 sch
= dev_queue
->qdisc_sleeping
;
251 if (gnet_stats_copy_basic(&sch
->running
, d
, NULL
, &sch
->bstats
) < 0 ||
252 qdisc_qstats_copy(d
, sch
) < 0)
257 static void mq_walk(struct Qdisc
*sch
, struct qdisc_walker
*arg
)
259 struct net_device
*dev
= qdisc_dev(sch
);
265 arg
->count
= arg
->skip
;
266 for (ntx
= arg
->skip
; ntx
< dev
->num_tx_queues
; ntx
++) {
267 if (arg
->fn(sch
, ntx
+ 1, arg
) < 0) {
275 static const struct Qdisc_class_ops mq_class_ops
= {
276 .select_queue
= mq_select_queue
,
281 .dump
= mq_dump_class
,
282 .dump_stats
= mq_dump_class_stats
,
285 struct Qdisc_ops mq_qdisc_ops __read_mostly
= {
286 .cl_ops
= &mq_class_ops
,
288 .priv_size
= sizeof(struct mq_sched
),
290 .destroy
= mq_destroy
,
293 .owner
= THIS_MODULE
,