From: Eric Dumazet Date: Sun, 10 May 2026 09:14:49 +0000 (+0000) Subject: net/sched: add qdisc_qlen_inc() and qdisc_qlen_dec() X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=863cd78652018576c60f1b73477a15bdb0ea7551;p=thirdparty%2Flinux.git net/sched: add qdisc_qlen_inc() and qdisc_qlen_dec() Helpers to increment or decrement sch->q.qlen, with appropriate WRITE_ONCE() to prevent store tearing. Add other WRITE_ONCE() when sch->q.qlen is changed. Signed-off-by: Eric Dumazet Acked-by: Toke Høiland-Jørgensen Link: https://patch.msgid.link/20260510091455.4039245-3-edumazet@google.com Signed-off-by: Jakub Kicinski --- diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index ccfabfac674e..dbb3ba6416e4 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -542,6 +542,16 @@ static inline int qdisc_qlen(const struct Qdisc *q) return q->q.qlen; } +static inline void qdisc_qlen_inc(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen + 1); +} + +static inline void qdisc_qlen_dec(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen - 1); +} + static inline int qdisc_qlen_sum(const struct Qdisc *q) { __u32 qlen = q->qstats.qlen; @@ -549,9 +559,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q) if (qdisc_is_percpu_stats(q)) { for_each_possible_cpu(i) - qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; + qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen); } else { - qlen += q->q.qlen; + qlen += READ_ONCE(q->q.qlen); } return qlen; @@ -1068,7 +1078,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb, qh->tail = skb; qh->head = skb; } - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) @@ -1086,7 +1096,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb, if (!qh->head) qh->tail = skb; qh->head = skb; - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) @@ -1095,7 +1105,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) if (likely(skb != NULL)) { qh->head = skb->next; - qh->qlen--; + WRITE_ONCE(qh->qlen, qh->qlen - 1); if (qh->head == NULL) qh->tail = NULL; skb->next = NULL; @@ -1110,7 +1120,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir skb = __skb_dequeue(&sch->gso_skb); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); return skb; } @@ -1266,7 +1276,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) __skb_queue_head(&sch->gso_skb, skb); /* it's still part of the queue */ qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); } } @@ -1283,7 +1293,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, } else { qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } @@ -1295,7 +1305,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, this_cpu_add(sch->cpu_qstats->backlog, pkt_len); } else { sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qdisc_qlen_inc(sch); } } @@ -1311,7 +1321,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) qdisc_qstats_cpu_qlen_dec(sch); } else { qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } else { skb = sch->dequeue(sch); @@ -1332,7 +1342,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) qh->head = NULL; qh->tail = NULL; - qh->qlen = 0; + WRITE_ONCE(qh->qlen, 0); } } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 6f7847c5536f..cefa2d8ac5ec 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -805,7 +805,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cl = cops->find(sch, parentid); cops->qlen_notify(sch, cl); } - sch->q.qlen -= n; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - n); sch->qstats.backlog -= len; __qdisc_qstats_drop(sch, drops); } diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index d931e8d51f72..7ab75a52f7d1 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1612,7 +1612,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) cake_advance_shaper(q, b, skb, now, true); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); - sch->q.qlen--; + qdisc_qlen_dec(sch); cake_heapify(q, 0); @@ -1822,7 +1822,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, segs); flow_queue_add(flow, segs); - sch->q.qlen++; + qdisc_qlen_inc(sch); numsegs++; slen += segs->len; q->buffer_used += segs->truesize; @@ -1861,7 +1861,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len); consume_skb(ack); } else { - sch->q.qlen++; + qdisc_qlen_inc(sch); q->buffer_used += skb->truesize; } @@ -1987,7 +1987,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; q->buffer_used -= skb->truesize; - sch->q.qlen--; + qdisc_qlen_dec(sch); if (q->overflow_timeout) cake_heapify(q, b->overflow_idx[q->cur_flow]); diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c index 8c9a0400c862..a75e58876797 100644 --- a/net/sched/sch_cbs.c +++ b/net/sched/sch_cbs.c @@ -97,7 +97,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch, return err; sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child) qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index 2875bcdb18a4..73d3e673dc7b 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx, if (idx == q->tail) choke_zap_tail_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb)); qdisc_drop(skb, sch, to_free); @@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (sch->q.qlen < q->limit) { q->tab[q->tail] = skb; q->tail = (q->tail + 1) & q->tab_mask; - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); return NET_XMIT_SUCCESS; } @@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch) skb = q->tab[q->head]; q->tab[q->head] = NULL; choke_zap_head_holes(q); - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); @@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt, } dropped += qdisc_pkt_len(skb); qdisc_qstats_backlog_dec(sch, skb); - --sch->q.qlen; + qdisc_qlen_dec(sch); rtnl_qdisc_drop(skb, sch); } qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped); diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 01335a49e091..925fa0cfd730 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -366,7 +366,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -399,7 +399,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) bstats_update(&cl->bstats, skb); qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index 241e6a46bd00..c6416f09dddd 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -415,7 +415,7 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch, dualpi2_skb_cb(skb)->apply_step = skb_apply_step(skb, q); /* Keep the overall qdisc stats consistent */ - ++sch->q.qlen; + qdisc_qlen_inc(sch); qdisc_qstats_backlog_inc(sch, skb); ++q->packets_in_l; if (!q->l_head_ts) @@ -530,7 +530,7 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch, qdisc_qstats_backlog_dec(q->l_queue, skb); /* Keep the global queue size consistent */ - --sch->q.qlen; + qdisc_qlen_dec(sch); q->memory_used -= skb->truesize; } else if (c_len) { skb = __qdisc_dequeue_head(&sch->q); @@ -888,7 +888,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, * l_queue on enqueue; qdisc_dequeue_internal() * handled l_queue, so we further account for sch. */ - --sch->q.qlen; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); q->memory_used -= skb->truesize; rtnl_qdisc_drop(skb, q->l_queue); diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c index c74d778c32a1..ada87a81da6a 100644 --- a/net/sched/sch_etf.c +++ b/net/sched/sch_etf.c @@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch, rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost); qdisc_qstats_backlog_inc(sch, nskb); - sch->q.qlen++; + qdisc_qlen_inc(sch); /* Now we may need to re-arm the qdisc watchdog for the next packet. */ reset_watchdog(sch); @@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb, qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, &to_free); qdisc_qstats_overlimit(sch); - sch->q.qlen--; + qdisc_qlen_dec(sch); } kfree_skb_list(to_free); @@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb) q->last = skb->tstamp; - sch->q.qlen--; + qdisc_qlen_dec(sch); } static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch) @@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch) rb_erase_cached(&skb->rbnode, &q->head); rtnl_kfree_skbs(skb, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index a4b07b661b77..c817e0a6c146 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -449,7 +449,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return err; } @@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index f2edcf872981..1e34ac136b15 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow, fq_erase_head(sch, flow, skb); skb_mark_not_on_list(skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } @@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch, flow_queue_add(f, skb); qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -801,7 +801,7 @@ static void fq_reset(struct Qdisc *sch) struct fq_flow *f; unsigned int idx; - sch->q.qlen = 0; + WRITE_ONCE(sch->q.qlen, 0); sch->qstats.backlog = 0; fq_flow_purge(&q->internal); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index ed42ce62a17f..cae8483fbb0c 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -178,7 +178,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, q->memory_usage -= mem; __qdisc_qstats_drop(sch, i); sch->qstats.backlog -= len; - sch->q.qlen -= i; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - i); return idx; } @@ -215,7 +215,8 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch, get_codel_cb(skb)->mem_usage = skb->truesize; q->memory_usage += get_codel_cb(skb)->mem_usage; memory_limited = q->memory_usage > q->memory_limit; - if (++sch->q.qlen <= sch->limit && !memory_limited) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit && !memory_limited) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -266,7 +267,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) WRITE_ONCE(q->backlogs[flow - q->flows], q->backlogs[flow - q->flows] - qdisc_pkt_len(skb)); q->memory_usage -= get_codel_cb(skb)->mem_usage; - sch->q.qlen--; + qdisc_qlen_dec(sch); sch->qstats.backlog -= qdisc_pkt_len(skb); } return skb; diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 7becbf5362b3..0a4eca4ab086 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -185,7 +185,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, q->stats.packets_in++; q->memory_usage += skb->truesize; sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qdisc_qlen_inc(sch); flow_queue_add(sel_flow, skb); if (list_empty(&sel_flow->flowchain)) { list_add_tail(&sel_flow->flowchain, &q->new_flows); @@ -263,7 +263,7 @@ begin: skb = dequeue_head(flow); pkt_len = qdisc_pkt_len(skb); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 05c250c483f0..7b4a5874ed1f 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q) qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = SKB_XOFF_MAGIC; @@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q, qdisc_qstats_cpu_qlen_inc(q); } else { qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } if (lock) @@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) } else { q->qstats.requeues++; qdisc_qstats_backlog_inc(q, skb); - q->q.qlen++; + qdisc_qlen_inc(q); } skb = next; @@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate, qdisc_qstats_cpu_qlen_dec(q); } else { qdisc_qstats_backlog_dec(q, skb); - q->q.qlen--; + qdisc_qlen_dec(q); } } else { skb = NULL; @@ -1062,7 +1062,7 @@ void qdisc_reset(struct Qdisc *qdisc) __skb_queue_purge(&qdisc->gso_skb); __skb_queue_purge(&qdisc->skb_bad_txq); - qdisc->q.qlen = 0; + WRITE_ONCE(qdisc->q.qlen, 0); qdisc->qstats.backlog = 0; } EXPORT_SYMBOL(qdisc_reset); diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 83b2ca2e37fc..e71a565100ed 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1561,7 +1561,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); if (first && !cl_in_el_or_vttree(cl)) { if (cl->cl_flags & HFSC_RSC) @@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch) qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 96021f52d835..1e25b75daae2 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free) if (bucket->head) { struct sk_buff *skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop(skb, sch, to_free); } @@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } bucket->deficit = weight * q->quantum; } - if (++sch->q.qlen <= sch->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= sch->limit) return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; @@ -443,7 +444,7 @@ begin: if (bucket->head) { skb = dequeue_head(bucket); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); } diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index eb12381795ce..c22ccd8eae8c 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -651,7 +651,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -951,7 +951,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch) ok: qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c index a0133a7b9d3b..ec8c91d3fde0 100644 --- a/net/sched/sch_mq.c +++ b/net/sched/sch_mq.c @@ -143,10 +143,10 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL"); void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) { struct net_device *dev = qdisc_dev(sch); + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -163,10 +163,11 @@ void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); } EXPORT_SYMBOL_NS_GPL(mq_dump_common, "NET_SCHED_INTERNAL"); diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index 002add5ce9e0..91a92992cd24 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -555,10 +555,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) struct mqprio_sched *priv = qdisc_priv(sch); struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb); struct tc_mqprio_qopt opt = { 0 }; + unsigned int qlen = 0; struct Qdisc *qdisc; unsigned int ntx; - sch->q.qlen = 0; + qlen = 0; gnet_stats_basic_sync_init(&sch->bstats); memset(&sch->qstats, 0, sizeof(sch->qstats)); @@ -575,10 +576,11 @@ static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb) &qdisc->bstats, false); gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } + WRITE_ONCE(sch->q.qlen, qlen); mqprio_qopt_reconstruct(dev, &opt); opt.hw = priv->hw_offload; @@ -663,12 +665,12 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, __acquires(d->lock) { if (cl >= TC_H_MIN_PRIORITY) { - int i; - __u32 qlen; - struct gnet_stats_queue qstats = {0}; - struct gnet_stats_basic_sync bstats; struct net_device *dev = qdisc_dev(sch); struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK]; + struct gnet_stats_queue qstats = {0}; + struct gnet_stats_basic_sync bstats; + u32 qlen = 0; + int i; gnet_stats_basic_sync_init(&bstats); /* Drop lock here it will be reclaimed before touching @@ -689,11 +691,11 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl, &qdisc->bstats, false); gnet_stats_add_queue(&qstats, qdisc->cpu_qstats, &qdisc->qstats); - sch->q.qlen += qdisc_qlen(qdisc); + qlen += qdisc_qlen(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); } - qlen = qdisc_qlen(sch) + qstats.qlen; + qlen = qlen + qstats.qlen; /* Reclaim root sleeping lock before completing stats */ if (d->lock) diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index 9f822fee113d..4e465d11e3d7 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -106,7 +106,7 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch) skb = qdisc->dequeue(qdisc); if (skb) { qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index bc18e1976b6e..57b12cbca453 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -416,7 +416,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) rb_insert_color(&nskb->rbnode, &q->t_root); } q->t_len++; - sch->q.qlen++; + qdisc_qlen_inc(sch); } /* netem can't properly corrupt a megapacket (like we get from GSO), so instead @@ -751,19 +751,19 @@ deliver: if (net_xmit_drop_count(err)) qdisc_qstats_drop(sch); sch->qstats.backlog -= pkt_len; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_tree_reduce_backlog(sch, 1, pkt_len); } goto tfifo_dequeue; } - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } @@ -776,7 +776,7 @@ deliver: if (q->qdisc) { skb = q->qdisc->ops->dequeue(q->qdisc); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); goto deliver; } } diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 9e2b9a490db2..fe42ae3d6b69 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -86,7 +86,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } if (net_xmit_drop_count(ret)) @@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } } diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 699e45873f86..195c434aae5f 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch) if (!skb) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); skb = agg_dequeue(in_serv_agg, cl, len); if (!skb) { - sch->q.qlen++; + qdisc_qlen_inc(sch); return NULL; } @@ -1265,7 +1265,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, _bstats_update(&cl->bstats, len, gso_segs); sch->qstats.backlog += len; - ++sch->q.qlen; + qdisc_qlen_inc(sch); agg = cl->agg; /* if the class is active, then done here */ diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 4d0e44a2e7c6..0719590dfd73 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -139,7 +139,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.pdrop, q->stats.pdrop + 1); @@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } else { if (!red_is_idling(&q->vars)) red_start_of_idle_period(&q->vars); diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index d3ee8e5479b3..efd9251c3add 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -416,7 +416,7 @@ enqueue: ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.childdrop, @@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch) if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); decrement_qlen(skb, q); } diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index f39822babf88..f9807ee2cf6c 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -302,7 +302,7 @@ drop: len = qdisc_pkt_len(skb); WRITE_ONCE(slot->backlog, slot->backlog - len); sfq_dec(q, x); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT); return len; @@ -456,7 +456,8 @@ enqueue: /* We could use a bigger initial quantum for new flows */ WRITE_ONCE(slot->allot, q->quantum); } - if (++sch->q.qlen <= q->limit) + qdisc_qlen_inc(sch); + if (sch->q.qlen <= q->limit) return NET_XMIT_SUCCESS; qlen = slot->qlen; @@ -497,7 +498,7 @@ next_slot: skb = slot_dequeue_head(slot); sfq_dec(q, a); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb)); /* Is the slot empty? */ @@ -596,7 +597,7 @@ drop: WRITE_ONCE(slot->allot, q->quantum); } } - sch->q.qlen -= dropped; + WRITE_ONCE(sch->q.qlen, sch->q.qlen - dropped); qdisc_tree_reduce_backlog(sch, dropped, drop_len); } diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c index f485f62ab721..52abfb4015a3 100644 --- a/net/sched/sch_skbprio.c +++ b/net/sched/sch_skbprio.c @@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (prio < q->lowest_prio) q->lowest_prio = prio; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch) if (unlikely(!skb)) return NULL; - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 71b690e1974d..d6b981e5df11 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, } qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); return qdisc_enqueue(skb, child, to_free); } @@ -755,7 +755,7 @@ skip_peek_checks: qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); return skb; } diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index f2340164f579..25edf11a7d67 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -231,7 +231,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, len += seg_len; } } - sch->q.qlen += nb; + WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb); sch->qstats.backlog += len; if (nb > 0) { qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len); @@ -264,7 +264,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, } sch->qstats.backlog += len; - sch->q.qlen++; + qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } @@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch) q->tokens = toks; q->ptokens = ptoks; qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); return skb; } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index ec4039a201a2..e7bbc9e5174d 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -107,7 +107,7 @@ teql_dequeue(struct Qdisc *sch) } else { qdisc_bstats_update(sch, skb); } - sch->q.qlen = dat->q.qlen + q->q.qlen; + WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen)); return skb; }