From: Eric Dumazet Date: Sun, 10 May 2026 09:14:50 +0000 (+0000) Subject: net/sched: annotate data-races around sch->qstats.backlog X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0e5b05f44d818d1efe5154832d2e3f3c44352c02;p=thirdparty%2Flinux.git net/sched: annotate data-races around sch->qstats.backlog Add qstats_backlog_sub() and qstats_backlog_add() helpers and use them instead of open-coding them. These helpers use WRITE_ONCE() to prevent store-tearing. Also use WRITE_ONCE() in fq_reset() and qdisc_reset() when sch->qstats.backlog is cleared. Signed-off-by: Eric Dumazet Acked-by: Toke Høiland-Jørgensen Link: https://patch.msgid.link/20260510091455.4039245-4-edumazet@google.com Signed-off-by: Jakub Kicinski --- diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index dbb3ba6416e4..391ee8530017 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -965,10 +965,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch, bstats_update(&sch->bstats, skb); } +static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val) +{ + WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val); +} + static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, const struct sk_buff *skb) { - sch->qstats.backlog -= qdisc_pkt_len(skb); + qstats_backlog_sub(sch, qdisc_pkt_len(skb)); } static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, @@ -977,10 +982,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); } +static inline void qstats_backlog_add(struct Qdisc *sch, u32 val) +{ + WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val); +} + static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, const struct sk_buff *skb) { - sch->qstats.backlog += qdisc_pkt_len(skb); + qstats_backlog_add(sch, qdisc_pkt_len(skb)); } static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, @@ -1304,7 +1314,7 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, qdisc_qstats_cpu_qlen_inc(sch); this_cpu_add(sch->cpu_qstats->backlog, pkt_len); } else { - sch->qstats.backlog += pkt_len; + qstats_backlog_add(sch, pkt_len); qdisc_qlen_inc(sch); } } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index cefa2d8ac5ec..3c779e5098ef 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -806,7 +806,7 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len) cops->qlen_notify(sch, cl); } WRITE_ONCE(sch->q.qlen, sch->q.qlen - n); - sch->qstats.backlog -= len; + qstats_backlog_sub(sch, len); __qdisc_qstats_drop(sch, drops); } rcu_read_unlock(); diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 7ab75a52f7d1..a3c185505afc 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1600,10 +1600,10 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) b->unresponsive_flow_count + 1); len = qdisc_pkt_len(skb); - q->buffer_used -= skb->truesize; + qstats_backlog_sub(sch, len); + q->buffer_used -= skb->truesize; WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] - len); - sch->qstats.backlog -= len; WRITE_ONCE(flow->dropped, flow->dropped + 1); WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1); @@ -1830,7 +1830,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, } /* stats */ - sch->qstats.backlog += slen; + qstats_backlog_add(sch, slen); q->avg_window_bytes += slen; WRITE_ONCE(b->bytes, b->bytes + slen); WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen); @@ -1867,7 +1867,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* stats */ WRITE_ONCE(b->packets, b->packets + 1); - sch->qstats.backlog += len - ack_pkt_len; + qstats_backlog_add(sch, len - ack_pkt_len); q->avg_window_bytes += len - ack_pkt_len; WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len); WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len); @@ -1985,7 +1985,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) len = qdisc_pkt_len(skb); WRITE_ONCE(b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len); WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); - sch->qstats.backlog -= len; + qstats_backlog_sub(sch, len); q->buffer_used -= skb->truesize; qdisc_qlen_dec(sch); diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c index a75e58876797..2cfa0fd92829 100644 --- a/net/sched/sch_cbs.c +++ b/net/sched/sch_cbs.c @@ -96,7 +96,7 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (err != NET_XMIT_SUCCESS) return err; - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 317aae0ec7bd..91dd2e629af8 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -42,7 +42,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) struct sk_buff *skb = __qdisc_dequeue_head(&sch->q); if (skb) { - sch->qstats.backlog -= qdisc_pkt_len(skb); + qstats_backlog_sub(sch, qdisc_pkt_len(skb)); prefetch(&skb->end); /* we'll need skb_shinfo() */ } return skb; diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 925fa0cfd730..3f6687fa9666 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -365,7 +365,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, cl->deficit = cl->quantum; } - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return err; } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index c817e0a6c146..1cc559634ed2 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -448,7 +448,7 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, cl->deficit = cl->quantum; } - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return err; } diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 1e34ac136b15..796cb8046a90 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -802,7 +802,7 @@ static void fq_reset(struct Qdisc *sch) unsigned int idx; WRITE_ONCE(sch->q.qlen, 0); - sch->qstats.backlog = 0; + WRITE_ONCE(sch->qstats.backlog, 0); fq_flow_purge(&q->internal); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index cae8483fbb0c..1b1de693d4c6 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -177,7 +177,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] - len); q->memory_usage -= mem; __qdisc_qstats_drop(sch, i); - sch->qstats.backlog -= len; + qstats_backlog_sub(sch, len); WRITE_ONCE(sch->q.qlen, sch->q.qlen - i); return idx; } @@ -268,7 +268,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) q->backlogs[flow - q->flows] - qdisc_pkt_len(skb)); q->memory_usage -= get_codel_cb(skb)->mem_usage; qdisc_qlen_dec(sch); - sch->qstats.backlog -= qdisc_pkt_len(skb); + qdisc_qstats_backlog_dec(sch, skb); } return skb; } diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 0a4eca4ab086..72f48fa4010b 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -184,7 +184,7 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, pkt_len = qdisc_pkt_len(skb); q->stats.packets_in++; q->memory_usage += skb->truesize; - sch->qstats.backlog += pkt_len; + qstats_backlog_add(sch, pkt_len); qdisc_qlen_inc(sch); flow_queue_add(sel_flow, skb); if (list_empty(&sel_flow->flowchain)) { @@ -262,7 +262,7 @@ begin: if (flow->head) { skb = dequeue_head(flow); pkt_len = qdisc_pkt_len(skb); - sch->qstats.backlog -= pkt_len; + qstats_backlog_sub(sch, pkt_len); qdisc_qlen_dec(sch); qdisc_bstats_update(sch, skb); } diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 7b4a5874ed1f..237ee1cd0136 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1063,7 +1063,7 @@ void qdisc_reset(struct Qdisc *qdisc) __skb_queue_purge(&qdisc->skb_bad_txq); WRITE_ONCE(qdisc->q.qlen, 0); - qdisc->qstats.backlog = 0; + WRITE_ONCE(qdisc->qstats.backlog, 0); } EXPORT_SYMBOL(qdisc_reset); diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index 8ae65572162c..fcc1a4c03636 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -388,7 +388,7 @@ static int gred_offload_dump_stats(struct Qdisc *sch) bytes += u64_stats_read(&hw_stats->stats.bstats[i].bytes); packets += u64_stats_read(&hw_stats->stats.bstats[i].packets); sch->qstats.qlen += hw_stats->stats.qstats[i].qlen; - sch->qstats.backlog += hw_stats->stats.qstats[i].backlog; + qstats_backlog_add(sch, hw_stats->stats.qstats[i].backlog); __qdisc_qstats_drop(sch, hw_stats->stats.qstats[i].drops); sch->qstats.requeues += hw_stats->stats.qstats[i].requeues; sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits; diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index e71a565100ed..59409ee2d2ff 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1560,7 +1560,7 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) return err; } - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); if (first && !cl_in_el_or_vttree(cl)) { diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index c22ccd8eae8c..1e600f65c876 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -650,7 +650,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, htb_activate(q, cl); } - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 57b12cbca453..ddbfea9dd32a 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -750,7 +750,7 @@ deliver: if (err != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(err)) qdisc_qstats_drop(sch); - sch->qstats.backlog -= pkt_len; + qstats_backlog_sub(sch, pkt_len); qdisc_qlen_dec(sch); qdisc_tree_reduce_backlog(sch, 1, pkt_len); } diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index fe42ae3d6b69..e4dd56a89072 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -85,7 +85,7 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) ret = qdisc_enqueue(skb, qdisc, to_free); if (ret == NET_XMIT_SUCCESS) { - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; } diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index 195c434aae5f..cb56787e1d25 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -1264,7 +1264,7 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, } _bstats_update(&cl->bstats, len, gso_segs); - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); agg = cl->agg; diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 0719590dfd73..d7598214270b 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -138,7 +138,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, len = qdisc_pkt_len(skb); ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); } else if (net_xmit_drop_count(ret)) { WRITE_ONCE(q->stats.pdrop, diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index efd9251c3add..b1d465094276 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -415,7 +415,7 @@ enqueue: memcpy(&cb, sfb_skb_cb(skb), sizeof(cb)); ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { @@ -592,7 +592,7 @@ static int sfb_dump(struct Qdisc *sch, struct sk_buff *skb) .penalty_burst = q->penalty_burst, }; - sch->qstats.backlog = q->qdisc->qstats.backlog; + WRITE_ONCE(sch->qstats.backlog, READ_ONCE(q->qdisc->qstats.backlog)); opts = nla_nest_start_noflag(skb, TCA_OPTIONS); if (opts == NULL) goto nla_put_failure; diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index f9807ee2cf6c..758b88f21865 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -427,7 +427,7 @@ congestion_drop: /* We know we have at least one packet in queue */ head = slot_dequeue_head(slot); delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb); - sch->qstats.backlog -= delta; + qstats_backlog_sub(sch, delta); WRITE_ONCE(slot->backlog, slot->backlog - delta); qdisc_drop_reason(head, sch, to_free, QDISC_DROP_FLOW_LIMIT); diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index 25edf11a7d67..67c7aaaf8f60 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -232,7 +232,7 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch, } } WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb); - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); if (nb > 0) { qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len); consume_skb(skb); @@ -263,7 +263,7 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch, return ret; } - sch->qstats.backlog += len; + qstats_backlog_add(sch, len); qdisc_qlen_inc(sch); return NET_XMIT_SUCCESS; }