net/sched: rename qstats_overlimit_inc() to qstats_cpu_overlimit_inc()

qstats_overlimit_inc() is only used to increment per cpu overlimits.

It can use this_cpu_inc() to avoid this_cpu_ptr() extra cost
and avoid potential store tearing.

Change qstats_overlimit_inc() name and its argument type.

Also add a WRITE_ONCE() in qdisc_qstats_overlimit() to prevent
store tearing.

$ scripts/bloat-o-meter -t vmlinux.0 vmlinux.1
add/remove: 0/0 grow/shrink: 0/7 up/down: 0/-91 (-91)
Function                                     old     new   delta
tcf_skbmod_act                               772     764      -8
tcf_police_act                               733     725      -8
tcf_gate_act                                 318     310      -8
tcf_pedit_act                               1295    1284     -11
tcf_mirred_to_dev                           1126    1114     -12
tcf_ife_act                                 1077    1061     -16
tcf_mirred_act                              1324    1296     -28
Total: Before=24274627, After=24274536, chg -0.00%

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jamal Hadi Salim <jhs@mojatatu.com>
Link: https://patch.msgid.link/20260428070919.3109557-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/act_api.h b/include/net/act_api.h
index d11b791079302f50c47e174979767e0b24afc59a..2ec4ef9a5d0c8e9110f92f135cc3c31a38af0479 100644 (file)
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -250,7 +250,7 @@ static inline void tcf_action_inc_drop_qstats(struct tc_action *a)
 static inline void tcf_action_inc_overlimit_qstats(struct tc_action *a)
 {
        if (likely(a->cpu_qstats)) {
-               qstats_overlimit_inc(this_cpu_ptr(a->cpu_qstats));
+               qstats_cpu_overlimit_inc(a->cpu_qstats);
                return;
        }
        atomic_inc(&a->tcfa_overlimits);

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 11159a50d6a14535d7c173c81c0de80dff108807..cbfe9ed435fd77422a181074980e5779190ab9c3 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -1004,9 +1004,9 @@ static inline void qstats_drop_inc(struct gnet_stats_queue *qstats)
        qstats->drops++;
 }
 
-static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats)
+static inline void qstats_cpu_overlimit_inc(struct gnet_stats_queue __percpu *qstats)
 {
-       qstats->overlimits++;
+       this_cpu_inc(qstats->overlimits);
 }
 
 static inline void qdisc_qstats_drop(struct Qdisc *sch)
@@ -1021,7 +1021,7 @@ static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)
 
 static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
 {
-       sch->qstats.overlimits++;
+       WRITE_ONCE(sch->qstats.overlimits, sch->qstats.overlimits + 1);
 }
 
 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)

diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c
index d5e8a91bb4eb9f1f1f084e199b5ada4e7f7e7205..e1b825e14900d6f46bbfd1b7f72ab6cd554d8a73 100644 (file)
--- a/net/sched/act_ife.c
+++ b/net/sched/act_ife.c
@@ -750,7 +750,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
                         */
                        pr_info_ratelimited("Unknown metaid %d dlen %d\n",
                                            mtype, dlen);
-                       qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
+                       qstats_cpu_overlimit_inc(ife->common.cpu_qstats);
                }
        }
 
@@ -814,7 +814,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,
                /* abuse overlimits to count when we allow packet
                 * with no metadata
                 */
-               qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
+               qstats_cpu_overlimit_inc(ife->common.cpu_qstats);
                return action;
        }
        /* could be stupid policy setup or mtu config

diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index 12ea9e5a600536b603ea73cc99b4c00381287219..8060f43e4d11c0a26e1475db06b76426f50c5975 100644 (file)
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -307,7 +307,7 @@ TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb,
        }
 
 inc_overlimits:
-       qstats_overlimit_inc(this_cpu_ptr(police->common.cpu_qstats));
+       qstats_cpu_overlimit_inc(police->common.cpu_qstats);
 inc_drops:
        if (ret == TC_ACT_SHOT)
                qstats_drop_inc(this_cpu_ptr(police->common.cpu_qstats));

diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c
index 23ca46138f040d38de37684439873921bc9c86af..a464b0a3c1b81dba6c28c1141aa38c5c7cad3acb 100644 (file)
--- a/net/sched/act_skbmod.c
+++ b/net/sched/act_skbmod.c
@@ -87,7 +87,7 @@ out:
        return p->action;
 
 drop:
-       qstats_overlimit_inc(this_cpu_ptr(d->common.cpu_qstats));
+       qstats_cpu_overlimit_inc(d->common.cpu_qstats);
        return TC_ACT_SHOT;
 }