net: make softnet_data.defer_count an atomic

This is preparation work to remove the softnet_data.defer_lock,
as it is contended on hosts with large number of cores.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jason Xing <kerneljasonxing@gmail.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20250928084934.3266948-2-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 1b85454116f666ced61a1450d3f899940f499c05..27e3fa69253f694b98d32b6138cf491da5a8b824 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -3538,7 +3538,7 @@ struct softnet_data {
 
        /* Another possibly contended cache line */
        spinlock_t              defer_lock ____cacheline_aligned_in_smp;
-       int                     defer_count;
+       atomic_t                defer_count;
        int                     defer_ipi_scheduled;
        struct sk_buff          *defer_list;
        call_single_data_t      defer_csd;

diff --git a/net/core/dev.c b/net/core/dev.c
index 8b54fdf0289ab223fc37d27a078536db37646b55..8566678d83444e8aacbfea4842878279cf28516f 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6726,7 +6726,7 @@ static void skb_defer_free_flush(struct softnet_data *sd)
        spin_lock(&sd->defer_lock);
        skb = sd->defer_list;
        sd->defer_list = NULL;
-       sd->defer_count = 0;
+       atomic_set(&sd->defer_count, 0);
        spin_unlock(&sd->defer_lock);
 
        while (skb != NULL) {

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 618afd59afff8a6b5997866feeee1b75ed0b7037..16cd357d62a6bdb13038e3a50b6d8eb3660bbc7f 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -7202,14 +7202,12 @@ nodefer:        kfree_skb_napi_cache(skb);
 
        sd = &per_cpu(softnet_data, cpu);
        defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
-       if (READ_ONCE(sd->defer_count) >= defer_max)
+       if (atomic_read(&sd->defer_count) >= defer_max)
                goto nodefer;
 
        spin_lock_bh(&sd->defer_lock);
        /* Send an IPI every time queue reaches half capacity. */
-       kick = sd->defer_count == (defer_max >> 1);
-       /* Paired with the READ_ONCE() few lines above */
-       WRITE_ONCE(sd->defer_count, sd->defer_count + 1);
+       kick = (atomic_inc_return(&sd->defer_count) - 1) == (defer_max >> 1);
 
        skb->next = sd->defer_list;
        /* Paired with READ_ONCE() in skb_defer_free_flush() */