net: napi: Skip last poll when arming gro timer in busy poll

Skip the extra call to napi->poll(), if the gro timer is armed at the
end of busy polling. This removes the need for having a separate
__busy_poll_stop() routine and its code is moved directly into the
relevant places in busy_poll_stop(). Remove obsolete comment about
ndo_busy_poll_stop().

This is a follow-up to commit 58e2330bd455 ("net: napi: Avoid gro timer
misfiring at end of busypoll"), which has deferred arming the gro timer
to the end of __busy_poll_stop() to eliminate a race condition between
a short timer and long poll that could leave the queue stuck with
interrupts disabled and no timer armed.

Co-developed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Martin Karsten <mkarsten@uwaterloo.ca>
Link: https://patch.msgid.link/20260523012247.1574691-1-mkarsten@uwaterloo.ca
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/net/core/dev.c b/net/core/dev.c
index 26ac8eb9b259d489159c7ab5a2b206d425110b3b..a50cd68c6bfa21ddade57c13bf7942892bf99a2c 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6863,22 +6863,6 @@ static void skb_defer_free_flush(void)
 
 #if defined(CONFIG_NET_RX_BUSY_POLL)
 
-static void __busy_poll_stop(struct napi_struct *napi, unsigned long timeout)
-{
-       if (!timeout) {
-               gro_normal_list(&napi->gro);
-               __napi_schedule(napi);
-               return;
-       }
-
-       /* Flush too old packets. If HZ < 1000, flush all packets */
-       gro_flush_normal(&napi->gro, HZ >= 1000);
-
-       clear_bit(NAPI_STATE_SCHED, &napi->state);
-       hrtimer_start(&napi->timer, ns_to_ktime(timeout),
-                     HRTIMER_MODE_REL_PINNED);
-}
-
 enum {
        NAPI_F_PREFER_BUSY_POLL = 1,
        NAPI_F_END_ON_RESCHED   = 2,
@@ -6894,8 +6878,8 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
        /* Busy polling means there is a high chance device driver hard irq
         * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
         * set in napi_schedule_prep().
-        * Since we are about to call napi->poll() once more, we can safely
-        * clear NAPI_STATE_MISSED.
+        * Since we either call napi->poll() once more or start the timer,
+        * we can safely clear NAPI_STATE_MISSED.
         *
         * Note: x86 could use a single "lock and ..." instruction
         * to perform these two clear_bit()
@@ -6908,27 +6892,35 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
 
        if (flags & NAPI_F_PREFER_BUSY_POLL) {
                napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
-               if (napi->defer_hard_irqs_count) {
-                       /* A short enough gro flush timeout and long enough
-                        * poll can result in timer firing too early.
-                        * Timer will be armed later if necessary.
-                        */
+               if (napi->defer_hard_irqs_count)
                        timeout = napi_get_gro_flush_timeout(napi);
+       }
+       if (timeout) {
+               netpoll_poll_unlock(have_poll_lock);
+
+               /* Drain aged GRO packets before clearing SCHED since the NAPI
+                * won't run again until after the timer fires. When HZ < 1000,
+                * GRO age comparison is too coarse, so flush everything.
+                */
+               gro_flush_normal(&napi->gro, HZ >= 1000);
+
+               clear_bit(NAPI_STATE_SCHED, &napi->state);
+               hrtimer_start(&napi->timer, ns_to_ktime(timeout),
+                             HRTIMER_MODE_REL_PINNED);
+       } else {
+               /* Use driver poll to re-enable device interrupts. */
+               rc = napi->poll(napi, budget);
+               /* Unless rc == budget we no longer own the NAPI instance,
+                * IRQ may fire on another CPU, poll this NAPI, and enter GRO.
+                */
+               trace_napi_poll(napi, rc, budget);
+               netpoll_poll_unlock(have_poll_lock);
+               if (rc == budget) {
+                       gro_normal_list(&napi->gro);
+                       __napi_schedule(napi);
                }
        }
 
-       /* All we really want here is to re-enable device interrupts.
-        * Ideally, a new ndo_busy_poll_stop() could avoid another round.
-        */
-       rc = napi->poll(napi, budget);
-       /* We can't gro_normal_list() here, because napi->poll() might have
-        * rearmed the napi (napi_complete_done()) in which case it could
-        * already be running on another CPU.
-        */
-       trace_napi_poll(napi, rc, budget);
-       netpoll_poll_unlock(have_poll_lock);
-       if (rc == budget)
-               __busy_poll_stop(napi, timeout);
        bpf_net_ctx_clear(bpf_net_ctx);
        local_bh_enable();
 }