]> git.ipfire.org Git - thirdparty/linux.git/commit
net: rps: softnet_data reorg to make enqueue_to_backlog() fast
authorEric Dumazet <edumazet@google.com>
Fri, 24 Oct 2025 09:12:40 +0000 (09:12 +0000)
committerJakub Kicinski <kuba@kernel.org>
Wed, 29 Oct 2025 00:41:17 +0000 (17:41 -0700)
commitc72568c21b97dbc48d02b769f4eec6667ad13d5a
treeff30418520f52173e8bbcb0c4ba251e52e25e66a
parenta086e9860ce6a751acd71dbec54d24a819dd6baa
net: rps: softnet_data reorg to make enqueue_to_backlog() fast

enqueue_to_backlog() is showing up in kernel profiles on hosts
with many cores, when RFS/RPS is used.

The following softnet_data fields need to be updated:

- input_queue_tail
- input_pkt_queue (next, prev, qlen, lock)
- backlog.state (if input_pkt_queue was empty)

Unfortunately they are currenly using two cache lines:

/* --- cacheline 3 boundary (192 bytes) --- */
call_single_data_t         csd __attribute__((__aligned__(64))); /*  0xc0  0x20 */
struct softnet_data *      rps_ipi_next;         /*  0xe0   0x8 */
unsigned int               cpu;                  /*  0xe8   0x4 */
unsigned int               input_queue_tail;     /*  0xec   0x4 */
struct sk_buff_head        input_pkt_queue;      /*  0xf0  0x18 */

/* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */

struct napi_struct         backlog __attribute__((__aligned__(8))); /* 0x108 0x1f0 */

Add one ____cacheline_aligned_in_smp to make sure they now are using
a single cache line.

Also, because napi_struct has written fields, make @state its first field.

We want to make sure that cpus adding packets to sd->input_pkt_queue
are not slowing down cpus processing their backlog because of
false sharing.

After this patch new layout is:

/* --- cacheline 5 boundary (320 bytes) --- */
long int                   pad[3] __attribute__((__aligned__(64))); /* 0x140  0x18 */
unsigned int               input_queue_tail;     /* 0x158   0x4 */

/* XXX 4 bytes hole, try to pack */

struct sk_buff_head        input_pkt_queue;      /* 0x160  0x18 */
struct napi_struct         backlog __attribute__((__aligned__(8))); /* 0x178 0x1f0 */

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20251024091240.3292546-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
include/linux/netdevice.h