rfs: annotate lockless accesses to RFS sock flow table

[ Upstream commit 5c3b74a92aa285a3df722bf6329ba7ccf70346d6 ]

Add READ_ONCE()/WRITE_ONCE() on accesses to the sock flow table.

This also prevents a (smart ?) compiler to remove the condition in:

if (table->ents[index] != newval)
        table->ents[index] = newval;

We need the condition to avoid dirtying a shared cache line.

Fixes: fec5e652e58f ("rfs: Receive Flow Steering")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Simon Horman <simon.horman@corigine.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 7db9f960221d39963cb8d765d1906069debc554f..2771aa046ab2a2c4499059aa8a214253d78502aa 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -760,8 +760,11 @@ static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
                /* We only give a hint, preemption can change CPU under us */
                val |= raw_smp_processor_id();
 
-               if (table->ents[index] != val)
-                       table->ents[index] = val;
+               /* The following WRITE_ONCE() is paired with the READ_ONCE()
+                * here, and another one in get_rps_cpu().
+                */
+               if (READ_ONCE(table->ents[index]) != val)
+                       WRITE_ONCE(table->ents[index], val);
        }
 }
 

diff --git a/net/core/dev.c b/net/core/dev.c
index b3d8e74fcaf0601816b8d50688bcafde580c6d03..6d46eb0402ccdd6e96fa07fafa720305f8899cb8 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4471,8 +4471,10 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
                u32 next_cpu;
                u32 ident;
 
-               /* First check into global flow table if there is a match */
-               ident = sock_flow_table->ents[hash & sock_flow_table->mask];
+               /* First check into global flow table if there is a match.
+                * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
+                */
+               ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]);
                if ((ident ^ hash) & ~rps_cpu_mask)
                        goto try_rps;