srcu: Make SRCU-fast readers enforce use of SRCU-fast definition/init

This commit makes CONFIG_PROVE_RCU=y kernels enforce the new rule
that srcu_struct structures that are passed to srcu_read_lock_fast()
and other SRCU-fast read-side markers be either initialized with
init_srcu_struct_fast() on the one hand or defined with DEFINE_SRCU_FAST()
or DEFINE_STATIC_SRCU_FAST() on the other.

This eliminates the read-side test that was formerly included in
srcu_read_lock_fast() and friends, speeding these primitives up by
about 25% (admittedly only about half of a nanosecond, but when tracing
on fastpaths...)

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: <bpf@vger.kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>

diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 2982b5a6930fa601346c7509fafab9d66dd7c2b5..41e27c1d917d3ec3964e58b1b33232c2c4dc51aa 100644 (file)
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -297,7 +297,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *
        struct srcu_ctr __percpu *retval;
 
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_fast().");
-       srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
+       srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        retval = __srcu_read_lock_fast(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
@@ -312,7 +312,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_
 {
        struct srcu_ctr __percpu *retval;
 
-       srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
+       srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        retval = __srcu_read_lock_fast(ssp);
        return retval;
 }
@@ -333,7 +333,7 @@ static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *
 {
        WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast().");
-       srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
+       srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        return __srcu_read_lock_fast(ssp);
 }
 

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 92e6ab53398fc00f452b98cf7703c69e4a5ea706..1ecc3393fb26bec6361855321eb2ccb5359e099c 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -112,7 +112,6 @@ static inline void srcu_barrier(struct srcu_struct *ssp)
 
 static inline void srcu_expedite_current(struct srcu_struct *ssp) { }
 #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
-#define srcu_check_read_flavor_force(ssp, read_flavor) do { } while (0)
 
 /* Defined here to avoid size increase for non-torture kernels. */
 static inline void srcu_torture_stats_print(struct srcu_struct *ssp,

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 7ff4a11bc5a34ecee399f10663b5c3d274f6e313..6080a909461854fe08887f07f648fc7b23c1b6e5 100644 (file)
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -307,21 +307,7 @@ __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
 
 void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor);
 
-// Record reader usage even for CONFIG_PROVE_RCU=n kernels.  This is
-// needed only for flavors that require grace-period smp_mb() calls to be
-// promoted to synchronize_rcu().
-static inline void srcu_check_read_flavor_force(struct srcu_struct *ssp, int read_flavor)
-{
-       struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);
-
-       if (likely(READ_ONCE(sdp->srcu_reader_flavor) & read_flavor))
-               return;
-
-       // Note that the cmpxchg() in __srcu_check_read_flavor() is fully ordered.
-       __srcu_check_read_flavor(ssp, read_flavor);
-}
-
-// Record non-_lite() usage only for CONFIG_PROVE_RCU=y kernels.
+// Record SRCU-reader usage type only for CONFIG_PROVE_RCU=y kernels.
 static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor)
 {
        if (IS_ENABLED(CONFIG_PROVE_RCU))