rcu/nocb: Atomic ->len field in rcu_segcblist structure

Upcoming ->nocb_lock contention-reduction work requires that the
rcu_segcblist structure's ->len field be concurrently manipulated,
but only if there are no-CBs CPUs in the kernel.  This commit
therefore makes this ->len field be an atomic_long_t, but only
in CONFIG_RCU_NOCB_CPU=y kernels.

Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>

diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h
index 8b684888f71d19344bd8e073c1be66c8b5fc5dc7..646759042333cfb045ef488555c595b8e85ddd07 100644 (file)
--- a/include/linux/rcu_segcblist.h
+++ b/include/linux/rcu_segcblist.h
@@ -68,7 +68,11 @@ struct rcu_segcblist {
        struct rcu_head *head;
        struct rcu_head **tails[RCU_CBLIST_NSEGS];
        unsigned long gp_seq[RCU_CBLIST_NSEGS];
+#ifdef CONFIG_RCU_NOCB_CPU
+       atomic_long_t len;
+#else
        long len;
+#endif
        long len_lazy;
        u8 enabled;
        u8 offloaded;

diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c
index 92968b8565936f02f6098d2936a5a35e7cc22aa8..ff431cc830378e1d048ccba4be7b60705bac5523 100644 (file)
--- a/kernel/rcu/rcu_segcblist.c
+++ b/kernel/rcu/rcu_segcblist.c
@@ -23,6 +23,19 @@ void rcu_cblist_init(struct rcu_cblist *rclp)
        rclp->len_lazy = 0;
 }
 
+/*
+ * Enqueue an rcu_head structure onto the specified callback list.
+ * This function assumes that the callback is non-lazy because it
+ * is intended for use by no-CBs CPUs, which do not distinguish
+ * between lazy and non-lazy RCU callbacks.
+ */
+void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp)
+{
+       *rclp->tail = rhp;
+       rclp->tail = &rhp->next;
+       WRITE_ONCE(rclp->len, rclp->len + 1);
+}
+
 /*
  * Dequeue the oldest rcu_head structure from the specified callback
  * list.  This function assumes that the callback is non-lazy, but
@@ -44,6 +57,67 @@ struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp)
        return rhp;
 }
 
+/* Set the length of an rcu_segcblist structure. */
+void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)
+{
+#ifdef CONFIG_RCU_NOCB_CPU
+       atomic_long_set(&rsclp->len, v);
+#else
+       WRITE_ONCE(rsclp->len, v);
+#endif
+}
+
+/*
+ * Increase the numeric length of an rcu_segcblist structure by the
+ * specified amount, which can be negative.  This can cause the ->len
+ * field to disagree with the actual number of callbacks on the structure.
+ * This increase is fully ordered with respect to the callers accesses
+ * both before and after.
+ */
+void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)
+{
+#ifdef CONFIG_RCU_NOCB_CPU
+       smp_mb__before_atomic(); /* Up to the caller! */
+       atomic_long_add(v, &rsclp->len);
+       smp_mb__after_atomic(); /* Up to the caller! */
+#else
+       smp_mb(); /* Up to the caller! */
+       WRITE_ONCE(rsclp->len, rsclp->len + v);
+       smp_mb(); /* Up to the caller! */
+#endif
+}
+
+/*
+ * Increase the numeric length of an rcu_segcblist structure by one.
+ * This can cause the ->len field to disagree with the actual number of
+ * callbacks on the structure.  This increase is fully ordered with respect
+ * to the callers accesses both before and after.
+ */
+void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)
+{
+       rcu_segcblist_add_len(rsclp, 1);
+}
+
+/*
+ * Exchange the numeric length of the specified rcu_segcblist structure
+ * with the specified value.  This can cause the ->len field to disagree
+ * with the actual number of callbacks on the structure.  This exchange is
+ * fully ordered with respect to the callers accesses both before and after.
+ */
+long rcu_segcblist_xchg_len(struct rcu_segcblist *rsclp, long v)
+{
+#ifdef CONFIG_RCU_NOCB_CPU
+       return atomic_long_xchg(&rsclp->len, v);
+#else
+       long ret = rsclp->len;
+
+       smp_mb(); /* Up to the caller! */
+       WRITE_ONCE(rsclp->len, v);
+       smp_mb(); /* Up to the caller! */
+       return ret;
+#endif
+}
+
 /*
  * Initialize an rcu_segcblist structure.
  */
@@ -56,7 +130,7 @@ void rcu_segcblist_init(struct rcu_segcblist *rsclp)
        rsclp->head = NULL;
        for (i = 0; i < RCU_CBLIST_NSEGS; i++)
                rsclp->tails[i] = &rsclp->head;
-       rsclp->len = 0;
+       rcu_segcblist_set_len(rsclp, 0);
        rsclp->len_lazy = 0;
        rsclp->enabled = 1;
 }
@@ -151,7 +225,7 @@ bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)
 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
                           struct rcu_head *rhp, bool lazy)
 {
-       WRITE_ONCE(rsclp->len, rsclp->len + 1); /* ->len sampled locklessly. */
+       rcu_segcblist_inc_len(rsclp);
        if (lazy)
                rsclp->len_lazy++;
        smp_mb(); /* Ensure counts are updated before callback is enqueued. */
@@ -177,7 +251,7 @@ bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
 
        if (rcu_segcblist_n_cbs(rsclp) == 0)
                return false;
-       WRITE_ONCE(rsclp->len, rsclp->len + 1);
+       rcu_segcblist_inc_len(rsclp);
        if (lazy)
                rsclp->len_lazy++;
        smp_mb(); /* Ensure counts are updated before callback is entrained. */
@@ -204,9 +278,8 @@ void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
                                               struct rcu_cblist *rclp)
 {
        rclp->len_lazy += rsclp->len_lazy;
-       rclp->len += rsclp->len;
        rsclp->len_lazy = 0;
-       WRITE_ONCE(rsclp->len, 0); /* ->len sampled locklessly. */
+       rclp->len = rcu_segcblist_xchg_len(rsclp, 0);
 }
 
 /*
@@ -259,8 +332,7 @@ void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
                                struct rcu_cblist *rclp)
 {
        rsclp->len_lazy += rclp->len_lazy;
-       /* ->len sampled locklessly. */
-       WRITE_ONCE(rsclp->len, rsclp->len + rclp->len);
+       rcu_segcblist_add_len(rsclp, rclp->len);
        rclp->len_lazy = 0;
        rclp->len = 0;
 }

diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h
index db38f0a512c4475b8ec4a5a5dbd3812eec7a075e..1ff996647d3cc17304747c291e0b54e0d4c52f23 100644 (file)
--- a/kernel/rcu/rcu_segcblist.h
+++ b/kernel/rcu/rcu_segcblist.h
@@ -9,6 +9,12 @@
 
 #include <linux/rcu_segcblist.h>
 
+/* Return number of callbacks in the specified callback list. */
+static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
+{
+       return READ_ONCE(rclp->len);
+}
+
 /*
  * Account for the fact that a previously dequeued callback turned out
  * to be marked as lazy.
@@ -42,7 +48,11 @@ static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
 /* Return number of callbacks in segmented callback list. */
 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
 {
+#ifdef CONFIG_RCU_NOCB_CPU
+       return atomic_long_read(&rsclp->len);
+#else
        return READ_ONCE(rsclp->len);
+#endif
 }
 
 /* Return number of lazy callbacks in segmented callback list. */
@@ -54,7 +64,7 @@ static inline long rcu_segcblist_n_lazy_cbs(struct rcu_segcblist *rsclp)
 /* Return number of lazy callbacks in segmented callback list. */
 static inline long rcu_segcblist_n_nonlazy_cbs(struct rcu_segcblist *rsclp)
 {
-       return rsclp->len - rsclp->len_lazy;
+       return rcu_segcblist_n_cbs(rsclp) - rsclp->len_lazy;
 }
 
 /*