git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob

   1 From 281d150c5f8892f158747594ab49ce2823fd8b8c Mon Sep 17 00:00:00 2001
   2 From: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
   3 Date: Mon, 2 Nov 2009 13:52:27 -0800
   4 Subject: rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handling of ->completed counter
   5
   6 From: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
   7
   8 commit 281d150c5f8892f158747594ab49ce2823fd8b8c upstream.
   9
  10 Impose a clear locking design on non-NO_HZ handling of the
  11 ->completed counter.  This increases the distance between the
  12 RCU and the CPU-hotplug mechanisms.
  13
  14 Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  15 Cc: laijs@cn.fujitsu.com
  16 Cc: dipankar@in.ibm.com
  17 Cc: mathieu.desnoyers@polymtl.ca
  18 Cc: josh@joshtriplett.org
  19 Cc: dvhltc@us.ibm.com
  20 Cc: niv@us.ibm.com
  21 Cc: peterz@infradead.org
  22 Cc: rostedt@goodmis.org
  23 Cc: Valdis.Kletnieks@vt.edu
  24 Cc: dhowells@redhat.com
  25 LKML-Reference: <12571987491353-git-send-email->
  26 Signed-off-by: Ingo Molnar <mingo@elte.hu>
  27 Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
  28
  29 ---
  30  kernel/rcutree.c |   67 ++++++++++++++++++++++++-------------------------------
  31  kernel/rcutree.h |    8 +++---
  32  2 files changed, 34 insertions(+), 41 deletions(-)
  33
  34 --- a/kernel/rcutree.c
  35 +++ b/kernel/rcutree.c
  36 @@ -176,9 +176,29 @@ static struct rcu_node *rcu_get_root(str
  37         return &rsp->node[0];
  38  }
  39
  40 +/*
  41 + * Record the specified "completed" value, which is later used to validate
  42 + * dynticks counter manipulations and CPU-offline checks.  Specify
  43 + * "rsp->completed - 1" to unconditionally invalidate any future dynticks
  44 + * manipulations and CPU-offline checks.  Such invalidation is useful at
  45 + * the beginning of a grace period.
  46 + */
  47 +static void dyntick_record_completed(struct rcu_state *rsp, long comp)
  48 +{
  49 +       rsp->dynticks_completed = comp;
  50 +}
  51 +
  52  #ifdef CONFIG_SMP
  53
  54  /*
  55 + * Recall the previously recorded value of the completion for dynticks.
  56 + */
  57 +static long dyntick_recall_completed(struct rcu_state *rsp)
  58 +{
  59 +       return rsp->dynticks_completed;
  60 +}
  61 +
  62 +/*
  63   * If the specified CPU is offline, tell the caller that it is in
  64   * a quiescent state.  Otherwise, whack it with a reschedule IPI.
  65   * Grace periods can end up waiting on an offline CPU when that
  66 @@ -335,28 +355,9 @@ void rcu_irq_exit(void)
  67                 set_need_resched();
  68  }
  69
  70 -/*
  71 - * Record the specified "completed" value, which is later used to validate
  72 - * dynticks counter manipulations.  Specify "rsp->completed - 1" to
  73 - * unconditionally invalidate any future dynticks manipulations (which is
  74 - * useful at the beginning of a grace period).
  75 - */
  76 -static void dyntick_record_completed(struct rcu_state *rsp, long comp)
  77 -{
  78 -       rsp->dynticks_completed = comp;
  79 -}
  80 -
  81  #ifdef CONFIG_SMP
  82
  83  /*
  84 - * Recall the previously recorded value of the completion for dynticks.
  85 - */
  86 -static long dyntick_recall_completed(struct rcu_state *rsp)
  87 -{
  88 -       return rsp->dynticks_completed;
  89 -}
  90 -
  91 -/*
  92   * Snapshot the specified CPU's dynticks counter so that we can later
  93   * credit them with an implicit quiescent state.  Return 1 if this CPU
  94   * is in dynticks idle mode, which is an extended quiescent state.
  95 @@ -419,24 +420,8 @@ static int rcu_implicit_dynticks_qs(stru
  96
  97  #else /* #ifdef CONFIG_NO_HZ */
  98
  99 -static void dyntick_record_completed(struct rcu_state *rsp, long comp)
 100 -{
 101 -}
 102 -
 103  #ifdef CONFIG_SMP
 104
 105 -/*
 106 - * If there are no dynticks, then the only way that a CPU can passively
 107 - * be in a quiescent state is to be offline.  Unlike dynticks idle, which
 108 - * is a point in time during the prior (already finished) grace period,
 109 - * an offline CPU is always in a quiescent state, and thus can be
 110 - * unconditionally applied.  So just return the current value of completed.
 111 - */
 112 -static long dyntick_recall_completed(struct rcu_state *rsp)
 113 -{
 114 -       return rsp->completed;
 115 -}
 116 -
 117  static int dyntick_save_progress_counter(struct rcu_data *rdp)
 118  {
 119         return 0;
 120 @@ -1144,6 +1129,7 @@ static void force_quiescent_state(struct
 121         long lastcomp;
 122         struct rcu_node *rnp = rcu_get_root(rsp);
 123         u8 signaled;
 124 +       u8 forcenow;
 125
 126         if (!rcu_gp_in_progress(rsp))
 127                 return;  /* No grace period in progress, nothing to force. */
 128 @@ -1180,16 +1166,23 @@ static void force_quiescent_state(struct
 129                 if (rcu_process_dyntick(rsp, lastcomp,
 130                                         dyntick_save_progress_counter))
 131                         goto unlock_ret;
 132 +               /* fall into next case. */
 133 +
 134 +       case RCU_SAVE_COMPLETED:
 135
 136                 /* Update state, record completion counter. */
 137 +               forcenow = 0;
 138                 spin_lock(&rnp->lock);
 139                 if (lastcomp == rsp->completed &&
 140 -                   rsp->signaled == RCU_SAVE_DYNTICK) {
 141 +                   rsp->signaled == signaled) {
 142                         rsp->signaled = RCU_FORCE_QS;
 143                         dyntick_record_completed(rsp, lastcomp);
 144 +                       forcenow = signaled == RCU_SAVE_COMPLETED;
 145                 }
 146                 spin_unlock(&rnp->lock);
 147 -               break;
 148 +               if (!forcenow)
 149 +                       break;
 150 +               /* fall into next case. */
 151
 152         case RCU_FORCE_QS:
 153
 154 --- a/kernel/rcutree.h
 155 +++ b/kernel/rcutree.h
 156 @@ -204,11 +204,12 @@ struct rcu_data {
 157  #define RCU_GP_IDLE            0       /* No grace period in progress. */
 158  #define RCU_GP_INIT            1       /* Grace period being initialized. */
 159  #define RCU_SAVE_DYNTICK       2       /* Need to scan dyntick state. */
 160 -#define RCU_FORCE_QS           3       /* Need to force quiescent state. */
 161 +#define RCU_SAVE_COMPLETED     3       /* Need to save rsp->completed. */
 162 +#define RCU_FORCE_QS           4       /* Need to force quiescent state. */
 163  #ifdef CONFIG_NO_HZ
 164  #define RCU_SIGNAL_INIT                RCU_SAVE_DYNTICK
 165  #else /* #ifdef CONFIG_NO_HZ */
 166 -#define RCU_SIGNAL_INIT                RCU_FORCE_QS
 167 +#define RCU_SIGNAL_INIT                RCU_SAVE_COMPLETED
 168  #endif /* #else #ifdef CONFIG_NO_HZ */
 169
 170  #define RCU_JIFFIES_TILL_FORCE_QS       3      /* for rsp->jiffies_force_qs */
 171 @@ -274,9 +275,8 @@ struct rcu_state {
 172         unsigned long jiffies_stall;            /* Time at which to check */
 173                                                 /*  for CPU stalls. */
 174  #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
 175 -#ifdef CONFIG_NO_HZ
 176         long dynticks_completed;                /* Value of completed @ snap. */
 177 -#endif /* #ifdef CONFIG_NO_HZ */
 178 +                                               /*  Protected by fqslock. */
 179  };
 180
 181  #ifdef RCU_TREE_NONCORE