add a tuneable to control how long we wait after a successful recovery

before we alow another recovery to be initiated

(This used to be ctdb commit f3b43519423b7a73e6a2dd986bdf11203b8653cf)

diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index bfad9cdea5a27311216c28b37aad121f865944bf..bce24343af77b58c43228a8a6df743a4599a11ce 100644 (file)
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -56,6 +56,7 @@ struct ctdb_tunable {
        uint32_t recovery_grace_period;
        uint32_t recovery_ban_period;
        uint32_t database_hash_size;
+       uint32_t rerecovery_timeout;
 };
 
 /*

diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index 074aa9100bdf95b028ee91e56a0ed1513f9a0c85..ef867efac122a2d4bf98e26133924f37c0ea986a 100644 (file)
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -594,6 +594,28 @@ static void unban_handler(struct ctdb_context *ctdb, uint64_t srvid,
 
 
 
+/*
+  called when ctdb_wait_timeout should finish
+ */
+static void ctdb_wait_handler(struct event_context *ev, struct timed_event *te, 
+                             struct timeval yt, void *p)
+{
+       uint32_t *timed_out = (uint32_t *)p;
+       (*timed_out) = 1;
+}
+
+/*
+  wait for a given number of seconds
+ */
+static void ctdb_wait_timeout(struct ctdb_context *ctdb, uint32_t secs)
+{
+       uint32_t timed_out = 0;
+       event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(secs, 0), ctdb_wait_handler, &timed_out);
+       while (!timed_out) {
+               event_loop_once(ctdb->ev);
+       }
+}
+
 /*
   we are the recmaster, and recovery is needed - start a recovery run
  */
@@ -802,6 +824,15 @@ static int do_recovery(struct ctdb_recoverd *rec,
        ctdb_send_message(ctdb, CTDB_BROADCAST_ALL, CTDB_SRVID_RECONFIGURE, tdb_null);
 
        DEBUG(0, (__location__ " Recovery complete\n"));
+
+       /* We just finished a recovery successfully. 
+          We now wait for rerecovery_timeout before we allow 
+          another recovery to take place.
+       */
+       DEBUG(0, (__location__ " New recoveries supressed for the rerecovery timeout\n"));
+       ctdb_wait_timeout(ctdb, ctdb->tunable.rerecovery_timeout);
+       DEBUG(0, (__location__ " Rerecovery timeout elapsed. Recovery reactivated.\n"));
+
        return 0;
 }
 
@@ -982,28 +1013,6 @@ static void election_handler(struct ctdb_context *ctdb, uint64_t srvid,
 }
 
 
-/*
-  called when ctdb_wait_timeout should finish
- */
-static void ctdb_wait_handler(struct event_context *ev, struct timed_event *te, 
-                             struct timeval yt, void *p)
-{
-       uint32_t *timed_out = (uint32_t *)p;
-       (*timed_out) = 1;
-}
-
-/*
-  wait for a given number of seconds
- */
-static void ctdb_wait_timeout(struct ctdb_context *ctdb, uint32_t secs)
-{
-       uint32_t timed_out = 0;
-       event_add_timed(ctdb->ev, ctdb, timeval_current_ofs(secs, 0), ctdb_wait_handler, &timed_out);
-       while (!timed_out) {
-               event_loop_once(ctdb->ev);
-       }
-}
-
 /*
   force the start of the election process
  */

diff --git a/ctdb/server/ctdb_tunables.c b/ctdb/server/ctdb_tunables.c
index 5dafc0aa9d59074d0e3641848d582d258770cde2..80d746ff00929aacbf0ea03636da5efce25a0a33 100644 (file)
--- a/ctdb/server/ctdb_tunables.c
+++ b/ctdb/server/ctdb_tunables.c
@@ -41,6 +41,7 @@ static const struct {
        { "RecoveryGracePeriod", 60,  offsetof(struct ctdb_tunable, recovery_grace_period) },
        { "RecoveryBanPeriod",  300,  offsetof(struct ctdb_tunable, recovery_ban_period) },
        { "DatabaseHashSize", 10000,  offsetof(struct ctdb_tunable, database_hash_size) },
+       { "RerecoveryTimeout",   10,  offsetof(struct ctdb_tunable, rerecovery_timeout) },
 };
 
 /*