ctdb-recoverd: Add handler for lost recovery lock

If the process holding the recovery lock terminates unexpectedly then
the recovery daemon needs to know that the lock is no longer held.

While here, rename hold_reclock_handler() to take_reclock_handler() so
there is a clear difference between the two handler names.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index 34b506a27d0828f3b8c0a01ca40893b9244b6549..1becc53f324c458d25738b0a138437caff9d4416 100644 (file)
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -1553,7 +1553,7 @@ struct hold_reclock_state {
        double latency;
 };
 
-static void hold_reclock_handler(char status,
+static void take_reclock_handler(char status,
                                 double latency,
                                 void *private_data)
 {
@@ -1578,6 +1578,22 @@ static void hold_reclock_handler(char status,
        s->locked = (status == '0') ;
 }
 
+static bool ctdb_recovery_lock(struct ctdb_recoverd *rec);
+
+static void lost_reclock_handler(void *private_data)
+{
+       struct ctdb_recoverd *rec = talloc_get_type_abort(
+               private_data, struct ctdb_recoverd);
+
+       DEBUG(DEBUG_ERR,
+             ("Recovery lock helper terminated unexpectedly - "
+              "trying to retake recovery lock\n"));
+       TALLOC_FREE(rec->recovery_lock_handle);
+       if (! ctdb_recovery_lock(rec)) {
+               DEBUG(DEBUG_ERR, ("Failed to take recovery lock\n"));
+       }
+}
+
 static bool ctdb_recovery_lock(struct ctdb_recoverd *rec)
 {
        struct ctdb_context *ctdb = rec->ctdb;
@@ -1589,7 +1605,8 @@ static bool ctdb_recovery_lock(struct ctdb_recoverd *rec)
        };
 
        h = ctdb_cluster_mutex(rec, ctdb, ctdb->recovery_lock, 0,
-                              hold_reclock_handler, &s, NULL, NULL);
+                              take_reclock_handler, &s,
+                              lost_reclock_handler, rec);
        if (h == NULL) {
                return false;
        }