ctdb-recovery: Recovery lock setting can now include helper command

The underlying change is to allow the cluster mutex argstring to
optionally contain a helper command.  When the argument string starts
with '!' then the first word is the helper command to run.  This is
now the standard way of changing the helper from the default.

CTDB_CLUSTER_MUTEX_HELPER show now only be used to change the location
of the default helper when testing.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

diff --git a/ctdb/config/events.d/01.reclock b/ctdb/config/events.d/01.reclock
index d3dd612471f3b91b30452d488b05f26ad0b0f944..e2d4d124bc3f224e1f77d3d1323a3c8bfe7cd8a4 100755 (executable)
--- a/ctdb/config/events.d/01.reclock
+++ b/ctdb/config/events.d/01.reclock
@@ -7,6 +7,12 @@
 . $CTDB_BASE/functions
 loadconfig
 
+# If CTDB_RECOVERY_LOCK specifies a helper then exit because this
+# script can't do anything useful.
+case "$CTDB_RECOVERY_LOCK" in
+!*) exit 0 ;;
+esac
+
 case "$1" in
     init)
        ctdb_counter_init

diff --git a/ctdb/doc/ctdb.7.xml b/ctdb/doc/ctdb.7.xml
index 51222ad48a4d7f4425a92f854dc4dd0f7dc9e294..14211435363dd761de47ed9e6336aba7c5355026 100644 (file)
--- a/ctdb/doc/ctdb.7.xml
+++ b/ctdb/doc/ctdb.7.xml
@@ -98,14 +98,26 @@
     </para>
 
     <para>
-      The recovery lock is implemented using a file residing in shared
-      storage (usually) on a cluster filesystem.  To support a
-      recovery lock the cluster filesystem must support lock
-      coherence.  See
+      By default, the recovery lock is implemented using a file
+      (specified by <parameter>CTDB_RECOVERY_LOCK</parameter>)
+      residing in shared storage (usually) on a cluster filesystem.
+      To support a recovery lock the cluster filesystem must support
+      lock coherence.  See
       <citerefentry><refentrytitle>ping_pong</refentrytitle>
       <manvolnum>1</manvolnum></citerefentry> for more details.
     </para>
 
+    <para>
+      The recovery lock can also be implemented using an arbitrary
+      cluster mutex call-out by using an exclamation point ('!') as
+      the first character of
+      <parameter>CTDB_RECOVERY_LOCK</parameter>.  For example, a value
+      of <command>!/usr/local/bin/myhelper recovery</command> would
+      run the given helper with the specified arguments.  See the
+      source code relating to cluster mutexes for clues about writing
+      call-outs.
+    </para>
+
     <para>
       If a cluster becomes partitioned (for example, due to a
       communication failure) and a different recovery master is

diff --git a/ctdb/doc/ctdbd.1.xml b/ctdb/doc/ctdbd.1.xml
index 0f75f77a5df44828019eb692e34524c9e055db11..7b8cc667afc59504682b1d9e39c764fc69bd9304 100644 (file)
--- a/ctdb/doc/ctdbd.1.xml
+++ b/ctdb/doc/ctdbd.1.xml
@@ -365,12 +365,11 @@
       </varlistentry>
 
       <varlistentry>
-       <term>--reclock=<parameter>FILE</parameter></term>
+       <term>--reclock=<parameter>LOCK</parameter></term>
        <listitem>
          <para>
-           FILE is the name of the recovery lock file, stored in
-           <emphasis>shared storage</emphasis>, that CTDB uses to
-           prevent split brains.
+           LOCK specifies the cluster-wide mutex used to detect and
+           prevent a partitioned cluster (or "split brain").
          </para>
          <para>
            For information about the recovery lock please see the

diff --git a/ctdb/doc/ctdbd.conf.5.xml b/ctdb/doc/ctdbd.conf.5.xml
index 324be050135c1702ce0daa5229e54ef0d49300f0..a364c9f0fc17f712277963ffbc80e8d1bd8eeb45 100644 (file)
--- a/ctdb/doc/ctdbd.conf.5.xml
+++ b/ctdb/doc/ctdbd.conf.5.xml
@@ -379,10 +379,14 @@
       </varlistentry>
 
       <varlistentry>
-       <term>CTDB_RECOVERY_LOCK=<parameter>FILENAME</parameter></term>
+       <term>CTDB_RECOVERY_LOCK=<parameter>LOCK</parameter></term>
        <listitem>
          <para>
-           Defaults to
+           LOCK specifies the cluster-wide mutex used to detect and
+           prevent a partitioned cluster (or "split brain").
+         </para>
+         <para>
+           No default, but the default configuration file specifies
            <filename>/some/place/on/shared/storage</filename>, which
            should be change to a useful value.  Corresponds to
            <option>--reclock</option>.

diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c
index 0699528d0cc1152b11d9395e6707695df2266558..b2037addc076b7897378c15f60d47eed80c6c5e5 100644 (file)
--- a/ctdb/server/ctdb_recover.c
+++ b/ctdb/server/ctdb_recover.c
@@ -906,11 +906,21 @@ static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx,
                                      const char *argstring, char ***argv)
 {
        int nargs, i, ret, n;
+       bool is_command = false;
        char **args = NULL;
        char *strv = NULL;
        char *t = NULL;
 
-       ret = strv_split(mem_ctx, &strv, argstring, " \t");
+       if (argstring != NULL && argstring[0] == '!') {
+               /* This is actually a full command */
+               is_command = true;
+               t = discard_const(&argstring[1]);
+       } else {
+               is_command = false;
+               t = discard_const(argstring);
+       }
+
+       ret = strv_split(mem_ctx, &strv, t, " \t");
        if (ret != 0) {
                DEBUG(DEBUG_ERR,
                      ("Unable to parse mutex helper string \"%s\" (%s)\n",
@@ -919,7 +929,8 @@ static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx,
        }
        n = strv_count(strv);
 
-       args = talloc_array(mem_ctx, char *, n + 2);
+       args = talloc_array(mem_ctx, char *, n + (is_command ? 1 : 2));
+
        if (args == NULL) {
                DEBUG(DEBUG_ERR,(__location__ " out of memory\n"));
                return false;
@@ -927,17 +938,21 @@ static bool cluster_mutex_helper_args(TALLOC_CTX *mem_ctx,
 
        nargs = 0;
 
-       if (!ctdb_set_helper("cluster mutex helper",
-                            cluster_mutex_helper,
-                            sizeof(cluster_mutex_helper),
-                            "CTDB_CLUSTER_MUTEX_HELPER",
-                            CTDB_HELPER_BINDIR, "ctdb_mutex_fcntl_helper")) {
-               DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n",
-                                __location__
-                                " Unable to set cluster mutex helper\n"));
-               exit(1);
+       if (! is_command) {
+               if (!ctdb_set_helper("cluster mutex helper",
+                                    cluster_mutex_helper,
+                                    sizeof(cluster_mutex_helper),
+                                    "CTDB_CLUSTER_MUTEX_HELPER",
+                                    CTDB_HELPER_BINDIR,
+                                    "ctdb_mutex_fcntl_helper")) {
+                       DEBUG(DEBUG_ERR,("ctdb exiting with error: %s\n",
+                                        __location__
+                                        " Unable to set cluster mutex helper\n"));
+                       exit(1);
+               }
+
+               args[nargs++] = cluster_mutex_helper;
        }
-       args[nargs++] = cluster_mutex_helper;
 
        t = NULL;
        for (i = 0; i < n; i++) {