added lockwait child code for entering recovery mode. A child processes holds lockall...

(This used to be ctdb commit f892f30def75b0d964c35eae38c4cf675597dd28)

diff --git a/ctdb/common/ctdb_client.c b/ctdb/common/ctdb_client.c
index 5a8ccc94bcd66e9bdf30f9127ef4eab472098f7c..e568acd2c7473424dd034d322028f46f74da0f80 100644 (file)
--- a/ctdb/common/ctdb_client.c
+++ b/ctdb/common/ctdb_client.c
@@ -883,7 +883,6 @@ int ctdb_ctrl_setrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint
        TDB_DATA data, outdata;
        int32_t res;
 
-       ZERO_STRUCT(data);
        data.dsize = sizeof(uint32_t);
        data.dptr = (unsigned char *)&recmode;
 

diff --git a/ctdb/common/ctdb_control.c b/ctdb/common/ctdb_control.c
index c64b873368e4fff8ffec917a3d6f57cca3bdc336..f921d4fa1bf1441026a4f2f79a4f54a398182e7a 100644 (file)
--- a/ctdb/common/ctdb_control.c
+++ b/ctdb/common/ctdb_control.c
@@ -227,38 +227,30 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
        }
 }
 
+
 /*
-  called when a CTDB_REQ_CONTROL packet comes in
-*/
-void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+  send a reply for a ctdb control
+ */
+void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_control *c,
+                               TDB_DATA *outdata, int32_t status)
 {
-       struct ctdb_req_control *c = (struct ctdb_req_control *)hdr;
-       TDB_DATA data, *outdata;
        struct ctdb_reply_control *r;
-       int32_t status;
        size_t len;
 
-       data.dptr = &c->data[0];
-       data.dsize = c->datalen;
-
-       outdata = talloc_zero(c, TDB_DATA);
-       status = ctdb_control_dispatch(ctdb, c->opcode, c->srvid, c->client_id,
-                                      data, outdata, hdr->srcnode);
-
        /* some controls send no reply */
        if (c->flags & CTDB_CTRL_FLAG_NOREPLY) {
                return;
        }
 
-       len = offsetof(struct ctdb_reply_control, data) + outdata->dsize;
+       len = offsetof(struct ctdb_reply_control, data) + (outdata?outdata->dsize:0);
        r = ctdb_transport_allocate(ctdb, ctdb, CTDB_REPLY_CONTROL, len, struct ctdb_reply_control);
        CTDB_NO_MEMORY_VOID(ctdb, r);
 
-       r->hdr.destnode     = hdr->srcnode;
-       r->hdr.reqid        = hdr->reqid;
+       r->hdr.destnode     = c->hdr.srcnode;
+       r->hdr.reqid        = c->hdr.reqid;
        r->status           = status;
-       r->datalen          = outdata->dsize;
-       if (outdata->dsize) {
+       r->datalen          = outdata?outdata->dsize:0;
+       if (outdata && outdata->dsize) {
                memcpy(&r->data[0], outdata->dptr, outdata->dsize);
        }
        
@@ -267,6 +259,31 @@ void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr
        talloc_free(r);
 }
 
+/*
+  called when a CTDB_REQ_CONTROL packet comes in
+*/
+void ctdb_request_control(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
+{
+       struct ctdb_req_control *c = (struct ctdb_req_control *)hdr;
+       TDB_DATA data, *outdata;
+       int32_t status;
+
+       data.dptr = &c->data[0];
+       data.dsize = c->datalen;
+
+       outdata = talloc_zero(c, TDB_DATA);
+
+       if (c->opcode == CTDB_CONTROL_SET_RECMODE) {
+               /* this function operates asynchronously */
+               ctdb_control_set_recmode(ctdb, c, data);
+               return;
+       }
+
+       status = ctdb_control_dispatch(ctdb, c->opcode, c->srvid, c->client_id,
+                                      data, outdata, hdr->srcnode);
+       ctdb_request_control_reply(ctdb, c, outdata, status);
+}
+
 /*
   called when a CTDB_REPLY_CONTROL packet comes in
 */
@@ -312,11 +329,14 @@ static void ctdb_control_timeout(struct event_context *ev, struct timed_event *t
                       struct timeval t, void *private_data)
 {
        struct ctdb_control_state *state = talloc_get_type(private_data, struct ctdb_control_state);
+       TALLOC_CTX *tmp_ctx = talloc_new(ev);
 
        state->ctdb->status.timeouts.control++;
 
+       talloc_steal(tmp_ctx, state);
+
        state->callback(state->ctdb, -1, tdb_null, state->private_data);
-       talloc_free(state);
+       talloc_free(tmp_ctx);
 }
 
 

diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c
index 7cce7ce471b3a5026ad1b0170f66f819a1c7e1e1..b2fb14e0e0a5bbbda525df2d56551ab28ab445b0 100644 (file)
--- a/ctdb/common/ctdb_daemon.c
+++ b/ctdb/common/ctdb_daemon.c
@@ -326,7 +326,7 @@ static void daemon_call_from_client_callback(struct ctdb_call_state *state)
 
        res = daemon_queue_send(client, &r->hdr);
        if (res != 0) {
-               DEBUG(0, (__location__ "Failed to queue packet from daemon to client\n"));
+               DEBUG(0, (__location__ " Failed to queue packet from daemon to client\n"));
        }
        ctdb_latency(&client->ctdb->status.max_call_latency, dstate->start_time);
        talloc_free(dstate);

diff --git a/ctdb/common/ctdb_recover.c b/ctdb/common/ctdb_recover.c
index 047543d4acd6214d3289e56cabb0918adb8997b7..419426f76fa7b44dea4a0009a0569d69a3d19d26 100644 (file)
--- a/ctdb/common/ctdb_recover.c
+++ b/ctdb/common/ctdb_recover.c
@@ -87,7 +87,7 @@ ctdb_control_getdbmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indat
        outdata->dsize = offsetof(struct ctdb_dbid_map, dbids) + 4*len;
        outdata->dptr  = (unsigned char *)talloc_zero_size(outdata, outdata->dsize);
        if (!outdata->dptr) {
-               DEBUG(0, (__location__ "Failed to allocate dbmap array\n"));
+               DEBUG(0, (__location__ " Failed to allocate dbmap array\n"));
                exit(1);
        }
 
@@ -113,7 +113,7 @@ ctdb_control_getnodemap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA ind
        outdata->dsize = offsetof(struct ctdb_node_map, nodes) + num_nodes*sizeof(struct ctdb_node_and_flags);
        outdata->dptr  = (unsigned char *)talloc_zero_size(outdata, outdata->dsize);
        if (!outdata->dptr) {
-               DEBUG(0, (__location__ "Failed to allocate nodemap array\n"));
+               DEBUG(0, (__location__ " Failed to allocate nodemap array\n"));
                exit(1);
        }
 
@@ -269,14 +269,14 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
 
                ret = ctdb_ltdb_fetch(ctdb_db, key, &header, NULL, NULL);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to fetch record\n"));
+                       DEBUG(0, (__location__ " Unable to fetch record\n"));
                        tdb_unlockall(ctdb_db->ltdb->tdb);
                        return -1;
                }
                if (header.rsn < hdr->rsn) {
                        ret = ctdb_ltdb_store(ctdb_db, key, hdr, data);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to store record\n"));
+                               DEBUG(0, (__location__ " Unable to store record\n"));
                                tdb_unlockall(ctdb_db->ltdb->tdb);
                                return -1;
                        }
@@ -301,7 +301,7 @@ static int traverse_setdmaster(struct tdb_context *tdb, TDB_DATA key, TDB_DATA d
 
        ret = tdb_store(tdb, key, data, TDB_REPLACE);
        if (ret) {
-               DEBUG(0,(__location__ "failed to write tdb data back  ret:%d\n",ret));
+               DEBUG(0,(__location__ " failed to write tdb data back  ret:%d\n",ret));
                return ret;
        }
        return 0;
@@ -338,7 +338,7 @@ static int traverse_cleardb(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data
 
        ret = tdb_delete(tdb, key);
        if (ret) {
-               DEBUG(0,(__location__ "failed to delete tdb record\n"));
+               DEBUG(0,(__location__ " failed to delete tdb record\n"));
                return ret;
        }
        return 0;
@@ -367,3 +367,232 @@ int32_t ctdb_control_clear_db(struct ctdb_context *ctdb, TDB_DATA indata)
 
        return 0;
 }
+
+
+/*
+  lock all databases
+ */
+static int ctdb_lock_all_databases(struct ctdb_context *ctdb)
+{
+       struct ctdb_db_context *ctdb_db;
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
+       }
+       return 0;
+}
+
+/*
+  lock all databases - mark only
+ */
+static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
+{
+       struct ctdb_db_context *ctdb_db;
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
+       }
+       return 0;
+}
+
+/*
+  lock all databases - unmark only
+ */
+static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb)
+{
+       struct ctdb_db_context *ctdb_db;
+       for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+               if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
+                       return -1;
+               }
+       }
+       return 0;
+}
+
+/*
+  a list of control requests waiting for a recovery lock child to gets
+  the database locks
+ */
+struct ctdb_recovery_waiter {
+       struct ctdb_recovery_waiter *next, *prev;
+       struct ctdb_context *ctdb;
+       struct ctdb_req_control *c;
+       int32_t status;
+};
+
+/* a handle to a recovery lock child process */
+struct ctdb_recovery_handle {
+       struct ctdb_context *ctdb;
+       pid_t child;
+       int fd;
+       struct ctdb_recovery_waiter *waiters;
+};
+
+/*
+  destroy a recovery handle
+ */    
+static int ctdb_recovery_handle_destructor(struct ctdb_recovery_handle *h)
+{
+       if (h->ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE) {
+               ctdb_lock_all_databases_unmark(h->ctdb);
+       }
+       kill(h->child, SIGKILL);
+       waitpid(h->child, NULL, 0);
+       return 0;
+}
+
+/*
+  called when the child writes its status to us
+ */
+static void ctdb_recovery_lock_handler(struct event_context *ev, struct fd_event *fde, 
+                                      uint16_t flags, void *private_data)
+{
+       struct ctdb_recovery_handle *h = talloc_get_type(private_data, struct ctdb_recovery_handle);
+       int32_t status;
+       struct ctdb_recovery_waiter *w;
+       int ret;
+
+       if (read(h->fd, &status, sizeof(status)) != sizeof(status)) {
+               DEBUG(0,("read error from recovery lock child\n"));
+               status = -1;
+       }
+
+       if (status == -1) {
+               DEBUG(0,("Failed to get locks in ctdb_recovery_child\n"));
+               /* we didn't get the locks - destroy the handle */
+               talloc_free(h);
+               return;
+       }
+
+       ret = ctdb_lock_all_databases_mark(h->ctdb);
+       if (ret == -1) {
+               DEBUG(0,("Failed to mark locks in ctdb_recovery\n"));
+               talloc_free(h);
+               return;
+       }
+
+       h->ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;  
+
+       /* notify the waiters */
+       while ((w = h->ctdb->recovery_handle->waiters)) {
+               w->status = status;
+               DLIST_REMOVE(h->ctdb->recovery_handle->waiters, w);
+               talloc_free(w);
+       }
+
+       talloc_free(fde);
+}
+
+/*
+  create a child which gets locks on all the open databases, then calls the callback telling the parent
+  that it is done
+ */
+static struct ctdb_recovery_handle *ctdb_recovery_lock(struct ctdb_context *ctdb)
+{
+       struct ctdb_recovery_handle *h;
+       int fd[2];
+       struct fd_event *fde;
+
+       h = talloc_zero(ctdb, struct ctdb_recovery_handle);
+       CTDB_NO_MEMORY_VOID(ctdb, h);
+
+       h->ctdb = ctdb;
+
+       /* use socketpair() instead of pipe() so we have bi-directional fds */
+       if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
+               DEBUG(0,("Failed to create pipe for ctdb_recovery_lock\n"));
+               talloc_free(h);
+               return NULL;
+       }
+       
+       h->child = fork();
+       if (h->child == -1) {
+               DEBUG(0,("Failed to fork child for ctdb_recovery_lock\n"));
+               talloc_free(h);
+               return NULL;
+       }
+
+       if (h->child == 0) {
+               int ret;
+               /* in the child */
+               close(fd[0]);
+               ret = ctdb_lock_all_databases(ctdb);
+               if (ret != 0) {
+                       _exit(0);
+               }
+               write(fd[1], &ret, sizeof(ret));
+               /* the read here means we will die if the parent exits */
+               read(fd[1], &ret, sizeof(ret));
+               _exit(0);
+       }
+
+       talloc_set_destructor(h, ctdb_recovery_handle_destructor);
+
+       close(fd[1]);
+
+       h->fd = fd[0];
+
+       fde = event_add_fd(ctdb->ev, h, h->fd, EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+                          ctdb_recovery_lock_handler, h);
+       if (fde == NULL) {
+               DEBUG(0,("Failed to setup fd event for ctdb_recovery_lock\n"));
+               close(fd[0]);
+               talloc_free(h);
+               return NULL;
+       }
+
+       return h;
+}
+
+/*
+  destroy a waiter for a recovery mode change
+ */
+static int ctdb_recovery_waiter_destructor(struct ctdb_recovery_waiter *w)
+{
+       DLIST_REMOVE(w->ctdb->recovery_handle->waiters, w);
+       ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status);
+       return 0;
+}
+
+/*
+  set the recovery mode
+ */
+void ctdb_control_set_recmode(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA data)
+{
+       uint32_t recmode = *(uint32_t *)data.dptr;
+       struct ctdb_recovery_waiter *w;
+
+       if (recmode == CTDB_RECOVERY_NORMAL) {
+               /* switching to normal mode is easy */
+               talloc_free(ctdb->recovery_handle);
+               ctdb->recovery_handle = NULL;
+               ctdb->recovery_mode = CTDB_RECOVERY_NORMAL;
+               ctdb_request_control_reply(ctdb, c, NULL, 0);
+               return;
+       }
+
+       if (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE) {
+               /* we're already active */
+               ctdb_request_control_reply(ctdb, c, NULL, 0);
+               return;
+       }
+
+       /* if there isn't a recovery lock child then create one */
+       if (!ctdb->recovery_handle) {
+               ctdb->recovery_handle = ctdb_recovery_lock(ctdb);
+               CTDB_NO_MEMORY_VOID(ctdb, ctdb->recovery_handle);
+       }
+
+       /* add ourselves to list of waiters */
+       w = talloc(ctdb->recovery_handle, struct ctdb_recovery_waiter);
+       CTDB_NO_MEMORY_VOID(ctdb, w);
+       w->ctdb   = ctdb;
+       w->c      = talloc_steal(w, c);
+       w->status = -1;
+       talloc_set_destructor(w, ctdb_recovery_waiter_destructor);
+       DLIST_ADD(ctdb->recovery_handle->waiters, w);
+}
+
+

diff --git a/ctdb/direct/ctdb_recoverd.c b/ctdb/direct/ctdb_recoverd.c
index d7de606b1481690ad50aeabba31aaf4de9981cd7..5ac4f635696f6fdbfd7c34cbc60876c31d945b2c 100644 (file)
--- a/ctdb/direct/ctdb_recoverd.c
+++ b/ctdb/direct/ctdb_recoverd.c
@@ -59,7 +59,7 @@ static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_node_map *no
 
                ret = ctdb_ctrl_setrecmode(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, rec_mode);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to set recmode on node %u\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Unable to set recmode on node %u\n", nodemap->nodes[j].vnn));
                        return -1;
                }
        }
@@ -80,7 +80,7 @@ static int set_recovery_master(struct ctdb_context *ctdb, struct ctdb_node_map *
 
                ret = ctdb_ctrl_setrecmaster(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, vnn);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to set recmaster on node %u\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Unable to set recmaster on node %u\n", nodemap->nodes[j].vnn));
                        return -1;
                }
        }
@@ -106,7 +106,7 @@ static int create_missing_remote_databases(struct ctdb_context *ctdb, struct ctd
 
                ret = ctdb_ctrl_getdbmap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_dbmap);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to get dbids from node %u\n", vnn));
+                       DEBUG(0, (__location__ " Unable to get dbids from node %u\n", vnn));
                        return -1;
                }
 
@@ -127,12 +127,12 @@ static int create_missing_remote_databases(struct ctdb_context *ctdb, struct ctd
                        /* ok so we need to create this database */
                        ctdb_ctrl_getdbname(ctdb, timeval_current_ofs(1, 0), vnn, dbmap->dbids[db], mem_ctx, &name);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to get dbname from node %u\n", vnn));
+                               DEBUG(0, (__location__ " Unable to get dbname from node %u\n", vnn));
                                return -1;
                        }
                        ctdb_ctrl_createdb(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, name);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to create remote db:%s\n", name));
+                               DEBUG(0, (__location__ " Unable to create remote db:%s\n", name));
                                return -1;
                        }
                }
@@ -160,7 +160,7 @@ static int create_missing_local_databases(struct ctdb_context *ctdb, struct ctdb
 
                ret = ctdb_ctrl_getdbmap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_dbmap);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to get dbids from node %u\n", vnn));
+                       DEBUG(0, (__location__ " Unable to get dbids from node %u\n", vnn));
                        return -1;
                }
 
@@ -182,17 +182,17 @@ static int create_missing_local_databases(struct ctdb_context *ctdb, struct ctdb
                         */
                        ctdb_ctrl_getdbname(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, remote_dbmap->dbids[db], mem_ctx, &name);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to get dbname from node %u\n", nodemap->nodes[j].vnn));
+                               DEBUG(0, (__location__ " Unable to get dbname from node %u\n", nodemap->nodes[j].vnn));
                                return -1;
                        }
                        ctdb_ctrl_createdb(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, name);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to create local db:%s\n", name));
+                               DEBUG(0, (__location__ " Unable to create local db:%s\n", name));
                                return -1;
                        }
                        ret = ctdb_ctrl_getdbmap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, dbmap);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to reread dbmap on node %u\n", vnn));
+                               DEBUG(0, (__location__ " Unable to reread dbmap on node %u\n", vnn));
                                return -1;
                        }
                }
@@ -221,7 +221,7 @@ static int pull_all_remote_databases(struct ctdb_context *ctdb, struct ctdb_node
                        }
                        ret = ctdb_ctrl_copydb(ctdb, timeval_current_ofs(2, 0), nodemap->nodes[j].vnn, vnn, dbmap->dbids[i], CTDB_LMASTER_ANY, mem_ctx);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to copy db from node %u to node %u\n", nodemap->nodes[j].vnn, vnn));
+                               DEBUG(0, (__location__ " Unable to copy db from node %u to node %u\n", nodemap->nodes[j].vnn, vnn));
                                return -1;
                        }
                }
@@ -245,7 +245,7 @@ static int update_dmaster_on_all_databases(struct ctdb_context *ctdb, struct ctd
                        }
                        ret = ctdb_ctrl_setdmaster(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, ctdb, dbmap->dbids[i], vnn);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to set dmaster for node %u db:0x%08x\n", nodemap->nodes[j].vnn, dbmap->dbids[i]));
+                               DEBUG(0, (__location__ " Unable to set dmaster for node %u db:0x%08x\n", nodemap->nodes[j].vnn, dbmap->dbids[i]));
                                return -1;
                        }
                }
@@ -272,7 +272,7 @@ static int push_all_local_databases(struct ctdb_context *ctdb, struct ctdb_node_
                        }
                        ret = ctdb_ctrl_copydb(ctdb, timeval_current_ofs(1, 0), vnn, nodemap->nodes[j].vnn, dbmap->dbids[i], CTDB_LMASTER_ANY, mem_ctx);
                        if (ret != 0) {
-                               DEBUG(0, (__location__ "Unable to copy db from node %u to node %u\n", vnn, nodemap->nodes[j].vnn));
+                               DEBUG(0, (__location__ " Unable to copy db from node %u to node %u\n", vnn, nodemap->nodes[j].vnn));
                                return -1;
                        }
                }
@@ -295,7 +295,7 @@ static int update_vnnmap_on_all_nodes(struct ctdb_context *ctdb, struct ctdb_nod
 
                ret = ctdb_ctrl_setvnnmap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, vnnmap);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to set vnnmap for node %u\n", vnn));
+                       DEBUG(0, (__location__ " Unable to set vnnmap for node %u\n", vnn));
                        return -1;
                }
        }
@@ -312,7 +312,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        uint32_t generation;
        struct ctdb_dbid_map *dbmap;
 
-       DEBUG(0, (__location__ "Recovery initiated\n"));
+       DEBUG(0, (__location__ " Recovery initiated\n"));
 
        /* pick a new generation number */
        generation = random();
@@ -330,7 +330,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        vnnmap->generation = generation;
        ret = ctdb_ctrl_setvnnmap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, vnnmap);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to set vnnmap for node %u\n", vnn));
+               DEBUG(0, (__location__ " Unable to set vnnmap for node %u\n", vnn));
                return -1;
        }
 
@@ -338,7 +338,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* set recovery mode to active on all nodes */
        ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
        if (ret!=0) {
-               DEBUG(0, (__location__ "Unable to set recovery mode to active on cluster\n"));
+               DEBUG(0, (__location__ " Unable to set recovery mode to active on cluster\n"));
                return -1;
        }
 
@@ -346,7 +346,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* get a list of all databases */
        ret = ctdb_ctrl_getdbmap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, &dbmap);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to get dbids from node :%d\n", vnn));
+               DEBUG(0, (__location__ " Unable to get dbids from node :%d\n", vnn));
                return -1;
        }
 
@@ -355,7 +355,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* verify that all other nodes have all our databases */
        ret = create_missing_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to create missing remote databases\n"));
+               DEBUG(0, (__location__ " Unable to create missing remote databases\n"));
                return -1;
        }
 
@@ -364,7 +364,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* verify that we have all the databases any other node has */
        ret = create_missing_local_databases(ctdb, nodemap, vnn, &dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to create missing local databases\n"));
+               DEBUG(0, (__location__ " Unable to create missing local databases\n"));
                return -1;
        }
 
@@ -373,7 +373,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* verify that all other nodes have all our databases */
        ret = create_missing_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to create missing remote databases\n"));
+               DEBUG(0, (__location__ " Unable to create missing remote databases\n"));
                return -1;
        }
 
@@ -384,7 +384,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
         */
        ret = update_dmaster_on_all_databases(ctdb, nodemap, 0xffffffff, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to update dmaster on all databases\n"));
+               DEBUG(0, (__location__ " Unable to update dmaster on all databases\n"));
                return -1;
        }
 
@@ -393,7 +393,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* pull all remote databases onto the local node */
        ret = pull_all_remote_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to pull remote databases\n"));
+               DEBUG(0, (__location__ " Unable to pull remote databases\n"));
                return -1;
        }
 
@@ -402,7 +402,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* push all local databases to the remote nodes */
        ret = push_all_local_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to push local databases\n"));
+               DEBUG(0, (__location__ " Unable to push local databases\n"));
                return -1;
        }
 
@@ -425,7 +425,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* update to the new vnnmap on all nodes */
        ret = update_vnnmap_on_all_nodes(ctdb, nodemap, vnn, vnnmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to update vnnmap on all nodes\n"));
+               DEBUG(0, (__location__ " Unable to update vnnmap on all nodes\n"));
                return -1;
        }
 
@@ -433,7 +433,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* update recmaster to point to us for all nodes */
        ret = set_recovery_master(ctdb, nodemap, vnn);
        if (ret!=0) {
-               DEBUG(0, (__location__ "Unable to set recovery master\n"));
+               DEBUG(0, (__location__ " Unable to set recovery master\n"));
                return -1;
        }
 
@@ -443,7 +443,7 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
         */
        ret = update_dmaster_on_all_databases(ctdb, nodemap, vnn, dbmap, mem_ctx);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to update dmaster on all databases\n"));
+               DEBUG(0, (__location__ " Unable to update dmaster on all databases\n"));
                return -1;
        }
 
@@ -452,12 +452,12 @@ static int do_recovery(struct ctdb_context *ctdb, struct event_context *ev,
        /* disable recovery mode */
        ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_NORMAL);
        if (ret!=0) {
-               DEBUG(0, (__location__ "Unable to set recovery mode to normal on cluster\n"));
+               DEBUG(0, (__location__ " Unable to set recovery mode to normal on cluster\n"));
                return -1;
        }
 
 
-       DEBUG(0, (__location__ "Recovery complete\n"));
+       DEBUG(0, (__location__ " Recovery complete\n"));
        return 0;
 }
 
@@ -487,7 +487,7 @@ static int send_election_request(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
         */
        ret = ctdb_ctrl_setrecmaster(ctdb, timeval_current_ofs(1, 0), vnn, vnn);
        if (ret != 0) {
-               DEBUG(0, (__location__ "failed to send recmaster election request"));
+               DEBUG(0, (__location__ " failed to send recmaster election request"));
                return -1;
        }
 
@@ -525,7 +525,7 @@ static void election_handler(struct ctdb_context *ctdb, uint64_t srvid,
        if (em->vnn > ctdb_get_vnn(ctdb)) {
                ret = send_election_request(ctdb, mem_ctx, ctdb_get_vnn(ctdb));
                if (ret!=0) {
-                       DEBUG(0, (__location__ "failed to initiate recmaster election"));
+                       DEBUG(0, (__location__ " failed to initiate recmaster election"));
                }
                talloc_free(mem_ctx);
                return;
@@ -534,7 +534,7 @@ static void election_handler(struct ctdb_context *ctdb, uint64_t srvid,
        /* ok, let that guy become recmaster then */
        ret = ctdb_ctrl_setrecmaster(ctdb, timeval_current_ofs(1, 0), ctdb_get_vnn(ctdb), em->vnn);
        if (ret != 0) {
-               DEBUG(0, (__location__ "failed to send recmaster election request"));
+               DEBUG(0, (__location__ " failed to send recmaster election request"));
                talloc_free(mem_ctx);
                return;
        }
@@ -551,13 +551,13 @@ static void force_election(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, uint3
        /* set all nodes to recovery mode to stop all internode traffic */
        ret = set_recovery_mode(ctdb, nodemap, CTDB_RECOVERY_ACTIVE);
        if (ret!=0) {
-               DEBUG(0, (__location__ "Unable to set recovery mode to active on cluster\n"));
+               DEBUG(0, (__location__ " Unable to set recovery mode to active on cluster\n"));
                return;
        }
        
        ret = send_election_request(ctdb, mem_ctx, vnn);
        if (ret!=0) {
-               DEBUG(0, (__location__ "failed to initiate recmaster election"));
+               DEBUG(0, (__location__ " failed to initiate recmaster election"));
                return;
        }
 
@@ -606,7 +606,7 @@ again:
        /* get the vnnmap */
        ret = ctdb_ctrl_getvnnmap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, &vnnmap);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to get vnnmap from node %u\n", vnn));
+               DEBUG(0, (__location__ " Unable to get vnnmap from node %u\n", vnn));
                goto again;
        }
 
@@ -614,7 +614,7 @@ again:
        /* get number of nodes */
        ret = ctdb_ctrl_getnodemap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, &nodemap);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to get nodemap from node %u\n", vnn));
+               DEBUG(0, (__location__ " Unable to get nodemap from node %u\n", vnn));
                goto again;
        }
 
@@ -631,7 +631,7 @@ again:
        /* check which node is the recovery master */
        ret = ctdb_ctrl_getrecmaster(ctdb, timeval_current_ofs(1, 0), vnn, &recmaster);
        if (ret != 0) {
-               DEBUG(0, (__location__ "Unable to get recmaster from node %u\n", vnn));
+               DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
                goto again;
        }
        
@@ -667,7 +667,7 @@ again:
 
                ret = ctdb_ctrl_getrecmaster(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, &recmaster);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to get recmaster from node %u\n", vnn));
+                       DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
                        goto again;
                }
 
@@ -693,7 +693,7 @@ again:
                        goto again;
                }
                if (recmode!=CTDB_RECOVERY_NORMAL) {
-                       DEBUG(0, (__location__ "Node:%d was in recovery mode. Restart recovery process\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Node:%d was in recovery mode. Restart recovery process\n", nodemap->nodes[j].vnn));
                        do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                        goto again;
                }
@@ -713,7 +713,7 @@ again:
 
                ret = ctdb_ctrl_getnodemap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_nodemap);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to get nodemap from remote node %u\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Unable to get nodemap from remote node %u\n", nodemap->nodes[j].vnn));
                        goto again;
                }
 
@@ -721,7 +721,7 @@ again:
                   then this is a good reason to try recovery
                 */
                if (remote_nodemap->num != nodemap->num) {
-                       DEBUG(0, (__location__ "Remote node:%d has different node count. %d vs %d of the local node\n", nodemap->nodes[j].vnn, remote_nodemap->num, nodemap->num));
+                       DEBUG(0, (__location__ " Remote node:%d has different node count. %d vs %d of the local node\n", nodemap->nodes[j].vnn, remote_nodemap->num, nodemap->num));
                        do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                        goto again;
                }
@@ -732,7 +732,7 @@ again:
                for (i=0;i<nodemap->num;i++) {
                        if ((remote_nodemap->nodes[i].vnn != nodemap->nodes[i].vnn)
                        ||  (remote_nodemap->nodes[i].flags != nodemap->nodes[i].flags)) {
-                               DEBUG(0, (__location__ "Remote node:%d has different nodemap.\n", nodemap->nodes[j].vnn));
+                               DEBUG(0, (__location__ " Remote node:%d has different nodemap.\n", nodemap->nodes[j].vnn));
                                do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                                goto again;
                        }
@@ -745,7 +745,7 @@ again:
           as there are active nodes or we will have to do a recovery
         */
        if (vnnmap->size != num_active) {
-               DEBUG(0, (__location__ "The vnnmap count is different from the number of active nodes. %d vs %d\n", vnnmap->size, num_active));
+               DEBUG(0, (__location__ " The vnnmap count is different from the number of active nodes. %d vs %d\n", vnnmap->size, num_active));
                do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                goto again;
        }
@@ -767,7 +767,7 @@ again:
                        }
                }
                if (i==vnnmap->size) {
-                       DEBUG(0, (__location__ "Node %d is active in the nodemap but did not exist in the vnnmap\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Node %d is active in the nodemap but did not exist in the vnnmap\n", nodemap->nodes[j].vnn));
                        do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                        goto again;
                }
@@ -787,20 +787,20 @@ again:
 
                ret = ctdb_ctrl_getvnnmap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_vnnmap);
                if (ret != 0) {
-                       DEBUG(0, (__location__ "Unable to get vnnmap from remote node %u\n", nodemap->nodes[j].vnn));
+                       DEBUG(0, (__location__ " Unable to get vnnmap from remote node %u\n", nodemap->nodes[j].vnn));
                        goto again;
                }
 
                /* verify the vnnmap generation is the same */
                if (vnnmap->generation != remote_vnnmap->generation) {
-                       DEBUG(0, (__location__ "Remote node %d has different generation of vnnmap. %d vs %d (ours)\n", nodemap->nodes[j].vnn, remote_vnnmap->generation, vnnmap->generation));
+                       DEBUG(0, (__location__ " Remote node %d has different generation of vnnmap. %d vs %d (ours)\n", nodemap->nodes[j].vnn, remote_vnnmap->generation, vnnmap->generation));
                        do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                        goto again;
                }
 
                /* verify the vnnmap size is the same */
                if (vnnmap->size != remote_vnnmap->size) {
-                       DEBUG(0, (__location__ "Remote node %d has different size of vnnmap. %d vs %d (ours)\n", nodemap->nodes[j].vnn, remote_vnnmap->size, vnnmap->size));
+                       DEBUG(0, (__location__ " Remote node %d has different size of vnnmap. %d vs %d (ours)\n", nodemap->nodes[j].vnn, remote_vnnmap->size, vnnmap->size));
                        do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                        goto again;
                }
@@ -808,7 +808,7 @@ again:
                /* verify the vnnmap is the same */
                for (i=0;i<vnnmap->size;i++) {
                        if (remote_vnnmap->map[i] != vnnmap->map[i]) {
-                               DEBUG(0, (__location__ "Remote node %d has different vnnmap.\n", nodemap->nodes[j].vnn));
+                               DEBUG(0, (__location__ " Remote node %d has different vnnmap.\n", nodemap->nodes[j].vnn));
                                do_recovery(ctdb, ev, mem_ctx, vnn, num_active, nodemap, vnnmap);
                                goto again;
                        }
@@ -869,7 +869,7 @@ int main(int argc, const char *argv[])
        /* initialise ctdb */
        ctdb = ctdb_cmdline_client(ev);
        if (ctdb == NULL) {
-               DEBUG(0, (__location__ "Failed to init ctdb\n"));
+               DEBUG(0, (__location__ " Failed to init ctdb\n"));
                exit(1);
        }
 

diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
index f5e02ebdee118514543c5952f424719ef5f9b8e2..4be519482924b6eb2d6ecba1dd3365ed3ce9b922 100644 (file)
--- a/ctdb/include/ctdb.h
+++ b/ctdb/include/ctdb.h
@@ -285,6 +285,7 @@ int ctdb_ctrl_write_record(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_
 
 #define CTDB_RECOVERY_NORMAL           0
 #define CTDB_RECOVERY_ACTIVE           1
+
 /*
   get the recovery mode of a remote node
  */

diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index a22c5c3a5e79a99347b69507ba3604afbf74a80a..f1948e121ad084981675e6e688bf4c710c6288d6 100644 (file)
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -217,11 +217,13 @@ struct ctdb_write_record {
        uint32_t datalen;
        unsigned char blob[1];
 };
-       
+
+
 /* main state of the ctdb daemon */
 struct ctdb_context {
        struct event_context *ev;
        uint32_t recovery_mode;
+       struct ctdb_recovery_handle *recovery_handle;
        struct ctdb_address address;
        const char *name;
        const char *db_directory;
@@ -781,4 +783,8 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata);
 int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata);
 int32_t ctdb_control_clear_db(struct ctdb_context *ctdb, TDB_DATA indata);
 
+void ctdb_control_set_recmode(struct ctdb_context *ctdb, struct ctdb_req_control *c, TDB_DATA data);
+void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_control *c,
+                               TDB_DATA *outdata, int32_t status);
+
 #endif