update getvnnmap control to take a timeout parameter

dont explicitely free the vnnmap pointer in the getvnnmap control  this
is freed by the mem_ctx instead

add code to the recoverd to detect when/if recovery is required
veiry that the number of active nodes, the nodemap and the vnn map is
consistent across the entire cluster and if not   trigger a recovery
(which right now just prints "we need to do recovery" to the screen.

(This used to be ctdb commit 2b0a207a3748bdb3394dc9fd0d1c344ee1bb0bb5)

diff --git a/ctdb/common/ctdb_client.c b/ctdb/common/ctdb_client.c
index 4e401728a10a6d28daa402df0b3b6cf2cc5340b8..c8480ec378629167bede011867668071ba77649e 100644 (file)
--- a/ctdb/common/ctdb_client.c
+++ b/ctdb/common/ctdb_client.c
@@ -791,7 +791,7 @@ int ctdb_ctrl_status(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_s
 /*
   get vnn map from a remote node
  */
-int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap)
+int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap)
 {
        int ret;
        TDB_DATA data, outdata;
@@ -800,16 +800,12 @@ int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX
        ZERO_STRUCT(data);
        ret = ctdb_control(ctdb, destnode, 0, 
                           CTDB_CONTROL_GETVNNMAP, 0, data, 
-                          ctdb, &outdata, &res, NULL);
+                          ctdb, &outdata, &res, &timeout);
        if (ret != 0 || res != 0) {
                DEBUG(0,(__location__ " ctdb_control for getvnnmap failed\n"));
                return -1;
        }
 
-       if (*vnnmap) {
-               talloc_free(*vnnmap);
-               *vnnmap=NULL;
-       }
        *vnnmap = (struct ctdb_vnn_map *)talloc_memdup(mem_ctx, outdata.dptr, outdata.dsize);
                    
        return 0;
@@ -910,10 +906,6 @@ int ctdb_ctrl_getnodemap(struct ctdb_context *ctdb,
                return -1;
        }
 
-       if (*nodemap) {
-               talloc_free(*nodemap);
-               *nodemap = NULL;
-       }
        *nodemap = (struct ctdb_node_map *)talloc_memdup(mem_ctx, outdata.dptr, outdata.dsize);
                    
        return 0;

diff --git a/ctdb/direct/recoverd.c b/ctdb/direct/recoverd.c
index 08ba651f973293a82c51c1e451f03afb0c641540..c427f077e52367a9456f54ac834f1bad7015f51b 100644 (file)
--- a/ctdb/direct/recoverd.c
+++ b/ctdb/direct/recoverd.c
@@ -45,15 +45,23 @@ void timeout_func(struct event_context *ev, struct timed_event *te,
        timed_out = 1;
 }
 
+void do_recovery(struct ctdb_context *ctdb, struct event_context *ev)
+{
+       printf("we need to do recovery !!!\n");
+}
 
 void recoverd(struct ctdb_context *ctdb, struct event_context *ev)
 {
-       uint32_t vnn;
+       uint32_t vnn, num_active;
        TALLOC_CTX *mem_ctx=NULL;
        struct ctdb_node_map *nodemap=NULL;
-       int ret;
+       struct ctdb_node_map *remote_nodemap=NULL;
+       struct ctdb_vnn_map *vnnmap=NULL;
+       struct ctdb_vnn_map *remote_vnnmap=NULL;
+       int i, j, ret;
        
 again:
+       printf("check if we need to do recovery\n");
        if (mem_ctx) {
                talloc_free(mem_ctx);
                mem_ctx = NULL;
@@ -75,7 +83,6 @@ again:
 
        /* get our vnn number */
        vnn = ctdb_get_vnn(ctdb);  
-printf("our node number is :%d\n",vnn);
 
        /* get number of nodes */
        ret = ctdb_ctrl_getnodemap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, &nodemap);
@@ -84,6 +91,130 @@ printf("our node number is :%d\n",vnn);
                goto again;
        }
 
+       /* count how many active nodes there are */
+       num_active = 0;
+       for (i=0; i<nodemap->num; i++) {
+               if (nodemap->nodes[i].flags&NODE_FLAGS_CONNECTED) {
+                       num_active++;
+               }
+       }
+
+
+       /* get the nodemap for all active remote nodes and verify
+          they are the same as for this node
+        */
+       for (j=0; j<nodemap->num; j++) {
+               if (!(nodemap->nodes[j].flags&NODE_FLAGS_CONNECTED)) {
+                       continue;
+               }
+               if (nodemap->nodes[j].vnn == vnn) {
+                       continue;
+               }
+
+               ret = ctdb_ctrl_getnodemap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_nodemap);
+               if (ret != 0) {
+                       printf("Unable to get nodemap from remote node %u\n", nodemap->nodes[j].vnn);
+                       goto again;
+               }
+
+               /* if the nodes disagree on how many nodes there are
+                  then this is a good reason to try recovery
+                */
+               if (remote_nodemap->num != nodemap->num) {
+                       printf("Remote node:%d has different node count. %d vs %d of the local node\n", nodemap->nodes[j].vnn, remote_nodemap->num, nodemap->num);
+                       do_recovery(ctdb, ev);
+                       goto again;
+               }
+
+               /* if the nodes disagree on which nodes exist and are
+                  active, then that is also a good reason to do recovery
+                */
+               for (i=0;i<nodemap->num;i++) {
+                       if ((remote_nodemap->nodes[i].vnn != nodemap->nodes[i].vnn)
+                       ||  (remote_nodemap->nodes[i].flags != nodemap->nodes[i].flags)) {
+                               printf("Remote node:%d has different nodemap.\n", nodemap->nodes[j].vnn);
+                               do_recovery(ctdb, ev);
+                               goto again;
+                       }
+               }
+
+       }
+
+       /* get the vnnmap */
+       ret = ctdb_ctrl_getvnnmap(ctdb, timeval_current_ofs(1, 0), vnn, mem_ctx, &vnnmap);
+       if (ret != 0) {
+               printf("Unable to get vnnmap from node %u\n", vnn);
+               goto again;
+       }
+
+       /* there better be the same number of lmasters in the vnn map
+          as there are active nodes or well have to do a recovery
+        */
+       if (vnnmap->size != num_active) {
+               printf("The vnnmap count is different from the number of active nodes. %d vs %d\n", vnnmap->size, num_active);
+               do_recovery(ctdb, ev);
+               goto again;
+       }
+
+       /* verify that all active nodes in the nodemap also exist in 
+          the vnnmap.
+        */
+       for (j=0; j<nodemap->num; j++) {
+               if (!(nodemap->nodes[j].flags&NODE_FLAGS_CONNECTED)) {
+                       continue;
+               }
+               if (nodemap->nodes[j].vnn == vnn) {
+                       continue;
+               }
+
+               for (i=0; i<vnnmap->size; i++) {
+                       if (vnnmap->map[i] == nodemap->nodes[j].vnn) {
+                               break;
+                       }
+               }
+               if (i==vnnmap->size) {
+                       printf("Node %d is active in the nodemap but did not exist in the vnnmap\n", nodemap->nodes[j].vnn);
+                       do_recovery(ctdb, ev);
+                       goto again;
+               }
+       }
+
+       
+       /* verify that all other nodes have the same vnnmap */
+       for (j=0; j<nodemap->num; j++) {
+               if (!(nodemap->nodes[j].flags&NODE_FLAGS_CONNECTED)) {
+                       continue;
+               }
+               if (nodemap->nodes[j].vnn == vnn) {
+                       continue;
+               }
+
+               ret = ctdb_ctrl_getvnnmap(ctdb, timeval_current_ofs(1, 0), nodemap->nodes[j].vnn, mem_ctx, &remote_vnnmap);
+               if (ret != 0) {
+                       printf("Unable to get vnnmap from remote node %u\n", nodemap->nodes[j].vnn);
+                       goto again;
+               }
+
+               /* verify the vnnmap size is the same */
+               if (vnnmap->size != remote_vnnmap->size) {
+                       printf("Remote node %d has different size of vnnmap. %d vs %d (ours)\n", nodemap->nodes[j].vnn, remote_vnnmap->size, vnnmap->size);
+                       do_recovery(ctdb, ev);
+                       goto again;
+               }
+
+               /* verify the vnnmap is the same */
+               for (i=0;i<vnnmap->size;i++) {
+                       if (remote_vnnmap->map[i] != vnnmap->map[i]) {
+                               printf("Remote node %d has different vnnmap.\n", nodemap->nodes[j].vnn);
+                               do_recovery(ctdb, ev);
+                               goto again;
+                       }
+               }
+       }
+
+       printf("no we did not need to do recovery\n");
+       goto again;
+
 }
 
 /*

diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
index 5b7c5e4f2c2db9b57bd527416b0f0e8b0aa49eaf..93e75af673746652597b0c22a001dbf5c34fa65a 100644 (file)
--- a/ctdb/include/ctdb.h
+++ b/ctdb/include/ctdb.h
@@ -201,7 +201,9 @@ struct ctdb_status;
 int ctdb_ctrl_status(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_status *status);
 
 struct ctdb_vnn_map;
-int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap);
+int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, 
+               struct timeval timeout, uint32_t destnode, 
+               TALLOC_CTX *mem_ctx, struct ctdb_vnn_map **vnnmap);
 int ctdb_ctrl_setvnnmap(struct ctdb_context *ctdb, uint32_t destnode, TALLOC_CTX *mem_ctx, struct ctdb_vnn_map *vnnmap);
 
 /* table that contains a list of all dbids on a node

diff --git a/ctdb/tools/ctdb_control.c b/ctdb/tools/ctdb_control.c
index 8bdee098e7e1af4edd591cf7e985c136e1cce950..8a8e8c3d8fdffa2cf3f140b67a899211f27c6fa7 100644 (file)
--- a/ctdb/tools/ctdb_control.c
+++ b/ctdb/tools/ctdb_control.c
@@ -445,7 +445,7 @@ static int control_getvnnmap(struct ctdb_context *ctdb, int argc, const char **a
 
        vnn = strtoul(argv[0], NULL, 0);
 
-       ret = ctdb_ctrl_getvnnmap(ctdb, vnn, ctdb, &vnnmap);
+       ret = ctdb_ctrl_getvnnmap(ctdb, timeval_current_ofs(1, 0), vnn, ctdb, &vnnmap);
        if (ret != 0) {
                printf("Unable to get vnnmap from node %u\n", vnn);
                return ret;