From: Ronnie Sahlberg <sahlberg@ronnie>
Date: Thu, 23 Aug 2007 09:27:09 +0000 (+1000)
Subject: try out a slightly different api for controls where you provide a
X-Git-Tag: tevent-0.9.20~348^2~2435^2~10
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=f854b5f87678efa1bbebd8a2c423fc35bb4649a0;p=thirdparty%2Fsamba.git

try out a slightly different api for controls where you provide a
callback function which is called upon completion (or timeout) of the
control.

modify scanning of recmaster in the monitoring_cluster code to try the
api out

(This used to be ctdb commit c37843f1d97b169afec910e7ddb4e5ac12c3015c)
---

diff --git a/ctdb/client/ctdb_client.c b/ctdb/client/ctdb_client.c
index 22fb0e319ce..cc1aa3fb8f5 100644
--- a/ctdb/client/ctdb_client.c
+++ b/ctdb/client/ctdb_client.c
@@ -665,8 +665,6 @@ int ctdb_fetch(struct ctdb_db_context *ctdb_db, TALLOC_CTX *mem_ctx,
 }
 
 
-enum control_state {CTDB_CONTROL_WAIT, CTDB_CONTROL_DONE, CTDB_CONTROL_ERROR, CTDB_CONTROL_TIMEOUT};
-
 struct ctdb_client_control_state {
 	struct ctdb_context *ctdb;
 	uint32_t reqid;
@@ -675,8 +673,60 @@ struct ctdb_client_control_state {
 	enum control_state state;
 	char *errormsg;
 	struct ctdb_req_control *c;
+
+	/* if we have a callback registered for the completion (or failure) of
+	   this control
+	   if a callback is used, it MUST talloc_free the cb_data passed to it
+	*/
+	control_callback callback;
+	void *cb_private;
 };
 
+/*
+   called when a control completes or timesout to invoke the callback
+   function the user provided
+*/
+static void invoke_control_callback(struct event_context *ev, struct timed_event *te, 
+	struct timeval t, void *private_data)
+{
+	struct ctdb_client_control_state *state;
+	struct ctdb_control_cb_data *cb_data;
+	struct ctdb_context *ctdb;
+	control_callback callback;
+	void *cb_private;
+	TALLOC_CTX *tmp_ctx = talloc_new(NULL);
+	int ret;
+
+	state = talloc_get_type(private_data, struct ctdb_client_control_state);
+	talloc_steal(tmp_ctx, state);
+
+	ctdb       = state->ctdb;
+	callback   = state->callback;
+	cb_private = state->cb_private;
+
+	cb_data = talloc_zero(tmp_ctx, struct ctdb_control_cb_data);
+	if (cb_data == NULL) {
+		talloc_free(tmp_ctx);
+		CTDB_NO_MEMORY_VOID(ctdb, cb_data);
+	}
+
+	cb_data->state = state->state;
+	cb_data->vnn   = state->c->hdr.destnode;
+
+	ret = ctdb_control_recv(ctdb, state, cb_data,
+			&cb_data->outdata, 
+			&cb_data->status, 
+			&cb_data->errormsg);
+	/* we dont check ret since we expect that ctdb_control_recv can fail
+	   for example if the control timedout
+
+	   state is always talloc_free()'d inside ctdb_control_recv
+	*/
+
+	callback(cb_data, cb_private);
+	talloc_free(tmp_ctx);
+}
+
 /*
   called when a CTDB_REPLY_CONTROL packet comes in in the client
 
@@ -713,6 +763,13 @@ static void ctdb_client_reply_control(struct ctdb_context *ctdb,
 	talloc_steal(state, c);
 
 	state->state = CTDB_CONTROL_DONE;
+
+	/* if we had a callback registered for this control, pull the response
+	   and call the callback.
+	*/
+	if (state->callback) {
+		event_add_timed(ctdb->ev, state, timeval_zero(), invoke_control_callback, state);
+	}
 }
 
 
@@ -735,6 +792,13 @@ static void control_timeout_func(struct event_context *ev, struct timed_event *t
 	DEBUG(0,("control timed out. reqid:%d opcode:%d dstnode:%d\n", state->reqid, state->c->opcode, state->c->hdr.destnode));
 
 	state->state = CTDB_CONTROL_TIMEOUT;
+
+	/* if we had a callback registered for this control, pull the response
+	   and call the callback.
+	*/
+	if (state->callback) {
+		event_add_timed(state->ctdb->ev, state, timeval_zero(), invoke_control_callback, state);
+	}
 }
 
 /* async version of send control request */
@@ -743,8 +807,8 @@ struct ctdb_client_control_state *ctdb_control_send(struct ctdb_context *ctdb,
 		uint32_t opcode, uint32_t flags, TDB_DATA data, 
 		TALLOC_CTX *mem_ctx, TDB_DATA *outdata,
 		struct timeval *timeout,
-		char **errormsg)
-
+		char **errormsg,
+		control_callback callback, void *cb_private)
 {
 	struct ctdb_client_control_state *state;
 	size_t len;
@@ -760,13 +824,15 @@ struct ctdb_client_control_state *ctdb_control_send(struct ctdb_context *ctdb,
 		ctdb_socket_connect(ctdb);
 	}
 
-	state = talloc_zero(ctdb, struct ctdb_client_control_state);
+	state = talloc_zero(mem_ctx, struct ctdb_client_control_state);
 	CTDB_NO_MEMORY_NULL(ctdb, state);
 
-	state->ctdb    = ctdb;
-	state->reqid   = ctdb_reqid_new(ctdb, state);
-	state->state   = CTDB_CONTROL_WAIT;
-	state->errormsg= NULL;
+	state->ctdb       = ctdb;
+	state->reqid      = ctdb_reqid_new(ctdb, state);
+	state->state      = CTDB_CONTROL_WAIT;
+	state->errormsg   = NULL;
+	state->callback   = callback;
+	state->cb_private = cb_private;
 
 	talloc_set_destructor(state, ctdb_control_destructor);
 
@@ -787,6 +853,11 @@ struct ctdb_client_control_state *ctdb_control_send(struct ctdb_context *ctdb,
 		memcpy(&c->data[0], data.dptr, data.dsize);
 	}
 
+	/* timeout */
+	if (timeout && !timeval_is_zero(timeout)) {
+		event_add_timed(ctdb->ev, state, *timeout, control_timeout_func, state);
+	}
+
 	ret = ctdb_client_queue_pkt(ctdb, &(c->hdr));
 	if (ret != 0) {
 		talloc_free(state);
@@ -798,11 +869,6 @@ struct ctdb_client_control_state *ctdb_control_send(struct ctdb_context *ctdb,
 		return NULL;
 	}
 
-	/* timeout */
-	if (timeout && !timeval_is_zero(timeout)) {
-		event_add_timed(ctdb->ev, state, *timeout, control_timeout_func, state);
-	}
-
 	return state;
 }
 
@@ -870,7 +936,8 @@ int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid,
 
 	state = ctdb_control_send(ctdb, destnode, srvid, opcode, 
 			flags, data, mem_ctx, outdata,
-			timeout, errormsg);
+			timeout, errormsg,
+			NULL, NULL);
 	return ctdb_control_recv(ctdb, state, mem_ctx, outdata, status, 
 			errormsg);
 }
@@ -988,12 +1055,16 @@ int ctdb_ctrl_getvnnmap(struct ctdb_context *ctdb, struct timeval timeout, uint3
 }
 
 
+/*
+  get the recovery mode of a remote node
+ */
 struct ctdb_client_control_state *
 ctdb_ctrl_getrecmode_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode)
 {
 	return ctdb_control_send(ctdb, destnode, 0, 
 			   CTDB_CONTROL_GET_RECMODE, 0, tdb_null, 
-			   mem_ctx, NULL, &timeout, NULL);
+			   mem_ctx, NULL, &timeout, NULL,
+			   NULL, NULL);
 }
 
 int ctdb_ctrl_getrecmode_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *recmode)
@@ -1014,9 +1085,6 @@ int ctdb_ctrl_getrecmode_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, st
 	return 0;
 }
 
-/*
-  get the recovery mode of a remote node
- */
 int ctdb_ctrl_getrecmode(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, uint32_t *recmode)
 {
 	struct ctdb_client_control_state *state;
@@ -1051,27 +1119,49 @@ int ctdb_ctrl_setrecmode(struct ctdb_context *ctdb, struct timeval timeout, uint
 	return 0;
 }
 
+
+
 /*
   get the recovery master of a remote node
  */
-int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmaster)
+struct ctdb_client_control_state *
+ctdb_ctrl_getrecmaster_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, 
+			struct timeval timeout, uint32_t destnode,
+			control_callback callback, void *cb_private)
+{
+	return ctdb_control_send(ctdb, destnode, 0, 
+			   CTDB_CONTROL_GET_RECMASTER, 0, tdb_null, 
+			   mem_ctx, NULL, &timeout, NULL,
+			   callback, cb_private);
+}
+
+int ctdb_ctrl_getrecmaster_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *recmaster)
 {
 	int ret;
 	int32_t res;
 
-	ret = ctdb_control(ctdb, destnode, 0, 
-			   CTDB_CONTROL_GET_RECMASTER, 0, tdb_null, 
-			   NULL, NULL, &res, &timeout, NULL);
+	ret = ctdb_control_recv(ctdb, state, mem_ctx, NULL, &res, NULL);
 	if (ret != 0) {
-		DEBUG(0,(__location__ " ctdb_control for getrecmaster failed\n"));
+		DEBUG(0,(__location__ " ctdb_ctrl_getrecmaster_recv failed\n"));
 		return -1;
 	}
 
-	*recmaster = res;
+	if (recmaster) {
+		*recmaster = (uint32_t)res;
+	}
 
 	return 0;
 }
 
+int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, uint32_t *recmaster)
+{
+	struct ctdb_client_control_state *state;
+
+	state = ctdb_ctrl_getrecmaster_send(ctdb, mem_ctx, timeout, destnode, NULL, NULL);
+	return ctdb_ctrl_getrecmaster_recv(ctdb, mem_ctx, state, recmaster);
+}
+
+
 /*
   set the recovery master of a remote node
  */
diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
index 2acde41dd52..17cf2c598f4 100644
--- a/ctdb/include/ctdb.h
+++ b/ctdb/include/ctdb.h
@@ -96,6 +96,19 @@ struct ctdb_call_info {
 #define CTDB_BROADCAST_CONNECTED 0xF0000004
 
 
+enum control_state {CTDB_CONTROL_WAIT, CTDB_CONTROL_DONE, CTDB_CONTROL_ERROR, CTDB_CONTROL_TIMEOUT};
+
+struct ctdb_control_cb_data {
+	enum control_state state;
+	uint32_t vnn;
+	int32_t status;
+	TDB_DATA outdata;
+	char *errormsg;
+};
+
+typedef int (*control_callback)(struct ctdb_control_cb_data *cb_data, void *cb_private);
+
+
 struct event_context;
 
 /*
@@ -325,7 +338,14 @@ int ctdb_ctrl_setmonmode(struct ctdb_context *ctdb, struct timeval timeout, uint
 /*
   get the recovery master of a remote node
  */
-int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode, uint32_t *recmaster);
+int ctdb_ctrl_getrecmaster(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, uint32_t *recmaster);
+
+struct ctdb_client_control_state *ctdb_ctrl_getrecmaster_send(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct timeval timeout, uint32_t destnode, control_callback callback, void *cb_private);
+
+int ctdb_ctrl_getrecmaster_recv(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx, struct ctdb_client_control_state *state, uint32_t *recmaster);
+
+
+
 /*
   set the recovery master of a remote node
  */
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 1338ce79a57..1c63daf469b 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -850,6 +850,19 @@ int ctdb_control(struct ctdb_context *ctdb, uint32_t destnode, uint64_t srvid,
 		 uint32_t opcode, uint32_t flags, TDB_DATA data, 
 		 TALLOC_CTX *mem_ctx, TDB_DATA *outdata, int32_t *status,
 		 struct timeval *timeout, char **errormsg);
+int ctdb_control_recv(struct ctdb_context *ctdb, 
+		struct ctdb_client_control_state *state, 
+		TALLOC_CTX *mem_ctx,
+		TDB_DATA *outdata, int32_t *status, char **errormsg);
+
+struct ctdb_client_control_state *
+ctdb_control_send(struct ctdb_context *ctdb, 
+		uint32_t destnode, uint64_t srvid, 
+		uint32_t opcode, uint32_t flags, TDB_DATA data, 
+		TALLOC_CTX *mem_ctx, TDB_DATA *outdata,
+		struct timeval *timeout,
+		char **errormsg,
+		control_callback callback, void *cb_private);
 
 
 
diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index df46f2fa3f1..c931efcf2ce 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -531,28 +531,33 @@ static void ban_handler(struct ctdb_context *ctdb, uint64_t srvid,
 {
 	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
 	struct ctdb_ban_info *b = (struct ctdb_ban_info *)data.dptr;
+	TALLOC_CTX *mem_ctx = talloc_new(ctdb);
 	uint32_t recmaster;
 	int ret;
 
 	if (data.dsize != sizeof(*b)) {
 		DEBUG(0,("Bad data in ban_handler\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, mem_ctx, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
 	if (ret != 0) {
 		DEBUG(0,(__location__ " Failed to find the recmaster\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 
 	if (recmaster != ctdb->vnn) {
 		DEBUG(0,("We are not the recmaster - ignoring ban request\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 
 	DEBUG(0,("Node %u has been banned for %u seconds by the administrator\n", 
 		 b->vnn, b->ban_time));
 	ctdb_ban_node(rec, b->vnn, b->ban_time);
+	talloc_free(mem_ctx);
 }
 
 /*
@@ -562,29 +567,34 @@ static void unban_handler(struct ctdb_context *ctdb, uint64_t srvid,
 			  TDB_DATA data, void *private_data)
 {
 	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+	TALLOC_CTX *mem_ctx = talloc_new(ctdb);
 	uint32_t vnn;
 	int ret;
 	uint32_t recmaster;
 
 	if (data.dsize != sizeof(uint32_t)) {
 		DEBUG(0,("Bad data in unban_handler\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 	vnn = *(uint32_t *)data.dptr;
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, mem_ctx, CONTROL_TIMEOUT(), CTDB_CURRENT_NODE, &recmaster);
 	if (ret != 0) {
 		DEBUG(0,(__location__ " Failed to find the recmaster\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 
 	if (recmaster != ctdb->vnn) {
 		DEBUG(0,("We are not the recmaster - ignoring unban request\n"));
+		talloc_free(mem_ctx);
 		return;
 	}
 
 	DEBUG(0,("Node %u has been unbanned by the administrator\n", vnn));
 	ctdb_unban_node(rec, vnn);
+	talloc_free(mem_ctx);
 }
 
 
@@ -1104,7 +1114,7 @@ static void monitor_handler(struct ctdb_context *ctdb, uint64_t srvid,
 
 	nodemap->nodes[i].flags = c->new_flags;
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), 
+	ret = ctdb_ctrl_getrecmaster(ctdb, tmp_ctx, CONTROL_TIMEOUT(), 
 				     CTDB_CURRENT_NODE, &ctdb->recovery_master);
 
 	if (ret == 0) {
@@ -1138,20 +1148,22 @@ static void monitor_handler(struct ctdb_context *ctdb, uint64_t srvid,
 }
 
 
-enum monitor_result { MONITOR_OK, MONITOR_RECOVERY_NEEDED, MONITOR_FAILED};
+enum monitor_result { MONITOR_OK, MONITOR_RECOVERY_NEEDED, MONITOR_ELECTION_NEEDED, MONITOR_FAILED};
 
 
 /* verify that all nodes are in recovery mode normal */
-static enum monitor_result verify_recmode(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, TALLOC_CTX *mem_ctx)
+static enum monitor_result verify_recmode(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
 {
 	struct ctdb_client_control_state **ctrl_states;
 	uint32_t recmode;
+	TALLOC_CTX *mem_ctx = talloc_new(ctdb);
 	int j, ret;
 	
 	ctrl_states = talloc_array(mem_ctx, struct ctdb_client_control_state *,
 				 nodemap->num);
 	if (!ctrl_states) {
 		DEBUG(0,(__location__ " Failed to allocate temporary ctrl state array\n"));
+		talloc_free(mem_ctx);
 		exit(-1);
 	}
 
@@ -1176,20 +1188,125 @@ static enum monitor_result verify_recmode(struct ctdb_context *ctdb, struct ctdb
 		ret = ctdb_ctrl_getrecmode_recv(ctdb, mem_ctx, ctrl_states[j], &recmode);
 		if (ret != 0) {
 			DEBUG(0, ("Unable to get recmode from node %u\n", nodemap->nodes[j].vnn));
-			talloc_free(ctrl_states);
+			talloc_free(mem_ctx);
 			return MONITOR_FAILED;
 		}
+
 		if (recmode != CTDB_RECOVERY_NORMAL) {
 			DEBUG(0, (__location__ " Node:%u was in recovery mode. Restart recovery process\n", nodemap->nodes[j].vnn));
-			talloc_free(ctrl_states);
+			talloc_free(mem_ctx);
 			return MONITOR_RECOVERY_NEEDED;
 		}
 	}
 
-	talloc_free(ctrl_states);
+	talloc_free(mem_ctx);
 	return MONITOR_OK;
 }
 
+struct verify_recmaster_data {
+	uint32_t count;
+	uint32_t vnn;
+	enum monitor_result status;
+};
+
+static int verify_recmaster_callback(struct ctdb_control_cb_data *cb_data, void *cb_private)
+{
+	struct verify_recmaster_data *rmdata = talloc_get_type(cb_private, struct verify_recmaster_data);
+
+
+	/* one more node has responded with recmaster data*/
+	rmdata->count--;
+
+	/* if we failed to get the recmaster, then return an error and let
+	   the main loop try again.
+	*/
+	if (cb_data->state != CTDB_CONTROL_DONE) {
+		if (rmdata->status == MONITOR_OK) {
+			rmdata->status = MONITOR_FAILED;
+		}
+		return 0;
+	}
+
+	/* if we got a response, then the recmaster will be stored in the
+	   status field
+	*/
+	if (cb_data->status != rmdata->vnn) {
+		DEBUG(0,("Node %d does not agree we are the recmaster. Need a new recmaster election\n",cb_data->vnn));
+		rmdata->status = MONITOR_ELECTION_NEEDED;
+	}
+
+	return 0;
+}
+
+static void verify_recmaster_timeout(struct event_context *ev, struct timed_event *te, struct timeval t, void *p)
+{
+	uint32_t *timedout = (uint32_t *)p;
+
+	*timedout = 1;
+}
+
+/* verify that all nodes agree that we are the recmaster */
+static enum monitor_result verify_recmaster(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap, uint32_t vnn)
+{
+	struct verify_recmaster_data *rmdata;
+	TALLOC_CTX *mem_ctx = talloc_new(ctdb);
+	struct ctdb_client_control_state *state;
+	enum monitor_result status;
+	uint32_t timedout;	
+	int j;
+	
+	rmdata = talloc(mem_ctx, struct verify_recmaster_data);
+	CTDB_NO_MEMORY_FATAL(ctdb, rmdata);
+	rmdata->count  = 0;
+	rmdata->vnn    = vnn;
+	rmdata->status = MONITOR_OK;
+
+	/* loop over all active nodes and send an async getrecmaster call to 
+	   them*/
+	for (j=0; j<nodemap->num; j++) {
+		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
+			continue;
+		}
+		state = ctdb_ctrl_getrecmaster_send(ctdb, mem_ctx, 
+					timeval_zero(),
+					nodemap->nodes[j].vnn,
+					verify_recmaster_callback, rmdata);
+		if (state == NULL) {
+			/* we failed to send the control, treat this as 
+			   an error and try again next iteration
+			*/			
+			DEBUG(0,("Failed to call ctdb_ctrl_getrecmaster_send during monitoring\n"));
+			talloc_free(mem_ctx);
+			return MONITOR_FAILED;
+		}
+
+		/* one more control to wait for to complete */
+		rmdata->count++;
+	}
+
+
+	/* now wait for up to the maximum number of seconds allowed
+	   or until all nodes we expect a response from has replied
+	*/
+	timedout = 0;
+	event_add_timed(ctdb->ev, rmdata, CONTROL_TIMEOUT(),
+			verify_recmaster_timeout, &timedout);
+
+	while ( (rmdata->count > 0)
+	&&	(timedout == 0) ) {
+		event_loop_once(ctdb->ev);
+	}
+	if (timedout) {
+		DEBUG(0,("Timedout while waiting for getrecmaster replies.\n"));
+		rmdata->status = MONITOR_FAILED;
+	}
+
+
+	status = rmdata->status;
+	talloc_free(mem_ctx);
+	return status;
+}
+
 
 /*
   the main monitoring loop
@@ -1287,7 +1404,7 @@ again:
 
 
 	/* check which node is the recovery master */
-	ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), vnn, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, mem_ctx, CONTROL_TIMEOUT(), vnn, &recmaster);
 	if (ret != 0) {
 		DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
 		goto again;
@@ -1328,39 +1445,31 @@ again:
 
 
 	/* verify that all active nodes agree that we are the recmaster */
-	for (j=0; j<nodemap->num; j++) {
-		if (nodemap->nodes[j].flags & NODE_FLAGS_INACTIVE) {
-			continue;
-		}
-		if (nodemap->nodes[j].vnn == vnn) {
-			continue;
-		}
-
-		ret = ctdb_ctrl_getrecmaster(ctdb, CONTROL_TIMEOUT(), nodemap->nodes[j].vnn, &recmaster);
-		if (ret != 0) {
-			DEBUG(0, (__location__ " Unable to get recmaster from node %u\n", vnn));
-			goto again;
-		}
-
-		if (recmaster!=vnn) {
-			DEBUG(0, ("Node %u does not agree we are the recmaster. Force reelection\n", 
-				  nodemap->nodes[j].vnn));
-			force_election(rec, mem_ctx, vnn, nodemap);
-			goto again;
-		}
+	switch (verify_recmaster(ctdb, nodemap, vnn)) {
+	case MONITOR_RECOVERY_NEEDED:
+		/* can not happen */
+		goto again;
+	case MONITOR_ELECTION_NEEDED:
+		force_election(rec, mem_ctx, vnn, nodemap);
+		goto again;
+	case MONITOR_OK:
+		break;
+	case MONITOR_FAILED:
+		goto again;
 	}
 
 
 	/* verify that all active nodes are in normal mode 
 	   and not in recovery mode 
 	 */
-	/* send a getrecmode call out to every node */
-	switch (verify_recmode(ctdb, nodemap, mem_ctx)) {
+	switch (verify_recmode(ctdb, nodemap)) {
 	case MONITOR_RECOVERY_NEEDED:
 		do_recovery(rec, mem_ctx, vnn, num_active, nodemap, vnnmap, nodemap->nodes[j].vnn);
 		goto again;
 	case MONITOR_FAILED:
 		goto again;
+	case MONITOR_ELECTION_NEEDED:
+		/* can not happen */
 	case MONITOR_OK:
 		break;
 	}
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index 9cfe966db92..7a6ff94233a 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -299,7 +299,7 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
 	}
 	printf("Recovery mode:%s (%d)\n",recmode==CTDB_RECOVERY_NORMAL?"NORMAL":"RECOVERY",recmode);
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.vnn, &recmaster);
 	if (ret != 0) {
 		DEBUG(0, ("Unable to get recmaster from node %u\n", options.vnn));
 		return ret;
@@ -544,7 +544,7 @@ static int control_ban(struct ctdb_context *ctdb, int argc, const char **argv)
 
 	ban_time = strtoul(argv[0], NULL, 0);
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.vnn, &recmaster);
 	if (ret != 0) {
 		DEBUG(0,("Failed to find the recmaster\n"));
 		return -1;
@@ -575,7 +575,7 @@ static int control_unban(struct ctdb_context *ctdb, int argc, const char **argv)
 	uint32_t recmaster;
 	TDB_DATA data;
 
-	ret = ctdb_ctrl_getrecmaster(ctdb, TIMELIMIT(), options.vnn, &recmaster);
+	ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.vnn, &recmaster);
 	if (ret != 0) {
 		DEBUG(0,("Failed to find the recmaster\n"));
 		return -1;