From: Ronnie Sahlberg <sahlberg@ronnie>
Date: Wed, 11 Jul 2007 07:43:51 +0000 (+1000)
Subject: first cut at a better and more scalable socketkiller
X-Git-Tag: tevent-0.9.20~348^2~2452^2~8
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=aa080f66d93c40094beb06906e5f9cc69fc2c129;p=thirdparty%2Fsamba.git

first cut at a better and more scalable socketkiller
that can kill multiple connections asynchronously using one listening
socket

(This used to be ctdb commit 22bb44f3d745aa354becd75d30774992f6c40b3a)
---

diff --git a/ctdb/common/system.c b/ctdb/common/system.c
index 1e536f5e8a4..c05898e267e 100644
--- a/ctdb/common/system.c
+++ b/ctdb/common/system.c
@@ -383,3 +383,99 @@ int ctdb_sys_kill_tcp(struct event_context *ev,
 
 	return -1;
 }
+
+
+
+/* This function is used to open a raw socket to capture from
+ */
+int ctdb_sys_open_capture_socket(void)
+{
+	int s;
+
+	/* Open a socket to capture all traffic */
+	s=socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
+	if (s == -1){
+		DEBUG(0,(__location__ " failed to open raw socket\n"));
+		return -1;
+	}
+
+	return s;
+}
+
+int sys_read_tcp_packet(struct ctdb_kill_tcp *killtcp)
+{
+	int ret;
+#define RCVPKTSIZE 100
+	char pkt[RCVPKTSIZE];
+	struct ether_header *eth;
+	struct iphdr *ip;
+	struct tcphdr *tcp;
+	struct ctdb_killtcp_connection *conn;
+
+	ret = recv(killtcp->fd, pkt, RCVPKTSIZE, MSG_TRUNC);
+	if (ret < sizeof(*eth)+sizeof(*ip)) {
+		return -1;
+	}
+
+	/* Ethernet */
+	eth = (struct ether_header *)pkt;
+	/* We only want IP packets */
+	if (ntohs(eth->ether_type) != ETHERTYPE_IP) {
+		return -1;
+	}
+	
+	/* IP */
+	ip = (struct iphdr *)(eth+1);
+
+	/* We only want IPv4 packets */
+	if (ip->version != 4) {
+		return -1;
+	}
+	/* Dont look at fragments */
+	if ((ntohs(ip->frag_off)&0x1fff) != 0) {
+		return -1;
+	}
+	/* we only want TCP */
+	if (ip->protocol != IPPROTO_TCP) {
+		return -1;
+	}
+	/* make sure its not a short packet */
+	if (offsetof(struct tcphdr, ack_seq) + 4 + 
+	    (ip->ihl*4) + sizeof(*eth) > ret) {
+		return -1;
+	}
+	/* TCP */
+	tcp = (struct tcphdr *)((ip->ihl*4) + (char *)ip);
+		
+
+	/* loop over all connections and see if we find one that matches */
+	for(conn = killtcp->connections; conn; conn = conn->next) {
+		/* We only want packets sent from a guy we have tickled */
+		if (ip->saddr != conn->dst.sin_addr.s_addr) {
+			continue;
+		}
+		/* We only want packets sent to us */
+		if (ip->daddr != conn->src.sin_addr.s_addr) {
+			continue;
+		}
+		/* We only want replies from a port we tickled */
+		if (tcp->source != conn->dst.sin_port) {
+			continue;
+		}
+		if (tcp->dest != conn->src.sin_port) {
+			continue;
+		}
+
+		/* This one has been tickled !
+		   now reset him and remove him from the list.
+		 */
+		ctdb_sys_send_tcp(&conn->dst, &conn->src, tcp->ack_seq, tcp->seq, 1);
+		talloc_free(conn);
+
+		return 0;
+	}
+
+	return -1;
+}
+
+
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index e41035f5cbc..13e3323b66e 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -307,6 +307,7 @@ struct ctdb_context {
 	struct ctdb_takeover takeover;
 	struct ctdb_tcp_list *tcp_list;
 	struct ctdb_client_ip *client_ip_list;
+	struct ctdb_kill_tcp *killtcp;
 };
 
 struct ctdb_db_context {
@@ -1032,11 +1033,29 @@ void ctdb_start_freeze(struct ctdb_context *ctdb);
 
 bool parse_ip_port(const char *s, struct sockaddr_in *ip);
 
-int ctdb_kill_tcp_callback(struct ctdb_context *ctdb, 
-			       struct timeval timeout,
-			       TALLOC_CTX *mem_ctx,
-			       void (*callback)(struct ctdb_context *, int, struct sockaddr_in *, struct sockaddr_in *),
-			       struct sockaddr_in *dst, 
-			       struct sockaddr_in *src);
+
+/*
+  list of tcp connections to kill
+ */
+struct ctdb_killtcp_connection {
+	struct ctdb_killtcp_connection *prev, *next;
+	struct ctdb_context *ctdb;
+	struct sockaddr_in src;
+	struct sockaddr_in dst;
+	int count;
+};
+
+/* structure containing the listening socket and the list of tcp connections
+   that the ctdb daemon is to kill
+*/
+struct ctdb_kill_tcp {
+	struct ctdb_context *ctdb;
+	int fd;
+	struct fd_event *fde;
+	struct ctdb_killtcp_connection *connections;
+};
+int ctdb_sys_open_capture_socket(void);
+int killtcp_add_connection(struct ctdb_context *ctdb, struct sockaddr_in *src, struct sockaddr_in *dst);
+int sys_read_tcp_packet(struct ctdb_kill_tcp *killtcp);
 
 #endif
diff --git a/ctdb/server/ctdb_takeover.c b/ctdb/server/ctdb_takeover.c
index 9b602ee04fc..76460facbdd 100644
--- a/ctdb/server/ctdb_takeover.c
+++ b/ctdb/server/ctdb_takeover.c
@@ -823,126 +823,133 @@ int32_t ctdb_control_get_public_ips(struct ctdb_context *ctdb, struct ctdb_req_c
 
 
 
+static void capture_tcp_handler(struct event_context *ev, struct fd_event *fde, 
+			     uint16_t flags, void *private_data)
+{
+	struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
 
-struct ctdb_kill_tcp_state {
-	struct ctdb_context *ctdb;
-	pid_t child;
-	void (*callback)(struct ctdb_context *, int, struct sockaddr_in *, struct sockaddr_in *);
-	int fd[2];
-	/* in case we need them in the callback */
-        struct sockaddr_in dst;
-        struct sockaddr_in src;
-};
+	if (flags & EVENT_FD_READ) {
+		sys_read_tcp_packet(killtcp);
+	}
+}
 
-/*
-  destroy a running kill tcp
+
+/* called every second until all sentenced connections have been reset
  */
-static int ctdb_kill_tcp_destructor(struct ctdb_kill_tcp_state *state)
+static void ctdb_tickle_sentenced_connections(struct event_context *ev, struct timed_event *te, 
+				      struct timeval t, void *private_data)
 {
-	DEBUG(0,("kill tcp destructor\n"));
-	kill(state->child, SIGKILL);
-	waitpid(state->child, NULL, 0);
-	return 0;
-}
+	struct ctdb_kill_tcp *killtcp = talloc_get_type(private_data, struct ctdb_kill_tcp);
+	struct ctdb_killtcp_connection *conn, tmpcon;
 
-/* called when the kill tcp child is finished */
-static void ctdb_kill_tcp_handler(struct event_context *ev, struct fd_event *fde, 
-				      uint16_t flags, void *p)
-{
-	struct ctdb_kill_tcp_state *state = 
-		talloc_get_type(p, struct ctdb_kill_tcp_state);
-	int status = -1;
-	void (*callback)(struct ctdb_context *, int, struct sockaddr_in *, struct sockaddr_in *) = state->callback;
-	struct ctdb_context *ctdb = state->ctdb;
 
-	waitpid(state->child, &status, 0);
-	if (status != -1) {
-		status = WEXITSTATUS(status);
+	/* loop over all connections and see if we find one that matches */
+	for(conn = killtcp->connections; conn; conn = conn->next) {
+		conn->count++;
+		if (conn->count > 5) {
+			tmpcon.next=conn->next;
+			talloc_free(conn);
+			conn=&tmpcon;
+
+			continue;
+		}
+		ctdb_sys_send_tcp(&conn->dst, &conn->src, 0, 0, 0);
 	}
 
-	DEBUG(0,("kill tcp handler  status %d\n",status));
-	talloc_set_destructor(state, NULL);
-	talloc_free(state);
-	if (callback) {
-		callback(ctdb, status, &state->dst, &state->src);
+	/* If there are no more connections to kill we can remove the
+	   entire killtcp structure
+	 */
+	if (killtcp->connections == NULL) {
+		talloc_free(killtcp);
+		return;
 	}
+
+	/* try tickling them again in a seconds time
+	 */
+	event_add_timed(killtcp->ctdb->ev, killtcp, timeval_current_ofs(1, 0), 
+			ctdb_tickle_sentenced_connections, killtcp);
 }
 
-/* called when kill tcp times out */
-static void ctdb_kill_tcp_timeout(struct event_context *ev, struct timed_event *te, 
-				      struct timeval t, void *p)
+/*
+  destroy the killtcp structure
+ */
+static int ctdb_killtcp_destructor(struct ctdb_kill_tcp *killtcp)
 {
-	struct ctdb_kill_tcp_state *state = talloc_get_type(p, struct ctdb_kill_tcp_state);
-	void (*callback)(struct ctdb_context *, int, struct sockaddr_in *, struct sockaddr_in *) = state->callback;
-	struct ctdb_context *ctdb = state->ctdb;
+	close(killtcp->fd);
+	killtcp->fd = -1;
+	killtcp->ctdb->killtcp = NULL;
 
-	DEBUG(0,("kill tcp timed out\n"));
-	talloc_free(state);
-	if (callback) {
-		callback(ctdb, -1, &state->dst, &state->src);
-	}
+	return 0;
 }
 
 /*
-  run killtcp in the background calling the callback once it has
-  finished
+  destroy a killtcp connection structure
  */
-int ctdb_kill_tcp_callback(struct ctdb_context *ctdb, 
-			       struct timeval timeout,
-			       TALLOC_CTX *mem_ctx,
-			       void (*callback)(struct ctdb_context *, int, struct sockaddr_in *, struct sockaddr_in *),
-			       struct sockaddr_in *dst, 
-			       struct sockaddr_in *src)
+static int ctdb_killtcp_connection_destructor(struct ctdb_killtcp_connection *conn)
 {
-	struct ctdb_kill_tcp_state *state;
-	int ret;
-
-	state = talloc(mem_ctx, struct ctdb_kill_tcp_state);
-	CTDB_NO_MEMORY(ctdb, state);
-
-	DEBUG(0,("kill tcp callback\n"));
+	DLIST_REMOVE(conn->ctdb->killtcp->connections, conn);
+	return 0;
+}
 
-	state->ctdb = ctdb;
-	state->callback = callback;
-	state->src = *src;
-	state->dst = *dst;
+int killtcp_add_connection(struct ctdb_context *ctdb, struct sockaddr_in *src, struct sockaddr_in *dst)
+{
+	struct ctdb_kill_tcp *killtcp=ctdb->killtcp;;
+	struct ctdb_killtcp_connection *conn;
 	
-	ret = pipe(state->fd);
-	if (ret != 0) {
-		talloc_free(state);
-		return -1;
+	/* If this is the first connection to kill we must allocate
+	   a new structure
+	 */
+	if (killtcp == NULL) {
+		killtcp = talloc(ctdb, struct ctdb_kill_tcp);
+		CTDB_NO_MEMORY(ctdb, killtcp);
+
+		killtcp->ctdb        = ctdb;
+		killtcp->fd          = -1;
+		killtcp->connections = NULL;
+		ctdb->killtcp        = killtcp;
+		talloc_set_destructor(killtcp, ctdb_killtcp_destructor);
 	}
 
-	state->child = fork();
+	/* If we dont have a socket to listen on yet we must create it
+	 */
+	if (killtcp->fd == -1) {
+		killtcp->fd = ctdb_sys_open_capture_socket();
+		if (killtcp->fd == -1) {
+			DEBUG(0,(__location__ " Failed to open listening socket for killtcp\n"));
+			goto failed;
+		}
 
-	if (state->child == (pid_t)-1) {
-		close(state->fd[0]);
-		close(state->fd[1]);
-		talloc_free(state);
-		return -1;
+		set_nonblocking(killtcp->fd);
+		set_close_on_exec(killtcp->fd);
 	}
 
-	if (state->child == 0) {
-		close(state->fd[0]);
-		ctdb_set_realtime(true);
-		set_close_on_exec(state->fd[1]);
-		ret = ctdb_sys_kill_tcp(ctdb->ev, dst, src);
-		_exit(ret);
-	}
 
-	talloc_set_destructor(state, ctdb_kill_tcp_destructor);
+	conn = talloc(killtcp, struct ctdb_killtcp_connection);
+	CTDB_NO_MEMORY(ctdb, conn);
+	conn->src   = *src;
+	conn->dst   = *dst;
+	conn->ctdb  = ctdb;
+	conn->count = 0;
+	talloc_set_destructor(conn, ctdb_killtcp_connection_destructor);
+	DLIST_ADD(killtcp->connections, conn);
 
-	close(state->fd[1]);
 
+	killtcp->fde = event_add_fd(ctdb->ev, killtcp, killtcp->fd, 
+				EVENT_FD_READ|EVENT_FD_AUTOCLOSE, 
+				capture_tcp_handler, killtcp);
 
-	event_add_fd(ctdb->ev, state, state->fd[0], EVENT_FD_READ|EVENT_FD_AUTOCLOSE,
-		     ctdb_kill_tcp_handler, state);
 
-	if (!timeval_is_zero(&timeout)) {
-		event_add_timed(ctdb->ev, state, timeout, ctdb_kill_tcp_timeout, state);
-	}
+	/* We also need to set up some events to tickle all these connections
+	   until they are all reset
+	 */
+	event_add_timed(ctdb->ev, killtcp, timeval_current_ofs(0, 0), 
+			ctdb_tickle_sentenced_connections, killtcp);
 
-	return 0;
-}
 
+	return 0;
 
+failed:
+	talloc_free(ctdb->killtcp);
+	ctdb->killtcp = NULL;
+	return -1;
+}