added code to ctdb to send a tcp 'tickle' ack when we takeover an

IP. A raw tcp ack is sent for each tcp connection held by clients
before the IP takeover.

These acks have a deliberately incorrect sequence number, and should
cause the windows client to send its own ack which will in turn cause
a tcp reset and thus cause windows clients to much more quickly
reconnect to the new node.

(This used to be ctdb commit eef38bfe8461b47489d169c61895d6bb8a8f79a1)

diff --git a/ctdb/common/ctdb_control.c b/ctdb/common/ctdb_control.c
index cb990d52c2bfeb3f7f292c20126a33d67b18088d..fd3dcf9988246cbad5b35f23f0e5b6b4762a3625 100644 (file)
--- a/ctdb/common/ctdb_control.c
+++ b/ctdb/common/ctdb_control.c
@@ -272,6 +272,18 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
                CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_delete_low_rsn));
                return ctdb_control_delete_low_rsn(ctdb, indata, outdata);
 
+       case CTDB_CONTROL_TCP_CLIENT: 
+               CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+               return ctdb_control_tcp_client(ctdb, client_id, indata);
+
+       case CTDB_CONTROL_TCP_ADD: 
+               CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+               return ctdb_control_tcp_add(ctdb, indata);
+
+       case CTDB_CONTROL_TCP_REMOVE: 
+               CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+               return ctdb_control_tcp_remove(ctdb, indata);
+
        default:
                DEBUG(0,(__location__ " Unknown CTDB control opcode %u\n", opcode));
                return -1;

diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c
index 40d5da6e6b21891e611e20e961074374adfc6847..4bf1d1a605d15a82171ad32b7784c0f6269590c9 100644 (file)
--- a/ctdb/common/ctdb_daemon.c
+++ b/ctdb/common/ctdb_daemon.c
@@ -29,17 +29,6 @@
 #include "../include/ctdb.h"
 #include "../include/ctdb_private.h"
 
-/*
-  structure describing a connected client in the daemon
- */
-struct ctdb_client {
-       struct ctdb_context *ctdb;
-       int fd;
-       struct ctdb_queue *queue;
-       uint32_t client_id;
-};
-
-
 static void daemon_incoming_packet(void *, struct ctdb_req_header *);
 
 static void ctdb_main_loop(struct ctdb_context *ctdb)
@@ -250,6 +239,7 @@ static void daemon_request_connect_wait(struct ctdb_client *client,
 */
 static int ctdb_client_destructor(struct ctdb_client *client)
 {
+       ctdb_takeover_client_destructor_hook(client);
        ctdb_reqid_remove(client->ctdb, client->client_id);
        client->ctdb->status.num_clients--;
        return 0;

diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 31751c24ae94c15098babf55805a81023212ec99..61b459108c88cd6152ce11fd4a496706f96a9bda 100644 (file)
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -80,6 +80,18 @@ typedef void (*ctdb_control_callback_fn_t)(struct ctdb_context *,
                                           const char *errormsg,
                                           void *private_data);
 
+/*
+  structure describing a connected client in the daemon
+ */
+struct ctdb_client {
+       struct ctdb_context *ctdb;
+       int fd;
+       struct ctdb_queue *queue;
+       uint32_t client_id;
+       struct ctdb_tcp_list *tcp_list;
+};
+
+
 /*
   state associated with one node
 */
@@ -291,6 +303,7 @@ struct ctdb_context {
        uint32_t recovery_master;
        struct ctdb_call_state *pending_calls;
        struct ctdb_takeover takeover;
+       struct ctdb_tcp_list *tcp_list;
 };
 
 struct ctdb_db_context {
@@ -398,6 +411,9 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS,
                    CTDB_CONTROL_DELETE_LOW_RSN,
                    CTDB_CONTROL_TAKEOVER_IP,
                    CTDB_CONTROL_RELEASE_IP,
+                   CTDB_CONTROL_TCP_CLIENT,
+                   CTDB_CONTROL_TCP_ADD,
+                   CTDB_CONTROL_TCP_REMOVE,
 };
 
 /*
@@ -425,6 +441,14 @@ struct ctdb_control_set_call {
        uint32_t id;
 };
 
+/*
+  struct for tcp_client, tcp_add and tcp_remove controls
+ */
+struct ctdb_control_tcp {
+       struct sockaddr_in src;
+       struct sockaddr_in dest;
+};
+
 enum call_state {CTDB_CALL_WAIT, CTDB_CALL_DONE, CTDB_CALL_ERROR};
 
 #define CTDB_LMASTER_ANY       0xffffffff
@@ -896,9 +920,18 @@ int ctdb_ctrl_release_ip(struct ctdb_context *ctdb, struct timeval timeout,
 int ctdb_sys_send_arp(const struct sockaddr_in *saddr, const char *iface);
 int ctdb_sys_take_ip(const char *ip, const char *interface);
 int ctdb_sys_release_ip(const char *ip, const char *interface);
+int ctdb_sys_send_ack(const struct sockaddr_in *dest, 
+                     const struct sockaddr_in *src);
 
 int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist);
 
 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap);
 
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+                               TDB_DATA indata);
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata);
+
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client);
+
 #endif

diff --git a/ctdb/takeover/ctdb_takeover.c b/ctdb/takeover/ctdb_takeover.c
index 63b5ed7dab63fe4c82b5beb521f952ffe055c148..220cd56ae7ff27f85e7200ac14c951538ea17d9b 100644 (file)
--- a/ctdb/takeover/ctdb_takeover.c
+++ b/ctdb/takeover/ctdb_takeover.c
@@ -29,15 +29,26 @@
 
 #define TAKEOVER_TIMEOUT() timeval_current_ofs(5,0)
 
-#define CTDB_ARP_INTERVAL 5
-#define CTDB_ARP_REPEAT  24
+#define CTDB_ARP_INTERVAL 1
+#define CTDB_ARP_REPEAT   3
 
 struct ctdb_takeover_arp {
        struct ctdb_context *ctdb;
        uint32_t count;
        struct sockaddr_in sin;
+       struct ctdb_tcp_list *tcp_list;
 };
 
+/*
+  lists of tcp endpoints
+ */
+struct ctdb_tcp_list {
+       struct ctdb_tcp_list *prev, *next;
+       struct sockaddr_in saddr;
+       struct sockaddr_in daddr;       
+};
+
+
 
 /*
   send a gratuitous arp
@@ -48,12 +59,25 @@ static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *
        struct ctdb_takeover_arp *arp = talloc_get_type(private_data, 
                                                        struct ctdb_takeover_arp);
        int ret;
+       struct ctdb_tcp_list *tcp;
 
        ret = ctdb_sys_send_arp(&arp->sin, arp->ctdb->takeover.interface);
        if (ret != 0) {
                DEBUG(0,(__location__ "sending of arp failed (%s)\n", strerror(errno)));
        }
 
+       for (tcp=arp->tcp_list;tcp;tcp=tcp->next) {
+               DEBUG(0,("sending tcp tickle ack for %u->%s:%u\n",
+                        (unsigned)tcp->daddr.sin_port, 
+                        inet_ntoa(tcp->saddr.sin_addr),
+                        (unsigned)tcp->saddr.sin_port));                        
+               ret = ctdb_sys_send_ack(&tcp->daddr, &tcp->saddr);
+               if (ret != 0) {
+                       DEBUG(0,(__location__ " Failed to send tcp tickle ack for %s\n",
+                                inet_ntoa(tcp->saddr.sin_addr)));
+               }
+       }
+
        arp->count++;
 
        if (arp->count == CTDB_ARP_REPEAT) {
@@ -66,6 +90,7 @@ static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *
                        ctdb_control_send_arp, arp);
 }
 
+
 /*
   take over an ip address
  */
@@ -75,6 +100,7 @@ int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, TDB_DATA indata)
        struct sockaddr_in *sin = (struct sockaddr_in *)indata.dptr;
        struct ctdb_takeover_arp *arp;
        char *ip = inet_ntoa(sin->sin_addr);
+       struct ctdb_tcp_list *tcp;
 
        DEBUG(0,("Takover of IP %s on interface %s\n", ip, ctdb->takeover.interface));
        ret = ctdb_sys_take_ip(ip, ctdb->takeover.interface);
@@ -95,6 +121,17 @@ int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, TDB_DATA indata)
        arp->ctdb = ctdb;
        arp->sin = *sin;
 
+       /* add all of the known tcp connections for this IP to the
+          list of tcp connections to send tickle acks for */
+       for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+               if (sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+                       struct ctdb_tcp_list *t2 = talloc(arp, struct ctdb_tcp_list);
+                       CTDB_NO_MEMORY(ctdb, t2);
+                       *t2 = *tcp;
+                       DLIST_ADD(arp->tcp_list, t2);
+               }
+       }
+
        event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
                        timeval_zero(), ctdb_control_send_arp, arp);
 
@@ -238,3 +275,129 @@ int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
 
        return 0;
 }
+
+
+/*
+  called by a client to inform us of a TCP connection that it is managing
+  that should tickled with an ACK when IP takeover is done
+ */
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+                               TDB_DATA indata)
+{
+       struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+       struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+       struct ctdb_tcp_list *tcp;
+       int ret;
+
+       tcp = talloc(client, struct ctdb_tcp_list);
+       CTDB_NO_MEMORY(ctdb, tcp);
+
+       tcp->saddr = p->src;
+       tcp->daddr = p->dest;
+
+       DLIST_ADD(client->tcp_list, tcp);
+
+       /* tell all nodes about this tcp connection */
+       ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+                                      CTDB_CONTROL_TCP_ADD,
+                                      0, CTDB_CTRL_FLAG_NOREPLY, indata, NULL, NULL);
+       if (ret != 0) {
+               DEBUG(0,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
+               return -1;
+       }
+
+       return 0;
+}
+
+/*
+  see if two sockaddr_in are the same
+ */
+static bool same_sockaddr_in(struct sockaddr_in *in1, struct sockaddr_in *in2)
+{
+       return in1->sin_family == in2->sin_family &&
+               in1->sin_port == in2->sin_port &&
+               in1->sin_addr.s_addr == in2->sin_addr.s_addr;
+}
+
+/*
+  find a tcp address on a list
+ */
+static struct ctdb_tcp_list *ctdb_tcp_find(struct ctdb_tcp_list *list, 
+                                          struct ctdb_tcp_list *tcp)
+{
+       while (list) {
+               if (same_sockaddr_in(&list->saddr, &tcp->saddr) &&
+                   same_sockaddr_in(&list->daddr, &tcp->daddr)) {
+                       return list;
+               }
+               list = list->next;
+       }
+       return NULL;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+       struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+       struct ctdb_tcp_list *tcp;
+
+       tcp = talloc(ctdb, struct ctdb_tcp_list);
+       CTDB_NO_MEMORY(ctdb, tcp);
+
+       tcp->saddr = p->src;
+       tcp->daddr = p->dest;
+
+       if (NULL == ctdb_tcp_find(ctdb->tcp_list, tcp)) {
+               DLIST_ADD(ctdb->tcp_list, tcp);
+       }
+
+       return 0;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+       struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+       struct ctdb_tcp_list t, *tcp;
+
+       t.saddr = p->src;
+       t.daddr = p->dest;
+
+       tcp = ctdb_tcp_find(ctdb->tcp_list, &t);
+       if (tcp) {
+               DLIST_REMOVE(ctdb->tcp_list, tcp);
+       }
+
+       return 0;
+}
+
+
+/*
+  called when a client structure goes away - hook to remove
+  elements from the tcp_list in all daemons
+ */
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
+{
+       while (client->tcp_list) {
+               TDB_DATA data;
+               struct ctdb_control_tcp p;
+               struct ctdb_tcp_list *tcp = client->tcp_list;
+               DLIST_REMOVE(client->tcp_list, tcp);
+               p.src = tcp->saddr;
+               p.dest = tcp->daddr;
+               data.dptr = (uint8_t *)&p;
+               data.dsize = sizeof(p);
+               ctdb_daemon_send_control(client->ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+                                        CTDB_CONTROL_TCP_REMOVE,
+                                        0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+               talloc_free(tcp);
+       }
+}

diff --git a/ctdb/takeover/system.c b/ctdb/takeover/system.c
index bddd6a636e3fb8467fb7e5b5cd5b48edaa3d3d19..b16b96343b4d611724ef011f4c0251a11fe62d50 100644 (file)
--- a/ctdb/takeover/system.c
+++ b/ctdb/takeover/system.c
@@ -195,7 +195,7 @@ int ctdb_sys_send_ack(const struct sockaddr_in *dest,
        pkt.tcp.source   = src->sin_port;
        pkt.tcp.dest     = dest->sin_port;
        pkt.tcp.ack      = 1;
-       pkt.tcp.check    = ip_checksum((uint16_t *)&pkt.tcp, sizeof(pkt.tcp)/2);
+       pkt.tcp.check    = 0;
 
        ret = sendto(3, &pkt, sizeof(pkt), 0, dest, sizeof(*dest));
        if (ret != 0) {