director: Avoid infinite SYNC loops if the originating director goes away for a long...

diff --git a/src/director/director-connection.c b/src/director/director-connection.c
index 9e4dde92f8bd79a1ce4f782ae46bb4120f9ebbda..0f396d00c33ccfdf6a8e24f785f1cea3d34462fc 100644 (file)
--- a/src/director/director-connection.c
+++ b/src/director/director-connection.c
@@ -73,6 +73,7 @@
 #define DIRECTOR_WAIT_DISCONNECT_SECS 10
 #define DIRECTOR_HANDSHAKE_WARN_SECS 29
 #define DIRECTOR_HANDSHAKE_BYTES_LOG_MIN_SECS (60*30)
+#define DIRECTOR_MAX_SYNC_SEQ_DUPLICATES 4
 
 #if DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS <= DIRECTOR_CONNECTION_PING_TIMEOUT_MSECS
 #  error DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS is too low
@@ -216,7 +217,7 @@ static void director_connection_assigned(struct director_connection *conn)
                dir->sync_seq++;
                director_set_ring_unsynced(dir);
                director_sync_send(dir, dir->self_host, dir->sync_seq,
-                                  DIRECTOR_VERSION_MINOR);
+                                  DIRECTOR_VERSION_MINOR, ioloop_time);
        }
        director_connection_set_ping_timeout(conn);
 }
@@ -1084,10 +1085,11 @@ director_connection_handle_handshake(struct director_connection *conn,
        return 0;
 }
 
-static void
+static bool
 director_connection_sync_host(struct director_connection *conn,
                              struct director_host *host,
-                             uint32_t seq, unsigned int minor_version)
+                             uint32_t seq, unsigned int minor_version,
+                             unsigned int timestamp)
 {
        struct director *dir = conn->dir;
 
@@ -1099,7 +1101,7 @@ director_connection_sync_host(struct director_connection *conn,
        if (host->self) {
                if (dir->sync_seq != seq) {
                        /* stale SYNC event */
-                       return;
+                       return FALSE;
                }
                /* sync_seq increases when we get disconnected, so we must be
                   successfully connected to both directions */
@@ -1117,10 +1119,28 @@ director_connection_sync_host(struct director_connection *conn,
                                  conn->name, seq);
                        director_set_ring_synced(dir);
                }
-       } else if (dir->right != NULL) {
-               /* forward it to the connection on right */
-               director_sync_send(dir, host, seq, minor_version);
+       } else {
+               if (seq < host->last_sync_seq) {
+                       /* stale SYNC event */
+                       return FALSE;
+               } else if (host->last_sync_seq != seq ||
+                          timestamp < host->last_sync_timestamp) {
+                       host->last_sync_seq = seq;
+                       host->last_sync_timestamp = timestamp;
+                       host->last_sync_seq_counter = 1;
+               } else if (++host->last_sync_seq_counter >
+                          DIRECTOR_MAX_SYNC_SEQ_DUPLICATES) {
+                       /* we've received this too many times already */
+                       return FALSE;
+               }
+
+               if (dir->right != NULL) {
+                       /* forward it to the connection on right */
+                       director_sync_send(dir, host, seq, minor_version,
+                                          timestamp);
+               }
        }
+       return TRUE;
 }
 
 static bool director_connection_sync(struct director_connection *conn,
@@ -1129,7 +1149,7 @@ static bool director_connection_sync(struct director_connection *conn,
        struct director *dir = conn->dir;
        struct director_host *host;
        struct ip_addr ip;
-       unsigned int port, seq, minor_version = 0;
+       unsigned int port, seq, minor_version = 0, timestamp = ioloop_time;
 
        if (str_array_length(args) < 3 ||
            !director_args_parse_ip_port(conn, args, &ip, &port) ||
@@ -1137,18 +1157,25 @@ static bool director_connection_sync(struct director_connection *conn,
                director_cmd_error(conn, "Invalid parameters");
                return FALSE;
        }
-       if (args[3] != NULL)
+       if (args[3] != NULL) {
                minor_version = atoi(args[3]);
+               if (args[4] != NULL && str_to_uint(args[4], &timestamp) < 0) {
+                       director_cmd_error(conn, "Invalid parameters");
+                       return FALSE;
+               }
+       }
 
        /* find the originating director. if we don't see it, it was already
           removed and we can ignore this sync. */
        host = director_host_lookup(dir, &ip, port);
        if (host != NULL) {
-               director_connection_sync_host(conn, host, seq,
-                                             minor_version);
+               if (!director_connection_sync_host(conn, host, seq,
+                                                  minor_version, timestamp))
+                       return TRUE;
        }
 
-       if (host == NULL || !host->self)
+       if ((host == NULL || !host->self) &&
+           dir->self_host->last_sync_timestamp != ioloop_time)
                (void)director_resend_sync(dir);
        return TRUE;
 }

diff --git a/src/director/director-host.h b/src/director/director-host.h
index b402375fc04f7e61157dac09ab6addb927c5ed99..e1af171d2952f5a9426acce48eb8c544eb6a780c 100644 (file)
--- a/src/director/director-host.h
+++ b/src/director/director-host.h
@@ -20,6 +20,9 @@ struct director_host {
           it can be ignored (or: it must be ignored to avoid potential command
           loops) */
        unsigned int last_seq;
+       /* use these to avoid infinitely sending SYNCs for directors that
+          aren't connected in the ring. */
+       unsigned int last_sync_seq, last_sync_seq_counter, last_sync_timestamp;
        /* Last time host was detected to be down */
        time_t last_network_failure;
        time_t last_protocol_failure;

diff --git a/src/director/director.c b/src/director/director.c
index 1f1d6d410d20c54c2e0afdc25f278d9afc01367c..520abc9a8c66369310593c2d9e49a3cdadecf35e 100644 (file)
--- a/src/director/director.c
+++ b/src/director/director.c
@@ -302,7 +302,8 @@ void director_set_ring_synced(struct director *dir)
 }
 
 void director_sync_send(struct director *dir, struct director_host *host,
-                       uint32_t seq, unsigned int minor_version)
+                       uint32_t seq, unsigned int minor_version,
+                       unsigned int timestamp)
 {
        string_t *str;
 
@@ -311,8 +312,8 @@ void director_sync_send(struct director *dir, struct director_host *host,
                    net_ip2addr(&host->ip), host->port, seq);
        if (minor_version > 0 &&
            director_connection_get_minor_version(dir->right) > 0) {
-               /* only minor_version>0 supports this parameter */
-               str_printfa(str, "\t%u", minor_version);
+               /* only minor_version>0 supports extra parameters */
+               str_printfa(str, "\t%u\t%u", minor_version, timestamp);
        }
        str_append_c(str, '\n');
        director_connection_send(dir->right, str_c(str));
@@ -329,8 +330,9 @@ bool director_resend_sync(struct director *dir)
 {
        if (!dir->ring_synced && dir->left != NULL && dir->right != NULL) {
                /* send a new SYNC in case the previous one got dropped */
+               dir->self_host->last_sync_timestamp = ioloop_time;
                director_sync_send(dir, dir->self_host, dir->sync_seq,
-                                  DIRECTOR_VERSION_MINOR);
+                                  DIRECTOR_VERSION_MINOR, ioloop_time);
                if (dir->to_sync != NULL)
                        timeout_reset(dir->to_sync);
                return TRUE;
@@ -393,7 +395,7 @@ static void director_sync(struct director *dir)
                director_connection_set_synced(dir->left, FALSE);
        director_connection_set_synced(dir->right, FALSE);
        director_sync_send(dir, dir->self_host, dir->sync_seq,
-                          DIRECTOR_VERSION_MINOR);
+                          DIRECTOR_VERSION_MINOR, ioloop_time);
 }
 
 void director_sync_freeze(struct director *dir)

diff --git a/src/director/director.h b/src/director/director.h
index 310b5a7dc4f70b891486f948ddd9376fbbdad307..140ed29269eb16be84ee5dcf6af883d52ce540ff 100644 (file)
--- a/src/director/director.h
+++ b/src/director/director.h
@@ -103,7 +103,8 @@ void director_set_ring_synced(struct director *dir);
 void director_set_ring_unsynced(struct director *dir);
 void director_set_state_changed(struct director *dir);
 void director_sync_send(struct director *dir, struct director_host *host,
-                       uint32_t seq, unsigned int minor_version);
+                       uint32_t seq, unsigned int minor_version,
+                       unsigned int timestamp);
 bool director_resend_sync(struct director *dir);
 
 void director_notify_ring_added(struct director_host *added_host,