director: Added director_consistent_hashing setting.

This should have been the default since the beginning. I didn't thik it was
worth the trouble originally because in theory it shouldn't matter much if
users jump between multiple backends. But this makes caching behavior worse,
especially for systems which are using local cache files.

diff --git a/src/director/director-connection.c b/src/director/director-connection.c
index aeab94ca2801a48765e07419eccf038df0c57f0b..d7534085d73ac7270804270467b2300315749284 100644 (file)
--- a/src/director/director-connection.c
+++ b/src/director/director-connection.c
@@ -90,6 +90,8 @@
 #define CMD_IS_USER_HANDHAKE(args) \
        (str_array_length(args) > 2)
 
+#define DIRECTOR_OPT_CONSISTENT_HASHING "consistent-hashing"
+
 struct director_connection {
        struct director *dir;
        char *name;
@@ -125,6 +127,7 @@ struct director_connection {
        unsigned int wrong_host:1;
        unsigned int verifying_left:1;
        unsigned int users_unsorted:1;
+       unsigned int done_pending:1;
 };
 
 static void director_connection_disconnected(struct director_connection **conn);
@@ -132,6 +135,7 @@ static void director_connection_reconnect(struct director_connection **conn,
                                          const char *reason);
 static void
 director_connection_log_disconnect(struct director_connection *conn, int err);
+static int director_connection_send_done(struct director_connection *conn);
 
 static void ATTR_FORMAT(2, 3)
 director_cmd_error(struct director_connection *conn, const char *fmt, ...)
@@ -1069,6 +1073,25 @@ static bool director_handshake_cmd_done(struct director_connection *conn)
        }
 }
 
+static int
+director_handshake_cmd_options(struct director_connection *conn,
+                              const char *const *args)
+{
+       bool consistent_hashing = FALSE;
+       unsigned int i;
+
+       for (i = 0; args[i] != NULL; i++) {
+               if (strcmp(args[i], DIRECTOR_OPT_CONSISTENT_HASHING) == 0)
+                       consistent_hashing = TRUE;
+       }
+       if (consistent_hashing != conn->dir->set->director_consistent_hashing) {
+               i_error("director(%s): director_consistent_hashing settings differ between directors",
+                       conn->name);
+               return -1;
+       }
+       return 1;
+}
+
 static int
 director_connection_handle_handshake(struct director_connection *conn,
                                     const char *cmd, const char *const *args)
@@ -1088,6 +1111,10 @@ director_connection_handle_handshake(struct director_connection *conn,
                }
                conn->minor_version = atoi(args[2]);
                conn->version_received = TRUE;
+               if (conn->done_pending) {
+                       if (director_connection_send_done(conn) < 0)
+                               return -1;
+               }
                return 1;
        }
        if (!conn->version_received) {
@@ -1114,6 +1141,8 @@ director_connection_handle_handshake(struct director_connection *conn,
                director_cmd_error(conn, "Unexpected command during host list");
                return -1;
        }
+       if (strcmp(cmd, "OPTIONS") == 0)
+               return director_handshake_cmd_options(conn, args);
        if (strcmp(cmd, "HOST-HAND-START") == 0) {
                if (!conn->in) {
                        director_cmd_error(conn,
@@ -1518,6 +1547,25 @@ director_connection_send_hosts(struct director_connection *conn, string_t *str)
        str_printfa(str, "HOST-HAND-END\t%u\n", conn->dir->ring_handshaked);
 }
 
+static int director_connection_send_done(struct director_connection *conn)
+{
+       i_assert(conn->version_received);
+
+       if (!conn->dir->set->director_consistent_hashing)
+               ;
+       else if (conn->minor_version >= DIRECTOR_VERSION_OPTIONS) {
+               director_connection_send(conn,
+                       "OPTIONS\t"DIRECTOR_OPT_CONSISTENT_HASHING"\n");
+       } else {
+               i_error("director(%s): Director version is too old for supporting director_consistent_hashing=yes",
+                       conn->name);
+               return -1;
+       }
+       director_connection_send(conn, "DONE\n");
+       conn->done_pending = FALSE;
+       return 0;
+}
+
 static int director_connection_send_users(struct director_connection *conn)
 {
        struct user *user;
@@ -1546,7 +1594,12 @@ static int director_connection_send_users(struct director_connection *conn)
                }
        }
        user_directory_iter_deinit(&conn->user_iter);
-       director_connection_send(conn, "DONE\n");
+       if (!conn->version_received)
+               conn->done_pending = TRUE;
+       else {
+               if (director_connection_send_done(conn) < 0)
+                       return -1;
+       }
 
        if (conn->users_unsorted && conn->handshake_received) {
                /* we received remote's list of users before sending ours */

diff --git a/src/director/director-settings.c b/src/director/director-settings.c
index 778d5fd0d61e1fa181c2f5515b2b82f3d5627d16..08a76c0408d18077a5f239b20df84cd733400b96 100644 (file)
--- a/src/director/director-settings.c
+++ b/src/director/director-settings.c
@@ -73,6 +73,7 @@ static const struct setting_define director_setting_defines[] = {
        DEF(SET_STR, director_username_hash),
        DEF(SET_TIME, director_user_expire),
        DEF(SET_UINT, director_doveadm_port),
+       DEF(SET_BOOL, director_consistent_hashing),
 
        SETTING_DEFINE_LIST_END
 };

diff --git a/src/director/director-settings.h b/src/director/director-settings.h
index 3cec8c790ccf93fce78734df91980b1fd965bdf2..58583d0b4474dc9f348e8e3d4661ab1a3a95add3 100644 (file)
--- a/src/director/director-settings.h
+++ b/src/director/director-settings.h
@@ -9,6 +9,7 @@ struct director_settings {
        const char *director_username_hash;
        unsigned int director_user_expire;
        unsigned int director_doveadm_port;
+       bool director_consistent_hashing;
 };
 
 extern const struct setting_parser_info director_setting_parser_info;

diff --git a/src/director/director.c b/src/director/director.c
index 901b504f0f21d3144f0aad6083f4b05091d7f784..eeb421a5058ac169a63b0948a144bb7c9fc55ea5 100644 (file)
--- a/src/director/director.c
+++ b/src/director/director.c
@@ -909,7 +909,7 @@ director_init(const struct director_settings *set,
        i_array_init(&dir->connections, 8);
        dir->users = user_directory_init(set->director_user_expire,
                                         set->director_username_hash);
-       dir->mail_hosts = mail_hosts_init();
+       dir->mail_hosts = mail_hosts_init(set->director_consistent_hashing);
 
        dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH);
        dir->ring_min_version = DIRECTOR_VERSION_MINOR;

diff --git a/src/director/director.h b/src/director/director.h
index c241afa1232e4c46796229cde78468e32118368b..dff62001d5c2d6bf953e4a9c0c70f586c4ca6a10 100644 (file)
--- a/src/director/director.h
+++ b/src/director/director.h
@@ -6,7 +6,7 @@
 
 #define DIRECTOR_VERSION_NAME "director"
 #define DIRECTOR_VERSION_MAJOR 1
-#define DIRECTOR_VERSION_MINOR 4
+#define DIRECTOR_VERSION_MINOR 5
 
 /* weak users supported in protocol */
 #define DIRECTOR_VERSION_WEAK_USERS 1
@@ -16,6 +16,8 @@
 #define DIRECTOR_VERSION_QUIT 3
 /* user-kick supported */
 #define DIRECTOR_VERSION_USER_KICK 4
+/* options supported in handshake */
+#define DIRECTOR_VERSION_OPTIONS 5
 
 /* Minimum time between even attempting to communicate with a director that
    failed due to a protocol error. */

diff --git a/src/director/doveadm-connection.c b/src/director/doveadm-connection.c
index 080ac305578a21f04eb6d06681b2183b247b7b5d..4c7fb82b5adb59c9d240cd4d45ad50ab910183d3 100644 (file)
--- a/src/director/doveadm-connection.c
+++ b/src/director/doveadm-connection.c
@@ -62,7 +62,7 @@ static void doveadm_cmd_host_list_removed(struct doveadm_connection *conn)
        string_t *str = t_str_new(1024);
        int ret;
 
-       orig_hosts_list = mail_hosts_init();
+       orig_hosts_list = mail_hosts_init(conn->dir->set->director_consistent_hashing);
        (void)mail_hosts_parse_and_add(orig_hosts_list,
                                       conn->dir->set->director_mail_servers);
 

diff --git a/src/director/mail-host.c b/src/director/mail-host.c
index f07edacb8b8f8eda9c6dc2d6b7cb5a9083991e49..1e31ee1d2d32c56955b49c679311f4c5b441eef7 100644 (file)
--- a/src/director/mail-host.c
+++ b/src/director/mail-host.c
@@ -2,14 +2,22 @@
 
 #include "lib.h"
 #include "array.h"
+#include "bsearch-insert-pos.h"
+#include "md5.h"
 #include "mail-host.h"
 
 #define VHOST_MULTIPLIER 100
 
+struct mail_vhost {
+       unsigned int hash;
+       struct mail_host *host;
+};
+
 struct mail_host_list {
        ARRAY_TYPE(mail_host) hosts;
-       ARRAY(struct mail_host *) vhosts;
+       ARRAY(struct mail_vhost) vhosts;
        bool hosts_unsorted;
+       bool consistent_hashing;
 };
 
 static int
@@ -18,8 +26,71 @@ mail_host_cmp(struct mail_host *const *h1, struct mail_host *const *h2)
        return net_ip_cmp(&(*h1)->ip, &(*h2)->ip);
 }
 
-static void mail_hosts_sort(struct mail_host_list *list)
+static int
+mail_vhost_cmp(const struct mail_vhost *h1, const struct mail_vhost *h2)
+{
+       if (h1->hash < h2->hash)
+               return -1;
+       else if (h1->hash > h2->hash)
+               return 1;
+       /* hash collision. not ideal, but we'll need to keep the order
+          consistent across directors so compare the IPs next. */
+       return net_ip_cmp(&h1->host->ip, &h2->host->ip);
+}
+
+static int
+mail_vhost_hash_cmp(const unsigned int *hash, const struct mail_vhost *vhost)
+{
+       if (vhost->hash < *hash)
+               return 1;
+       else if (vhost->hash > *hash)
+               return -1;
+       else
+               return 0;
+}
+
+static void mail_vhost_add(struct mail_host_list *list, struct mail_host *host)
+{
+       struct mail_vhost *vhost;
+       struct md5_context md5_ctx, md5_ctx2;
+       unsigned char md5[MD5_RESULTLEN];
+       const char *ip_str;
+       char num_str[MAX_INT_STRLEN];
+       unsigned int i, j;
+
+       ip_str = net_ip2addr(&host->ip);
+
+       md5_init(&md5_ctx);
+       md5_update(&md5_ctx, ip_str, strlen(ip_str));
+
+       for (i = 0; i < host->vhost_count; i++) {
+               md5_ctx2 = md5_ctx;
+               i_snprintf(num_str, sizeof(num_str), "-%u", i);
+               md5_update(&md5_ctx2, num_str, strlen(num_str));
+               md5_final(&md5_ctx2, md5);
+
+               vhost = array_append_space(&list->vhosts);
+               vhost->host = host;
+               for (j = 0; j < sizeof(vhost->hash); j++)
+                       vhost->hash = (vhost->hash << CHAR_BIT) | md5[j];
+       }
+}
+
+static void mail_hosts_sort_ring(struct mail_host_list *list)
+{
+       struct mail_host *const *hostp;
+
+       /* rebuild vhosts */
+       array_clear(&list->vhosts);
+       array_foreach(&list->hosts, hostp)
+               mail_vhost_add(list, *hostp);
+       array_sort(&list->vhosts, mail_vhost_cmp);
+       list->hosts_unsorted = FALSE;
+}
+
+static void mail_hosts_sort_direct(struct mail_host_list *list)
 {
+       struct mail_vhost *vhost;
        struct mail_host *const *hostp;
        unsigned int i;
 
@@ -28,12 +99,22 @@ static void mail_hosts_sort(struct mail_host_list *list)
        /* rebuild vhosts */
        array_clear(&list->vhosts);
        array_foreach(&list->hosts, hostp) {
-               for (i = 0; i < (*hostp)->vhost_count; i++)
-                       array_append(&list->vhosts, hostp, 1);
+               for (i = 0; i < (*hostp)->vhost_count; i++) {
+                       vhost = array_append_space(&list->vhosts);
+                       vhost->host = *hostp;
+               }
        }
        list->hosts_unsorted = FALSE;
 }
 
+static void mail_hosts_sort(struct mail_host_list *list)
+{
+       if (list->consistent_hashing)
+               mail_hosts_sort_ring(list);
+       else
+               mail_hosts_sort_direct(list);
+}
+
 struct mail_host *
 mail_host_add_ip(struct mail_host_list *list, const struct ip_addr *ip)
 {
@@ -205,20 +286,47 @@ mail_host_lookup(struct mail_host_list *list, const struct ip_addr *ip)
        return NULL;
 }
 
-struct mail_host *
-mail_host_get_by_hash(struct mail_host_list *list, unsigned int hash)
+static struct mail_host *
+mail_host_get_by_hash_ring(struct mail_host_list *list, unsigned int hash)
 {
-       struct mail_host *const *vhosts;
-       unsigned int count;
+       const struct mail_vhost *vhosts;
+       unsigned int count, idx;
 
-       if (list->hosts_unsorted)
-               mail_hosts_sort(list);
+       vhosts = array_get(&list->vhosts, &count);
+       array_bsearch_insert_pos(&list->vhosts, &hash,
+                                mail_vhost_hash_cmp, &idx);
+       if (idx == count) {
+               if (count == 0)
+                       return NULL;
+               idx = 0;
+       }
+
+       return vhosts[idx].host;
+}
+
+static struct mail_host *
+mail_host_get_by_hash_direct(struct mail_host_list *list, unsigned int hash)
+{
+       const struct mail_vhost *vhosts;
+       unsigned int count;
 
        vhosts = array_get(&list->vhosts, &count);
        if (count == 0)
                return NULL;
 
-       return vhosts[hash % count];
+       return vhosts[hash % count].host;
+}
+
+struct mail_host *
+mail_host_get_by_hash(struct mail_host_list *list, unsigned int hash)
+{
+       if (list->hosts_unsorted)
+               mail_hosts_sort(list);
+
+       if (list->consistent_hashing)
+               return mail_host_get_by_hash_ring(list, hash);
+       else
+               return mail_host_get_by_hash_direct(list, hash);
 }
 
 const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list)
@@ -228,11 +336,12 @@ const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list)
        return &list->hosts;
 }
 
-struct mail_host_list *mail_hosts_init(void)
+struct mail_host_list *mail_hosts_init(bool consistent_hashing)
 {
        struct mail_host_list *list;
 
        list = i_new(struct mail_host_list, 1);
+       list->consistent_hashing = consistent_hashing;
        i_array_init(&list->hosts, 16);
        i_array_init(&list->vhosts, 16*VHOST_MULTIPLIER);
        return list;
@@ -266,7 +375,7 @@ struct mail_host_list *mail_hosts_dup(const struct mail_host_list *src)
        struct mail_host_list *dest;
        struct mail_host *const *hostp, *dest_host;
 
-       dest = mail_hosts_init();
+       dest = mail_hosts_init(src->consistent_hashing);
        array_foreach(&src->hosts, hostp) {
                dest_host = mail_host_dup(*hostp);
                array_append(&dest->hosts, &dest_host, 1);

diff --git a/src/director/mail-host.h b/src/director/mail-host.h
index 4a6cc78b77b280421c510b8611ef435666b0e205..fbd97b1d05a846368a4a644de4ea452bee15e0ec 100644 (file)
--- a/src/director/mail-host.h
+++ b/src/director/mail-host.h
@@ -29,7 +29,7 @@ void mail_host_remove(struct mail_host_list *list, struct mail_host *host);
 
 const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list);
 
-struct mail_host_list *mail_hosts_init(void);
+struct mail_host_list *mail_hosts_init(bool consistent_hashing);
 void mail_hosts_deinit(struct mail_host_list **list);
 
 struct mail_host_list *mail_hosts_dup(const struct mail_host_list *src);