dsync: Add support for features

Add empty_header_workaround as first feature

diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
index 94e1599c47c420ca8d28076264f88c6ac1216588..22a48c25f593795318e32976f296cc44dbe0d74e 100644 (file)
--- a/src/doveadm/doveadm-dsync.c
+++ b/src/doveadm/doveadm-dsync.c
@@ -17,6 +17,7 @@
 #include "settings-parser.h"
 #include "imap-util.h"
 #include "master-service.h"
+#include "master-service-settings.h"
 #include "master-service-ssl-settings.h"
 #include "mail-storage.h"
 #include "mail-storage-service.h"
@@ -103,6 +104,7 @@ struct dsync_cmd_context {
        bool local_location_from_arg:1;
        bool replicator_notify:1;
        bool exited:1;
+       bool empty_hdr_workaround:1;
 };
 
 static bool legacy_dsync = FALSE;
@@ -573,6 +575,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
        set.lock_timeout_secs = ctx->lock_timeout;
        set.state = ctx->state_input;
        set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0];
+
        if (array_count(&ctx->exclude_mailboxes) > 0) {
                /* array is NULL-terminated in init() */
                set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0);
@@ -622,6 +625,8 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
                brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC;
        if (ctx->oneway)
                brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
+       if (ctx->empty_hdr_workaround)
+               brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
        if (doveadm_debug)
                brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
 
@@ -1064,6 +1069,8 @@ static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void)
                             DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
        p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4);
        p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4);
+        if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0)
+                ctx->empty_hdr_workaround = TRUE;
        return &ctx->ctx;
 }
 
@@ -1179,6 +1186,7 @@ static struct doveadm_mail_cmd_context *cmd_dsync_server_alloc(void)
        ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg;
        ctx->ctx.v.run = cmd_dsync_server_run;
        ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+
        ctx->fd_in = STDIN_FILENO;
        ctx->fd_out = STDOUT_FILENO;
        return &ctx->ctx;

diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c
index df1228415d6782dea3bb2079994a2483416b956e..0923f3799ae0347568580d6b48d9f1453e3514b7 100644 (file)
--- a/src/doveadm/doveadm-settings.c
+++ b/src/doveadm/doveadm-settings.c
@@ -71,6 +71,7 @@ static const struct setting_define doveadm_setting_defines[] = {
        DEF(SET_STR, ssl_client_ca_file),
        DEF(SET_STR, director_username_hash),
        DEF(SET_STR, doveadm_api_key),
+       DEF(SET_STR, dsync_features),
 
        { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL },
 
@@ -92,6 +93,7 @@ const struct doveadm_settings doveadm_default_settings = {
        .doveadm_allowed_commands = "",
        .dsync_alt_char = "_",
        .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U",
+       .dsync_features = "",
        .ssl_client_ca_dir = "",
        .ssl_client_ca_file = "",
        .director_username_hash = "%Lu",
@@ -129,6 +131,43 @@ fix_base_path(struct doveadm_settings *set, pool_t pool, const char **str)
 }
 
 /* <settings checks> */
+struct dsync_feature_list {
+       const char *name;
+       enum dsync_features num;
+};
+
+static const struct dsync_feature_list dsync_feature_list[] = {
+       { "empty_header_workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND },
+       { NULL, 0 }
+};
+
+static int
+dsync_settings_parse_features(struct doveadm_settings *set,
+                             const char **error_r)
+{
+       enum dsync_features features = 0;
+       const struct dsync_feature_list *list;
+       const char *const *str;
+
+       str = t_strsplit_spaces(set->dsync_features, " ,");
+       for (; *str != NULL; str++) {
+               list = dsync_feature_list;
+               for (; list->name != NULL; list++) {
+                       if (strcasecmp(*str, list->name) == 0) {
+                               features |= list->num;
+                               break;
+                       }
+               }
+               if (list->name == NULL) {
+                       *error_r = t_strdup_printf("dsync_features: "
+                               "Unknown feature: %s", *str);
+                       return -1;
+               }
+       }
+       set->parsed_features = features;
+       return 0;
+}
+
 static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
                                   const char **error_r)
 {
@@ -142,6 +181,8 @@ static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
                *error_r = "dsync_alt_char must not be empty";
                return FALSE;
        }
+       if (dsync_settings_parse_features(set, error_r) != 0)
+               return FALSE;
        return TRUE;
 }
 /* </settings checks> */

diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h
index 07ecd3d9409073382e288598aad82738dd38e3d3..6ea049a4d889eec7fa754d26c850f11a76f9def2 100644 (file)
--- a/src/doveadm/doveadm-settings.h
+++ b/src/doveadm/doveadm-settings.h
@@ -3,6 +3,12 @@
 
 #include "net.h"
 
+/* <settings checks> */
+enum dsync_features {
+       DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1,
+};
+/* </settings checks> */
+
 struct doveadm_settings {
        const char *base_dir;
        const char *libexec_dir;
@@ -22,7 +28,8 @@ struct doveadm_settings {
        const char *ssl_client_ca_file;
        const char *director_username_hash;
        const char *doveadm_api_key;
-
+       const char *dsync_features;
+       enum dsync_features parsed_features;
        ARRAY(const char *) plugin_envs;
 };
 

diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c
index 67a8034238c7bb1ebc3398eef79e6313deab9032..b73fa76fdc599cf88fad86987c68f2f3e65342ae 100644 (file)
--- a/src/doveadm/doveadm.c
+++ b/src/doveadm/doveadm.c
@@ -269,6 +269,8 @@ static void doveadm_read_settings(void)
        set = master_service_settings_get_others(master_service)[0];
        doveadm_settings = settings_dup(&doveadm_setting_parser_info, set,
                                        pool_datastack_create());
+
+       doveadm_settings->parsed_features = set->parsed_features; /* copy this value by hand */
 }
 
 static struct doveadm_cmd *doveadm_cmdline_commands[] = {

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index bd380645bbfd8a5d23788cfa75946988d0585004..c7ffcd080b475ec6d60f4720fa529788a7785d9f 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -220,6 +220,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
                import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY;
        if (brain->hdr_hash_v2)
                import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2;
+       if (brain->empty_hdr_workaround)
+               import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND;
 
        brain->box_importer = brain->backup_send ? NULL :
                dsync_mailbox_import_init(brain->box, brain->virtual_all_box,

diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index 67e9e9cac44f11633a64682a4acf41eb7d6c1e21..dc9938bd33219a918fef6776e749da3451fbf12b 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -114,6 +114,7 @@ struct dsync_brain {
        bool no_notify:1;
        bool hdr_hash_v2:1;
        bool failed:1;
+       bool empty_hdr_workaround:1;
 };
 
 extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 8e43d3d102e940e862e82f7ad075403e8f0e0984..1b1e7823ae74026e296394077294b11bcd308565 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -143,6 +143,7 @@ dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
        brain->no_mailbox_renames =
                (flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0;
        brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
+       brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
 }
 
 static void

diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 168e92f13815b216803afe06afeaaf7a78f13254..3f8db7e3a621df8fdeca0020fe1b58b4c4bbe8dc 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -32,7 +32,9 @@ enum dsync_brain_flags {
           be removed once the renaming logic has no more bugs.. */
        DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES     = 0x200,
        /* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */
-       DSYNC_BRAIN_FLAG_NO_NOTIFY              = 0x400
+       DSYNC_BRAIN_FLAG_NO_NOTIFY              = 0x400,
+       /* Workaround missing Date/Message-ID headers */
+       DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND   = 0x800,
 };
 
 enum dsync_brain_sync_type {

diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 76bd1552859ebaf40b10020bd59cff080bf82abd..c03cfe6e22cf55e8742659455c9726665e04c945 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -731,6 +731,8 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "purge_remote", "");
        if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0)
                dsync_serializer_encode_add(encoder, "no_notify", "");
+       if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
+               dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
 
        dsync_serializer_encode_finish(&encoder, str);
        dsync_ibc_stream_send_string(ibc, str);
@@ -839,6 +841,8 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
                set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE;
        if (dsync_deserializer_decode_try(decoder, "no_notify", &value))
                set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
+       if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
+               set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
        set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
 
        *set_r = set;

diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
index 78a1a25084551763b2f90d6e7d22d7c6e861b40f..89cc143dd0fb136ad29c0393403c6c94533e45c7 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.h
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -88,6 +88,12 @@ dsync_mail_get_hash_headers(struct mailbox *box);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
                            const char **hdr_hash_r);
+static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
+{
+       /* md5(\n) */
+       return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0;
+}
+
 int dsync_mail_fill(struct mail *mail, bool minimal_fill,
                    struct dsync_mail *dmail_r, const char **error_field_r);
 int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 877305cd5a1a19093813a5e60adc872156c6056e..c8192a8541ebb3c6b3f00d8fe2bd934f86de58af 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -122,6 +122,7 @@ struct dsync_mailbox_importer {
        bool mails_have_guids:1;
        bool mails_use_guid128:1;
        bool delete_mailbox:1;
+       bool empty_hdr_workaround:1;
 };
 
 static const char *dsync_mail_change_type_names[] = {
@@ -278,6 +279,8 @@ dsync_mailbox_import_init(struct mailbox *box,
                (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
        importer->hdr_hash_version =
                (flags & DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2) != 0 ? 2 : 1;
+       importer->empty_hdr_workaround =
+               (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
 
        mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
                                STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
@@ -782,7 +785,16 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
                i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
        }
 
-       diff = importer_mail_cmp(&m1, &m2);
+       if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
+           importer->cur_mail != NULL && save_change != NULL &&
+           (dsync_mail_hdr_hash_is_empty(m1.guid) ||
+            dsync_mail_hdr_hash_is_empty(m2.guid))) {
+               /* one of the headers is empty. assume it's broken and that
+                  the header matches what we have currently. */
+               diff = 0;
+       } else {
+               diff = importer_mail_cmp(&m1, &m2);
+       }
        if (diff < 0) {
                /* add a record for local mail */
                i_assert(importer->cur_mail != NULL);

diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index 8139d650edef4c790d13c56d251f5db6a9a47175..a0492495d4241486dfca1e25eb735ebf5322d8b5 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -11,7 +11,8 @@ enum dsync_mailbox_import_flags {
        DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS      = 0x10,
        DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128     = 0x20,
        DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY             = 0x40,
-       DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2           = 0x80
+       DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2           = 0x80,
+       DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND  = 0x100
 };
 
 struct mailbox;