doveadm: Add dsync_features=no-header-hashes

When this setting is enabled and one dsync side doesn't support mail GUIDs,
there is no fallback to using header hashes. Instead, dsync assumes that
all mails with identical IMAP UIDs contains the same mail contents. This
can significantly improve dsync performance with some IMAP servers that
don't support caching Date/Message-ID headers.

diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
index e490def451d40cb94e34690f0436d14058931bb0..b0b896c38f2dffca8d571262d3d563dda756b897 100644 (file)
--- a/src/doveadm/doveadm-dsync.c
+++ b/src/doveadm/doveadm-dsync.c
@@ -109,6 +109,7 @@ struct dsync_cmd_context {
        bool replicator_notify:1;
        bool exited:1;
        bool empty_hdr_workaround:1;
+       bool no_header_hashes:1;
 };
 
 static bool legacy_dsync = FALSE;
@@ -729,6 +730,8 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
                brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
        if (ctx->empty_hdr_workaround)
                brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+       if (ctx->no_header_hashes)
+               brain_flags |= DSYNC_BRAIN_FLAG_NO_HEADER_HASHES;
        if (doveadm_debug)
                brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
 
@@ -1210,6 +1213,8 @@ static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void)
        p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4);
         if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0)
                 ctx->empty_hdr_workaround = TRUE;
+        if ((doveadm_settings->parsed_features & DSYNC_FEATURE_NO_HEADER_HASHES) != 0)
+                ctx->no_header_hashes = TRUE;
        ctx->import_commit_msgs_interval = doveadm_settings->dsync_commit_msgs_interval;
        return &ctx->ctx;
 }

diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c
index 54d6631a9da57ca2aa23a01d22242d23492c8541..b6b31cfaf08875997ffc45f533e5a5a68befa5c5 100644 (file)
--- a/src/doveadm/doveadm-settings.c
+++ b/src/doveadm/doveadm-settings.c
@@ -155,6 +155,7 @@ struct dsync_feature_list {
 
 static const struct dsync_feature_list dsync_feature_list[] = {
        { "empty-header-workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND },
+       { "no-header-hashes", DSYNC_FEATURE_NO_HEADER_HASHES },
        { NULL, 0 }
 };
 

diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h
index b3c2184401f2a7cd7ed20c172f59c6382ff17421..c718b8677a56acffa1ca68069425d04420ce830b 100644 (file)
--- a/src/doveadm/doveadm-settings.h
+++ b/src/doveadm/doveadm-settings.h
@@ -8,6 +8,7 @@ struct ssl_iostream_settings;
 /* <settings checks> */
 enum dsync_features {
        DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1,
+       DSYNC_FEATURE_NO_HEADER_HASHES = 0x2,
 };
 /* </settings checks> */
 

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index 759018fe4f471354b8e1bf71bfecc92cb56de3e4..79667865c6967f85f46f70c57551cee1d545f9f0 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -222,6 +222,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
                import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY;
        if (brain->empty_hdr_workaround)
                import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND;
+       if (brain->no_header_hashes)
+               import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES;
 
        brain->box_importer = brain->backup_send ? NULL :
                dsync_mailbox_import_init(brain->box, brain->virtual_all_box,
@@ -337,7 +339,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
        if (brain->sync_max_size > 0)
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;
-       if (remote_dsync_box->messages_count == 0) {
+       if (remote_dsync_box->messages_count == 0 ||
+           brain->no_header_hashes) {
                /* remote mailbox is empty - we don't really need to export
                   header hashes since they're not going to match anything
                   anyway. */

diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index 984970349b5fbb4bd2747c842aafb260314ffa1a..6680a9bb06fccfeb806ad3c464152afd67de43cc 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -124,6 +124,7 @@ struct dsync_brain {
        bool no_notify:1;
        bool failed:1;
        bool empty_hdr_workaround:1;
+       bool no_header_hashes:1;
 };
 
 extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 8ff7247630b175eb5ee8db391b875c2bbb88a136..d847169f543f48f29041cba931e72c05cbaeb03f 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -165,6 +165,7 @@ dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
                (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
        brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
        brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
+       brain->no_header_hashes = (flags & DSYNC_BRAIN_FLAG_NO_HEADER_HASHES) != 0;
 }
 
 static void

diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 5813148cdae45f21fa531442fbeee114faa890bc..7677168144935bf6b5812315a72b17710a609faf 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -34,6 +34,12 @@ enum dsync_brain_flags {
        DSYNC_BRAIN_FLAG_NO_NOTIFY              = 0x400,
        /* Workaround missing Date/Message-ID headers */
        DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND   = 0x800,
+       /* If mail GUIDs aren't supported, don't emulate them with header
+          hashes either. This trusts that all the existing mails with
+          identical UIDs have the same email content. This makes it slightly
+          less safe, but can have huge performance improvement with imapc
+          if the remote server doesn't have a fast header cache. */
+       DSYNC_BRAIN_FLAG_NO_HEADER_HASHES       = 0x1000,
 };
 
 enum dsync_brain_sync_type {

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 8a8fbe427dc6ed1773e91741a47d9dc4266e1e69..19442495ae1666f57f3ce4d417ef3dc67f569b19 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -130,6 +130,7 @@ struct dsync_mailbox_importer {
        bool mails_use_guid128:1;
        bool delete_mailbox:1;
        bool empty_hdr_workaround:1;
+       bool no_header_hashes:1;
 };
 
 static const char *dsync_mail_change_type_names[] = {
@@ -300,7 +301,8 @@ dsync_mailbox_import_init(struct mailbox *box,
        importer->hdr_hash_version = hdr_hash_version;
        importer->empty_hdr_workaround =
                (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
-
+       importer->no_header_hashes =
+               (flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES) != 0;
        mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
                                STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
                                &status);
@@ -815,6 +817,11 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
                /* one of the headers is empty. assume it's broken and that
                   the header matches what we have currently. */
                diff = 0;
+       } else if (importer->no_header_hashes && !importer->mails_have_guids &&
+                  importer->cur_mail != NULL && save_change != NULL &&
+                  (m1.guid[0] == '\0' || m2.guid[0] == '\0')) {
+               /* Header hashes aren't known. Assume that the mails match. */
+               diff = 0;
        } else {
                diff = importer_mail_cmp(&m1, &m2);
        }
@@ -1598,6 +1605,10 @@ dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
             dsync_mail_hdr_hash_is_empty(hdr_hash))) {
                *result_r = "Empty headers found with workaround enabled - assuming a match";
                return 1;
+       } else if (importer->no_header_hashes &&
+                  (change->hdr_hash[0] == '\0' || hdr_hash[0] == '\0')) {
+               *result_r = "Header hashing disabled - assuming a match";
+               return 1;
        } else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
                *result_r = "Headers hashes match";
                return 1;

diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index 6a2020eb406ca8d1814c77e8f42baedbef21d747..c860e065dbbbe1819088ed685a05bd0e3c438368 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -11,7 +11,8 @@ enum dsync_mailbox_import_flags {
        DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS      = 0x10,
        DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128     = 0x20,
        DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY             = 0x40,
-       DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND  = 0x100
+       DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND  = 0x100,
+       DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES      = 0x200,
 };
 
 struct mailbox;