From 9708531c0428634045ae28f28e46d41bab914e40 Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Thu, 19 May 2022 11:24:23 +0200 Subject: [PATCH] doveadm: Add dsync_features=no-header-hashes When this setting is enabled and one dsync side doesn't support mail GUIDs, there is no fallback to using header hashes. Instead, dsync assumes that all mails with identical IMAP UIDs contains the same mail contents. This can significantly improve dsync performance with some IMAP servers that don't support caching Date/Message-ID headers. --- src/doveadm/doveadm-dsync.c | 5 +++++ src/doveadm/doveadm-settings.c | 1 + src/doveadm/doveadm-settings.h | 1 + src/doveadm/dsync/dsync-brain-mailbox.c | 5 ++++- src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 1 + src/doveadm/dsync/dsync-brain.h | 6 ++++++ src/doveadm/dsync/dsync-mailbox-import.c | 13 ++++++++++++- src/doveadm/dsync/dsync-mailbox-import.h | 3 ++- 9 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c index e490def451..b0b896c38f 100644 --- a/src/doveadm/doveadm-dsync.c +++ b/src/doveadm/doveadm-dsync.c @@ -109,6 +109,7 @@ struct dsync_cmd_context { bool replicator_notify:1; bool exited:1; bool empty_hdr_workaround:1; + bool no_header_hashes:1; }; static bool legacy_dsync = FALSE; @@ -729,6 +730,8 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; if (ctx->empty_hdr_workaround) brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; + if (ctx->no_header_hashes) + brain_flags |= DSYNC_BRAIN_FLAG_NO_HEADER_HASHES; if (doveadm_debug) brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; @@ -1210,6 +1213,8 @@ static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void) p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4); if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0) ctx->empty_hdr_workaround = TRUE; + if ((doveadm_settings->parsed_features & DSYNC_FEATURE_NO_HEADER_HASHES) != 0) + ctx->no_header_hashes = TRUE; ctx->import_commit_msgs_interval = doveadm_settings->dsync_commit_msgs_interval; return &ctx->ctx; } diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c index 54d6631a9d..b6b31cfaf0 100644 --- a/src/doveadm/doveadm-settings.c +++ b/src/doveadm/doveadm-settings.c @@ -155,6 +155,7 @@ struct dsync_feature_list { static const struct dsync_feature_list dsync_feature_list[] = { { "empty-header-workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND }, + { "no-header-hashes", DSYNC_FEATURE_NO_HEADER_HASHES }, { NULL, 0 } }; diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h index b3c2184401..c718b8677a 100644 --- a/src/doveadm/doveadm-settings.h +++ b/src/doveadm/doveadm-settings.h @@ -8,6 +8,7 @@ struct ssl_iostream_settings; /* */ enum dsync_features { DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1, + DSYNC_FEATURE_NO_HEADER_HASHES = 0x2, }; /* */ diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index 759018fe4f..79667865c6 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -222,6 +222,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY; if (brain->empty_hdr_workaround) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND; + if (brain->no_header_hashes) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES; brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->virtual_all_box, @@ -337,7 +339,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS; if (brain->sync_max_size > 0) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES; - if (remote_dsync_box->messages_count == 0) { + if (remote_dsync_box->messages_count == 0 || + brain->no_header_hashes) { /* remote mailbox is empty - we don't really need to export header hashes since they're not going to match anything anyway. */ diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h index 984970349b..6680a9bb06 100644 --- a/src/doveadm/dsync/dsync-brain-private.h +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -124,6 +124,7 @@ struct dsync_brain { bool no_notify:1; bool failed:1; bool empty_hdr_workaround:1; + bool no_header_hashes:1; }; extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1]; diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index 8ff7247630..d847169f54 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -165,6 +165,7 @@ dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags) (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0; brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0; + brain->no_header_hashes = (flags & DSYNC_BRAIN_FLAG_NO_HEADER_HASHES) != 0; } static void diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h index 5813148cda..7677168144 100644 --- a/src/doveadm/dsync/dsync-brain.h +++ b/src/doveadm/dsync/dsync-brain.h @@ -34,6 +34,12 @@ enum dsync_brain_flags { DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400, /* Workaround missing Date/Message-ID headers */ DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND = 0x800, + /* If mail GUIDs aren't supported, don't emulate them with header + hashes either. This trusts that all the existing mails with + identical UIDs have the same email content. This makes it slightly + less safe, but can have huge performance improvement with imapc + if the remote server doesn't have a fast header cache. */ + DSYNC_BRAIN_FLAG_NO_HEADER_HASHES = 0x1000, }; enum dsync_brain_sync_type { diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c index 8a8fbe427d..19442495ae 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.c +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -130,6 +130,7 @@ struct dsync_mailbox_importer { bool mails_use_guid128:1; bool delete_mailbox:1; bool empty_hdr_workaround:1; + bool no_header_hashes:1; }; static const char *dsync_mail_change_type_names[] = { @@ -300,7 +301,8 @@ dsync_mailbox_import_init(struct mailbox *box, importer->hdr_hash_version = hdr_hash_version; importer->empty_hdr_workaround = (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0; - + importer->no_header_hashes = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES) != 0; mailbox_get_open_status(importer->box, STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, &status); @@ -815,6 +817,11 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer, /* one of the headers is empty. assume it's broken and that the header matches what we have currently. */ diff = 0; + } else if (importer->no_header_hashes && !importer->mails_have_guids && + importer->cur_mail != NULL && save_change != NULL && + (m1.guid[0] == '\0' || m2.guid[0] == '\0')) { + /* Header hashes aren't known. Assume that the mails match. */ + diff = 0; } else { diff = importer_mail_cmp(&m1, &m2); } @@ -1598,6 +1605,10 @@ dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer, dsync_mail_hdr_hash_is_empty(hdr_hash))) { *result_r = "Empty headers found with workaround enabled - assuming a match"; return 1; + } else if (importer->no_header_hashes && + (change->hdr_hash[0] == '\0' || hdr_hash[0] == '\0')) { + *result_r = "Header hashing disabled - assuming a match"; + return 1; } else if (strcmp(change->hdr_hash, hdr_hash) == 0) { *result_r = "Headers hashes match"; return 1; diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h index 6a2020eb40..c860e065db 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.h +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -11,7 +11,8 @@ enum dsync_mailbox_import_flags { DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10, DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20, DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY = 0x40, - DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100 + DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100, + DSYNC_MAILBOX_IMPORT_FLAG_NO_HEADER_HASHES = 0x200, }; struct mailbox; -- 2.47.3