dsync: Add hashed_headers setting

This makes it possible to configure them

diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
index 5ccf3d2fca5f65bf578da87318f8475dc4bac172..b2806f5c80c1ccd3886d9b1b6e618d73ea4378c3 100644 (file)
--- a/src/doveadm/doveadm-dsync.c
+++ b/src/doveadm/doveadm-dsync.c
@@ -595,7 +595,13 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
        set.import_commit_msgs_interval = ctx->import_commit_msgs_interval;
        set.state = ctx->state_input;
        set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0];
-
+       if (*doveadm_settings->dsync_hashed_headers == '\0') {
+               i_error("dsync_hashed_headers must not be empty");
+               ctx->ctx.exit_code = EX_USAGE;
+               return -1;
+       }
+       set.hashed_headers =
+               t_strsplit_spaces(doveadm_settings->dsync_hashed_headers, " ,");
        if (array_count(&ctx->exclude_mailboxes) > 0) {
                /* array is NULL-terminated in init() */
                set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0);

diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c
index 9c031b72882ccfded2e9d526d963d70944e28041..be14e435a10d0df8365ff87b3a182acc1d5d876c 100644 (file)
--- a/src/doveadm/doveadm-settings.c
+++ b/src/doveadm/doveadm-settings.c
@@ -74,6 +74,7 @@ static const struct setting_define doveadm_setting_defines[] = {
        DEF(SET_STR, dsync_features),
        DEF(SET_UINT, dsync_commit_msgs_interval),
        DEF(SET_STR, doveadm_http_rawlog_dir),
+       DEF(SET_STR, dsync_hashed_headers),
 
        { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL },
 
@@ -96,6 +97,7 @@ const struct doveadm_settings doveadm_default_settings = {
        .dsync_alt_char = "_",
        .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U",
        .dsync_features = "",
+       .dsync_hashed_headers = "Date Message-ID",
        .dsync_commit_msgs_interval = 100,
        .ssl_client_ca_dir = "",
        .ssl_client_ca_file = "",
@@ -181,6 +183,10 @@ static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
        fix_base_path(set, pool, &set->auth_socket_path);
        fix_base_path(set, pool, &set->doveadm_socket_path);
 #endif
+       if (*set->dsync_hashed_headers == '\0') {
+               *error_r = "dsync_hashed_headers must not be empty";
+               return FALSE;
+       }
        if (*set->dsync_alt_char == '\0') {
                *error_r = "dsync_alt_char must not be empty";
                return FALSE;

diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h
index 2990a91e8ffebd2f55b5f1af935aed82acd75afc..0ad88ff18f21f41435f18826589dd9e95eea2b69 100644 (file)
--- a/src/doveadm/doveadm-settings.h
+++ b/src/doveadm/doveadm-settings.h
@@ -29,6 +29,7 @@ struct doveadm_settings {
        const char *director_username_hash;
        const char *doveadm_api_key;
        const char *dsync_features;
+       const char *dsync_hashed_headers;
        unsigned int dsync_commit_msgs_interval;
        const char *doveadm_http_rawlog_dir;
        enum dsync_features parsed_features;

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index 954a513dbb522c38fc572c1f9dcb62e220d11d81..b6eee57276d0af2d7262bed86329dd93974cb8cc 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -235,7 +235,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
                                          brain->sync_max_size,
                                          brain->sync_flag,
                                          brain->import_commit_msgs_interval,
-                                         import_flags, brain->hdr_hash_version);
+                                         import_flags, brain->hdr_hash_version,
+                                         brain->hashed_headers);
 }
 
 int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
@@ -345,7 +346,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
                dsync_mailbox_export_init(brain->box, brain->log_scan,
                                          last_common_uid,
                                          exporter_flags,
-                                         brain->hdr_hash_version);
+                                         brain->hdr_hash_version,
+                                         brain->hashed_headers);
        dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
        return 1;
 }

diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index 7840c8c780191db1c60c0371a3432df83d515fd3..2abcf8f60f428563d4d983c45b607a628a90ff79 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -102,6 +102,8 @@ struct dsync_brain {
        const char *changes_during_sync;
        enum mail_error mail_error;
 
+       const char *const *hashed_headers;
+
        bool master_brain:1;
        bool mail_requests:1;
        bool backup_send:1;

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 505259f42e6adac876bb70a7924f2912471483b0..06106ab3e945f807f4bc7195738a31a5d5461f5c 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -223,6 +223,8 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
        brain->lock_timeout = set->lock_timeout_secs;
        brain->import_commit_msgs_interval = set->import_commit_msgs_interval;
        brain->master_brain = TRUE;
+       brain->hashed_headers =
+               (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers);
        dsync_brain_set_flags(brain, flags);
 
        if (set->virtual_all_box != NULL)
@@ -262,6 +264,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
        ibc_set.hdr_hash_v2 = TRUE;
        ibc_set.lock_timeout = set->lock_timeout_secs;
        ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval;
+       ibc_set.hashed_headers = set->hashed_headers;
        /* reverse the backup direction for the slave */
        ibc_set.brain_flags = flags & ~(DSYNC_BRAIN_FLAG_BACKUP_SEND |
                                        DSYNC_BRAIN_FLAG_BACKUP_RECV);
@@ -511,6 +514,9 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
        brain->sync_type = ibc_set->sync_type;
 
        dsync_brain_set_flags(brain, ibc_set->brain_flags);
+       if (ibc_set->hashed_headers != NULL)
+               brain->hashed_headers =
+                       p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers);
        /* this flag is only set on the remote slave brain */
        brain->purge = (ibc_set->brain_flags &
                        DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0;

diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 929b80344bda9a3572f52e68be34979cce1143ca..5bfa9345338579a3df329f322afb8393bf3bb7b7 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -76,6 +76,8 @@ struct dsync_brain_settings {
        /* Sync only mails which contains / doesn't contain this flag.
           '-' at the beginning means this flag must not exist. */
        const char *sync_flag;
+       /* Headers to hash (defaults to Date, Message-ID) */
+       const char *const *hashed_headers;
 
        /* If non-zero, use dsync lock file for this user */
        unsigned int lock_timeout_secs;

diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 661258aa7807e559fb7b619bf70e716e5d4db5cc..7f6cc53dcbf171005caa9ea8f736c10a8e16c438 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -79,7 +79,8 @@ static const struct {
                "send_mail_requests backup_send backup_recv lock_timeout "
                "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote "
                "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp "
-               "virtual_all_box empty_hdr_workaround import_commit_msgs_interval"
+               "virtual_all_box empty_hdr_workaround import_commit_msgs_interval "
+               "hashed_headers"
        },
        { .name = "mailbox_state",
          .chr = 'S',
@@ -747,7 +748,15 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "no_notify", "");
        if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
                dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
-
+       /* this can be NULL in slave */
+       string_t *str2 = t_str_new(32);
+       if (set->hashed_headers != NULL) {
+               for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) {
+                       str_append_tabescaped(str2, *ptr);
+                       str_append_c(str2, '\t');
+               }
+       }
+       dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2));
        dsync_serializer_encode_finish(&encoder, str);
        dsync_ibc_stream_send_string(ibc, str);
 }
@@ -881,6 +890,8 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
                set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
        if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
                set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+       if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value))
+               set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value);
        set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
        set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3;
 

diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
index c1f0a2ef9bac8dd7298070e2bd4fb020f82a4c2f..5c6fe6baf2c5610c745b8b7f1d65c92526a8ff68 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc.h
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -64,6 +64,8 @@ struct dsync_ibc_settings {
        uoff_t sync_max_size;
        /* Sync only mails with specified flags. */
        const char *sync_flags;
+       /* Hashed headers */
+       const char *const *hashed_headers;
 
        enum dsync_brain_sync_type sync_type;
        enum dsync_brain_flags brain_flags;

diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
index 9ac8175c1fa0bc0f20a23fdd6bd3501d7dcd2011..2271bd870406458ac55538ee3bc91e97941dcfa6 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.c
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -11,22 +11,14 @@
 #include "mail-storage.h"
 #include "dsync-mail.h"
 
-/* These should be good enough to identify all normal mails. Received: header
-   would make it even better, but those can be somewhat large. Also these
-   fields can be looked up using IMAP ENVELOPE, which is more efficient in
-   some IMAP servers. */
-static const char *hashed_headers[] = {
-       "Date", "Message-ID", NULL
-};
-
 struct mailbox_header_lookup_ctx *
-dsync_mail_get_hash_headers(struct mailbox *box)
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers)
 {
        return mailbox_header_lookup_init(box, hashed_headers);
 }
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
-                           const char **hdr_hash_r)
+                           const char *const *hashed_headers, const char **hdr_hash_r)
 {
        struct istream *hdr_input, *input;
        struct mailbox_header_lookup_ctx *hdr_ctx;

diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
index fbf602048a5c2c35279524e918f78a9b774f4a01..a18a7db7b711acda12fe88442c5a2dc5c3523d09 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.h
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -87,10 +87,10 @@ struct dsync_mail_change {
 };
 
 struct mailbox_header_lookup_ctx *
-dsync_mail_get_hash_headers(struct mailbox *box);
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
-                           const char **hdr_hash_r);
+                           const char *const *hashed_headers, const char **hdr_hash_r);
 static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
 {
        /* md5(\n) */

diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
index 9eed9dcd37bbb9fa772ddab82b4e57a527732b37..bf23dc1ff6477490beb75a594bfccdaa13b79637 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.c
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -30,6 +30,8 @@ struct dsync_mailbox_exporter {
        unsigned int search_pos, search_count;
        unsigned int hdr_hash_version;
 
+       const char *const *hashed_headers;
+
        /* GUID => instances */
        HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
        ARRAY_TYPE(seq_range) requested_uids;
@@ -169,7 +171,8 @@ exporter_get_guids(struct dsync_mailbox_exporter *exporter,
                        *hdr_hash_r = "";
                        return 1;
                }
-               if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, hdr_hash_r) < 0)
+               if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
+                                           exporter->hashed_headers, hdr_hash_r) < 0)
                        return dsync_mail_error(exporter, mail, "hdr-stream");
                return 1;
        } else if (**guid_r == '\0') {
@@ -500,7 +503,8 @@ dsync_mailbox_export_init(struct mailbox *box,
                          struct dsync_transaction_log_scan *log_scan,
                          uint32_t last_common_uid,
                          enum dsync_mailbox_exporter_flags flags,
-                         unsigned int hdr_hash_version)
+                         unsigned int hdr_hash_version,
+                         const char *const *hashed_headers)
 {
        struct dsync_mailbox_exporter *exporter;
        pool_t pool;
@@ -525,6 +529,8 @@ dsync_mailbox_export_init(struct mailbox *box,
        exporter->hdr_hash_version = hdr_hash_version;
        exporter->no_hdr_hashes =
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
+       exporter->hashed_headers = hashed_headers;
+
        p_array_init(&exporter->requested_uids, pool, 16);
        p_array_init(&exporter->search_uids, pool, 16);
        hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
@@ -532,7 +538,8 @@ dsync_mailbox_export_init(struct mailbox *box,
        p_array_init(&exporter->expunged_guids, pool, 16);
 
        if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
-               exporter->wanted_headers = dsync_mail_get_hash_headers(box);
+               exporter->wanted_headers =
+                       dsync_mail_get_hash_headers(box, exporter->hashed_headers);
 
        /* first scan transaction log and save any expunges and flag changes */
        dsync_mailbox_export_log_scan(exporter, log_scan);

diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
index 11fa95423036f8da08a4bf548c51bb46c51d83e3..67e3216ad4b4101b665bf4a5f9ffa62b8c08638f 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.h
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -15,7 +15,8 @@ dsync_mailbox_export_init(struct mailbox *box,
                          struct dsync_transaction_log_scan *log_scan,
                          uint32_t last_common_uid,
                          enum dsync_mailbox_exporter_flags flags,
-                         unsigned int hdr_hash_version);
+                         unsigned int hdr_hash_version,
+                         const char *const *hashed_headers);
 /* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */
 int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
                                   const struct dsync_mailbox_attribute **attr_r);

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 1cfdda18c7334613b862a6aee2a6c3e2c8c9018f..ca07eb9b90b116b9a5e2c4ed9fb7f7e1e43ad2ae 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -69,6 +69,8 @@ struct dsync_mailbox_importer {
        unsigned int hdr_hash_version;
        unsigned int commit_msgs_interval;
 
+       const char *const *hashed_headers;
+
        enum mail_flags sync_flag;
        const char *sync_keyword;
        bool sync_flag_dontwant;
@@ -229,7 +231,8 @@ dsync_mailbox_import_init(struct mailbox *box,
                          const char *sync_flag,
                          unsigned int commit_msgs_interval,
                          enum dsync_mailbox_import_flags flags,
-                         unsigned int hdr_hash_version)
+                         unsigned int hdr_hash_version,
+                         const char *const *hashed_headers)
 {
        struct dsync_mailbox_importer *importer;
        struct mailbox_status status;
@@ -254,6 +257,8 @@ dsync_mailbox_import_init(struct mailbox *box,
        importer->sync_until_timestamp = sync_until_timestamp;
        importer->sync_max_size = sync_max_size;
        importer->stateful_import = importer->last_common_uid_found;
+       importer->hashed_headers = hashed_headers;
+
        if (sync_flag != NULL) {
                if (sync_flag[0] == '-') {
                        importer->sync_flag_dontwant = TRUE;
@@ -659,6 +664,7 @@ importer_try_next_mail(struct dsync_mailbox_importer *importer,
        } else {
                if (dsync_mail_get_hdr_hash(importer->cur_mail,
                                            importer->hdr_hash_version,
+                                           importer->hashed_headers,
                                            &hdr_hash) < 0) {
                        dsync_mail_error(importer, importer->cur_mail,
                                         "header hash");
@@ -1575,7 +1581,8 @@ dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
        }
 
        if (dsync_mail_get_hdr_hash(importer->cur_mail,
-                                   importer->hdr_hash_version, &hdr_hash) < 0) {
+                                   importer->hdr_hash_version,
+                                   importer->hashed_headers, &hdr_hash) < 0) {
                dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
                *result_r = "Error fetching header stream";
                return -1;

diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index 69505d373c406bc119832898eb892e355012d38e..6a2020eb406ca8d1814c77e8f42baedbef21d747 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -37,7 +37,8 @@ dsync_mailbox_import_init(struct mailbox *box,
                          const char *sync_flag,
                          unsigned int commit_msgs_interval,
                          enum dsync_mailbox_import_flags flags,
-                         unsigned int hdr_hash_version);
+                         unsigned int hdr_hash_version,
+                         const char *const *hashed_headers);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
                                   const struct dsync_mailbox_attribute *attr);
 int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,