From: Aki Tuomi Date: Fri, 22 Sep 2017 10:30:43 +0000 (+0300) Subject: dsync: Add hashed_headers setting X-Git-Tag: 2.3.0.rc1~954 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=69a71891361b2b27ff68ed84b29278486628464a;p=thirdparty%2Fdovecot%2Fcore.git dsync: Add hashed_headers setting This makes it possible to configure them --- diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c index 5ccf3d2fca..b2806f5c80 100644 --- a/src/doveadm/doveadm-dsync.c +++ b/src/doveadm/doveadm-dsync.c @@ -595,7 +595,13 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) set.import_commit_msgs_interval = ctx->import_commit_msgs_interval; set.state = ctx->state_input; set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0]; - + if (*doveadm_settings->dsync_hashed_headers == '\0') { + i_error("dsync_hashed_headers must not be empty"); + ctx->ctx.exit_code = EX_USAGE; + return -1; + } + set.hashed_headers = + t_strsplit_spaces(doveadm_settings->dsync_hashed_headers, " ,"); if (array_count(&ctx->exclude_mailboxes) > 0) { /* array is NULL-terminated in init() */ set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0); diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c index 9c031b7288..be14e435a1 100644 --- a/src/doveadm/doveadm-settings.c +++ b/src/doveadm/doveadm-settings.c @@ -74,6 +74,7 @@ static const struct setting_define doveadm_setting_defines[] = { DEF(SET_STR, dsync_features), DEF(SET_UINT, dsync_commit_msgs_interval), DEF(SET_STR, doveadm_http_rawlog_dir), + DEF(SET_STR, dsync_hashed_headers), { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL }, @@ -96,6 +97,7 @@ const struct doveadm_settings doveadm_default_settings = { .dsync_alt_char = "_", .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U", .dsync_features = "", + .dsync_hashed_headers = "Date Message-ID", .dsync_commit_msgs_interval = 100, .ssl_client_ca_dir = "", .ssl_client_ca_file = "", @@ -181,6 +183,10 @@ static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED, fix_base_path(set, pool, &set->auth_socket_path); fix_base_path(set, pool, &set->doveadm_socket_path); #endif + if (*set->dsync_hashed_headers == '\0') { + *error_r = "dsync_hashed_headers must not be empty"; + return FALSE; + } if (*set->dsync_alt_char == '\0') { *error_r = "dsync_alt_char must not be empty"; return FALSE; diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h index 2990a91e8f..0ad88ff18f 100644 --- a/src/doveadm/doveadm-settings.h +++ b/src/doveadm/doveadm-settings.h @@ -29,6 +29,7 @@ struct doveadm_settings { const char *director_username_hash; const char *doveadm_api_key; const char *dsync_features; + const char *dsync_hashed_headers; unsigned int dsync_commit_msgs_interval; const char *doveadm_http_rawlog_dir; enum dsync_features parsed_features; diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index 954a513dbb..b6eee57276 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -235,7 +235,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, brain->sync_max_size, brain->sync_flag, brain->import_commit_msgs_interval, - import_flags, brain->hdr_hash_version); + import_flags, brain->hdr_hash_version, + brain->hashed_headers); } int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, @@ -345,7 +346,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, dsync_mailbox_export_init(brain->box, brain->log_scan, last_common_uid, exporter_flags, - brain->hdr_hash_version); + brain->hdr_hash_version, + brain->hashed_headers); dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box); return 1; } diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h index 7840c8c780..2abcf8f60f 100644 --- a/src/doveadm/dsync/dsync-brain-private.h +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -102,6 +102,8 @@ struct dsync_brain { const char *changes_during_sync; enum mail_error mail_error; + const char *const *hashed_headers; + bool master_brain:1; bool mail_requests:1; bool backup_send:1; diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index 505259f42e..06106ab3e9 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -223,6 +223,8 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, brain->lock_timeout = set->lock_timeout_secs; brain->import_commit_msgs_interval = set->import_commit_msgs_interval; brain->master_brain = TRUE; + brain->hashed_headers = + (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers); dsync_brain_set_flags(brain, flags); if (set->virtual_all_box != NULL) @@ -262,6 +264,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, ibc_set.hdr_hash_v2 = TRUE; ibc_set.lock_timeout = set->lock_timeout_secs; ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval; + ibc_set.hashed_headers = set->hashed_headers; /* reverse the backup direction for the slave */ ibc_set.brain_flags = flags & ~(DSYNC_BRAIN_FLAG_BACKUP_SEND | DSYNC_BRAIN_FLAG_BACKUP_RECV); @@ -511,6 +514,9 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) brain->sync_type = ibc_set->sync_type; dsync_brain_set_flags(brain, ibc_set->brain_flags); + if (ibc_set->hashed_headers != NULL) + brain->hashed_headers = + p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers); /* this flag is only set on the remote slave brain */ brain->purge = (ibc_set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0; diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h index 929b80344b..5bfa934533 100644 --- a/src/doveadm/dsync/dsync-brain.h +++ b/src/doveadm/dsync/dsync-brain.h @@ -76,6 +76,8 @@ struct dsync_brain_settings { /* Sync only mails which contains / doesn't contain this flag. '-' at the beginning means this flag must not exist. */ const char *sync_flag; + /* Headers to hash (defaults to Date, Message-ID) */ + const char *const *hashed_headers; /* If non-zero, use dsync lock file for this user */ unsigned int lock_timeout_secs; diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c index 661258aa78..7f6cc53dcb 100644 --- a/src/doveadm/dsync/dsync-ibc-stream.c +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -79,7 +79,8 @@ static const struct { "send_mail_requests backup_send backup_recv lock_timeout " "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote " "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp " - "virtual_all_box empty_hdr_workaround import_commit_msgs_interval" + "virtual_all_box empty_hdr_workaround import_commit_msgs_interval " + "hashed_headers" }, { .name = "mailbox_state", .chr = 'S', @@ -747,7 +748,15 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, dsync_serializer_encode_add(encoder, "no_notify", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0) dsync_serializer_encode_add(encoder, "empty_hdr_workaround", ""); - + /* this can be NULL in slave */ + string_t *str2 = t_str_new(32); + if (set->hashed_headers != NULL) { + for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) { + str_append_tabescaped(str2, *ptr); + str_append_c(str2, '\t'); + } + } + dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2)); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); } @@ -881,6 +890,8 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc, set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY; if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; + if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value)) + set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value); set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2; set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3; diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h index c1f0a2ef9b..5c6fe6baf2 100644 --- a/src/doveadm/dsync/dsync-ibc.h +++ b/src/doveadm/dsync/dsync-ibc.h @@ -64,6 +64,8 @@ struct dsync_ibc_settings { uoff_t sync_max_size; /* Sync only mails with specified flags. */ const char *sync_flags; + /* Hashed headers */ + const char *const *hashed_headers; enum dsync_brain_sync_type sync_type; enum dsync_brain_flags brain_flags; diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c index 9ac8175c1f..2271bd8704 100644 --- a/src/doveadm/dsync/dsync-mail.c +++ b/src/doveadm/dsync/dsync-mail.c @@ -11,22 +11,14 @@ #include "mail-storage.h" #include "dsync-mail.h" -/* These should be good enough to identify all normal mails. Received: header - would make it even better, but those can be somewhat large. Also these - fields can be looked up using IMAP ENVELOPE, which is more efficient in - some IMAP servers. */ -static const char *hashed_headers[] = { - "Date", "Message-ID", NULL -}; - struct mailbox_header_lookup_ctx * -dsync_mail_get_hash_headers(struct mailbox *box) +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers) { return mailbox_header_lookup_init(box, hashed_headers); } int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, - const char **hdr_hash_r) + const char *const *hashed_headers, const char **hdr_hash_r) { struct istream *hdr_input, *input; struct mailbox_header_lookup_ctx *hdr_ctx; diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h index fbf602048a..a18a7db7b7 100644 --- a/src/doveadm/dsync/dsync-mail.h +++ b/src/doveadm/dsync/dsync-mail.h @@ -87,10 +87,10 @@ struct dsync_mail_change { }; struct mailbox_header_lookup_ctx * -dsync_mail_get_hash_headers(struct mailbox *box); +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers); int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, - const char **hdr_hash_r); + const char *const *hashed_headers, const char **hdr_hash_r); static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash) { /* md5(\n) */ diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c index 9eed9dcd37..bf23dc1ff6 100644 --- a/src/doveadm/dsync/dsync-mailbox-export.c +++ b/src/doveadm/dsync/dsync-mailbox-export.c @@ -30,6 +30,8 @@ struct dsync_mailbox_exporter { unsigned int search_pos, search_count; unsigned int hdr_hash_version; + const char *const *hashed_headers; + /* GUID => instances */ HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids; ARRAY_TYPE(seq_range) requested_uids; @@ -169,7 +171,8 @@ exporter_get_guids(struct dsync_mailbox_exporter *exporter, *hdr_hash_r = ""; return 1; } - if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, hdr_hash_r) < 0) + if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, + exporter->hashed_headers, hdr_hash_r) < 0) return dsync_mail_error(exporter, mail, "hdr-stream"); return 1; } else if (**guid_r == '\0') { @@ -500,7 +503,8 @@ dsync_mailbox_export_init(struct mailbox *box, struct dsync_transaction_log_scan *log_scan, uint32_t last_common_uid, enum dsync_mailbox_exporter_flags flags, - unsigned int hdr_hash_version) + unsigned int hdr_hash_version, + const char *const *hashed_headers) { struct dsync_mailbox_exporter *exporter; pool_t pool; @@ -525,6 +529,8 @@ dsync_mailbox_export_init(struct mailbox *box, exporter->hdr_hash_version = hdr_hash_version; exporter->no_hdr_hashes = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0; + exporter->hashed_headers = hashed_headers; + p_array_init(&exporter->requested_uids, pool, 16); p_array_init(&exporter->search_uids, pool, 16); hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp); @@ -532,7 +538,8 @@ dsync_mailbox_export_init(struct mailbox *box, p_array_init(&exporter->expunged_guids, pool, 16); if (!exporter->mails_have_guids && !exporter->no_hdr_hashes) - exporter->wanted_headers = dsync_mail_get_hash_headers(box); + exporter->wanted_headers = + dsync_mail_get_hash_headers(box, exporter->hashed_headers); /* first scan transaction log and save any expunges and flag changes */ dsync_mailbox_export_log_scan(exporter, log_scan); diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h index 11fa954230..67e3216ad4 100644 --- a/src/doveadm/dsync/dsync-mailbox-export.h +++ b/src/doveadm/dsync/dsync-mailbox-export.h @@ -15,7 +15,8 @@ dsync_mailbox_export_init(struct mailbox *box, struct dsync_transaction_log_scan *log_scan, uint32_t last_common_uid, enum dsync_mailbox_exporter_flags flags, - unsigned int hdr_hash_version); + unsigned int hdr_hash_version, + const char *const *hashed_headers); /* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */ int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter, const struct dsync_mailbox_attribute **attr_r); diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c index 1cfdda18c7..ca07eb9b90 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.c +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -69,6 +69,8 @@ struct dsync_mailbox_importer { unsigned int hdr_hash_version; unsigned int commit_msgs_interval; + const char *const *hashed_headers; + enum mail_flags sync_flag; const char *sync_keyword; bool sync_flag_dontwant; @@ -229,7 +231,8 @@ dsync_mailbox_import_init(struct mailbox *box, const char *sync_flag, unsigned int commit_msgs_interval, enum dsync_mailbox_import_flags flags, - unsigned int hdr_hash_version) + unsigned int hdr_hash_version, + const char *const *hashed_headers) { struct dsync_mailbox_importer *importer; struct mailbox_status status; @@ -254,6 +257,8 @@ dsync_mailbox_import_init(struct mailbox *box, importer->sync_until_timestamp = sync_until_timestamp; importer->sync_max_size = sync_max_size; importer->stateful_import = importer->last_common_uid_found; + importer->hashed_headers = hashed_headers; + if (sync_flag != NULL) { if (sync_flag[0] == '-') { importer->sync_flag_dontwant = TRUE; @@ -659,6 +664,7 @@ importer_try_next_mail(struct dsync_mailbox_importer *importer, } else { if (dsync_mail_get_hdr_hash(importer->cur_mail, importer->hdr_hash_version, + importer->hashed_headers, &hdr_hash) < 0) { dsync_mail_error(importer, importer->cur_mail, "header hash"); @@ -1575,7 +1581,8 @@ dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer, } if (dsync_mail_get_hdr_hash(importer->cur_mail, - importer->hdr_hash_version, &hdr_hash) < 0) { + importer->hdr_hash_version, + importer->hashed_headers, &hdr_hash) < 0) { dsync_mail_error(importer, importer->cur_mail, "hdr-stream"); *result_r = "Error fetching header stream"; return -1; diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h index 69505d373c..6a2020eb40 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.h +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -37,7 +37,8 @@ dsync_mailbox_import_init(struct mailbox *box, const char *sync_flag, unsigned int commit_msgs_interval, enum dsync_mailbox_import_flags flags, - unsigned int hdr_hash_version); + unsigned int hdr_hash_version, + const char *const *hashed_headers); int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, const struct dsync_mailbox_attribute *attr); int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,