From: Aki Tuomi Date: Mon, 10 Oct 2016 07:36:52 +0000 (+0300) Subject: doveadm-sync: Add end-date support X-Git-Tag: 2.2.26~170 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=69031c6abb8f016fdfee4ca3019c73f6412b2759;p=thirdparty%2Fdovecot%2Fcore.git doveadm-sync: Add end-date support --- diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c index 494217d580..a0db38d53c 100644 --- a/src/doveadm/doveadm-dsync.c +++ b/src/doveadm/doveadm-dsync.c @@ -39,7 +39,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:T:Ux:" +#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:e:T:Ux:" #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 /* The broken_char is mainly set to get a proper error message when trying to convert a mailbox with a name that can't be used properly translated between @@ -67,6 +67,7 @@ struct dsync_cmd_context { ARRAY_TYPE(const_string) exclude_mailboxes; ARRAY_TYPE(const_string) namespace_prefixes; time_t sync_since_timestamp; + time_t sync_until_timestamp; uoff_t sync_max_size; unsigned int io_timeout_secs; @@ -567,6 +568,13 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) "%s ", net_ip2addr(&_ctx->cur_client_ip)); } set.sync_since_timestamp = ctx->sync_since_timestamp; + set.sync_until_timestamp = ctx->sync_until_timestamp; + + if (set.sync_since_timestamp > 0 && set.sync_until_timestamp > 0 && + set.sync_since_timestamp > set.sync_until_timestamp) { + i_fatal("start date is later than end date"); + } + set.sync_max_size = ctx->sync_max_size; set.sync_box = ctx->mailbox; set.sync_flag = ctx->sync_flags; @@ -1038,6 +1046,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0) i_fatal("Invalid -t parameter: %s", optarg); break; + case 'e': + if (mail_parse_human_timestamp(optarg, &ctx->sync_until_timestamp) < 0) + i_fatal("Invalid -e parameter: %s", optarg); + break; case 'I': if (settings_get_size(optarg, &ctx->sync_max_size, &error) < 0) i_fatal("Invalid -I parameter '%s': %s", optarg, error); diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index 37e3da65aa..54bc07a1da 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -233,6 +233,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, remote_dsync_box->highest_modseq, remote_dsync_box->highest_pvt_modseq, brain->sync_since_timestamp, + brain->sync_until_timestamp, brain->sync_max_size, brain->sync_flag, import_flags); @@ -321,7 +322,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; if (brain->no_mail_prefetch) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; - if (brain->sync_since_timestamp > 0) + if (brain->sync_since_timestamp > 0 || + brain->sync_until_timestamp > 0) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS; if (brain->sync_max_size > 0) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES; diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h index 17834a91ce..69fda3eb77 100644 --- a/src/doveadm/dsync/dsync-brain-private.h +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -58,6 +58,7 @@ struct dsync_brain { const char *const *exclude_mailboxes; enum dsync_brain_sync_type sync_type; time_t sync_since_timestamp; + time_t sync_until_timestamp; uoff_t sync_max_size; const char *sync_flag; char alt_char; diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index cc0b67b607..3935b5f88c 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -192,6 +192,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : set->mailbox_alt_char; brain->sync_since_timestamp = set->sync_since_timestamp; + brain->sync_until_timestamp = set->sync_until_timestamp; brain->sync_max_size = set->sync_max_size; brain->sync_flag = p_strdup(brain->pool, set->sync_flag); brain->sync_box = p_strdup(brain->pool, set->sync_box); @@ -231,6 +232,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, ibc_set.virtual_all_box = set->virtual_all_box; ibc_set.exclude_mailboxes = set->exclude_mailboxes; ibc_set.sync_since_timestamp = set->sync_since_timestamp; + ibc_set.sync_until_timestamp = set->sync_until_timestamp; ibc_set.sync_max_size = set->sync_max_size; ibc_set.sync_flags = set->sync_flag; memcpy(ibc_set.sync_box_guid, set->sync_box_guid, @@ -487,6 +489,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes); brain->sync_since_timestamp = ibc_set->sync_since_timestamp; + brain->sync_until_timestamp = ibc_set->sync_until_timestamp; brain->sync_max_size = ibc_set->sync_max_size; brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags); memcpy(brain->sync_box_guid, ibc_set->sync_box_guid, diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h index 430cdf8d04..259ade1c4f 100644 --- a/src/doveadm/dsync/dsync-brain.h +++ b/src/doveadm/dsync/dsync-brain.h @@ -69,6 +69,8 @@ struct dsync_brain_settings { char mailbox_alt_char; /* Sync only mails with received timestamp at least this high. */ time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; /* Don't sync mails larger than this. */ uoff_t sync_max_size; /* Sync only mails which contains / doesn't contain this flag. diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c index da9cc332cc..6f8d12ac47 100644 --- a/src/doveadm/dsync/dsync-ibc-pipe.c +++ b/src/doveadm/dsync/dsync-ibc-pipe.c @@ -180,6 +180,7 @@ dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc, memcpy(item->u.set.sync_box_guid, set->sync_box_guid, sizeof(item->u.set.sync_box_guid)); item->u.set.sync_since_timestamp = set->sync_since_timestamp; + item->u.set.sync_until_timestamp = set->sync_until_timestamp; item->u.set.sync_max_size = set->sync_max_size; item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags); } diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c index bff77c07d0..650bc3b62f 100644 --- a/src/doveadm/dsync/dsync-ibc-stream.c +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -77,7 +77,7 @@ static const struct { "debug sync_visible_namespaces exclude_mailboxes " "send_mail_requests backup_send backup_recv lock_timeout " "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote " - "no_notify sync_since_timestamp sync_max_size sync_flags " + "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp" "virtual_all_box" }, { .name = "mailbox_state", @@ -711,6 +711,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, dsync_serializer_encode_add(encoder, "sync_since_timestamp", t_strdup_printf("%ld", (long)set->sync_since_timestamp)); } + if (set->sync_until_timestamp > 0) { + dsync_serializer_encode_add(encoder, "sync_until_timestamp", + t_strdup_printf("%ld", (long)set->sync_since_timestamp)); + } if (set->sync_max_size > 0) { dsync_serializer_encode_add(encoder, "sync_max_size", t_strdup_printf("%llu", (unsigned long long)set->sync_max_size)); @@ -827,6 +831,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc, return DSYNC_IBC_RECV_RET_TRYAGAIN; } } + if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) { + if (str_to_time(value, &set->sync_until_timestamp) < 0 || + set->sync_until_timestamp == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_until_timestamp: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) { if (str_to_uoff(value, &set->sync_max_size) < 0 || set->sync_max_size == 0) { diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h index ec7266a8fd..db02a04aae 100644 --- a/src/doveadm/dsync/dsync-ibc.h +++ b/src/doveadm/dsync/dsync-ibc.h @@ -58,6 +58,8 @@ struct dsync_ibc_settings { const char *const *exclude_mailboxes; /* Sync only mails with received timestamp at least this high. */ time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; /* Don't sync mails larger than this. */ uoff_t sync_max_size; /* Sync only mails with specified flags. */ diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h index 918d81cfd5..fbf602048a 100644 --- a/src/doveadm/dsync/dsync-mail.h +++ b/src/doveadm/dsync/dsync-mail.h @@ -79,7 +79,7 @@ struct dsync_mail_change { /* +add, -remove, =final, &add_and_final. */ ARRAY_TYPE(const_string) keyword_changes; - /* Received timestamp for saves, if brain.sync_since_timestamp is set */ + /* Received timestamp for saves, if brain.sync_since/until_timestamp is set */ time_t received_timestamp; /* Mail's size for saves if brain.sync_max_size is set, (uoff_t)-1 otherwise. */ diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c index 78fb2f7d54..c7e853f147 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.c +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -62,6 +62,7 @@ struct dsync_mailbox_importer { uint32_t remote_first_recent_uid; uint64_t remote_highest_modseq, remote_highest_pvt_modseq; time_t sync_since_timestamp; + time_t sync_until_timestamp; uoff_t sync_max_size; enum mailbox_transaction_flags transaction_flags; unsigned int hdr_hash_version; @@ -217,7 +218,9 @@ dsync_mailbox_import_init(struct mailbox *box, uint32_t remote_first_recent_uid, uint64_t remote_highest_modseq, uint64_t remote_highest_pvt_modseq, - time_t sync_since_timestamp, uoff_t sync_max_size, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, const char *sync_flag, enum dsync_mailbox_import_flags flags) { @@ -241,6 +244,7 @@ dsync_mailbox_import_init(struct mailbox *box, importer->remote_highest_modseq = remote_highest_modseq; importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq; importer->sync_since_timestamp = sync_since_timestamp; + importer->sync_until_timestamp = sync_until_timestamp; importer->sync_max_size = sync_max_size; importer->stateful_import = importer->last_common_uid_found; if (sync_flag != NULL) { @@ -1352,6 +1356,14 @@ dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer, return FALSE; } } + if (importer->sync_until_timestamp > 0) { + i_assert(change->received_timestamp > 0); + if (change->received_timestamp > importer->sync_until_timestamp) { + /* mail has too new timestamp - skip it */ + *result_r = "Ignoring missing local mail with too new timestamp"; + return FALSE; + } + } if (importer->sync_max_size > 0) { i_assert(change->virtual_size != (uoff_t)-1); if (change->virtual_size < importer->sync_max_size) { @@ -1646,7 +1658,9 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, int ret; i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || - ((change->received_timestamp > 0 || importer->sync_since_timestamp == 0) && + ((change->received_timestamp > 0 || + (importer->sync_since_timestamp == 0 && + importer->sync_until_timestamp == 0)) && (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0))); /* try to find the matching local mail */ diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h index ce6d09c4f4..28f388b11a 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.h +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -32,7 +32,9 @@ dsync_mailbox_import_init(struct mailbox *box, uint32_t remote_first_recent_uid, uint64_t remote_highest_modseq, uint64_t remote_highest_pvt_modseq, - time_t sync_since_timestamp, uoff_t sync_max_size, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, const char *sync_flag, enum dsync_mailbox_import_flags flags); int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,