From: Timo Sirainen <tss@iki.fi>
Date: Mon, 19 Jan 2015 21:43:37 +0000 (+0200)
Subject: dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
X-Git-Tag: 2.2.16.rc1~129
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3561c7bb472a78af74d755219cc0fc71c85ff5c2;p=thirdparty%2Fdovecot%2Fcore.git

dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
If one side has old mails that don't exist on the other side, they are
ignored (not synced and not deleted).
---

diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
index abbc01f42f..46f673fd30 100644
--- a/src/doveadm/doveadm-dsync.c
+++ b/src/doveadm/doveadm-dsync.c
@@ -16,6 +16,7 @@
 #include "settings-parser.h"
 #include "master-service.h"
 #include "master-service-ssl-settings.h"
+#include "mail-storage.h"
 #include "mail-storage-service.h"
 #include "mail-user.h"
 #include "mail-namespace.h"
@@ -36,7 +37,7 @@
 #include <ctype.h>
 #include <sys/wait.h>
 
-#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:"
+#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:t:Ux:"
 #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
 /* The broken_char is mainly set to get a proper error message when trying to
    convert a mailbox with a name that can't be used properly translated between
@@ -59,6 +60,7 @@ struct dsync_cmd_context {
 	const char *state_input, *rawlog_path;
 	ARRAY_TYPE(const_string) exclude_mailboxes;
 	ARRAY_TYPE(const_string) namespace_prefixes;
+	time_t sync_since_timestamp;
 
 	const char *remote_name;
 	const char *local_location;
@@ -539,6 +541,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
 		set.process_title_prefix = t_strdup_printf(
 			"%s ", net_ip2addr(&_ctx->cur_client_ip));
 	}
+	set.sync_since_timestamp = ctx->sync_since_timestamp;
 	set.sync_box = ctx->mailbox;
 	memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid));
 	set.lock_timeout_secs = ctx->lock_timeout;
@@ -960,6 +963,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
 			ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
 		ctx->state_input = optarg;
 		break;
+	case 't':
+		if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
+			i_fatal("Invalid -t parameter: %s", optarg);
+		break;
 	case 'U':
 		ctx->replicator_notify = TRUE;
 		break;
diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index 0a78fbc04a..fad0c715b0 100644
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -220,6 +220,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
 					  remote_dsync_box->first_recent_uid,
 					  remote_dsync_box->highest_modseq,
 					  remote_dsync_box->highest_pvt_modseq,
+					  brain->sync_since_timestamp,
 					  import_flags);
 }
 
@@ -306,6 +307,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
 	if (brain->no_mail_prefetch)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+	if (brain->sync_since_timestamp > 0)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
 
 	brain->box_exporter = brain->backup_recv ? NULL :
 		dsync_mailbox_export_init(brain->box, brain->log_scan,
diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index 3114969915..84f96acf7b 100644
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -55,6 +55,7 @@ struct dsync_brain {
 	guid_128_t sync_box_guid;
 	const char *const *exclude_mailboxes;
 	enum dsync_brain_sync_type sync_type;
+	time_t sync_since_timestamp;
 	char alt_char;
 
 	unsigned int lock_timeout;
diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index cd7a925302..902b9a5b7f 100644
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -175,6 +175,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
 	}
 	brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
 		set->mailbox_alt_char;
+	brain->sync_since_timestamp = set->sync_since_timestamp;
 	brain->sync_box = p_strdup(brain->pool, set->sync_box);
 	brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
 		p_strarray_dup(brain->pool, set->exclude_mailboxes);
@@ -207,6 +208,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
 		NULL : str_c(sync_ns_str);
 	ibc_set.sync_box = set->sync_box;
 	ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+	ibc_set.sync_since_timestamp = set->sync_since_timestamp;
 	memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
 	       sizeof(ibc_set.sync_box_guid));
 	ibc_set.sync_type = sync_type;
@@ -452,6 +454,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
 	brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
 	brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
 		p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+	brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
 	memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
 	       sizeof(brain->sync_box_guid));
 	i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 8ccdf1179b..a2ac204b52 100644
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -55,6 +55,8 @@ struct dsync_brain_settings {
 	/* Alternative character to use in mailbox names where the original
 	   character cannot be used. */
 	char mailbox_alt_char;
+	/* Sync only mails with received timestamp at least this high. */
+	time_t sync_since_timestamp;
 
 	/* If non-zero, use dsync lock file for this user */
 	unsigned int lock_timeout_secs;
diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 78807bb542..0c12a38a33 100644
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -73,9 +73,10 @@ static const struct {
 	  .chr = 'H',
 	  .required_keys = "hostname",
 	  .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
-	  	"debug sync_visible_namespaces exclude_mailboxes "
+	  	"debug sync_visible_namespaces exclude_mailboxes  "
 	  	"send_mail_requests backup_send backup_recv lock_timeout "
-	  	"no_mail_sync no_backup_overwrite purge_remote"
+	  	"no_mail_sync no_backup_overwrite purge_remote "
+		"sync_since_timestamp"
 	},
 	{ .name = "mailbox_state",
 	  .chr = 'S',
@@ -649,6 +650,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
 		dsync_serializer_encode_add(encoder, "lock_timeout",
 			t_strdup_printf("%u", set->lock_timeout));
 	}
+	if (set->sync_since_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+	}
 	if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
 		dsync_serializer_encode_add(encoder, "send_mail_requests", "");
 	if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
@@ -741,6 +746,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
 			return DSYNC_IBC_RECV_RET_TRYAGAIN;
 		}
 	}
+	if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+		if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+		    set->sync_since_timestamp == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_since_timestamp: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
 	if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
 		set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
 	if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
@@ -1539,6 +1552,10 @@ dsync_ibc_stream_send_change(struct dsync_ibc *_ibc,
 		dsync_serializer_encode_add(encoder, "keyword_changes",
 					    str_c(kw_str));
 	}
+	if (change->received_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "received_timestamp",
+			t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
+	}
 
 	dsync_serializer_encode_finish(&encoder, str);
 	dsync_ibc_stream_send_string(ibc, str);
@@ -1619,6 +1636,11 @@ dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc,
 			array_append(&change->keyword_changes, &value, 1);
 		}
 	}
+	if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value) &&
+	    str_to_time(value, &change->received_timestamp) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
 
 	*change_r = change;
 	return DSYNC_IBC_RECV_RET_OK;
diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
index 587754aabb..ead460ab71 100644
--- a/src/doveadm/dsync/dsync-ibc.h
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -52,6 +52,8 @@ struct dsync_ibc_settings {
 	/* Exclude these mailboxes from the sync. They can contain '*'
 	   wildcards and be \special-use flags. */
 	const char *const *exclude_mailboxes;
+	/* Sync only mails with received timestamp at least this high. */
+	time_t sync_since_timestamp;
 
 	enum dsync_brain_sync_type sync_type;
 	enum dsync_brain_flags brain_flags;
diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
index 12636d2033..d966791272 100644
--- a/src/doveadm/dsync/dsync-mail.c
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -156,4 +156,5 @@ void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
 	dest_r->keywords_reset = src->keywords_reset;
 	const_string_array_dup(pool, &src->keyword_changes,
 			       &dest_r->keyword_changes);
+	dest_r->received_timestamp = src->received_timestamp;
 }
diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
index 00ead4b3cd..cdd4167c5c 100644
--- a/src/doveadm/dsync/dsync-mail.h
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -77,6 +77,9 @@ struct dsync_mail_change {
 	bool keywords_reset;
 	/* +add, -remove, =final, &add_and_final. */
 	ARRAY_TYPE(const_string) keyword_changes;
+
+	/* Received timestamp for saves, if brain.sync_since_timestamp is set */
+	time_t received_timestamp;
 };
 
 struct mailbox_header_lookup_ctx *
diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
index 2dc524108e..95bc370c77 100644
--- a/src/doveadm/dsync/dsync-mailbox-export.c
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -59,6 +59,7 @@ struct dsync_mailbox_exporter {
 	unsigned int mails_have_guids:1;
 	unsigned int minimal_dmail_fill:1;
 	unsigned int return_all_mails:1;
+	unsigned int export_received_timestamps:1;
 };
 
 static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
@@ -271,21 +272,36 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
 {
 	struct dsync_mail_change *change;
 	const char *guid, *hdr_hash;
+	enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+	time_t received_timestamp = 0;
 	int ret;
 
 	/* update wanted fields in case we didn't already set them for the
 	   search */
-	mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID,
+	if (exporter->export_received_timestamps)
+		wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+	mail_add_temp_wanted_fields(mail, wanted_fields,
 				    exporter->wanted_headers);
 
 	/* If message is already expunged here, just skip it */
 	if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
 		return ret;
 
+	if (exporter->export_received_timestamps) {
+		if (mail_get_received_date(mail, &received_timestamp) < 0)
+			return dsync_mail_error(exporter, mail, "received-time");
+		if (received_timestamp == 0) {
+			/* don't allow timestamps to be zero. we want to have
+			   asserts verify that the timestamp is set properly. */
+			received_timestamp = 1;
+		}
+	}
+
 	change = export_save_change_get(exporter, mail->uid);
 	change->guid = *guid == '\0' ? "" :
 		p_strdup(exporter->pool, guid);
 	change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+	change->received_timestamp = received_timestamp;
 	search_update_flag_changes(exporter, mail, change);
 
 	export_add_mail_instance(exporter, change, mail->seq);
@@ -478,6 +494,8 @@ dsync_mailbox_export_init(struct mailbox *box,
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
 	exporter->minimal_dmail_fill =
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+	exporter->export_received_timestamps =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
 	p_array_init(&exporter->requested_uids, pool, 16);
 	p_array_init(&exporter->search_uids, pool, 16);
 	hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
index 8b8e04cf1a..0e76c29ec6 100644
--- a/src/doveadm/dsync/dsync-mailbox-export.h
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -4,7 +4,8 @@
 enum dsync_mailbox_exporter_flags {
 	DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS	= 0x01,
 	DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS	= 0x02,
-	DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL	= 0x04
+	DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL	= 0x04,
+	DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS		= 0x08
 };
 
 struct dsync_mailbox_exporter *
diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 2c73cf33db..f8ccffb344 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -58,6 +58,7 @@ struct dsync_mailbox_importer {
 	uint32_t remote_uid_next;
 	uint32_t remote_first_recent_uid;
 	uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+	time_t sync_since_timestamp;
 
 	struct mailbox_transaction_context *trans, *ext_trans;
 	struct mail_search_context *search_ctx;
@@ -177,6 +178,7 @@ dsync_mailbox_import_init(struct mailbox *box,
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
+			  time_t sync_since_timestamp,
 			  enum dsync_mailbox_import_flags flags)
 {
 	struct dsync_mailbox_importer *importer;
@@ -197,6 +199,7 @@ dsync_mailbox_import_init(struct mailbox *box,
 	importer->remote_first_recent_uid = remote_first_recent_uid;
 	importer->remote_highest_modseq = remote_highest_modseq;
 	importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+	importer->sync_since_timestamp = sync_since_timestamp;
 	importer->stateful_import = importer->last_common_uid_found;
 
 	hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
@@ -1228,6 +1231,13 @@ dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
 		dsync_mailbox_import_flag_change(importer, change);
 		return;
 	}
+	if (importer->sync_since_timestamp > 0) {
+		i_assert(change->received_timestamp > 0);
+		if (change->received_timestamp < importer->sync_since_timestamp) {
+			/* mail has too old timestamp - skip it */
+			return;
+		}
+	}
 
 	save = p_new(importer->pool, struct dsync_mail_change, 1);
 	dsync_mail_change_dup(importer->pool, change, save);
@@ -1460,6 +1470,10 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 {
 	int ret;
 
+	i_assert(importer->sync_since_timestamp == 0 ||
+		 change->received_timestamp > 0 ||
+		 change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+
 	/* try to find the matching local mail */
 	if (!importer_next_mail(importer, change->uid)) {
 		/* no more local mails. we can still try to match
@@ -1471,7 +1485,10 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 			return;
 		}
 		i_assert(change->guid != NULL);
-		if (importer->local_uid_next <= change->uid) {
+		if (importer->sync_since_timestamp > 0 &&
+		    change->received_timestamp < importer->sync_since_timestamp) {
+			*result_r = "Ignoring missing local mail with too old timestamp";
+		} else if (importer->local_uid_next <= change->uid) {
 			dsync_mailbox_common_uid_found(importer);
 			*result_r = "Mail's UID is above local UIDNEXT";
 		} else if (importer->revert_local_changes) {
@@ -1510,6 +1527,12 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 		}
 		return;
 	}
+	/* mail exists remotely, but doesn't exist locally. */
+	if (importer->sync_since_timestamp > 0 &&
+	    change->received_timestamp < importer->sync_since_timestamp) {
+		*result_r = "Ignoring missing local mail with too old timestamp";
+		return;
+	}
 	if (importer->revert_local_changes &&
 	    change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
 		dsync_mailbox_revert_missing(importer, change);
diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index ef983cbc2e..69d0d27886 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -26,6 +26,7 @@ dsync_mailbox_import_init(struct mailbox *box,
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
+			  time_t sync_since_timestamp,
 			  enum dsync_mailbox_import_flags flags);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
 				   const struct dsync_mailbox_attribute *attr);