#include "settings-parser.h"
#include "master-service.h"
#include "master-service-ssl-settings.h"
+#include "mail-storage.h"
#include "mail-storage-service.h"
#include "mail-user.h"
#include "mail-namespace.h"
#include <ctype.h>
#include <sys/wait.h>
-#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:"
+#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:t:Ux:"
#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
/* The broken_char is mainly set to get a proper error message when trying to
convert a mailbox with a name that can't be used properly translated between
const char *state_input, *rawlog_path;
ARRAY_TYPE(const_string) exclude_mailboxes;
ARRAY_TYPE(const_string) namespace_prefixes;
+ time_t sync_since_timestamp;
const char *remote_name;
const char *local_location;
set.process_title_prefix = t_strdup_printf(
"%s ", net_ip2addr(&_ctx->cur_client_ip));
}
+ set.sync_since_timestamp = ctx->sync_since_timestamp;
set.sync_box = ctx->mailbox;
memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid));
set.lock_timeout_secs = ctx->lock_timeout;
ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
ctx->state_input = optarg;
break;
+ case 't':
+ if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
+ i_fatal("Invalid -t parameter: %s", optarg);
+ break;
case 'U':
ctx->replicator_notify = TRUE;
break;
remote_dsync_box->first_recent_uid,
remote_dsync_box->highest_modseq,
remote_dsync_box->highest_pvt_modseq,
+ brain->sync_since_timestamp,
import_flags);
}
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
if (brain->no_mail_prefetch)
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+ if (brain->sync_since_timestamp > 0)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
brain->box_exporter = brain->backup_recv ? NULL :
dsync_mailbox_export_init(brain->box, brain->log_scan,
guid_128_t sync_box_guid;
const char *const *exclude_mailboxes;
enum dsync_brain_sync_type sync_type;
+ time_t sync_since_timestamp;
char alt_char;
unsigned int lock_timeout;
}
brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
set->mailbox_alt_char;
+ brain->sync_since_timestamp = set->sync_since_timestamp;
brain->sync_box = p_strdup(brain->pool, set->sync_box);
brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
p_strarray_dup(brain->pool, set->exclude_mailboxes);
NULL : str_c(sync_ns_str);
ibc_set.sync_box = set->sync_box;
ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+ ibc_set.sync_since_timestamp = set->sync_since_timestamp;
memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
sizeof(ibc_set.sync_box_guid));
ibc_set.sync_type = sync_type;
brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+ brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
sizeof(brain->sync_box_guid));
i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
/* Alternative character to use in mailbox names where the original
character cannot be used. */
char mailbox_alt_char;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
/* If non-zero, use dsync lock file for this user */
unsigned int lock_timeout_secs;
.chr = 'H',
.required_keys = "hostname",
.optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
- "debug sync_visible_namespaces exclude_mailboxes "
+ "debug sync_visible_namespaces exclude_mailboxes "
"send_mail_requests backup_send backup_recv lock_timeout "
- "no_mail_sync no_backup_overwrite purge_remote"
+ "no_mail_sync no_backup_overwrite purge_remote "
+ "sync_since_timestamp"
},
{ .name = "mailbox_state",
.chr = 'S',
dsync_serializer_encode_add(encoder, "lock_timeout",
t_strdup_printf("%u", set->lock_timeout));
}
+ if (set->sync_since_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+ t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+ }
if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
dsync_serializer_encode_add(encoder, "send_mail_requests", "");
if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
}
+ if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+ if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+ set->sync_since_timestamp == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_since_timestamp: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
dsync_serializer_encode_add(encoder, "keyword_changes",
str_c(kw_str));
}
+ if (change->received_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "received_timestamp",
+ t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
+ }
dsync_serializer_encode_finish(&encoder, str);
dsync_ibc_stream_send_string(ibc, str);
array_append(&change->keyword_changes, &value, 1);
}
}
+ if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value) &&
+ str_to_time(value, &change->received_timestamp) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
*change_r = change;
return DSYNC_IBC_RECV_RET_OK;
/* Exclude these mailboxes from the sync. They can contain '*'
wildcards and be \special-use flags. */
const char *const *exclude_mailboxes;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
enum dsync_brain_sync_type sync_type;
enum dsync_brain_flags brain_flags;
dest_r->keywords_reset = src->keywords_reset;
const_string_array_dup(pool, &src->keyword_changes,
&dest_r->keyword_changes);
+ dest_r->received_timestamp = src->received_timestamp;
}
bool keywords_reset;
/* +add, -remove, =final, &add_and_final. */
ARRAY_TYPE(const_string) keyword_changes;
+
+ /* Received timestamp for saves, if brain.sync_since_timestamp is set */
+ time_t received_timestamp;
};
struct mailbox_header_lookup_ctx *
unsigned int mails_have_guids:1;
unsigned int minimal_dmail_fill:1;
unsigned int return_all_mails:1;
+ unsigned int export_received_timestamps:1;
};
static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
{
struct dsync_mail_change *change;
const char *guid, *hdr_hash;
+ enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+ time_t received_timestamp = 0;
int ret;
/* update wanted fields in case we didn't already set them for the
search */
- mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID,
+ if (exporter->export_received_timestamps)
+ wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+ mail_add_temp_wanted_fields(mail, wanted_fields,
exporter->wanted_headers);
/* If message is already expunged here, just skip it */
if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
return ret;
+ if (exporter->export_received_timestamps) {
+ if (mail_get_received_date(mail, &received_timestamp) < 0)
+ return dsync_mail_error(exporter, mail, "received-time");
+ if (received_timestamp == 0) {
+ /* don't allow timestamps to be zero. we want to have
+ asserts verify that the timestamp is set properly. */
+ received_timestamp = 1;
+ }
+ }
+
change = export_save_change_get(exporter, mail->uid);
change->guid = *guid == '\0' ? "" :
p_strdup(exporter->pool, guid);
change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+ change->received_timestamp = received_timestamp;
search_update_flag_changes(exporter, mail, change);
export_add_mail_instance(exporter, change, mail->seq);
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
exporter->minimal_dmail_fill =
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+ exporter->export_received_timestamps =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
p_array_init(&exporter->requested_uids, pool, 16);
p_array_init(&exporter->search_uids, pool, 16);
hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
enum dsync_mailbox_exporter_flags {
DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01,
DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02,
- DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04
+ DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04,
+ DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08
};
struct dsync_mailbox_exporter *
uint32_t remote_uid_next;
uint32_t remote_first_recent_uid;
uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+ time_t sync_since_timestamp;
struct mailbox_transaction_context *trans, *ext_trans;
struct mail_search_context *search_ctx;
uint32_t remote_first_recent_uid,
uint64_t remote_highest_modseq,
uint64_t remote_highest_pvt_modseq,
+ time_t sync_since_timestamp,
enum dsync_mailbox_import_flags flags)
{
struct dsync_mailbox_importer *importer;
importer->remote_first_recent_uid = remote_first_recent_uid;
importer->remote_highest_modseq = remote_highest_modseq;
importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+ importer->sync_since_timestamp = sync_since_timestamp;
importer->stateful_import = importer->last_common_uid_found;
hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
dsync_mailbox_import_flag_change(importer, change);
return;
}
+ if (importer->sync_since_timestamp > 0) {
+ i_assert(change->received_timestamp > 0);
+ if (change->received_timestamp < importer->sync_since_timestamp) {
+ /* mail has too old timestamp - skip it */
+ return;
+ }
+ }
save = p_new(importer->pool, struct dsync_mail_change, 1);
dsync_mail_change_dup(importer->pool, change, save);
{
int ret;
+ i_assert(importer->sync_since_timestamp == 0 ||
+ change->received_timestamp > 0 ||
+ change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+
/* try to find the matching local mail */
if (!importer_next_mail(importer, change->uid)) {
/* no more local mails. we can still try to match
return;
}
i_assert(change->guid != NULL);
- if (importer->local_uid_next <= change->uid) {
+ if (importer->sync_since_timestamp > 0 &&
+ change->received_timestamp < importer->sync_since_timestamp) {
+ *result_r = "Ignoring missing local mail with too old timestamp";
+ } else if (importer->local_uid_next <= change->uid) {
dsync_mailbox_common_uid_found(importer);
*result_r = "Mail's UID is above local UIDNEXT";
} else if (importer->revert_local_changes) {
}
return;
}
+ /* mail exists remotely, but doesn't exist locally. */
+ if (importer->sync_since_timestamp > 0 &&
+ change->received_timestamp < importer->sync_since_timestamp) {
+ *result_r = "Ignoring missing local mail with too old timestamp";
+ return;
+ }
if (importer->revert_local_changes &&
change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
dsync_mailbox_revert_missing(importer, change);
uint32_t remote_first_recent_uid,
uint64_t remote_highest_modseq,
uint64_t remote_highest_pvt_modseq,
+ time_t sync_since_timestamp,
enum dsync_mailbox_import_flags flags);
int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
const struct dsync_mailbox_attribute *attr);