]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
authorTimo Sirainen <tss@iki.fi>
Mon, 19 Jan 2015 21:43:37 +0000 (23:43 +0200)
committerTimo Sirainen <tss@iki.fi>
Mon, 19 Jan 2015 21:43:37 +0000 (23:43 +0200)
If one side has old mails that don't exist on the other side, they are
ignored (not synced and not deleted).

13 files changed:
src/doveadm/doveadm-dsync.c
src/doveadm/dsync/dsync-brain-mailbox.c
src/doveadm/dsync/dsync-brain-private.h
src/doveadm/dsync/dsync-brain.c
src/doveadm/dsync/dsync-brain.h
src/doveadm/dsync/dsync-ibc-stream.c
src/doveadm/dsync/dsync-ibc.h
src/doveadm/dsync/dsync-mail.c
src/doveadm/dsync/dsync-mail.h
src/doveadm/dsync/dsync-mailbox-export.c
src/doveadm/dsync/dsync-mailbox-export.h
src/doveadm/dsync/dsync-mailbox-import.c
src/doveadm/dsync/dsync-mailbox-import.h

index abbc01f42fa3d71e2b64cad75794d1890178bdcd..46f673fd30472877d8face176672e66f9b6c8790 100644 (file)
@@ -16,6 +16,7 @@
 #include "settings-parser.h"
 #include "master-service.h"
 #include "master-service-ssl-settings.h"
+#include "mail-storage.h"
 #include "mail-storage-service.h"
 #include "mail-user.h"
 #include "mail-namespace.h"
@@ -36,7 +37,7 @@
 #include <ctype.h>
 #include <sys/wait.h>
 
-#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:"
+#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:t:Ux:"
 #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
 /* The broken_char is mainly set to get a proper error message when trying to
    convert a mailbox with a name that can't be used properly translated between
@@ -59,6 +60,7 @@ struct dsync_cmd_context {
        const char *state_input, *rawlog_path;
        ARRAY_TYPE(const_string) exclude_mailboxes;
        ARRAY_TYPE(const_string) namespace_prefixes;
+       time_t sync_since_timestamp;
 
        const char *remote_name;
        const char *local_location;
@@ -539,6 +541,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
                set.process_title_prefix = t_strdup_printf(
                        "%s ", net_ip2addr(&_ctx->cur_client_ip));
        }
+       set.sync_since_timestamp = ctx->sync_since_timestamp;
        set.sync_box = ctx->mailbox;
        memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid));
        set.lock_timeout_secs = ctx->lock_timeout;
@@ -960,6 +963,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
                        ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
                ctx->state_input = optarg;
                break;
+       case 't':
+               if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
+                       i_fatal("Invalid -t parameter: %s", optarg);
+               break;
        case 'U':
                ctx->replicator_notify = TRUE;
                break;
index 0a78fbc04a10d943136b0f4a108078702942efe8..fad0c715b041a313c145bab45eeffc76bc8665d3 100644 (file)
@@ -220,6 +220,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
                                          remote_dsync_box->first_recent_uid,
                                          remote_dsync_box->highest_modseq,
                                          remote_dsync_box->highest_pvt_modseq,
+                                         brain->sync_since_timestamp,
                                          import_flags);
 }
 
@@ -306,6 +307,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
        if (brain->no_mail_prefetch)
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+       if (brain->sync_since_timestamp > 0)
+               exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
 
        brain->box_exporter = brain->backup_recv ? NULL :
                dsync_mailbox_export_init(brain->box, brain->log_scan,
index 3114969915eb07d5f7cdff45a0047dc2ee65bc94..84f96acf7b7d0b27f0a0a779876f2ff904cd3a9c 100644 (file)
@@ -55,6 +55,7 @@ struct dsync_brain {
        guid_128_t sync_box_guid;
        const char *const *exclude_mailboxes;
        enum dsync_brain_sync_type sync_type;
+       time_t sync_since_timestamp;
        char alt_char;
 
        unsigned int lock_timeout;
index cd7a925302eaa4fbd56a197e2342d27037cc056c..902b9a5b7f28c2407180fae65fc2acdf265017b5 100644 (file)
@@ -175,6 +175,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
        }
        brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
                set->mailbox_alt_char;
+       brain->sync_since_timestamp = set->sync_since_timestamp;
        brain->sync_box = p_strdup(brain->pool, set->sync_box);
        brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
                p_strarray_dup(brain->pool, set->exclude_mailboxes);
@@ -207,6 +208,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
                NULL : str_c(sync_ns_str);
        ibc_set.sync_box = set->sync_box;
        ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+       ibc_set.sync_since_timestamp = set->sync_since_timestamp;
        memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
               sizeof(ibc_set.sync_box_guid));
        ibc_set.sync_type = sync_type;
@@ -452,6 +454,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
        brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
        brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
                p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+       brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
        memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
               sizeof(brain->sync_box_guid));
        i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
index 8ccdf1179b1b8fa45e2905b41dc2c14a116c6a80..a2ac204b52e7f9265b9b89a8f970214825e17c35 100644 (file)
@@ -55,6 +55,8 @@ struct dsync_brain_settings {
        /* Alternative character to use in mailbox names where the original
           character cannot be used. */
        char mailbox_alt_char;
+       /* Sync only mails with received timestamp at least this high. */
+       time_t sync_since_timestamp;
 
        /* If non-zero, use dsync lock file for this user */
        unsigned int lock_timeout_secs;
index 78807bb54217cf6f4eb1f62b4791d7b67c4474e3..0c12a38a33223de3168c2e01cb6e6a7fd9876b2d 100644 (file)
@@ -73,9 +73,10 @@ static const struct {
          .chr = 'H',
          .required_keys = "hostname",
          .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
-               "debug sync_visible_namespaces exclude_mailboxes "
+               "debug sync_visible_namespaces exclude_mailboxes  "
                "send_mail_requests backup_send backup_recv lock_timeout "
-               "no_mail_sync no_backup_overwrite purge_remote"
+               "no_mail_sync no_backup_overwrite purge_remote "
+               "sync_since_timestamp"
        },
        { .name = "mailbox_state",
          .chr = 'S',
@@ -649,6 +650,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "lock_timeout",
                        t_strdup_printf("%u", set->lock_timeout));
        }
+       if (set->sync_since_timestamp > 0) {
+               dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+                       t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+       }
        if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
                dsync_serializer_encode_add(encoder, "send_mail_requests", "");
        if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
@@ -741,6 +746,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
                        return DSYNC_IBC_RECV_RET_TRYAGAIN;
                }
        }
+       if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+               if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+                   set->sync_since_timestamp == 0) {
+                       dsync_ibc_input_error(ibc, decoder,
+                               "Invalid sync_since_timestamp: %s", value);
+                       return DSYNC_IBC_RECV_RET_TRYAGAIN;
+               }
+       }
        if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
                set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
        if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
@@ -1539,6 +1552,10 @@ dsync_ibc_stream_send_change(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "keyword_changes",
                                            str_c(kw_str));
        }
+       if (change->received_timestamp > 0) {
+               dsync_serializer_encode_add(encoder, "received_timestamp",
+                       t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
+       }
 
        dsync_serializer_encode_finish(&encoder, str);
        dsync_ibc_stream_send_string(ibc, str);
@@ -1619,6 +1636,11 @@ dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc,
                        array_append(&change->keyword_changes, &value, 1);
                }
        }
+       if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value) &&
+           str_to_time(value, &change->received_timestamp) < 0) {
+               dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+               return DSYNC_IBC_RECV_RET_TRYAGAIN;
+       }
 
        *change_r = change;
        return DSYNC_IBC_RECV_RET_OK;
index 587754aabb6aa1db78a224bc38e19fb7d155071b..ead460ab71450e017001ad58d4b03c0b0f130f14 100644 (file)
@@ -52,6 +52,8 @@ struct dsync_ibc_settings {
        /* Exclude these mailboxes from the sync. They can contain '*'
           wildcards and be \special-use flags. */
        const char *const *exclude_mailboxes;
+       /* Sync only mails with received timestamp at least this high. */
+       time_t sync_since_timestamp;
 
        enum dsync_brain_sync_type sync_type;
        enum dsync_brain_flags brain_flags;
index 12636d2033cf1acc6601b19f3e26acb6684c6f9d..d9667912720efa89f033cc0ad303c9465bcad6ad 100644 (file)
@@ -156,4 +156,5 @@ void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
        dest_r->keywords_reset = src->keywords_reset;
        const_string_array_dup(pool, &src->keyword_changes,
                               &dest_r->keyword_changes);
+       dest_r->received_timestamp = src->received_timestamp;
 }
index 00ead4b3cd9476cec57c2a8b7cc87adb45f95bc5..cdd4167c5c718b1f8ba03fa799d5ecf405548a17 100644 (file)
@@ -77,6 +77,9 @@ struct dsync_mail_change {
        bool keywords_reset;
        /* +add, -remove, =final, &add_and_final. */
        ARRAY_TYPE(const_string) keyword_changes;
+
+       /* Received timestamp for saves, if brain.sync_since_timestamp is set */
+       time_t received_timestamp;
 };
 
 struct mailbox_header_lookup_ctx *
index 2dc524108e7a68e86124c3a290ac02080d60ffc3..95bc370c778eb2f85c25834e6a0b769c1db69684 100644 (file)
@@ -59,6 +59,7 @@ struct dsync_mailbox_exporter {
        unsigned int mails_have_guids:1;
        unsigned int minimal_dmail_fill:1;
        unsigned int return_all_mails:1;
+       unsigned int export_received_timestamps:1;
 };
 
 static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
@@ -271,21 +272,36 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
 {
        struct dsync_mail_change *change;
        const char *guid, *hdr_hash;
+       enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+       time_t received_timestamp = 0;
        int ret;
 
        /* update wanted fields in case we didn't already set them for the
           search */
-       mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID,
+       if (exporter->export_received_timestamps)
+               wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+       mail_add_temp_wanted_fields(mail, wanted_fields,
                                    exporter->wanted_headers);
 
        /* If message is already expunged here, just skip it */
        if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
                return ret;
 
+       if (exporter->export_received_timestamps) {
+               if (mail_get_received_date(mail, &received_timestamp) < 0)
+                       return dsync_mail_error(exporter, mail, "received-time");
+               if (received_timestamp == 0) {
+                       /* don't allow timestamps to be zero. we want to have
+                          asserts verify that the timestamp is set properly. */
+                       received_timestamp = 1;
+               }
+       }
+
        change = export_save_change_get(exporter, mail->uid);
        change->guid = *guid == '\0' ? "" :
                p_strdup(exporter->pool, guid);
        change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+       change->received_timestamp = received_timestamp;
        search_update_flag_changes(exporter, mail, change);
 
        export_add_mail_instance(exporter, change, mail->seq);
@@ -478,6 +494,8 @@ dsync_mailbox_export_init(struct mailbox *box,
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
        exporter->minimal_dmail_fill =
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+       exporter->export_received_timestamps =
+               (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
        p_array_init(&exporter->requested_uids, pool, 16);
        p_array_init(&exporter->search_uids, pool, 16);
        hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
index 8b8e04cf1a0c0a8d3886a8756cdc0a15e89906ff..0e76c29ec6a8002eb91b53ba3ae65e153655d9f0 100644 (file)
@@ -4,7 +4,8 @@
 enum dsync_mailbox_exporter_flags {
        DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS   = 0x01,
        DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS    = 0x02,
-       DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL  = 0x04
+       DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL  = 0x04,
+       DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS          = 0x08
 };
 
 struct dsync_mailbox_exporter *
index 2c73cf33db8e07b4a3014cecbd9099698ef235fb..f8ccffb3442ead64e13ef8abc692060331b85669 100644 (file)
@@ -58,6 +58,7 @@ struct dsync_mailbox_importer {
        uint32_t remote_uid_next;
        uint32_t remote_first_recent_uid;
        uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+       time_t sync_since_timestamp;
 
        struct mailbox_transaction_context *trans, *ext_trans;
        struct mail_search_context *search_ctx;
@@ -177,6 +178,7 @@ dsync_mailbox_import_init(struct mailbox *box,
                          uint32_t remote_first_recent_uid,
                          uint64_t remote_highest_modseq,
                          uint64_t remote_highest_pvt_modseq,
+                         time_t sync_since_timestamp,
                          enum dsync_mailbox_import_flags flags)
 {
        struct dsync_mailbox_importer *importer;
@@ -197,6 +199,7 @@ dsync_mailbox_import_init(struct mailbox *box,
        importer->remote_first_recent_uid = remote_first_recent_uid;
        importer->remote_highest_modseq = remote_highest_modseq;
        importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+       importer->sync_since_timestamp = sync_since_timestamp;
        importer->stateful_import = importer->last_common_uid_found;
 
        hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
@@ -1228,6 +1231,13 @@ dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
                dsync_mailbox_import_flag_change(importer, change);
                return;
        }
+       if (importer->sync_since_timestamp > 0) {
+               i_assert(change->received_timestamp > 0);
+               if (change->received_timestamp < importer->sync_since_timestamp) {
+                       /* mail has too old timestamp - skip it */
+                       return;
+               }
+       }
 
        save = p_new(importer->pool, struct dsync_mail_change, 1);
        dsync_mail_change_dup(importer->pool, change, save);
@@ -1460,6 +1470,10 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 {
        int ret;
 
+       i_assert(importer->sync_since_timestamp == 0 ||
+                change->received_timestamp > 0 ||
+                change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+
        /* try to find the matching local mail */
        if (!importer_next_mail(importer, change->uid)) {
                /* no more local mails. we can still try to match
@@ -1471,7 +1485,10 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
                        return;
                }
                i_assert(change->guid != NULL);
-               if (importer->local_uid_next <= change->uid) {
+               if (importer->sync_since_timestamp > 0 &&
+                   change->received_timestamp < importer->sync_since_timestamp) {
+                       *result_r = "Ignoring missing local mail with too old timestamp";
+               } else if (importer->local_uid_next <= change->uid) {
                        dsync_mailbox_common_uid_found(importer);
                        *result_r = "Mail's UID is above local UIDNEXT";
                } else if (importer->revert_local_changes) {
@@ -1510,6 +1527,12 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
                }
                return;
        }
+       /* mail exists remotely, but doesn't exist locally. */
+       if (importer->sync_since_timestamp > 0 &&
+           change->received_timestamp < importer->sync_since_timestamp) {
+               *result_r = "Ignoring missing local mail with too old timestamp";
+               return;
+       }
        if (importer->revert_local_changes &&
            change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
                dsync_mailbox_revert_missing(importer, change);
index ef983cbc2eb681835f5b3091204681dee5251bef..69d0d278865c800a2400b48244e6ec186b8615e8 100644 (file)
@@ -26,6 +26,7 @@ dsync_mailbox_import_init(struct mailbox *box,
                          uint32_t remote_first_recent_uid,
                          uint64_t remote_highest_modseq,
                          uint64_t remote_highest_pvt_modseq,
+                         time_t sync_since_timestamp,
                          enum dsync_mailbox_import_flags flags);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
                                   const struct dsync_mailbox_attribute *attr);