doveadm sync/backup: Added -S <max size> parameter to skip too large mails.

diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
index a0a0b3287cb3ac7d953033574ecf0798f1778a35..10577d4b55eeb0a5c7371a399cf0866d4be0afc6 100644 (file)
--- a/src/doveadm/doveadm-dsync.c
+++ b/src/doveadm/doveadm-dsync.c
@@ -67,6 +67,7 @@ struct dsync_cmd_context {
        ARRAY_TYPE(const_string) exclude_mailboxes;
        ARRAY_TYPE(const_string) namespace_prefixes;
        time_t sync_since_timestamp;
+       uoff_t sync_max_size;
        unsigned int io_timeout_secs;
 
        const char *remote_name;
@@ -566,6 +567,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
                        "%s ", net_ip2addr(&_ctx->cur_client_ip));
        }
        set.sync_since_timestamp = ctx->sync_since_timestamp;
+       set.sync_max_size = ctx->sync_max_size;
        set.sync_box = ctx->mailbox;
        set.sync_flag = ctx->sync_flags;
        set.virtual_all_box = ctx->virtual_all_box;
@@ -1036,6 +1038,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
                if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
                        i_fatal("Invalid -t parameter: %s", optarg);
                break;
+       case 'S':
+               if (settings_parse_size(optarg, &ctx->sync_max_size) < 0)
+                       i_fatal("Invalid -S parameter: %s", optarg);
+               break;
        case 'T':
                if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0)
                        i_fatal("Invalid -T parameter: %s", optarg);

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index c7ffcd080b475ec6d60f4720fa529788a7785d9f..37e3da65aa40e99a8be036cd2aa3898739f94d59 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -233,6 +233,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
                                          remote_dsync_box->highest_modseq,
                                          remote_dsync_box->highest_pvt_modseq,
                                          brain->sync_since_timestamp,
+                                         brain->sync_max_size,
                                          brain->sync_flag,
                                          import_flags);
 }
@@ -322,6 +323,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
        if (brain->sync_since_timestamp > 0)
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
+       if (brain->sync_max_size > 0)
+               exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;
        if (brain->hdr_hash_v2)
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2;
        if (remote_dsync_box->messages_count == 0) {

diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index d1650d28a45a818108217503f81d5caba0b7a081..17834a91ce130773a66eeb1691de09a1996be4af 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -58,6 +58,7 @@ struct dsync_brain {
        const char *const *exclude_mailboxes;
        enum dsync_brain_sync_type sync_type;
        time_t sync_since_timestamp;
+       uoff_t sync_max_size;
        const char *sync_flag;
        char alt_char;
 

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index d69f486462f49684cbde2ba9433505ec65d4d119..cc0b67b607cfba2b354348ba038c9a38def806be 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -192,6 +192,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
        brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
                set->mailbox_alt_char;
        brain->sync_since_timestamp = set->sync_since_timestamp;
+       brain->sync_max_size = set->sync_max_size;
        brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
        brain->sync_box = p_strdup(brain->pool, set->sync_box);
        brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
@@ -230,6 +231,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
        ibc_set.virtual_all_box = set->virtual_all_box;
        ibc_set.exclude_mailboxes = set->exclude_mailboxes;
        ibc_set.sync_since_timestamp = set->sync_since_timestamp;
+       ibc_set.sync_max_size = set->sync_max_size;
        ibc_set.sync_flags = set->sync_flag;
        memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
               sizeof(ibc_set.sync_box_guid));
@@ -485,6 +487,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
        brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
                p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
        brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
+       brain->sync_max_size = ibc_set->sync_max_size;
        brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
        memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
               sizeof(brain->sync_box_guid));

diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 3f8db7e3a621df8fdeca0020fe1b58b4c4bbe8dc..430cdf8d04ecf1143f41d99ad504ce8c8c1a6c33 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -69,6 +69,8 @@ struct dsync_brain_settings {
        char mailbox_alt_char;
        /* Sync only mails with received timestamp at least this high. */
        time_t sync_since_timestamp;
+       /* Don't sync mails larger than this. */
+       uoff_t sync_max_size;
        /* Sync only mails which contains / doesn't contain this flag.
           '-' at the beginning means this flag must not exist. */
        const char *sync_flag;

diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c
index c8a1841d5672ff858a678e02cbad1d9aecf626d7..da9cc332cc9745ce7fa139017d02631336e20d59 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc-pipe.c
+++ b/src/doveadm/dsync/dsync-ibc-pipe.c
@@ -180,6 +180,7 @@ dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc,
        memcpy(item->u.set.sync_box_guid, set->sync_box_guid,
               sizeof(item->u.set.sync_box_guid));
        item->u.set.sync_since_timestamp = set->sync_since_timestamp;
+       item->u.set.sync_max_size = set->sync_max_size;
        item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags);
 }
 

diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index aaee86ebc4db9e2ba75aa194692b58062a2e38c8..b605e043df408b74d6bbb7c5999b5d9ed7ff7059 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -77,7 +77,8 @@ static const struct {
                "debug sync_visible_namespaces exclude_mailboxes  "
                "send_mail_requests backup_send backup_recv lock_timeout "
                "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote "
-               "no_notify sync_since_timestamp sync_flags virtual_all_box"
+               "no_notify sync_since_timestamp sync_max_size sync_flags "
+               "virtual_all_box"
        },
        { .name = "mailbox_state",
          .chr = 'S',
@@ -710,6 +711,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "sync_since_timestamp",
                        t_strdup_printf("%ld", (long)set->sync_since_timestamp));
        }
+       if (set->sync_max_size > 0) {
+               dsync_serializer_encode_add(encoder, "sync_max_size",
+                       t_strdup_printf("%llu", (unsigned long long)set->sync_max_size));
+       }
        if (set->sync_flags != NULL) {
                dsync_serializer_encode_add(encoder, "sync_flags",
                                            set->sync_flags);
@@ -822,6 +827,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
                        return DSYNC_IBC_RECV_RET_TRYAGAIN;
                }
        }
+       if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) {
+               if (str_to_uoff(value, &set->sync_max_size) < 0 ||
+                   set->sync_max_size == 0) {
+                       dsync_ibc_input_error(ibc, decoder,
+                               "Invalid sync_max_size: %s", value);
+                       return DSYNC_IBC_RECV_RET_TRYAGAIN;
+               }
+       }
        if (dsync_deserializer_decode_try(decoder, "sync_flags", &value))
                set->sync_flags = p_strdup(pool, value);
        if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
@@ -1634,6 +1647,10 @@ dsync_ibc_stream_send_change(struct dsync_ibc *_ibc,
                dsync_serializer_encode_add(encoder, "received_timestamp",
                        t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
        }
+       if (change->virtual_size > 0) {
+               dsync_serializer_encode_add(encoder, "virtual_size",
+                       t_strdup_printf("%llx", (unsigned long long)change->virtual_size));
+       }
 
        dsync_serializer_encode_finish(&encoder, str);
        dsync_ibc_stream_send_string(ibc, str);
@@ -1741,6 +1758,11 @@ dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc,
                dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
                return DSYNC_IBC_RECV_RET_TRYAGAIN;
        }
+       if (dsync_deserializer_decode_try(decoder, "virtual_size", &value) &&
+           str_to_uoff(value, &change->virtual_size) < 0) {
+               dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size");
+               return DSYNC_IBC_RECV_RET_TRYAGAIN;
+       }
 
        *change_r = change;
        return DSYNC_IBC_RECV_RET_OK;

diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
index 66b4026ceaf666643efe56dfd23b0a992070e905..ec7266a8fd274351829ac68e286f5d0fe32762de 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc.h
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -58,6 +58,8 @@ struct dsync_ibc_settings {
        const char *const *exclude_mailboxes;
        /* Sync only mails with received timestamp at least this high. */
        time_t sync_since_timestamp;
+       /* Don't sync mails larger than this. */
+       uoff_t sync_max_size;
        /* Sync only mails with specified flags. */
        const char *sync_flags;
 

diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
index 0d5ce53dbfeaad2bcbe9eab79fa8b50197ab30fa..7e93684c05dae4d49d0474c252a7dbda5139d183 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.c
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -160,4 +160,5 @@ void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
        const_string_array_dup(pool, &src->keyword_changes,
                               &dest_r->keyword_changes);
        dest_r->received_timestamp = src->received_timestamp;
+       dest_r->virtual_size = src->virtual_size;
 }

diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
index 89cc143dd0fb136ad29c0393403c6c94533e45c7..918d81cfd5980e3b9af5a1323840f6cf07e1610d 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.h
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -81,6 +81,9 @@ struct dsync_mail_change {
 
        /* Received timestamp for saves, if brain.sync_since_timestamp is set */
        time_t received_timestamp;
+       /* Mail's size for saves if brain.sync_max_size is set,
+          (uoff_t)-1 otherwise. */
+       uoff_t virtual_size;
 };
 
 struct mailbox_header_lookup_ctx *

diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
index 0267f86f2a406eb54a136c6cbbc8cdbdc687bed2..572b43ea188d90276cd94e0fc48af811abcff0e4 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.c
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -63,6 +63,7 @@ struct dsync_mailbox_exporter {
        unsigned int minimal_dmail_fill:1;
        unsigned int return_all_mails:1;
        unsigned int export_received_timestamps:1;
+       unsigned int export_virtual_sizes:1;
        unsigned int no_hdr_hashes:1;
 };
 
@@ -284,12 +285,15 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
        const char *guid, *hdr_hash;
        enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
        time_t received_timestamp = 0;
+       uoff_t virtual_size = (uoff_t)-1;
        int ret;
 
        /* update wanted fields in case we didn't already set them for the
           search */
        if (exporter->export_received_timestamps)
                wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+       if (exporter->export_virtual_sizes)
+               wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
        mail_add_temp_wanted_fields(mail, wanted_fields,
                                    exporter->wanted_headers);
 
@@ -306,12 +310,18 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
                        received_timestamp = 1;
                }
        }
+       if (exporter->export_received_timestamps) {
+               if (mail_get_virtual_size(mail, &virtual_size) < 0)
+                       return dsync_mail_error(exporter, mail, "virtual-size");
+               i_assert(virtual_size != (uoff_t)-1);
+       }
 
        change = export_save_change_get(exporter, mail->uid);
        change->guid = *guid == '\0' ? "" :
                p_strdup(exporter->pool, guid);
        change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
        change->received_timestamp = received_timestamp;
+       change->virtual_size = virtual_size;
        search_update_flag_changes(exporter, mail, change);
 
        export_add_mail_instance(exporter, change, mail->seq);
@@ -508,6 +518,8 @@ dsync_mailbox_export_init(struct mailbox *box,
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
        exporter->export_received_timestamps =
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
+       exporter->export_virtual_sizes =
+               (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
        exporter->hdr_hash_version =
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2) ? 2 : 1;
        exporter->no_hdr_hashes =

diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
index 02c6aa917d9f99d5370e912a140239b18439e3bd..364500ef54f53a73a19afd4e1cf818486c224f68 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.h
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -7,7 +7,8 @@ enum dsync_mailbox_exporter_flags {
        DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL  = 0x04,
        DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS          = 0x08,
        DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2         = 0x10,
-       DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES       = 0x20
+       DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES       = 0x20,
+       DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES              = 0x40,
 };
 
 struct dsync_mailbox_exporter *

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 52858ab2293a61349d9e3d29724a6917ece2323c..78fb2f7d54d2b34a90c336eca73d30683fbd5ee1 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -62,6 +62,7 @@ struct dsync_mailbox_importer {
        uint32_t remote_first_recent_uid;
        uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
        time_t sync_since_timestamp;
+       uoff_t sync_max_size;
        enum mailbox_transaction_flags transaction_flags;
        unsigned int hdr_hash_version;
 
@@ -216,7 +217,8 @@ dsync_mailbox_import_init(struct mailbox *box,
                          uint32_t remote_first_recent_uid,
                          uint64_t remote_highest_modseq,
                          uint64_t remote_highest_pvt_modseq,
-                         time_t sync_since_timestamp, const char *sync_flag,
+                         time_t sync_since_timestamp, uoff_t sync_max_size,
+                         const char *sync_flag,
                          enum dsync_mailbox_import_flags flags)
 {
        struct dsync_mailbox_importer *importer;
@@ -239,6 +241,7 @@ dsync_mailbox_import_init(struct mailbox *box,
        importer->remote_highest_modseq = remote_highest_modseq;
        importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
        importer->sync_since_timestamp = sync_since_timestamp;
+       importer->sync_max_size = sync_max_size;
        importer->stateful_import = importer->last_common_uid_found;
        if (sync_flag != NULL) {
                if (sync_flag[0] == '-') {
@@ -1349,6 +1352,14 @@ dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
                        return FALSE;
                }
        }
+       if (importer->sync_max_size > 0) {
+               i_assert(change->virtual_size != (uoff_t)-1);
+               if (change->virtual_size < importer->sync_max_size) {
+                       /* mail is too large - skip it */
+                       *result_r = "Ignoring missing local mail with too large size";
+                       return FALSE;
+               }
+       }
        if (importer->sync_flag != 0) {
                bool have_flag = (change->final_flags & importer->sync_flag) != 0;
 
@@ -1634,9 +1645,9 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 {
        int ret;
 
-       i_assert(importer->sync_since_timestamp == 0 ||
-                change->received_timestamp > 0 ||
-                change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+       i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+                ((change->received_timestamp > 0 || importer->sync_since_timestamp == 0) &&
+                 (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0)));
 
        /* try to find the matching local mail */
        if (!importer_next_mail(importer, change->uid)) {

diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index a0492495d4241486dfca1e25eb735ebf5322d8b5..ce6d09c4f467f1ce37fc5242e81069d01fa2d745 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -32,7 +32,8 @@ dsync_mailbox_import_init(struct mailbox *box,
                          uint32_t remote_first_recent_uid,
                          uint64_t remote_highest_modseq,
                          uint64_t remote_highest_pvt_modseq,
-                         time_t sync_since_timestamp, const char *sync_flag,
+                         time_t sync_since_timestamp, uoff_t sync_max_size,
+                         const char *sync_flag,
                          enum dsync_mailbox_import_flags flags);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
                                   const struct dsync_mailbox_attribute *attr);