From: Timo Sirainen Date: Fri, 1 Jul 2016 11:30:24 +0000 (+0300) Subject: doveadm sync/backup: Added -S parameter to skip too large mails. X-Git-Tag: 2.2.26~368 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dd053332ed3c3528715c33802eb9673a831b636f;p=thirdparty%2Fdovecot%2Fcore.git doveadm sync/backup: Added -S parameter to skip too large mails. --- diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c index a0a0b3287c..10577d4b55 100644 --- a/src/doveadm/doveadm-dsync.c +++ b/src/doveadm/doveadm-dsync.c @@ -67,6 +67,7 @@ struct dsync_cmd_context { ARRAY_TYPE(const_string) exclude_mailboxes; ARRAY_TYPE(const_string) namespace_prefixes; time_t sync_since_timestamp; + uoff_t sync_max_size; unsigned int io_timeout_secs; const char *remote_name; @@ -566,6 +567,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) "%s ", net_ip2addr(&_ctx->cur_client_ip)); } set.sync_since_timestamp = ctx->sync_since_timestamp; + set.sync_max_size = ctx->sync_max_size; set.sync_box = ctx->mailbox; set.sync_flag = ctx->sync_flags; set.virtual_all_box = ctx->virtual_all_box; @@ -1036,6 +1038,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0) i_fatal("Invalid -t parameter: %s", optarg); break; + case 'S': + if (settings_parse_size(optarg, &ctx->sync_max_size) < 0) + i_fatal("Invalid -S parameter: %s", optarg); + break; case 'T': if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0) i_fatal("Invalid -T parameter: %s", optarg); diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index c7ffcd080b..37e3da65aa 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -233,6 +233,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, remote_dsync_box->highest_modseq, remote_dsync_box->highest_pvt_modseq, brain->sync_since_timestamp, + brain->sync_max_size, brain->sync_flag, import_flags); } @@ -322,6 +323,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; if (brain->sync_since_timestamp > 0) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS; + if (brain->sync_max_size > 0) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES; if (brain->hdr_hash_v2) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2; if (remote_dsync_box->messages_count == 0) { diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h index d1650d28a4..17834a91ce 100644 --- a/src/doveadm/dsync/dsync-brain-private.h +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -58,6 +58,7 @@ struct dsync_brain { const char *const *exclude_mailboxes; enum dsync_brain_sync_type sync_type; time_t sync_since_timestamp; + uoff_t sync_max_size; const char *sync_flag; char alt_char; diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index d69f486462..cc0b67b607 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -192,6 +192,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : set->mailbox_alt_char; brain->sync_since_timestamp = set->sync_since_timestamp; + brain->sync_max_size = set->sync_max_size; brain->sync_flag = p_strdup(brain->pool, set->sync_flag); brain->sync_box = p_strdup(brain->pool, set->sync_box); brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : @@ -230,6 +231,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, ibc_set.virtual_all_box = set->virtual_all_box; ibc_set.exclude_mailboxes = set->exclude_mailboxes; ibc_set.sync_since_timestamp = set->sync_since_timestamp; + ibc_set.sync_max_size = set->sync_max_size; ibc_set.sync_flags = set->sync_flag; memcpy(ibc_set.sync_box_guid, set->sync_box_guid, sizeof(ibc_set.sync_box_guid)); @@ -485,6 +487,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes); brain->sync_since_timestamp = ibc_set->sync_since_timestamp; + brain->sync_max_size = ibc_set->sync_max_size; brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags); memcpy(brain->sync_box_guid, ibc_set->sync_box_guid, sizeof(brain->sync_box_guid)); diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h index 3f8db7e3a6..430cdf8d04 100644 --- a/src/doveadm/dsync/dsync-brain.h +++ b/src/doveadm/dsync/dsync-brain.h @@ -69,6 +69,8 @@ struct dsync_brain_settings { char mailbox_alt_char; /* Sync only mails with received timestamp at least this high. */ time_t sync_since_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; /* Sync only mails which contains / doesn't contain this flag. '-' at the beginning means this flag must not exist. */ const char *sync_flag; diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c index c8a1841d56..da9cc332cc 100644 --- a/src/doveadm/dsync/dsync-ibc-pipe.c +++ b/src/doveadm/dsync/dsync-ibc-pipe.c @@ -180,6 +180,7 @@ dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc, memcpy(item->u.set.sync_box_guid, set->sync_box_guid, sizeof(item->u.set.sync_box_guid)); item->u.set.sync_since_timestamp = set->sync_since_timestamp; + item->u.set.sync_max_size = set->sync_max_size; item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags); } diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c index aaee86ebc4..b605e043df 100644 --- a/src/doveadm/dsync/dsync-ibc-stream.c +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -77,7 +77,8 @@ static const struct { "debug sync_visible_namespaces exclude_mailboxes " "send_mail_requests backup_send backup_recv lock_timeout " "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote " - "no_notify sync_since_timestamp sync_flags virtual_all_box" + "no_notify sync_since_timestamp sync_max_size sync_flags " + "virtual_all_box" }, { .name = "mailbox_state", .chr = 'S', @@ -710,6 +711,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, dsync_serializer_encode_add(encoder, "sync_since_timestamp", t_strdup_printf("%ld", (long)set->sync_since_timestamp)); } + if (set->sync_max_size > 0) { + dsync_serializer_encode_add(encoder, "sync_max_size", + t_strdup_printf("%llu", (unsigned long long)set->sync_max_size)); + } if (set->sync_flags != NULL) { dsync_serializer_encode_add(encoder, "sync_flags", set->sync_flags); @@ -822,6 +827,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc, return DSYNC_IBC_RECV_RET_TRYAGAIN; } } + if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) { + if (str_to_uoff(value, &set->sync_max_size) < 0 || + set->sync_max_size == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_max_size: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } if (dsync_deserializer_decode_try(decoder, "sync_flags", &value)) set->sync_flags = p_strdup(pool, value); if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value)) @@ -1634,6 +1647,10 @@ dsync_ibc_stream_send_change(struct dsync_ibc *_ibc, dsync_serializer_encode_add(encoder, "received_timestamp", t_strdup_printf("%lx", (unsigned long)change->received_timestamp)); } + if (change->virtual_size > 0) { + dsync_serializer_encode_add(encoder, "virtual_size", + t_strdup_printf("%llx", (unsigned long long)change->virtual_size)); + } dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); @@ -1741,6 +1758,11 @@ dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc, dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + if (dsync_deserializer_decode_try(decoder, "virtual_size", &value) && + str_to_uoff(value, &change->virtual_size) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } *change_r = change; return DSYNC_IBC_RECV_RET_OK; diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h index 66b4026cea..ec7266a8fd 100644 --- a/src/doveadm/dsync/dsync-ibc.h +++ b/src/doveadm/dsync/dsync-ibc.h @@ -58,6 +58,8 @@ struct dsync_ibc_settings { const char *const *exclude_mailboxes; /* Sync only mails with received timestamp at least this high. */ time_t sync_since_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; /* Sync only mails with specified flags. */ const char *sync_flags; diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c index 0d5ce53dbf..7e93684c05 100644 --- a/src/doveadm/dsync/dsync-mail.c +++ b/src/doveadm/dsync/dsync-mail.c @@ -160,4 +160,5 @@ void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, const_string_array_dup(pool, &src->keyword_changes, &dest_r->keyword_changes); dest_r->received_timestamp = src->received_timestamp; + dest_r->virtual_size = src->virtual_size; } diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h index 89cc143dd0..918d81cfd5 100644 --- a/src/doveadm/dsync/dsync-mail.h +++ b/src/doveadm/dsync/dsync-mail.h @@ -81,6 +81,9 @@ struct dsync_mail_change { /* Received timestamp for saves, if brain.sync_since_timestamp is set */ time_t received_timestamp; + /* Mail's size for saves if brain.sync_max_size is set, + (uoff_t)-1 otherwise. */ + uoff_t virtual_size; }; struct mailbox_header_lookup_ctx * diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c index 0267f86f2a..572b43ea18 100644 --- a/src/doveadm/dsync/dsync-mailbox-export.c +++ b/src/doveadm/dsync/dsync-mailbox-export.c @@ -63,6 +63,7 @@ struct dsync_mailbox_exporter { unsigned int minimal_dmail_fill:1; unsigned int return_all_mails:1; unsigned int export_received_timestamps:1; + unsigned int export_virtual_sizes:1; unsigned int no_hdr_hashes:1; }; @@ -284,12 +285,15 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail) const char *guid, *hdr_hash; enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID; time_t received_timestamp = 0; + uoff_t virtual_size = (uoff_t)-1; int ret; /* update wanted fields in case we didn't already set them for the search */ if (exporter->export_received_timestamps) wanted_fields |= MAIL_FETCH_RECEIVED_DATE; + if (exporter->export_virtual_sizes) + wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE; mail_add_temp_wanted_fields(mail, wanted_fields, exporter->wanted_headers); @@ -306,12 +310,18 @@ search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail) received_timestamp = 1; } } + if (exporter->export_received_timestamps) { + if (mail_get_virtual_size(mail, &virtual_size) < 0) + return dsync_mail_error(exporter, mail, "virtual-size"); + i_assert(virtual_size != (uoff_t)-1); + } change = export_save_change_get(exporter, mail->uid); change->guid = *guid == '\0' ? "" : p_strdup(exporter->pool, guid); change->hdr_hash = p_strdup(exporter->pool, hdr_hash); change->received_timestamp = received_timestamp; + change->virtual_size = virtual_size; search_update_flag_changes(exporter, mail, change); export_add_mail_instance(exporter, change, mail->seq); @@ -508,6 +518,8 @@ dsync_mailbox_export_init(struct mailbox *box, (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0; exporter->export_received_timestamps = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0; + exporter->export_virtual_sizes = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0; exporter->hdr_hash_version = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2) ? 2 : 1; exporter->no_hdr_hashes = diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h index 02c6aa917d..364500ef54 100644 --- a/src/doveadm/dsync/dsync-mailbox-export.h +++ b/src/doveadm/dsync/dsync-mailbox-export.h @@ -7,7 +7,8 @@ enum dsync_mailbox_exporter_flags { DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04, DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08, DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2 = 0x10, - DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES = 0x20 + DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES = 0x20, + DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES = 0x40, }; struct dsync_mailbox_exporter * diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c index 52858ab229..78fb2f7d54 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.c +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -62,6 +62,7 @@ struct dsync_mailbox_importer { uint32_t remote_first_recent_uid; uint64_t remote_highest_modseq, remote_highest_pvt_modseq; time_t sync_since_timestamp; + uoff_t sync_max_size; enum mailbox_transaction_flags transaction_flags; unsigned int hdr_hash_version; @@ -216,7 +217,8 @@ dsync_mailbox_import_init(struct mailbox *box, uint32_t remote_first_recent_uid, uint64_t remote_highest_modseq, uint64_t remote_highest_pvt_modseq, - time_t sync_since_timestamp, const char *sync_flag, + time_t sync_since_timestamp, uoff_t sync_max_size, + const char *sync_flag, enum dsync_mailbox_import_flags flags) { struct dsync_mailbox_importer *importer; @@ -239,6 +241,7 @@ dsync_mailbox_import_init(struct mailbox *box, importer->remote_highest_modseq = remote_highest_modseq; importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq; importer->sync_since_timestamp = sync_since_timestamp; + importer->sync_max_size = sync_max_size; importer->stateful_import = importer->last_common_uid_found; if (sync_flag != NULL) { if (sync_flag[0] == '-') { @@ -1349,6 +1352,14 @@ dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer, return FALSE; } } + if (importer->sync_max_size > 0) { + i_assert(change->virtual_size != (uoff_t)-1); + if (change->virtual_size < importer->sync_max_size) { + /* mail is too large - skip it */ + *result_r = "Ignoring missing local mail with too large size"; + return FALSE; + } + } if (importer->sync_flag != 0) { bool have_flag = (change->final_flags & importer->sync_flag) != 0; @@ -1634,9 +1645,9 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, { int ret; - i_assert(importer->sync_since_timestamp == 0 || - change->received_timestamp > 0 || - change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + ((change->received_timestamp > 0 || importer->sync_since_timestamp == 0) && + (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0))); /* try to find the matching local mail */ if (!importer_next_mail(importer, change->uid)) { diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h index a0492495d4..ce6d09c4f4 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.h +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -32,7 +32,8 @@ dsync_mailbox_import_init(struct mailbox *box, uint32_t remote_first_recent_uid, uint64_t remote_highest_modseq, uint64_t remote_highest_pvt_modseq, - time_t sync_since_timestamp, const char *sync_flag, + time_t sync_since_timestamp, uoff_t sync_max_size, + const char *sync_flag, enum dsync_mailbox_import_flags flags); int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, const struct dsync_mailbox_attribute *attr);