dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index aef7db5a46172b07c66d2bbe08ca297fdc7b1abf..cbbe8161c02fb7231690ff778b30e2204a0df5d6 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -304,6 +304,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
            (brain->local_dsync_box.have_save_guids ||
             (brain->backup_send && brain->local_dsync_box.have_guids)))
                exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
+       if (brain->no_mail_prefetch)
+               exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
 
        brain->box_exporter = brain->backup_recv ? NULL :
                dsync_mailbox_export_init(brain->box, brain->log_scan,

diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
index f01267becd27fea5bdf85f2f9b010f3e6d5cba6e..3114969915eb07d5f7cdff45a0047dc2ee65bc94 100644 (file)
--- a/src/doveadm/dsync/dsync-brain-private.h
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -100,6 +100,7 @@ struct dsync_brain {
        unsigned int sync_visible_namespaces:1;
        unsigned int no_mail_sync:1;
        unsigned int no_backup_overwrite:1;
+       unsigned int no_mail_prefetch:1;
        unsigned int changes_during_sync:1;
        unsigned int require_full_resync:1;
        unsigned int verbose_proctitle:1;

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 3db4b5f4280575ba5dbc4e0dda148b3af931a39b..2c2c78152249d0c80ad2997aa4621b886dd02ce1 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -137,6 +137,8 @@ dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
        brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
        brain->no_backup_overwrite =
                (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
+       brain->no_mail_prefetch =
+               (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
 }
 
 struct dsync_brain *

diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
index 3326eaa6c69cc37f461190ad7f7081bf87ffd013..8ccdf1179b1b8fa45e2905b41dc2c14a116c6a80 100644 (file)
--- a/src/doveadm/dsync/dsync-brain.h
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -22,7 +22,11 @@ enum dsync_brain_flags {
        DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE    = 0x40,
        /* Run storage purge on the remote after syncing.
           Useful with e.g. a nightly doveadm backup. */
-       DSYNC_BRAIN_FLAG_PURGE_REMOTE           = 0x80
+       DSYNC_BRAIN_FLAG_PURGE_REMOTE           = 0x80,
+       /* Don't prefetch mail bodies until they're actually needed. This works
+          only with pipe ibc. It's useful if most of the mails can be copied
+          directly within filesystem without having to read them. */
+       DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH       = 0x100
 };
 
 enum dsync_brain_sync_type {

diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 312df2a5cd36837a93c34c0c1a900407d0db1fcd..f7c7339cbf53fc4c7157469f7387c9fefed09982 100644 (file)
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -1681,6 +1681,7 @@ dsync_ibc_stream_send_mail(struct dsync_ibc *_ibc,
        struct dsync_serializer_encoder *encoder;
        string_t *str = t_str_new(128);
 
+       i_assert(!mail->minimal_fields);
        i_assert(ibc->value_output == NULL);
 
        str_append_c(str, items[ITEM_MAIL].chr);

diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
index 5fb986f6a795137b3ae7ec2848384476f3d09cbf..49fcb999ab7568123a5085da0106c2cc3f55c560 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.c
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -60,10 +60,10 @@ int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r)
        return ret;
 }
 
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
-                   const char **error_field_r)
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+                   struct dsync_mail *dmail_r, const char **error_field_r)
 {
-       const char *guid, *str;
+       const char *guid;
 
        memset(dmail_r, 0, sizeof(*dmail_r));
 
@@ -76,6 +76,22 @@ int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
 
        dmail_r->input_mail = mail;
        dmail_r->input_mail_uid = mail->uid;
+
+       if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
+               *error_field_r = "saved-date";
+               return -1;
+       }
+       if (!minimal_fill)
+               return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r);
+       dmail_r->minimal_fields = TRUE;
+       return 0;
+}
+
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+                              const char **error_field_r)
+{
+       const char *str;
+
        if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
                *error_field_r = "body";
                return -1;
@@ -97,10 +113,6 @@ int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
                *error_field_r = "received-date";
                return -1;
        }
-       if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
-               *error_field_r = "saved-date";
-               return -1;
-       }
        return 0;
 }
 

diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
index 70ccd0c5234e4077dc1a6fa86eb44eccfc12aab7..00ead4b3cd9476cec57c2a8b7cc87adb45f95bc5 100644 (file)
--- a/src/doveadm/dsync/dsync-mail.h
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -10,21 +10,24 @@ struct dsync_mail {
        /* either GUID="" or uid=0 */
        const char *guid;
        uint32_t uid;
-
-       const char *pop3_uidl;
-       unsigned int pop3_order;
-       time_t received_date;
        time_t saved_date;
 
-       /* Input stream containing the message text, or NULL if all instances
-          of the message were already expunged from this mailbox. */
-       struct istream *input;
-
        /* If non-NULL, we're syncing within the dsync process using ibc-pipe.
           This mail can be used to mailbox_copy() the mail. */
        struct mail *input_mail;
        /* Verify that this equals to input_mail->uid */
        uint32_t input_mail_uid;
+
+       /* TRUE if the following fields aren't set, because minimal_fill=TRUE
+          parameter was used. */
+       bool minimal_fields;
+
+       const char *pop3_uidl;
+       unsigned int pop3_order;
+       time_t received_date;
+       /* Input stream containing the message text, or NULL if all instances
+          of the message were already expunged from this mailbox. */
+       struct istream *input;
 };
 
 struct dsync_mail_request {
@@ -80,8 +83,10 @@ struct mailbox_header_lookup_ctx *
 dsync_mail_get_hash_headers(struct mailbox *box);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r);
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
-                   const char **error_field_r);
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+                   struct dsync_mail *dmail_r, const char **error_field_r);
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+                              const char **error_field_r);
 
 void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
                           struct dsync_mail_change *dest_r);

diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
index 31ce03c2fdf9b26528d5b51842db3bf051007c96..aff3c0549dcd7687ccd6f8046c645bfb55fd04f2 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.c
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -57,6 +57,7 @@ struct dsync_mailbox_exporter {
        unsigned int body_search_initialized:1;
        unsigned int auto_export_mails:1;
        unsigned int mails_have_guids:1;
+       unsigned int minimal_dmail_fill:1;
        unsigned int return_all_mails:1;
 };
 
@@ -474,6 +475,8 @@ dsync_mailbox_export_init(struct mailbox *box,
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
        exporter->mails_have_guids =
                (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
+       exporter->minimal_dmail_fill =
+               (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
        p_array_init(&exporter->requested_uids, pool, 16);
        p_array_init(&exporter->search_uids, pool, 16);
        hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
@@ -657,6 +660,7 @@ dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
        const struct seq_range *uids;
        char *guid;
        const char *const_guid;
+       enum mail_fetch_field wanted_fields;
        struct dsync_mail_guid_instances *instances;
        const struct seq_range *range;
        unsigned int i, count;
@@ -714,16 +718,16 @@ dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
        array_append_array(&exporter->search_uids, &exporter->requested_uids);
        array_clear(&exporter->requested_uids);
 
+       wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
+       if (!exporter->minimal_dmail_fill) {
+               wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
+                       MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
+                       MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
+       }
        exporter->search_count += seq_range_count(&sarg->value.seqset);
        exporter->search_ctx =
                mailbox_search_init(exporter->trans, search_args, NULL,
-                                   MAIL_FETCH_GUID |
-                                   MAIL_FETCH_UIDL_BACKEND |
-                                   MAIL_FETCH_POP3_ORDER |
-                                   MAIL_FETCH_RECEIVED_DATE |
-                                   MAIL_FETCH_SAVE_DATE |
-                                   MAIL_FETCH_STREAM_HEADER |
-                                   MAIL_FETCH_STREAM_BODY, NULL);
+                                   wanted_fields, NULL);
        mail_search_args_unref(&search_args);
        return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
 }
@@ -748,7 +752,8 @@ static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
        struct dsync_mail_guid_instances *instances;
        const char *error_field;
 
-       if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0)
+       if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
+                           &exporter->dsync_mail, &error_field) < 0)
                return dsync_mail_error(exporter, mail, error_field);
 
        instances = *exporter->dsync_mail.guid == '\0' ? NULL :

diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
index 5777b32f215ce52117d0bdf04b1c6171bc8f0852..8b8e04cf1a0c0a8d3886a8756cdc0a15e89906ff 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-export.h
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -3,7 +3,8 @@
 
 enum dsync_mailbox_exporter_flags {
        DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS   = 0x01,
-       DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS    = 0x02
+       DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS    = 0x02,
+       DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL  = 0x04
 };
 
 struct dsync_mailbox_exporter *

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 1d209e36ecc12d48b4c7fe2d40261d239a13401c..25ed801b3a228bc5da27831a84744416dbe2bda0 100644 (file)
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -1544,7 +1544,7 @@ dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
        if (!mail_set_uid(importer->mail, uid))
                return 0;
 
-       if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) {
+       if (dsync_mail_fill(importer->mail, TRUE, dmail_r, &error_field) < 0) {
                errstr = mailbox_get_last_error(importer->mail->box, &error);
                if (error == MAIL_ERROR_EXPUNGED)
                        return 0;
@@ -1905,6 +1905,17 @@ dsync_msg_try_copy(struct dsync_mailbox_importer *importer,
        return 0;
 }
 
+static void
+dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
+                                 const struct dsync_mail *mail)
+{
+       if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
+               mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
+       if (mail->pop3_order > 0)
+               mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
+       mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+}
+
 static struct mail_save_context *
 dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
                        const struct dsync_mail *mail,
@@ -1919,11 +1930,9 @@ dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
        if (mail->saved_date != 0)
                mailbox_save_set_save_date(save_ctx, mail->saved_date);
        dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
-       if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
-               mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
-       if (mail->pop3_order > 0)
-               mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
-       mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+
+       if (!mail->minimal_fields)
+               dsync_mailbox_save_set_nonminimal(save_ctx, mail);
        return save_ctx;
 }
 
@@ -1934,6 +1943,7 @@ dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
                        struct importer_new_mail **all_newmails_forcopy)
 {
        struct mail_save_context *save_ctx;
+       struct istream *input;
        ssize_t ret;
        bool save_failed = FALSE;
 
@@ -1960,22 +1970,42 @@ dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
                return;
        }
        /* fallback to saving from remote stream */
+       if (mail->minimal_fields) {
+               struct dsync_mail mail2;
+               const char *error_field;
+
+               i_assert(mail->input_mail != NULL);
+
+               if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
+                                              &error_field) < 0) {
+                       i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
+                               mailbox_get_vname(importer->box),
+                               error_field, mail->uid,
+                               mailbox_get_last_error(importer->box, NULL));
+                       importer->failed = TRUE;
+                       return;
+               }
+               dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
+               input = mail2.input;
+       } else {
+               input = mail->input;
+       }
 
-       if (mail->input == NULL) {
+       if (input == NULL) {
                /* it was just expunged in remote, skip it */
                mailbox_save_cancel(&save_ctx);
                return;
        }
 
-       i_stream_seek(mail->input, 0);
-       if (mailbox_save_begin(&save_ctx, mail->input) < 0) {
+       i_stream_seek(input, 0);
+       if (mailbox_save_begin(&save_ctx, input) < 0) {
                i_error("Mailbox %s: Saving failed: %s",
                        mailbox_get_vname(importer->box),
                        mailbox_get_last_error(importer->box, NULL));
                importer->failed = TRUE;
                return;
        }
-       while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) {
+       while ((ret = i_stream_read(input)) > 0 || ret == -2) {
                if (mailbox_save_continue(save_ctx) < 0) {
                        save_failed = TRUE;
                        ret = -1;
@@ -1984,10 +2014,10 @@ dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
        }
        i_assert(ret == -1);
 
-       if (mail->input->stream_errno != 0) {
+       if (input->stream_errno != 0) {
                i_error("Mailbox %s: read(msg input) failed: %s",
                        mailbox_get_vname(importer->box),
-                       i_stream_get_error(mail->input));
+                       i_stream_get_error(input));
                mailbox_save_cancel(&save_ctx);
                importer->failed = TRUE;
        } else if (save_failed) {
@@ -1997,7 +2027,7 @@ dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
                mailbox_save_cancel(&save_ctx);
                importer->failed = TRUE;
        } else {
-               i_assert(mail->input->eof);
+               i_assert(input->eof);
                if (mailbox_save_finish(&save_ctx) < 0) {
                        i_error("Mailbox %s: Saving failed: %s",
                                mailbox_get_vname(importer->box),