lib-storage: Added support for plugins to specify message's physical size when saving.

Changed dbox to save this value to metadata, and use when present.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c
index b4af99223247bb8519c271ca4e968c96ecab690e..e93f2aa4e59f1187918420c36ad029c3d1d0c41d 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-file.c
+++ b/src/lib-storage/index/dbox-common/dbox-file.c
@@ -310,7 +310,6 @@ int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r)
 }
 
 int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
-                             uoff_t *physical_size_r,
                              struct istream **stream_r)
 {
        uoff_t size;
@@ -334,7 +333,6 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
                *stream_r = i_stream_create_limit(file->input,
                                                  file->cur_physical_size);
        }
-       *physical_size_r = file->cur_physical_size;
        return 1;
 }
 
@@ -366,7 +364,7 @@ void dbox_file_seek_rewind(struct dbox_file *file)
 
 int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
 {
-       uoff_t offset, size;
+       uoff_t offset;
        int ret;
 
        i_assert(file->input != NULL);
@@ -391,7 +389,7 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
        }
        *last_r = FALSE;
 
-       ret = dbox_file_get_mail_stream(file, offset, &size, NULL);
+       ret = dbox_file_get_mail_stream(file, offset, NULL);
        if (*offset_r == 0)
                *offset_r = file->file_header_size;
        return ret;

diff --git a/src/lib-storage/index/dbox-common/dbox-file.h b/src/lib-storage/index/dbox-common/dbox-file.h
index c8f1030c9c8698684dc0e0f0985d81c7447ccff5..2607f01cd6b9be549a66e9612c5e7ca92c37b5c7 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-file.h
+++ b/src/lib-storage/index/dbox-common/dbox-file.h
@@ -41,6 +41,10 @@ enum dbox_metadata_key {
        DBOX_METADATA_RECEIVED_TIME     = 'R',
        /* Saved UNIX timestamp in hex */
        DBOX_METADATA_SAVE_TIME         = 'S',
+       /* Physical message size in hex. Necessary only if it differs from
+          the dbox_message_header.message_size_hex, for example because the
+          message is compressed. */
+       DBOX_METADATA_PHYSICAL_SIZE     = 'Z',
        /* Virtual message size in hex (line feeds counted as CRLF) */
        DBOX_METADATA_VIRTUAL_SIZE      = 'V',
        /* Pointer to external message data. Format is:
@@ -135,11 +139,9 @@ void dbox_file_close(struct dbox_file *file);
 int dbox_file_try_lock(struct dbox_file *file);
 void dbox_file_unlock(struct dbox_file *file);
 
-/* Seek to given offset in file and return the message's input stream
-   and physical size. Returns 1 if ok/expunged, 0 if file/offset is corrupted,
-   -1 if I/O error. */
+/* Seek to given offset in file and return the message's input stream.
+   Returns 1 if ok/expunged, 0 if file/offset is corrupted, -1 if I/O error. */
 int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
-                             uoff_t *physical_size_r,
                              struct istream **input_r);
 /* Start seeking at the beginning of the file. */
 void dbox_file_seek_rewind(struct dbox_file *file);

diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c
index e7bebceba42f54cde2f06e72fd7de409fdad2aec..e0441c7b513c542fb6ff9bdcfe1c464b1ce971ea 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-mail.c
+++ b/src/lib-storage/index/dbox-common/dbox-mail.c
@@ -40,31 +40,59 @@ int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r)
 {
        struct dbox_storage *storage =
                (struct dbox_storage *)mail->imail.mail.mail.box->storage;
-       uoff_t offset, size;
+       uoff_t offset;
 
        if (storage->v.mail_open(mail, &offset, file_r) < 0)
                return -1;
 
-       if (dbox_file_get_mail_stream(*file_r, offset, &size, NULL) <= 0)
+       if (dbox_file_get_mail_stream(*file_r, offset, NULL) <= 0)
                return -1;
        if (dbox_file_metadata_read(*file_r) <= 0)
                return -1;
+
+       if (mail->imail.data.stream != NULL) {
+               /* we just messed up mail's input stream by reading metadata */
+               i_stream_seek((*file_r)->input, offset);
+               i_stream_sync(mail->imail.data.stream);
+       }
+       return 0;
+}
+
+static int
+dbox_mail_metadata_get(struct dbox_mail *mail, enum dbox_metadata_key key,
+                      const char **value_r)
+{
+       struct dbox_file *file;
+
+       if (dbox_mail_metadata_read(mail, &file) < 0)
+               return -1;
+
+       *value_r = dbox_file_metadata_get(file, key);
        return 0;
 }
 
 int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 {
-       struct index_mail *mail = (struct index_mail *)_mail;
-       struct index_mail_data *data = &mail->data;
+       struct dbox_mail *mail = (struct dbox_mail *)_mail;
+       struct index_mail_data *data = &mail->imail.data;
        struct istream *input;
+       const char *value;
 
        if (index_mail_get_physical_size(_mail, size_r) == 0)
                return 0;
 
-       if (mail_get_stream(_mail, NULL, NULL, &input) < 0)
+       /* see if we have it in metadata */
+       if (dbox_mail_metadata_get(mail, DBOX_METADATA_PHYSICAL_SIZE,
+                                  &value) < 0)
                return -1;
 
-       i_assert(data->physical_size != (uoff_t)-1);
+       if (value != NULL)
+               data->physical_size = strtoul(value, NULL, 16);
+       else {
+               if (mail_get_stream(_mail, NULL, NULL, &input) < 0)
+                       return -1;
+               i_assert(data->physical_size != (uoff_t)-1);
+       }
        *size_r = data->physical_size;
        return 0;
 }
@@ -73,16 +101,14 @@ int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
 {
        struct dbox_mail *mail = (struct dbox_mail *)_mail;
        struct index_mail_data *data = &mail->imail.data;
-       struct dbox_file *file;
        const char *value;
 
        if (index_mail_get_cached_virtual_size(&mail->imail, size_r))
                return 0;
 
-       if (dbox_mail_metadata_read(mail, &file) < 0)
+       if (dbox_mail_metadata_get(mail, DBOX_METADATA_VIRTUAL_SIZE,
+                                  &value) < 0)
                return -1;
-
-       value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE);
        if (value == NULL)
                return index_mail_get_virtual_size(_mail, size_r);
 
@@ -95,16 +121,15 @@ int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
        struct dbox_mail *mail = (struct dbox_mail *)_mail;
        struct index_mail_data *data = &mail->imail.data;
-       struct dbox_file *file;
        const char *value;
 
        if (index_mail_get_received_date(_mail, date_r) == 0)
                return 0;
 
-       if (dbox_mail_metadata_read(mail, &file) < 0)
+       if (dbox_mail_metadata_get(mail, DBOX_METADATA_RECEIVED_TIME,
+                                  &value) < 0)
                return -1;
 
-       value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME);
        data->received_date = value == NULL ? 0 : strtoul(value, NULL, 16);
        *date_r = data->received_date;
        return 0;
@@ -150,7 +175,6 @@ dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key,
        struct index_mail *imail = &mail->imail;
        struct index_mailbox_context *ibox =
                INDEX_STORAGE_CONTEXT(imail->mail.mail.box);
-       struct dbox_file *file;
        const char *value;
        string_t *str;
 
@@ -162,10 +186,9 @@ dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key,
                return 0;
        }
 
-       if (dbox_mail_metadata_read(mail, &file) < 0)
+       if (dbox_mail_metadata_get(mail, key, &value) < 0)
                return -1;
 
-       value = dbox_file_metadata_get(file, key);
        if (value == NULL)
                value = "";
        index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx,
@@ -205,7 +228,7 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
        struct dbox_mail *mail = (struct dbox_mail *)_mail;
        struct index_mail_data *data = &mail->imail.data;
        struct istream *input;
-       uoff_t offset, size;
+       uoff_t offset;
        int ret;
 
        if (data->stream == NULL) {
@@ -213,7 +236,7 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
                        return -1;
 
                ret = dbox_file_get_mail_stream(mail->open_file, offset,
-                                               &size, &input);
+                                               &input);
                if (ret <= 0) {
                        if (ret < 0)
                                return -1;
@@ -222,7 +245,6 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
                                "%"PRIuUOFF_T, _mail->uid, offset);
                        return -1;
                }
-               data->physical_size = size;
                data->stream = input;
        }
 

diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c
index bab4c0b26904ad820229cfc2c320a240f17a6c5b..afd7ec0db56358b5674b18ab778bf320f2a66a5c 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-save.c
+++ b/src/lib-storage/index/dbox-common/dbox-save.c
@@ -94,12 +94,13 @@ void dbox_save_end(struct dbox_save_context *ctx)
 {
        struct ostream *dbox_output = ctx->dbox_output;
 
-       if (ctx->ctx.output != dbox_output) {
-               /* e.g. zlib plugin had changed this */
-               o_stream_ref(dbox_output);
-               o_stream_destroy(&ctx->ctx.output);
-               ctx->ctx.output = dbox_output;
-       }
+       if (ctx->ctx.output == dbox_output)
+               return;
+
+       /* e.g. zlib plugin had changed this */
+       o_stream_ref(dbox_output);
+       o_stream_destroy(&ctx->ctx.output);
+       ctx->ctx.output = dbox_output;
 }
 
 void dbox_save_write_metadata(struct mail_save_context *ctx,
@@ -118,6 +119,10 @@ void dbox_save_write_metadata(struct mail_save_context *ctx,
        o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr));
 
        str = t_str_new(256);
+       if (ctx->saved_physical_size != 0) {
+               str_printfa(str, "%c%llx\n", DBOX_METADATA_PHYSICAL_SIZE,
+                           (unsigned long long)ctx->saved_physical_size);
+       }
        str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME,
                    (unsigned long)ctx->received_date);
        str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME,

diff --git a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c
index c51080fb45479b29c7a196c5462f474c0d576ba0..718fe14027d10675cbb00ddefac50c99ced24d3b 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c
@@ -135,10 +135,9 @@ int mdbox_file_purge(struct dbox_file *file)
        i_array_init(&expunged_map_uids, I_MIN(count, 1));
        offset = file->file_header_size;
        for (i = 0; i < count; i++) {
-               if ((ret = dbox_file_get_mail_stream(file, offset,
-                                                    &physical_size,
-                                                    NULL)) <= 0)
+               if ((ret = dbox_file_get_mail_stream(file, offset, NULL)) <= 0)
                        break;
+               physical_size = file->cur_physical_size;
                msg_size = file->msg_header_size + physical_size;
 
                if (msgs[i].offset != offset) {

diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
index 15ecc2d70e2c5fdd141945d10927c2d5900b34fa..51c4f68f1d632987fdb3605551344863ce7cb962 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
@@ -133,7 +133,7 @@ static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
 {
        const char *guid;
        struct mdbox_rebuild_msg *rec;
-       uoff_t offset, prev_offset, size;
+       uoff_t offset, prev_offset;
        bool last, first, fixed = FALSE;
        int ret;
 
@@ -162,7 +162,7 @@ static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx,
                        if (!first) {
                                /* seek to the offset where we last left off */
                                ret = dbox_file_get_mail_stream(file,
-                                       prev_offset, &size, NULL);
+                                       prev_offset, NULL);
                                if (ret <= 0)
                                        break;
                        }
@@ -508,7 +508,6 @@ static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
        struct mdbox_mailbox *mbox;
        enum mail_error error;
        bool deleted, created;
-       uoff_t size;
        int ret;
        uint32_t seq;
 
@@ -517,7 +516,7 @@ static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx,
        file = mdbox_file_init(ctx->storage, msg->file_id);
        ret = dbox_file_open(file, &deleted);
        if (ret > 0)
-               ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL);
+               ret = dbox_file_get_mail_stream(file, msg->offset, NULL);
        if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) {
                mailbox = dbox_file_metadata_get(file,
                                                 DBOX_METADATA_ORIG_MAILBOX);

diff --git a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c
index 6f39bd91cba74cd4fa4addd6bc7ac152aa21b929..3014fca957a5cb4574f38aa20ee08c6364df31f9 100644 (file)
--- a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c
+++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c
@@ -28,14 +28,13 @@ static int sdbox_sync_add_file_index(struct dbox_sync_rebuild_context *ctx,
                                     struct dbox_file *file, uint32_t uid)
 {
        uint32_t seq;
-       uoff_t size;
        bool deleted;
        int ret;
 
        if ((ret = dbox_file_open(file, &deleted)) > 0) {
                if (deleted)
                        return 0;
-               ret = dbox_file_get_mail_stream(file, 0, &size, NULL);
+               ret = dbox_file_get_mail_stream(file, 0, NULL);
        }
 
        if (ret <= 0) {

diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 5ed1cc0584d6bc918509af346955ededbd749888..a9576ea79b0d53a6756ae17f7b5f8f8a0886d81a 100644 (file)
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -410,6 +410,11 @@ struct mail_save_context {
        char *guid, *pop3_uidl, *from_envelope;
        struct ostream *output;
 
+       /* if non-zero, overrides the physical size that should be saved.
+          for example when using zlib plugin, this would contain the mail's
+          uncompressed size. */
+       uoff_t saved_physical_size;
+
        /* we came here from mailbox_copy() */
        unsigned int copying:1;
 };