From b58aafbd21b365117538f73f306d22f75acd91f1 Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Fri, 19 Feb 2010 07:15:46 +0200 Subject: [PATCH] lib-storage: Added support for plugins to specify message's physical size when saving. Changed dbox to save this value to metadata, and use when present. --HG-- branch : HEAD --- src/lib-storage/index/dbox-common/dbox-file.c | 6 +- src/lib-storage/index/dbox-common/dbox-file.h | 10 ++-- src/lib-storage/index/dbox-common/dbox-mail.c | 60 +++++++++++++------ src/lib-storage/index/dbox-common/dbox-save.c | 17 ++++-- .../index/dbox-multi/mdbox-file-purge.c | 5 +- .../index/dbox-multi/mdbox-storage-rebuild.c | 7 +-- .../index/dbox-single/sdbox-sync-rebuild.c | 3 +- src/lib-storage/mail-storage-private.h | 5 ++ 8 files changed, 71 insertions(+), 42 deletions(-) diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c index b4af992232..e93f2aa4e5 100644 --- a/src/lib-storage/index/dbox-common/dbox-file.c +++ b/src/lib-storage/index/dbox-common/dbox-file.c @@ -310,7 +310,6 @@ int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) } int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, struct istream **stream_r) { uoff_t size; @@ -334,7 +333,6 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); } - *physical_size_r = file->cur_physical_size; return 1; } @@ -366,7 +364,7 @@ void dbox_file_seek_rewind(struct dbox_file *file) int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) { - uoff_t offset, size; + uoff_t offset; int ret; i_assert(file->input != NULL); @@ -391,7 +389,7 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) } *last_r = FALSE; - ret = dbox_file_get_mail_stream(file, offset, &size, NULL); + ret = dbox_file_get_mail_stream(file, offset, NULL); if (*offset_r == 0) *offset_r = file->file_header_size; return ret; diff --git a/src/lib-storage/index/dbox-common/dbox-file.h b/src/lib-storage/index/dbox-common/dbox-file.h index c8f1030c9c..2607f01cd6 100644 --- a/src/lib-storage/index/dbox-common/dbox-file.h +++ b/src/lib-storage/index/dbox-common/dbox-file.h @@ -41,6 +41,10 @@ enum dbox_metadata_key { DBOX_METADATA_RECEIVED_TIME = 'R', /* Saved UNIX timestamp in hex */ DBOX_METADATA_SAVE_TIME = 'S', + /* Physical message size in hex. Necessary only if it differs from + the dbox_message_header.message_size_hex, for example because the + message is compressed. */ + DBOX_METADATA_PHYSICAL_SIZE = 'Z', /* Virtual message size in hex (line feeds counted as CRLF) */ DBOX_METADATA_VIRTUAL_SIZE = 'V', /* Pointer to external message data. Format is: @@ -135,11 +139,9 @@ void dbox_file_close(struct dbox_file *file); int dbox_file_try_lock(struct dbox_file *file); void dbox_file_unlock(struct dbox_file *file); -/* Seek to given offset in file and return the message's input stream - and physical size. Returns 1 if ok/expunged, 0 if file/offset is corrupted, - -1 if I/O error. */ +/* Seek to given offset in file and return the message's input stream. + Returns 1 if ok/expunged, 0 if file/offset is corrupted, -1 if I/O error. */ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, struct istream **input_r); /* Start seeking at the beginning of the file. */ void dbox_file_seek_rewind(struct dbox_file *file); diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c index e7bebceba4..e0441c7b51 100644 --- a/src/lib-storage/index/dbox-common/dbox-mail.c +++ b/src/lib-storage/index/dbox-common/dbox-mail.c @@ -40,31 +40,59 @@ int dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r) { struct dbox_storage *storage = (struct dbox_storage *)mail->imail.mail.mail.box->storage; - uoff_t offset, size; + uoff_t offset; if (storage->v.mail_open(mail, &offset, file_r) < 0) return -1; - if (dbox_file_get_mail_stream(*file_r, offset, &size, NULL) <= 0) + if (dbox_file_get_mail_stream(*file_r, offset, NULL) <= 0) return -1; if (dbox_file_metadata_read(*file_r) <= 0) return -1; + + if (mail->imail.data.stream != NULL) { + /* we just messed up mail's input stream by reading metadata */ + i_stream_seek((*file_r)->input, offset); + i_stream_sync(mail->imail.data.stream); + } + return 0; +} + +static int +dbox_mail_metadata_get(struct dbox_mail *mail, enum dbox_metadata_key key, + const char **value_r) +{ + struct dbox_file *file; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + *value_r = dbox_file_metadata_get(file, key); return 0; } int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) { - struct index_mail *mail = (struct index_mail *)_mail; - struct index_mail_data *data = &mail->data; + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; struct istream *input; + const char *value; if (index_mail_get_physical_size(_mail, size_r) == 0) return 0; - if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + /* see if we have it in metadata */ + if (dbox_mail_metadata_get(mail, DBOX_METADATA_PHYSICAL_SIZE, + &value) < 0) return -1; - i_assert(data->physical_size != (uoff_t)-1); + if (value != NULL) + data->physical_size = strtoul(value, NULL, 16); + else { + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + i_assert(data->physical_size != (uoff_t)-1); + } *size_r = data->physical_size; return 0; } @@ -73,16 +101,14 @@ int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) { struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; const char *value; if (index_mail_get_cached_virtual_size(&mail->imail, size_r)) return 0; - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, DBOX_METADATA_VIRTUAL_SIZE, + &value) < 0) return -1; - - value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE); if (value == NULL) return index_mail_get_virtual_size(_mail, size_r); @@ -95,16 +121,15 @@ int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r) { struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; const char *value; if (index_mail_get_received_date(_mail, date_r) == 0) return 0; - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, DBOX_METADATA_RECEIVED_TIME, + &value) < 0) return -1; - value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME); data->received_date = value == NULL ? 0 : strtoul(value, NULL, 16); *date_r = data->received_date; return 0; @@ -150,7 +175,6 @@ dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, struct index_mail *imail = &mail->imail; struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(imail->mail.mail.box); - struct dbox_file *file; const char *value; string_t *str; @@ -162,10 +186,9 @@ dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, return 0; } - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, key, &value) < 0) return -1; - value = dbox_file_metadata_get(file, key); if (value == NULL) value = ""; index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx, @@ -205,7 +228,7 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; struct istream *input; - uoff_t offset, size; + uoff_t offset; int ret; if (data->stream == NULL) { @@ -213,7 +236,7 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, return -1; ret = dbox_file_get_mail_stream(mail->open_file, offset, - &size, &input); + &input); if (ret <= 0) { if (ret < 0) return -1; @@ -222,7 +245,6 @@ int dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, "%"PRIuUOFF_T, _mail->uid, offset); return -1; } - data->physical_size = size; data->stream = input; } diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c index bab4c0b269..afd7ec0db5 100644 --- a/src/lib-storage/index/dbox-common/dbox-save.c +++ b/src/lib-storage/index/dbox-common/dbox-save.c @@ -94,12 +94,13 @@ void dbox_save_end(struct dbox_save_context *ctx) { struct ostream *dbox_output = ctx->dbox_output; - if (ctx->ctx.output != dbox_output) { - /* e.g. zlib plugin had changed this */ - o_stream_ref(dbox_output); - o_stream_destroy(&ctx->ctx.output); - ctx->ctx.output = dbox_output; - } + if (ctx->ctx.output == dbox_output) + return; + + /* e.g. zlib plugin had changed this */ + o_stream_ref(dbox_output); + o_stream_destroy(&ctx->ctx.output); + ctx->ctx.output = dbox_output; } void dbox_save_write_metadata(struct mail_save_context *ctx, @@ -118,6 +119,10 @@ void dbox_save_write_metadata(struct mail_save_context *ctx, o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr)); str = t_str_new(256); + if (ctx->saved_physical_size != 0) { + str_printfa(str, "%c%llx\n", DBOX_METADATA_PHYSICAL_SIZE, + (unsigned long long)ctx->saved_physical_size); + } str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME, (unsigned long)ctx->received_date); str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME, diff --git a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c index c51080fb45..718fe14027 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c +++ b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c @@ -135,10 +135,9 @@ int mdbox_file_purge(struct dbox_file *file) i_array_init(&expunged_map_uids, I_MIN(count, 1)); offset = file->file_header_size; for (i = 0; i < count; i++) { - if ((ret = dbox_file_get_mail_stream(file, offset, - &physical_size, - NULL)) <= 0) + if ((ret = dbox_file_get_mail_stream(file, offset, NULL)) <= 0) break; + physical_size = file->cur_physical_size; msg_size = file->msg_header_size + physical_size; if (msgs[i].offset != offset) { diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c index 15ecc2d70e..51c4f68f1d 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c @@ -133,7 +133,7 @@ static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx, { const char *guid; struct mdbox_rebuild_msg *rec; - uoff_t offset, prev_offset, size; + uoff_t offset, prev_offset; bool last, first, fixed = FALSE; int ret; @@ -162,7 +162,7 @@ static int rebuild_file_mails(struct mdbox_storage_rebuild_context *ctx, if (!first) { /* seek to the offset where we last left off */ ret = dbox_file_get_mail_stream(file, - prev_offset, &size, NULL); + prev_offset, NULL); if (ret <= 0) break; } @@ -508,7 +508,6 @@ static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx, struct mdbox_mailbox *mbox; enum mail_error error; bool deleted, created; - uoff_t size; int ret; uint32_t seq; @@ -517,7 +516,7 @@ static int rebuild_restore_msg(struct mdbox_storage_rebuild_context *ctx, file = mdbox_file_init(ctx->storage, msg->file_id); ret = dbox_file_open(file, &deleted); if (ret > 0) - ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL); + ret = dbox_file_get_mail_stream(file, msg->offset, NULL); if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { mailbox = dbox_file_metadata_get(file, DBOX_METADATA_ORIG_MAILBOX); diff --git a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c index 6f39bd91cb..3014fca957 100644 --- a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c @@ -28,14 +28,13 @@ static int sdbox_sync_add_file_index(struct dbox_sync_rebuild_context *ctx, struct dbox_file *file, uint32_t uid) { uint32_t seq; - uoff_t size; bool deleted; int ret; if ((ret = dbox_file_open(file, &deleted)) > 0) { if (deleted) return 0; - ret = dbox_file_get_mail_stream(file, 0, &size, NULL); + ret = dbox_file_get_mail_stream(file, 0, NULL); } if (ret <= 0) { diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index 5ed1cc0584..a9576ea79b 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -410,6 +410,11 @@ struct mail_save_context { char *guid, *pop3_uidl, *from_envelope; struct ostream *output; + /* if non-zero, overrides the physical size that should be saved. + for example when using zlib plugin, this would contain the mail's + uncompressed size. */ + uoff_t saved_physical_size; + /* we came here from mailbox_copy() */ unsigned int copying:1; }; -- 2.47.3