lib-storage: Verify that cached message size matches actually read size.

diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am
index 0046bfd55c70a5f8fa004bb93d2f8110175d8a8a..435ae8f097e5a395f6eef3c113647400b6ccea15 100644 (file)
--- a/src/lib-storage/index/Makefile.am
+++ b/src/lib-storage/index/Makefile.am
@@ -13,7 +13,7 @@ AM_CPPFLAGS = \
 
 libstorage_index_la_SOURCES = \
        istream-attachment.c \
-       istream-mail-stats.c \
+       istream-mail.c \
        index-attachment.c \
        index-mail.c \
        index-mail-headers.c \
@@ -37,7 +37,7 @@ libstorage_index_la_DEPENDENCIES = @LINKED_STORAGE_LIBS@
 
 headers = \
        istream-attachment.h \
-       istream-mail-stats.h \
+       istream-mail.h \
        index-attachment.h \
        index-mail.h \
        index-search-private.h \

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index ca51d84fcd7c9272cb1fbdabbd5401979a96c08d..ea9406b3fe857f0b058206d69861c539d2a2ce0c 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -266,6 +266,7 @@ void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body)
                imail->data.virtual_size = size;
        }
 
+       imail->data.stream_has_only_header = !have_body;
        if (index_mail_init_stream(imail, NULL, NULL, &input) < 0)
                i_stream_unref(&imail->data.stream);
 }

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 02e8cde7a46451de869b55eaea25237838c8c95d..4312f8536500d37bf2b1b042d07e470c41767d32 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -14,7 +14,7 @@
 #include "mail-cache.h"
 #include "mail-index-modseq.h"
 #include "index-storage.h"
-#include "istream-mail-stats.h"
+#include "istream-mail.h"
 #include "index-mail.h"
 
 #include <fcntl.h>
@@ -837,8 +837,8 @@ int index_mail_init_stream(struct index_mail *mail,
 
        if (!data->initialized_wrapper_stream &&
            _mail->transaction->stats_track) {
-               input = i_stream_create_mail_stats_counter(_mail->transaction,
-                                                          data->stream);
+               input = i_stream_create_mail(_mail, data->stream,
+                                            !data->stream_has_only_header);
                i_stream_unref(&data->stream);
                data->stream = input;
                data->initialized_wrapper_stream = TRUE;
@@ -1616,6 +1616,12 @@ void index_mail_set_cache_corrupted(struct mail *mail,
        case 0:
                field_name = "fields";
                break;
+       case MAIL_FETCH_PHYSICAL_SIZE:
+               field_name = "physical size";
+               imail->data.physical_size = (uoff_t)-1;
+               imail->data.virtual_size = (uoff_t)-1;
+               imail->data.parts = NULL;
+               break;
        case MAIL_FETCH_VIRTUAL_SIZE:
                field_name = "virtual size";
                imail->data.physical_size = (uoff_t)-1;

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 802375ac6c0f8e1a31e957d0c7c8eb6fd16c27d5..9e854b388c4c45f4f4e587e4dacbb579e0dbbc94 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -105,6 +105,7 @@ struct index_mail_data {
        unsigned int save_bodystructure_header:1;
        unsigned int save_bodystructure_body:1;
        unsigned int save_message_parts:1;
+       unsigned int stream_has_only_header:1;
        unsigned int parsed_bodystructure:1;
        unsigned int hdr_size_set:1;
        unsigned int body_size_set:1;

diff --git a/src/lib-storage/index/istream-mail-stats.c b/src/lib-storage/index/istream-mail-stats.c
deleted file mode 100644 (file)
index b3dd41a..0000000
--- a/src/lib-storage/index/istream-mail-stats.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/* Copyright (c) 2009-2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "mail-storage-private.h"
-#include "istream-private.h"
-#include "istream-mail-stats.h"
-
-struct mail_stats_istream {
-       struct istream_private istream;
-
-       struct mailbox_transaction_context *trans;
-       unsigned int files_read_increased:1;
-};
-
-static ssize_t
-i_stream_mail_stats_read_mail_stats(struct istream_private *stream)
-{
-       struct mail_stats_istream *mstream =
-               (struct mail_stats_istream *)stream;
-       ssize_t ret;
-
-       i_stream_seek(stream->parent, stream->parent_start_offset +
-                     stream->istream.v_offset);
-
-       ret = i_stream_read_copy_from_parent(&stream->istream);
-       if (ret > 0) {
-               mstream->trans->stats.files_read_bytes += ret;
-               if (!mstream->files_read_increased) {
-                       mstream->files_read_increased = TRUE;
-                       mstream->trans->stats.files_read_count++;
-               }
-       }
-       return ret;
-}
-
-static void
-i_stream_mail_stats_seek(struct istream_private *stream,
-                        uoff_t v_offset, bool mark ATTR_UNUSED)
-{
-       stream->istream.v_offset = v_offset;
-       stream->skip = stream->pos = 0;
-}
-
-static const struct stat *
-i_stream_mail_stats_stat(struct istream_private *stream, bool exact)
-{
-       return i_stream_stat(stream->parent, exact);
-}
-
-struct istream *
-i_stream_create_mail_stats_counter(struct mailbox_transaction_context *trans,
-                                  struct istream *input)
-{
-       struct mail_stats_istream *mstream;
-
-       mstream = i_new(struct mail_stats_istream, 1);
-       mstream->trans = trans;
-       mstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
-
-       mstream->istream.parent = input;
-       mstream->istream.read = i_stream_mail_stats_read_mail_stats;
-       mstream->istream.seek = i_stream_mail_stats_seek;
-       mstream->istream.stat = i_stream_mail_stats_stat;
-
-       mstream->istream.istream.blocking = input->blocking;
-       mstream->istream.istream.seekable = input->seekable;
-       return i_stream_create(&mstream->istream, input,
-                              i_stream_get_fd(input));
-}

diff --git a/src/lib-storage/index/istream-mail-stats.h b/src/lib-storage/index/istream-mail-stats.h
deleted file mode 100644 (file)
index 7052f41..0000000
--- a/src/lib-storage/index/istream-mail-stats.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef ISTREAM_MAIL_STATS_H
-#define ISTREAM_MAIL_STATS_H
-
-struct istream *
-i_stream_create_mail_stats_counter(struct mailbox_transaction_context *trans,
-                                  struct istream *input);
-
-#endif

diff --git a/src/lib-storage/index/istream-mail.c b/src/lib-storage/index/istream-mail.c
new file mode 100644 (file)
index 0000000..8bdd953
--- /dev/null
+++ b/src/lib-storage/index/istream-mail.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2009-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage-private.h"
+#include "istream-private.h"
+#include "index-mail.h"
+#include "istream-mail.h"
+
+struct mail_istream {
+       struct istream_private istream;
+
+       struct mail *mail;
+       uoff_t expected_size;
+       unsigned int files_read_increased:1;
+       unsigned int input_has_body:1;
+};
+
+static bool i_stream_mail_try_get_cached_size(struct mail_istream *mstream)
+{
+       struct mail *mail = mstream->mail;
+       enum mail_lookup_abort orig_lookup_abort;
+
+       if (mstream->expected_size != (uoff_t)-1)
+               return TRUE;
+
+       orig_lookup_abort = mail->lookup_abort;
+       mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+       if (mail_get_physical_size(mail, &mstream->expected_size) < 0)
+               mstream->expected_size = (uoff_t)-1;
+       mail->lookup_abort = orig_lookup_abort;
+       return mstream->expected_size != (uoff_t)-1;
+}
+
+static void
+i_stream_mail_set_size_corrupted(struct mail_istream *mstream, size_t size)
+{
+       uoff_t cur_size = mstream->istream.istream.v_offset + size;
+       const char *str;
+       char chr;
+
+       if (mstream->expected_size < cur_size) {
+               str = "smaller";
+               chr = '<';
+       } else {
+               str = "larger";
+               chr = '>';
+       }
+
+       mail_storage_set_critical(mstream->mail->box->storage,
+               "Cached message size %s than expected "
+               "(%"PRIuUOFF_T" %c %"PRIuUOFF_T")", str,
+               mstream->expected_size, chr, cur_size);
+       index_mail_set_cache_corrupted(mstream->mail, MAIL_FETCH_PHYSICAL_SIZE);
+       mstream->istream.istream.stream_errno = EIO;
+}
+
+static ssize_t
+i_stream_mail_read(struct istream_private *stream)
+{
+       struct mail_istream *mstream = (struct mail_istream *)stream;
+       size_t size;
+       ssize_t ret;
+
+       i_stream_seek(stream->parent, stream->parent_start_offset +
+                     stream->istream.v_offset);
+
+       ret = i_stream_read_copy_from_parent(&stream->istream);
+       (void)i_stream_get_data(&stream->istream, &size);
+       if (ret > 0) {
+               mstream->mail->transaction->stats.files_read_bytes += ret;
+               if (!mstream->files_read_increased) {
+                       mstream->files_read_increased = TRUE;
+                       mstream->mail->transaction->stats.files_read_count++;
+               }
+               if (mstream->expected_size < stream->istream.v_offset + size) {
+                       i_stream_mail_set_size_corrupted(mstream, size);
+                       return -1;
+               }
+       } else if (ret < 0 && stream->istream.eof) {
+               if (!mstream->input_has_body) {
+                       /* trying to read past the header, but this stream
+                          doesn't have the body */
+                       return -1;
+               }
+               if (i_stream_mail_try_get_cached_size(mstream) &&
+                   mstream->expected_size > stream->istream.v_offset + size) {
+                       i_stream_mail_set_size_corrupted(mstream, size);
+                       return -1;
+               }
+       }
+       return ret;
+}
+
+static void
+i_stream_mail_seek(struct istream_private *stream,
+                  uoff_t v_offset, bool mark ATTR_UNUSED)
+{
+       stream->istream.v_offset = v_offset;
+       stream->skip = stream->pos = 0;
+}
+
+static const struct stat *
+i_stream_mail_stat(struct istream_private *stream, bool exact)
+{
+       return i_stream_stat(stream->parent, exact);
+}
+
+struct istream *i_stream_create_mail(struct mail *mail, struct istream *input,
+                                    bool input_has_body)
+{
+       struct mail_istream *mstream;
+
+       mstream = i_new(struct mail_istream, 1);
+       mstream->mail = mail;
+       mstream->input_has_body = input_has_body;
+       mstream->expected_size = (uoff_t)-1;
+       (void)i_stream_mail_try_get_cached_size(mstream);
+       mstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+
+       mstream->istream.parent = input;
+       mstream->istream.read = i_stream_mail_read;
+       mstream->istream.seek = i_stream_mail_seek;
+       mstream->istream.stat = i_stream_mail_stat;
+
+       mstream->istream.istream.blocking = input->blocking;
+       mstream->istream.istream.seekable = input->seekable;
+       return i_stream_create(&mstream->istream, input,
+                              i_stream_get_fd(input));
+}

diff --git a/src/lib-storage/index/istream-mail.h b/src/lib-storage/index/istream-mail.h
new file mode 100644 (file)
index 0000000..24bd567
--- /dev/null
+++ b/src/lib-storage/index/istream-mail.h
@@ -0,0 +1,7 @@
+#ifndef ISTREAM_MAIL_H
+#define ISTREAM_MAIL_H
+
+struct istream *i_stream_create_mail(struct mail *mail, struct istream *input,
+                                    bool input_has_body);
+
+#endif