imapc: Don't use RFC822.SIZE values for message body size calculation

If the RFC822.SIZE doesn't match the exact stream size, the body size
calculation will become wrong. The only downside to this patch is that
now the body size will need to be calculated by parsing the stream, but
there shouldn't be any need to do that unless the body was already
FETCHed, so it shouldn't cause any extra IMAP traffic.

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index 496026ad97c1b0f09a2e9c5349b4aa62c292becb..f6b3d1c29fdc5d015ec9a8a76524afa7e2a339d5 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -589,6 +589,8 @@ void imapc_mail_init_stream(struct imapc_mail *mail)
           smaller than the fetched message header. In this case change the
           size as well, otherwise reading via istream-mail will fail. */
        if (mail->body_fetched || imail->data.physical_size < size) {
+               if (mail->body_fetched)
+                       imail->data.inexact_total_sizes = FALSE;
                imail->data.physical_size = size;
                /* we'll assume that the remote server is working properly and
                   sending CRLF linefeeds */
@@ -839,8 +841,11 @@ void imapc_mail_fetch_update(struct imapc_mail *mail,
                } else if (strcasecmp(key, "RFC822.SIZE") == 0) {
                        if (imap_arg_get_atom(&args[i+1], &value) &&
                            str_to_uoff(value, &size) == 0 &&
-                           IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
+                           IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
                                mail->imail.data.physical_size = size;
+                               mail->imail.data.virtual_size = size;
+                               mail->imail.data.inexact_total_sizes = TRUE;
+                       }
                        match = TRUE;
                } else if (strcasecmp(key, "X-GM-MSGID") == 0 ||
                           strcasecmp(key, "X-GUID") == 0) {

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 002d709b7d22a5aaae2afa9baa52de3372e8c2d6..4c730dd97a40b2d5741b0586e24da089a0fe49e9 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -169,7 +169,9 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 
        if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) &&
            data->stream == NULL) {
-               /* trust RFC822.SIZE to be correct */
+               /* Trust RFC822.SIZE to be correct enough to present to the
+                  IMAP client. However, it can be wrong in some implementation
+                  so try not to trust it too much. */
                if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
                        return -1;
                if (data->physical_size == (uoff_t)-1) {
@@ -406,8 +408,14 @@ static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
 {
        struct imapc_mail *imail = IMAPC_MAIL(_mail);
        struct index_mail *mail = &imail->imail;
+       struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
 
        index_mail_set_seq(_mail, seq, saving);
+       if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+               /* RFC822.SIZE may be read from vsize record or cache. It may
+                  not be exactly correct. */
+               mail->data.inexact_total_sizes = TRUE;
+       }
 
        /* searching code handles prefetching internally,
           elsewhere we want to do it immediately */

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 0a6e9e05a8c61ce8aced27846e085529ca29120b..52d74d4c199eb5b41878c2a4b07946566f177732 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -466,9 +466,14 @@ static void index_mail_try_set_body_size(struct index_mail *mail)
 {
        struct index_mail_data *data = &mail->data;
 
-       if (data->hdr_size_set &&
+       if (data->hdr_size_set && !data->inexact_total_sizes &&
            data->physical_size != (uoff_t)-1 &&
            data->virtual_size != (uoff_t)-1) {
+               /* We know the total size of this mail and we know the
+                  header size, so we can calculate also the body size.
+                  However, don't do this if there's a possibility that
+                  physical_size or virtual_size don't actually match the
+                  mail stream's size (e.g. buggy imapc servers). */
                data->body_size.physical_size = data->physical_size -
                        data->hdr_size.physical_size;
                data->body_size.virtual_size = data->virtual_size -
@@ -1230,7 +1235,7 @@ int index_mail_init_stream(struct index_mail *mail,
        struct mail *_mail = &mail->mail.mail;
        struct index_mail_data *data = &mail->data;
        struct istream *input;
-       bool has_nuls;
+       bool has_nuls, body_size_from_stream = FALSE;
        int ret;
 
        if (mail->mail.get_stream_reason != NULL &&
@@ -1301,6 +1306,7 @@ int index_mail_init_stream(struct index_mail *mail,
                                }
                                data->body_size_set = TRUE;
                        }
+                       body_size_from_stream = TRUE;
                }
 
                *body_size = data->body_size;
@@ -1311,6 +1317,10 @@ int index_mail_init_stream(struct index_mail *mail,
                        data->body_size.virtual_size;
                data->physical_size = data->hdr_size.physical_size +
                        data->body_size.physical_size;
+               if (body_size_from_stream) {
+                       /* the sizes were just calculated */
+                       data->inexact_total_sizes = FALSE;
+               }
        }
        ret = index_mail_stream_check_failure(mail);
 

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 2cd80c6aee7e43c0019ebe5da77a7648dbd45ebf..8053704df2a4a88b22ce5fa0c7d3599bc55eb9be 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -129,6 +129,9 @@ struct index_mail_data {
        bool destroy_callback_set:1;
        bool prefetch_sent:1;
        bool header_parser_initialized:1;
+       /* virtual_size and physical_size may not match the stream size.
+          Try to avoid trusting them too much. */
+       bool inexact_total_sizes:1;
 };
 
 struct index_mail {