imapc: Don't use RFC822.SIZE values for message body size calculation

If the RFC822.SIZE doesn't match the exact stream size, the body size
calculation will become wrong. The only downside to this patch is that
now the body size will need to be calculated by parsing the stream, but
there shouldn't be any need to do that unless the body was already
FETCHed, so it shouldn't cause any extra IMAP traffic.

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index a71497d70094aaddf0c6bd05d9c10093b30cb132..b543a92a0e81ba301cbfa607857baeb6786fc596 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -589,6 +589,8 @@ void imapc_mail_init_stream(struct imapc_mail *mail)
           smaller than the fetched message header. In this case change the
           size as well, otherwise reading via istream-mail will fail. */
        if (mail->body_fetched || imail->data.physical_size < size) {
+               if (mail->body_fetched)
+                       imail->data.inexact_total_sizes = FALSE;
                imail->data.physical_size = size;
                /* we'll assume that the remote server is working properly and
                   sending CRLF linefeeds */
@@ -839,8 +841,11 @@ void imapc_mail_fetch_update(struct imapc_mail *mail,
                } else if (strcasecmp(key, "RFC822.SIZE") == 0) {
                        if (imap_arg_get_atom(&args[i+1], &value) &&
                            str_to_uoff(value, &size) == 0 &&
-                           IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
+                           IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
                                mail->imail.data.physical_size = size;
+                               mail->imail.data.virtual_size = size;
+                               mail->imail.data.inexact_total_sizes = TRUE;
+                       }
                        match = TRUE;
                } else if (strcasecmp(key, "X-GM-MSGID") == 0 ||
                           strcasecmp(key, "X-GUID") == 0) {

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 38f12db4608a689d6a624691e3a2980fc4cbf6bd..27a20fcb97a5df30069499102a3b144d838e3c91 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -169,7 +169,9 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 
        if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) &&
            data->stream == NULL) {
-               /* trust RFC822.SIZE to be correct */
+               /* Trust RFC822.SIZE to be correct enough to present to the
+                  IMAP client. However, it can be wrong in some implementation
+                  so try not to trust it too much. */
                if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
                        return -1;
                if (data->physical_size == (uoff_t)-1) {
@@ -400,8 +402,14 @@ static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
 {
        struct imapc_mail *imail = IMAPC_MAIL(_mail);
        struct index_mail *mail = &imail->imail;
+       struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
 
        index_mail_set_seq(_mail, seq, saving);
+       if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+               /* RFC822.SIZE may be read from vsize record or cache. It may
+                  not be exactly correct. */
+               mail->data.inexact_total_sizes = TRUE;
+       }
 
        /* searching code handles prefetching internally,
           elsewhere we want to do it immediately */

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 76c3358c47d31b29677ac8f46c07b0f72fffde85..a6382b7055b0b6073f5b34b482a3fbfaf2c18cfe 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -466,9 +466,14 @@ static void index_mail_try_set_body_size(struct index_mail *mail)
 {
        struct index_mail_data *data = &mail->data;
 
-       if (data->hdr_size_set &&
+       if (data->hdr_size_set && !data->inexact_total_sizes &&
            data->physical_size != (uoff_t)-1 &&
            data->virtual_size != (uoff_t)-1) {
+               /* We know the total size of this mail and we know the
+                  header size, so we can calculate also the body size.
+                  However, don't do this if there's a possibility that
+                  physical_size or virtual_size don't actually match the
+                  mail stream's size (e.g. buggy imapc servers). */
                data->body_size.physical_size = data->physical_size -
                        data->hdr_size.physical_size;
                data->body_size.virtual_size = data->virtual_size -
@@ -1230,7 +1235,7 @@ int index_mail_init_stream(struct index_mail *mail,
        struct mail *_mail = &mail->mail.mail;
        struct index_mail_data *data = &mail->data;
        struct istream *input;
-       bool has_nuls;
+       bool has_nuls, body_size_from_stream = FALSE;
        int ret;
 
        if (_mail->box->storage->user->mail_debug &&
@@ -1302,6 +1307,7 @@ int index_mail_init_stream(struct index_mail *mail,
                                }
                                data->body_size_set = TRUE;
                        }
+                       body_size_from_stream = TRUE;
                }
 
                *body_size = data->body_size;
@@ -1312,6 +1318,10 @@ int index_mail_init_stream(struct index_mail *mail,
                        data->body_size.virtual_size;
                data->physical_size = data->hdr_size.physical_size +
                        data->body_size.physical_size;
+               if (body_size_from_stream) {
+                       /* the sizes were just calculated */
+                       data->inexact_total_sizes = FALSE;
+               }
        }
        ret = index_mail_stream_check_failure(mail);
 

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 2cd80c6aee7e43c0019ebe5da77a7648dbd45ebf..8053704df2a4a88b22ce5fa0c7d3599bc55eb9be 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -129,6 +129,9 @@ struct index_mail_data {
        bool destroy_callback_set:1;
        bool prefetch_sent:1;
        bool header_parser_initialized:1;
+       /* virtual_size and physical_size may not match the stream size.
+          Try to avoid trusting them too much. */
+       bool inexact_total_sizes:1;
 };
 
 struct index_mail {