If BODY/BODYSTRUCTURE is requested with some other headers, parse the

headers only once. If body contains multiple MIME parts, cache the internal
MIME structure so BODY[part] fetching doesn't need to parse it again.

--HG--
branch : HEAD

diff --git a/src/lib-imap/imap-bodystructure.c b/src/lib-imap/imap-bodystructure.c
index f3b34e12f68f43dc5ca25f4e4c36ab866497859c..1c29bbf973023df2a313542de1a242c3158faa48 100644 (file)
--- a/src/lib-imap/imap-bodystructure.c
+++ b/src/lib-imap/imap-bodystructure.c
@@ -20,7 +20,7 @@
 
 struct message_part_body_data {
        pool_t pool;
-       string_t *str;
+       string_t *str; /* temporary */
        char *content_type, *content_subtype;
        char *content_type_params;
        char *content_transfer_encoding;
@@ -36,6 +36,12 @@ struct message_part_body_data {
        unsigned int charset_found:1;
 };
 
+struct imap_bodystructure_parse_ctx {
+       pool_t pool;
+       int extended;
+        struct message_part *root;
+};
+
 static void part_write_bodystructure(struct message_part *part,
                                     string_t *str, int extended);
 
@@ -235,10 +241,9 @@ static void parse_content_header(struct message_part_body_data *d,
        }
 }
 
-static void parse_header(struct message_part *part,
-                         struct message_header_line *hdr, void *context)
+void imap_bodystructure_parse_header(pool_t pool, struct message_part *part,
+                                    struct message_header_line *hdr)
 {
-       pool_t pool = context;
        struct message_part_body_data *part_data;
        struct message_part_envelope_data *envelope;
        int parent_rfc822;
@@ -284,26 +289,6 @@ static void parse_header(struct message_part *part,
        t_pop();
 }
 
-static void part_parse_headers(struct message_part *part, struct istream *input,
-                              uoff_t start_offset, pool_t pool)
-{
-       while (part != NULL) {
-               /* note that we want to parse the header of all
-                  the message parts, multiparts too. */
-               i_assert(part->physical_pos >= input->v_offset - start_offset);
-               i_stream_skip(input, part->physical_pos -
-                             (input->v_offset - start_offset));
-
-               message_parse_header(part, input, NULL, parse_header, pool);
-               if (part->children != NULL) {
-                       part_parse_headers(part->children, input,
-                                          start_offset, pool);
-               }
-
-               part = part->next;
-       }
-}
-
 static void part_write_body_multipart(struct message_part *part,
                                      string_t *str, int extended)
 {
@@ -501,21 +486,13 @@ static void part_write_bodystructure(struct message_part *part,
        }
 }
 
-const char *imap_part_get_bodystructure(pool_t pool, struct message_part **part,
-                                       struct istream *input, int extended)
+const char *imap_bodystructure_parse_finish(struct message_part *root,
+                                           int extended)
 {
        string_t *str;
-       uoff_t start_offset;
-
-       if (*part == NULL)
-               *part = message_parse(pool, input, parse_header, pool);
-       else {
-               start_offset = input->v_offset;
-               part_parse_headers(*part, input, start_offset, pool);
-       }
 
        str = t_str_new(2048);
-       part_write_bodystructure(*part, str, extended);
+       part_write_bodystructure(root, str, extended);
        return str_c(str);
 }
 

diff --git a/src/lib-imap/imap-bodystructure.h b/src/lib-imap/imap-bodystructure.h
index 8d92456a48acb67757cdcc0500884eb0d47636ef..40196f9c100ebc7d8f9f6b893b8f246547ed97f7 100644 (file)
--- a/src/lib-imap/imap-bodystructure.h
+++ b/src/lib-imap/imap-bodystructure.h
@@ -2,13 +2,16 @@
 #define __IMAP_BODYSTRUCTURE_H
 
 struct message_part;
+struct message_header_line;
 
-/* If *part is non-NULL, it's used as base for building the body structure.
-   Otherwise it's set to the root message_part and parsed. pool is used only
-   for allocating message_part, not the return value which is from data
-   stack. */
-const char *imap_part_get_bodystructure(pool_t pool, struct message_part **part,
-                                       struct istream *input, int extended);
+struct imap_bodystructure_parse_ctx;
+
+/* Parse a single header. Note that this modifies part->context. */
+void imap_bodystructure_parse_header(pool_t pool, struct message_part *part,
+                                    struct message_header_line *hdr);
+
+const char *imap_bodystructure_parse_finish(struct message_part *root,
+                                           int extended);
 
 /* Return BODY part from BODYSTRUCTURE */
 const char *imap_body_parse_from_bodystructure(const char *bodystructure);

diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c
index b9d5de6a927ce304da22c9afe050536ddac0c02c..d8346b7a96e11904b1ad3dbdef6c1f37b1e73167 100644 (file)
--- a/src/lib-mail/message-parser.c
+++ b/src/lib-mail/message-parser.c
@@ -564,6 +564,35 @@ struct message_part *message_parse(pool_t pool, struct istream *input,
        return part;
 }
 
+static void part_parse_headers(struct message_part *part, struct istream *input,
+                              uoff_t start_offset,
+                              message_header_callback_t *callback,
+                              void *context)
+{
+       while (part != NULL) {
+               /* note that we want to parse the header of all
+                  the message parts, multiparts too. */
+               i_assert(part->physical_pos >= input->v_offset - start_offset);
+               i_stream_skip(input, part->physical_pos -
+                             (input->v_offset - start_offset));
+
+               message_parse_header(part, input, NULL, callback, context);
+               if (part->children != NULL) {
+                       part_parse_headers(part->children, input,
+                                          start_offset, callback, context);
+               }
+
+               part = part->next;
+       }
+}
+
+void message_parse_from_parts(struct message_part *part, struct istream *input,
+                             message_header_callback_t *callback,
+                             void *context)
+{
+       part_parse_headers(part, input, input->v_offset, callback, context);
+}
+
 void message_parse_header(struct message_part *part, struct istream *input,
                          struct message_size *hdr_size,
                          message_header_callback_t *callback, void *context)

diff --git a/src/lib-mail/message-parser.h b/src/lib-mail/message-parser.h
index e44f3042b97f99fd6add4652d9e9cc4a7818ae72..dc92890586818c00d60e8fda6d526135df14145d 100644 (file)
--- a/src/lib-mail/message-parser.h
+++ b/src/lib-mail/message-parser.h
@@ -65,6 +65,9 @@ typedef void message_header_callback_t(struct message_part *part,
 struct message_part *message_parse(pool_t pool, struct istream *input,
                                   message_header_callback_t *callback,
                                   void *context);
+void message_parse_from_parts(struct message_part *part, struct istream *input,
+                             message_header_callback_t *callback,
+                             void *context);
 void message_parse_header(struct message_part *part, struct istream *input,
                          struct message_size *hdr_size,
                          message_header_callback_t *callback, void *context);

diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c
index bb1a14258bf6c1bbbed13bb1f90c56c936996151..f4b32b2233174733d3aef25e83a17d81a1da0c80 100644 (file)
--- a/src/lib-storage/index/index-mail-headers.c
+++ b/src/lib-storage/index/index-mail-headers.c
@@ -40,6 +40,7 @@
 #include "str.h"
 #include "message-date.h"
 #include "imap-envelope.h"
+#include "imap-bodystructure.h"
 #include "index-storage.h"
 #include "index-mail.h"
 
@@ -258,13 +259,19 @@ void index_mail_parse_header_init(struct index_mail *mail,
        }
 }
 
-void index_mail_parse_header(struct message_part *part __attr_unused__,
+void index_mail_parse_header(struct message_part *part,
                             struct message_header_line *hdr, void *context)
 {
        struct index_mail *mail = context;
        struct index_mail_data *data = &mail->data;
        struct cached_header *cached_hdr;
 
+       if (data->bodystructure_header_parse)
+               imap_bodystructure_parse_header(mail->pool, part, hdr);
+
+       if (part != NULL && part->parent != NULL)
+               return;
+
        if (data->save_envelope) {
                imap_envelope_parse_header(mail->pool,
                                           &data->envelope_data, hdr);
@@ -413,7 +420,7 @@ static int parse_cached_headers(struct index_mail *mail, int idx)
        return TRUE;
 }
 
-int index_mail_parse_headers(struct index_mail *mail)
+int index_mail_parse_headers(struct index_mail *mail, int get_parts)
 {
        struct mail_cache *cache = mail->ibox->index->cache;
        struct index_mail_data *data = &mail->data;
@@ -481,9 +488,42 @@ int index_mail_parse_headers(struct index_mail *mail)
                data->header_save_idx = idx;
        }
 
+       data->bodystructure_header_parse = data->bodystructure_header_want;
        index_mail_parse_header_init(mail, NULL);
-       message_parse_header(NULL, data->stream, &data->hdr_size,
-                            index_mail_parse_header, mail);
+
+       if ((mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) != 0)
+               get_parts = TRUE;
+       if (data->parts != NULL)
+               get_parts = FALSE;
+
+       if (!data->bodystructure_header_want && !get_parts) {
+               message_parse_header(data->parts, data->stream, &data->hdr_size,
+                                    index_mail_parse_header, mail);
+       } else if (data->parts == NULL) {
+               data->parts = message_parse(mail->pool, data->stream,
+                                           index_mail_parse_header, mail);
+       } else {
+               message_parse_from_parts(data->parts, data->stream,
+                                        index_mail_parse_header, mail);
+       }
+
+       if (data->bodystructure_header_want) {
+               data->bodystructure_header_want = FALSE;
+               data->bodystructure_header_parse = FALSE;
+               data->bodystructure_header_parsed = TRUE;
+       }
+
+       if (get_parts) {
+               /* we know the NULs now, update them */
+               if ((data->parts->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) {
+                       mail->mail.has_nuls = TRUE;
+                       mail->mail.has_no_nuls = FALSE;
+               } else {
+                       mail->mail.has_nuls = FALSE;
+                       mail->mail.has_no_nuls = TRUE;
+               }
+       }
+
        data->parse_header = FALSE;
        data->hdr_size_set = TRUE;
        data->header_fully_parsed = TRUE;
@@ -514,7 +554,7 @@ const char *index_mail_get_header(struct mail *_mail, const char *field)
                }
 
                if (idx < 0) {
-                       index_mail_parse_headers(mail);
+                       index_mail_parse_headers(mail, FALSE);
 
                        /* might have been moved in memory, get it again */
                        hdr = cached_header_find(mail, field, NULL);
@@ -567,7 +607,7 @@ struct istream *index_mail_get_headers(struct mail *_mail,
                }
 
                if (!all_saved)
-                       index_mail_parse_headers(mail);
+                       index_mail_parse_headers(mail, FALSE);
        }
 
        return i_stream_create_from_data(mail->pool,

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index f48bdfe9d446131c23bc59b71be482fe7cf24ec1..38b49e42a259066886feba40b47db3e0b55ab6ee 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -210,14 +210,29 @@ static const struct mail_full_flags *get_flags(struct mail *_mail)
        return &data->flags;
 }
 
-static const struct message_part *get_parts(struct mail *_mail)
+static void cache_parts(struct index_mail *mail)
 {
-       struct index_mail *mail = (struct index_mail *) _mail;
-       struct index_mail_data *data = &mail->data;
        buffer_t *buffer;
        const void *buf_data;
        size_t buf_size;
 
+       if (!index_mail_cache_can_add(mail, MAIL_CACHE_MESSAGEPART))
+               return;
+
+       t_push();
+       buffer = buffer_create_dynamic(data_stack_pool, 1024, (size_t)-1);
+       message_part_serialize(mail->data.parts, buffer);
+
+       buf_data = buffer_get_data(buffer, &buf_size);
+       index_mail_cache_add(mail, MAIL_CACHE_MESSAGEPART, buf_data, buf_size);
+       t_pop();
+}
+
+static const struct message_part *get_parts(struct mail *_mail)
+{
+       struct index_mail *mail = (struct index_mail *) _mail;
+       struct index_mail_data *data = &mail->data;
+
        if (data->parts != NULL)
                return data->parts;
 
@@ -227,32 +242,10 @@ static const struct message_part *get_parts(struct mail *_mail)
                        return data->parts;
        }
 
-       if (!index_mail_open_stream(mail, 0))
+       if (!index_mail_parse_headers(mail, TRUE))
                return NULL;
 
-       data->parts = message_parse(mail->pool, data->stream,
-                                   index_mail_parse_header, mail);
-
-       /* we know the NULs now, update them */
-       if ((data->parts->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) {
-               _mail->has_nuls = TRUE;
-               _mail->has_no_nuls = FALSE;
-       } else {
-               _mail->has_nuls = FALSE;
-               _mail->has_no_nuls = TRUE;
-       }
-
-       if (index_mail_cache_can_add(mail, MAIL_CACHE_MESSAGEPART)) {
-               t_push();
-               buffer = buffer_create_dynamic(data_stack_pool,
-                                              1024, (size_t)-1);
-               message_part_serialize(data->parts, buffer);
-
-               buf_data = buffer_get_data(buffer, &buf_size);
-               index_mail_cache_add(mail, MAIL_CACHE_MESSAGEPART,
-                                    buf_data, buf_size);
-               t_pop();
-       }
+        cache_parts(mail);
        return data->parts;
 }
 
@@ -398,7 +391,7 @@ static uoff_t get_size(struct mail *_mail)
        if (!get_msgpart_sizes(mail)) {
                /* this gives us header size for free */
                if (data->parse_header)
-                       index_mail_parse_headers(mail);
+                       index_mail_parse_headers(mail, FALSE);
        }
 
        hdr_size = data->hdr_size_set ?
@@ -534,16 +527,15 @@ static const char *get_special(struct mail *_mail, enum mail_fetch_field field)
                        return data->body;
                }
 
-               if (!index_mail_open_stream(mail, 0))
-                       return NULL;
-
-               if (data->parts == NULL)
-                       data->parts = get_cached_parts(mail);
+               if (!data->bodystructure_header_parsed) {
+                       data->bodystructure_header_want = TRUE;
+                       if (!index_mail_parse_headers(mail, FALSE))
+                               return NULL;
+               }
 
                t_push();
-               str = p_strdup(mail->pool, imap_part_get_bodystructure(
-                               mail->pool, &data->parts, data->stream,
-                               field == MAIL_FETCH_IMAP_BODYSTRUCTURE));
+                str = p_strdup(mail->pool, imap_bodystructure_parse_finish(
+                       data->parts, field == MAIL_FETCH_IMAP_BODYSTRUCTURE));
                t_pop();
 
                /* should never fail */
@@ -553,6 +545,12 @@ static const char *get_special(struct mail *_mail, enum mail_fetch_field field)
                        MAIL_CACHE_BODYSTRUCTURE : MAIL_CACHE_BODY;
                index_mail_cache_add(mail, cache_field, str, strlen(str)+1);
 
+               if (data->parts->children != NULL) {
+                       /* cache the message parts only if this is a
+                          multipart message. it's pretty useless otherwise. */
+                       cache_parts(mail);
+               }
+
                if (field == MAIL_FETCH_IMAP_BODYSTRUCTURE)
                        data->bodystructure = str;
                else
@@ -671,12 +669,14 @@ int index_mail_next(struct index_mail *mail, struct mail_index_record *rec,
                        data->parts = get_cached_parts(mail);
                open_mail = TRUE;
                data->parse_header = data->parts == NULL;
+                data->bodystructure_header_want = TRUE;
        } else if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODY) &&
                   data->body == NULL && data->bodystructure == NULL) {
                if (data->parts == NULL)
                        data->parts = get_cached_parts(mail);
                open_mail = TRUE;
                data->parse_header = data->parts == NULL;
+                data->bodystructure_header_want = TRUE;
        } else if (mail->wanted_fields & (MAIL_FETCH_STREAM_HEADER |
                                          MAIL_FETCH_STREAM_BODY))
                open_mail = TRUE;

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index a430756585be549f33ccdb5d66d32ca14b2b6fb5..769003e3f2c8d6bd5453c5e2511ddebb22219bec 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -23,7 +23,7 @@ struct index_mail_data {
 
        struct message_part *parts;
        const char *envelope, *body, *bodystructure;
-        struct message_part_envelope_data *envelope_data;
+       struct message_part_envelope_data *envelope_data;
 
        struct mail_index_record *rec;
        unsigned int idx_seq;
@@ -32,6 +32,9 @@ struct index_mail_data {
         struct message_size hdr_size, body_size;
 
        unsigned int parse_header:1;
+       unsigned int bodystructure_header_want:1;
+       unsigned int bodystructure_header_parse:1;
+       unsigned int bodystructure_header_parsed:1;
        unsigned int save_envelope:1;
        unsigned int save_sent_date:1;
        unsigned int hdr_size_set:1;
@@ -73,7 +76,7 @@ void index_mail_cache_add(struct index_mail *mail, enum mail_cache_field field,
                          const void *data, size_t size);
 
 int index_mail_open_stream(struct index_mail *mail, uoff_t position);
-int index_mail_parse_headers(struct index_mail *mail);
+int index_mail_parse_headers(struct index_mail *mail, int get_parts);
 
 void index_mail_headers_init(struct index_mail *mail);
 void index_mail_headers_init_next(struct index_mail *mail);