Use mail_get_*stream_because() wherever possible.

diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c
index 7eeabb0fafed978fb983dea5d88993b0598c0288..421b4c609340fedd52eaaf7b48d03a882d5325f6 100644 (file)
--- a/src/lib-imap-storage/imap-msgpart.c
+++ b/src/lib-imap-storage/imap-msgpart.c
@@ -575,7 +575,7 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart,
 
        if (*msgpart->section_number != '\0') {
                /* find the MIME part */
-               if (mail_get_stream(mail, NULL, NULL, &input) < 0)
+               if (mail_get_stream_because(mail, NULL, NULL, "MIME part", &input) < 0)
                        return -1;
 
                i_stream_seek(input, part->physical_pos);
@@ -584,7 +584,7 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart,
        } else switch (msgpart->fetch_type) {
        case FETCH_FULL:
                /* fetch the whole message */
-               if (mail_get_stream(mail, NULL, NULL, &input) < 0 ||
+               if (mail_get_stream_because(mail, NULL, NULL, "full mail", &input) < 0 ||
                    mail_get_virtual_size(mail, &body_size.virtual_size) < 0)
                        return -1;
                result_r->size_field = MAIL_FETCH_VIRTUAL_SIZE;
@@ -619,7 +619,8 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart,
                break;
        case FETCH_BODY:
                /* fetch the message's body */
-               if (mail_get_stream(mail, &hdr_size, &body_size, &input) < 0)
+               if (mail_get_stream_because(mail, &hdr_size, &body_size,
+                                           "mail body", &input) < 0)
                        return -1;
                result_r->size_field = MAIL_FETCH_MESSAGE_PARTS;
                break;

diff --git a/src/lib-storage/index/index-mail-binary.c b/src/lib-storage/index/index-mail-binary.c
index a9ac1feb73cdff03e23bb79b2042199f47a36798..32cd818fae1502be673688f7d2a6fd9a11642864 100644 (file)
--- a/src/lib-storage/index/index-mail-binary.c
+++ b/src/lib-storage/index/index-mail-binary.c
@@ -362,8 +362,8 @@ blocks_count_lines(struct binary_ctx *ctx, struct istream *full_input)
 static int
 index_mail_read_binary_to_cache(struct mail *_mail,
                                const struct message_part *part,
-                               bool include_hdr, bool *binary_r,
-                               bool *converted_r)
+                               bool include_hdr, const char *reason,
+                               bool *binary_r, bool *converted_r)
 {
        struct index_mail *mail = (struct index_mail *)_mail;
        struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
@@ -374,7 +374,7 @@ index_mail_read_binary_to_cache(struct mail *_mail,
        t_array_init(&ctx.blocks, 8);
 
        mail_storage_free_binary_cache(_mail->box->storage);
-       if (mail_get_stream(_mail, NULL, NULL, &ctx.input) < 0)
+       if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0)
                return -1;
 
        if (add_binary_part(&ctx, part, include_hdr) < 0) {
@@ -491,7 +491,7 @@ index_mail_get_binary_size(struct mail *_mail,
        if (!get_cached_binary_parts(mail)) {
                /* not found. parse the whole message */
                if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE,
-                                                   &binary, &converted) < 0)
+                                                   "binary.size", &binary, &converted) < 0)
                        return -1;
        }
 
@@ -568,7 +568,7 @@ int index_mail_get_binary_stream(struct mail *_mail,
                converted = TRUE;
        } else {
                if (index_mail_read_binary_to_cache(_mail, part, include_hdr,
-                                                   &binary, &converted) < 0)
+                                                   "binary stream", &binary, &converted) < 0)
                        return -1;
                mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY;
        }

diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c
index 5636e1840631c6cada89570683be8b187fff3c57..2f286e2eca0ed9af5d7d3060b408bb05608fc797 100644 (file)
--- a/src/lib-storage/index/index-mail-headers.c
+++ b/src/lib-storage/index/index-mail-headers.c
@@ -427,7 +427,8 @@ static void index_mail_init_parser(struct index_mail *mail)
 }
 
 int index_mail_parse_headers(struct index_mail *mail,
-                            struct mailbox_header_lookup_ctx *headers)
+                            struct mailbox_header_lookup_ctx *headers,
+                            const char *reason)
 {
        struct index_mail_data *data = &mail->data;
        struct istream *input;
@@ -435,7 +436,7 @@ int index_mail_parse_headers(struct index_mail *mail,
 
        old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
 
-       if (mail_get_hdr_stream(&mail->mail.mail, NULL, &input) < 0)
+       if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0)
                return -1;
 
        index_mail_parse_header_init(mail, headers);
@@ -638,10 +639,12 @@ index_mail_get_raw_headers(struct index_mail *mail, const char *field,
                if (mail->header_seq != mail->data.seq ||
                    index_mail_header_is_parsed(mail, field_idx) < 0) {
                        /* parse */
+                       const char *reason = index_mail_cache_reason(_mail,
+                               t_strdup_printf("header %s", field));
                        headers[0] = field; headers[1] = NULL;
                        headers_ctx = mailbox_header_lookup_init(_mail->box,
                                                                 headers);
-                       ret = index_mail_parse_headers(mail, headers_ctx);
+                       ret = index_mail_parse_headers(mail, headers_ctx, reason);
                        mailbox_header_lookup_unref(&headers_ctx);
                        if (ret < 0)
                                return -1;
@@ -860,11 +863,14 @@ int index_mail_get_header_stream(struct mail *_mail,
        struct istream *input;
        string_t *dest;
 
+       i_assert(headers->count > 0);
        i_assert(headers->box == _mail->box);
 
        if (mail->data.save_bodystructure_header) {
                /* we have to parse the header. */
-               if (index_mail_parse_headers(mail, headers) < 0)
+               const char *reason =
+                       index_mail_cache_reason(_mail, "bodystructure");
+               if (index_mail_parse_headers(mail, headers, reason) < 0)
                        return -1;
        }
 
@@ -885,7 +891,25 @@ int index_mail_get_header_stream(struct mail *_mail,
        /* not in cache / error */
        p_free(mail->mail.data_pool, dest);
 
-       if (mail_get_hdr_stream(_mail, NULL, &input) < 0)
+       unsigned int first_not_found = UINT_MAX, not_found_count = 0;
+       for (unsigned int i = 0; i < headers->count; i++) {
+               if (mail_cache_field_exists(_mail->transaction->cache_view,
+                                           _mail->seq, headers->idx[i]) <= 0) {
+                       if (not_found_count++ == 0)
+                               first_not_found = i;
+               }
+       }
+
+       const char *reason;
+       if (not_found_count == 0)
+               reason = "BUG: all headers seem to exist in cache";
+       else {
+               i_assert(first_not_found != UINT_MAX);
+               reason = index_mail_cache_reason(_mail, t_strdup_printf(
+                       "%u/%u headers not cached (first=%s)",
+                       not_found_count, headers->count, headers->name[first_not_found]));
+       }
+       if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0)
                return -1;
 
        if (mail->data.filter_stream != NULL)

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index d1e44708c1287921ff08b27293418ffa46cb06a9..7ffabe36e28048ca560275ace4a2469dc99f4290 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -321,7 +321,9 @@ int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r)
        }
 
        if (data->parser_ctx == NULL) {
-               if (index_mail_parse_headers(mail, NULL) < 0)
+               const char *reason =
+                       index_mail_cache_reason(_mail, "mime parts");
+               if (index_mail_parse_headers(mail, NULL, reason) < 0)
                        return -1;
        }
 
@@ -510,7 +512,8 @@ int index_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
                return 0;
 
        old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
-       if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0)
+       if (mail_get_stream_because(_mail, &hdr_size, &body_size,
+                       index_mail_cache_reason(_mail, "virtual size"), &input) < 0)
                return -1;
        i_stream_seek(data->stream, old_offset);
 
@@ -945,7 +948,8 @@ static int index_mail_write_body_snippet(struct index_mail *mail)
        }
 
        old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset;
-       if (mail_get_stream(&mail->mail.mail, NULL, NULL, &input) < 0)
+       const char *reason = index_mail_cache_reason(&mail->mail.mail, "snippet");
+       if (mail_get_stream_because(&mail->mail.mail, NULL, NULL, reason, &input) < 0)
                return -1;
        i_assert(mail->data.stream != NULL);
 
@@ -1177,7 +1181,7 @@ int index_mail_init_stream(struct index_mail *mail,
                if (!data->hdr_size_set) {
                        if ((data->access_part & PARSE_HDR) != 0) {
                                (void)get_cached_parts(mail);
-                               if (index_mail_parse_headers(mail, NULL) < 0)
+                               if (index_mail_parse_headers(mail, NULL, "parse header") < 0)
                                        return -1;
                        } else {
                                if (message_get_header_size(data->stream,
@@ -1247,10 +1251,12 @@ static int index_mail_parse_bodystructure(struct index_mail *mail,
                    !data->save_bodystructure_body ||
                    field == MAIL_CACHE_BODY_SNIPPET) {
                        /* we haven't parsed the header yet */
+                       const char *reason =
+                               index_mail_cache_reason(&mail->mail.mail, "bodystructure");
                        data->save_bodystructure_header = TRUE;
                        data->save_bodystructure_body = TRUE;
                        (void)get_cached_parts(mail);
-                       if (index_mail_parse_headers(mail, NULL) < 0) {
+                       if (index_mail_parse_headers(mail, NULL, reason) < 0) {
                                data->save_bodystructure_header = TRUE;
                                return -1;
                        }
@@ -1745,7 +1751,7 @@ void index_mail_update_access_parts_post(struct mail *_mail)
                hdr = mail_index_get_header(_mail->transaction->view);
                if (!_mail->saving && _mail->uid < hdr->next_uid) {
                        if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
-                               (void)mail_get_stream(_mail, NULL, NULL, &input);
+                               (void)mail_get_stream_because(_mail, NULL, NULL, "access", &input);
                        else
                                (void)mail_get_hdr_stream(_mail, NULL, &input);
                }
@@ -1808,7 +1814,7 @@ bool index_mail_prefetch(struct mail *_mail)
        }
 
        if (mail->data.stream == NULL) {
-               (void)mail_get_stream(_mail, NULL, NULL, &input);
+               (void)mail_get_stream_because(_mail, NULL, NULL, "prefetch", &input);
                if (mail->data.stream == NULL)
                        return TRUE;
        }
@@ -2107,7 +2113,7 @@ static void index_mail_parse(struct mail *mail, bool parse_body)
        struct index_mail *imail = (struct index_mail *)mail;
 
        imail->data.access_part |= PARSE_HDR;
-       if (index_mail_parse_headers(imail, NULL) == 0) {
+       if (index_mail_parse_headers(imail, NULL, "precache") == 0) {
                if (parse_body) {
                        imail->data.access_part |= PARSE_BODY;
                        (void)index_mail_parse_body(imail, 0);
@@ -2231,3 +2237,10 @@ void index_mail_save_finish(struct mail_save_context *ctx)
                        p_strdup(imail->mail.data_pool, ctx->data.from_envelope);
        }
 }
+
+const char *index_mail_cache_reason(struct mail *mail, const char *reason)
+{
+       const char *cache_reason =
+               mail_cache_get_missing_reason(mail->transaction->cache_view, mail->seq);
+       return t_strdup_printf("%s (%s)", reason, cache_reason);
+}

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 392420f652c17923bbf5d2fd52d833ade397a3e6..6b0df554180bd4b5d2775db023997351cefd2744 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -188,7 +188,8 @@ void index_mail_parse_header(struct message_part *part,
                             struct message_header_line *hdr,
                             struct index_mail *mail) ATTR_NULL(1);
 int index_mail_parse_headers(struct index_mail *mail,
-                            struct mailbox_header_lookup_ctx *headers)
+                            struct mailbox_header_lookup_ctx *headers,
+                            const char *reason)
        ATTR_NULL(2);
 int index_mail_headers_get_envelope(struct index_mail *mail);
 
@@ -266,4 +267,6 @@ int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf,
                                  unsigned int field_idx);
 void index_mail_save_finish(struct mail_save_context *ctx);
 
+const char *index_mail_cache_reason(struct mail *mail, const char *reason);
+
 #endif

diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c
index 46d0f198579693929c1130a238f0263e8c6c1d3a..7dfdbda73b4e7028951d034f658901911e270387 100644 (file)
--- a/src/lib-storage/index/index-search.c
+++ b/src/lib-storage/index/index-search.c
@@ -713,8 +713,8 @@ static int search_arg_match_text(struct mail_search_arg *args,
        } else if (have_headers) {
                /* we need to read the entire header */
                ret = have_body ?
-                       mail_get_stream(ctx->cur_mail, NULL, NULL, &input) :
-                       mail_get_hdr_stream(ctx->cur_mail, NULL, &input);
+                       mail_get_stream_because(ctx->cur_mail, NULL, NULL, "search", &input) :
+                       mail_get_hdr_stream_because(ctx->cur_mail, NULL, "search", &input);
                if (ret < 0)
                        failed = TRUE;
                else {
@@ -766,7 +766,7 @@ static int search_arg_match_text(struct mail_search_arg *args,
                /* we didn't search headers. */
                struct message_size hdr_size;
 
-               if (mail_get_stream(ctx->cur_mail, &hdr_size, NULL, &input) < 0)
+               if (mail_get_stream_because(ctx->cur_mail, &hdr_size, NULL, "search", &input) < 0)
                        return -1;
                i_stream_seek(input, hdr_size.physical_size);
        }

diff --git a/src/lib-storage/mail-copy.c b/src/lib-storage/mail-copy.c
index db71052f1a395256649fefb5c301104fe26eaf3b..db499724d36a8e8980030be3dd2815d1a58a2948 100644 (file)
--- a/src/lib-storage/mail-copy.c
+++ b/src/lib-storage/mail-copy.c
@@ -66,7 +66,7 @@ mail_storage_try_copy(struct mail_save_context **_ctx, struct mail *mail)
           to help anything. */
        pmail->v.set_uid_cache_updates(mail, TRUE);
 
-       if (mail_get_stream(mail, NULL, NULL, &input) < 0) {
+       if (mail_get_stream_because(mail, NULL, NULL, "copying", &input) < 0) {
                mail_copy_set_failed(ctx, mail, "stream");
                return -1;
        }

diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c
index 2a334afa170432409da5f2408fa20da40efcc534..abac1d8f59e0dd123085fa324baaf3950c784875 100644 (file)
--- a/src/plugins/fts/fts-build-mail.c
+++ b/src/plugins/fts/fts-build-mail.c
@@ -468,7 +468,7 @@ fts_build_mail_real(struct fts_backend_update_context *update_ctx,
        const char *error;
        int ret;
 
-       if (mail_get_stream(mail, NULL, NULL, &input) < 0) {
+       if (mail_get_stream_because(mail, NULL, NULL, "fts indexing", &input) < 0) {
                if (mail->expunged)
                        return 0;
                i_error("Failed to read mailbox %s mail UID=%u stream: %s",

diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c
index 5e0e2b33a3922c0d495c4728f6e9a0e6dd568069..82c78e1a60f22d04104df848716789805081a63b 100644 (file)
--- a/src/plugins/pop3-migration/pop3-migration-plugin.c
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c
@@ -276,7 +276,7 @@ get_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN
           So we'll try to avoid this by falling back to full FETCH BODY[]
           (and/or RETR) and we'll parse the header ourself from it. This
           should work around any similar bugs in all IMAP/POP3 servers. */
-       if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
+       if (mail_get_stream_because(mail, &hdr_size, NULL, "pop3-migration", &input) < 0) {
                errstr = mailbox_get_last_error(mail->box, &error);
                i_error("pop3_migration: Failed to get body for msg %u: %s",
                        mail->seq, errstr);

diff --git a/src/pop3/pop3-commands.c b/src/pop3/pop3-commands.c
index adfc8fec86e35560b23f9095925856477e977b0d..09360d84d5adb3b482adcce38fa452c4367f1adf 100644 (file)
--- a/src/pop3/pop3-commands.c
+++ b/src/pop3/pop3-commands.c
@@ -438,7 +438,7 @@ static int client_reply_msg_expunged(struct client *client, unsigned int msgnum)
 }
 
 static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines,
-                uoff_t *byte_counter)
+                const char *reason, uoff_t *byte_counter)
 {
         struct fetch_context *ctx;
        int ret;
@@ -451,7 +451,7 @@ static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines,
                               MAIL_FETCH_STREAM_BODY, NULL);
        mail_set_seq(ctx->mail, msgnum_to_seq(client, msgnum));
 
-       if (mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) {
+       if (mail_get_stream_because(ctx->mail, NULL, NULL, reason, &ctx->stream) < 0) {
                ret = client_reply_msg_expunged(client, msgnum);
                fetch_deinit(ctx);
                return ret;
@@ -495,7 +495,7 @@ static int cmd_retr(struct client *client, const char *args)
                client->last_seen_pop3_msn = msgnum+1;
 
        client->retr_count++;
-       return fetch(client, msgnum, (uoff_t)-1, &client->retr_bytes);
+       return fetch(client, msgnum, (uoff_t)-1, "RETR", &client->retr_bytes);
 }
 
 static int cmd_rset(struct client *client, const char *args ATTR_UNUSED)
@@ -556,7 +556,7 @@ static int cmd_top(struct client *client, const char *args)
                return -1;
 
        client->top_count++;
-       return fetch(client, msgnum, max_lines, &client->top_bytes);
+       return fetch(client, msgnum, max_lines, "TOP", &client->top_bytes);
 }
 
 struct cmd_uidl_context {