From: Timo Sirainen Date: Mon, 25 Jul 2016 18:20:05 +0000 (-0400) Subject: Use mail_get_*stream_because() wherever possible. X-Git-Tag: 2.3.0.rc1~3264 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=02b78558dc03daa2e7da2010b63f247b49936a38;p=thirdparty%2Fdovecot%2Fcore.git Use mail_get_*stream_because() wherever possible. --- diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c index 08f78254ce..2f28f83cc7 100644 --- a/src/lib-imap-storage/imap-msgpart.c +++ b/src/lib-imap-storage/imap-msgpart.c @@ -578,7 +578,7 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart, if (*msgpart->section_number != '\0') { /* find the MIME part */ - if (mail_get_stream(mail, NULL, NULL, &input) < 0) + if (mail_get_stream_because(mail, NULL, NULL, "MIME part", &input) < 0) return -1; i_stream_seek(input, part->physical_pos); @@ -587,7 +587,7 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart, } else switch (msgpart->fetch_type) { case FETCH_FULL: /* fetch the whole message */ - if (mail_get_stream(mail, NULL, NULL, &input) < 0 || + if (mail_get_stream_because(mail, NULL, NULL, "full mail", &input) < 0 || mail_get_virtual_size(mail, &body_size.virtual_size) < 0) return -1; result_r->size_field = MAIL_FETCH_VIRTUAL_SIZE; @@ -622,7 +622,8 @@ imap_msgpart_open_normal(struct mail *mail, struct imap_msgpart *msgpart, break; case FETCH_BODY: /* fetch the message's body */ - if (mail_get_stream(mail, &hdr_size, &body_size, &input) < 0) + if (mail_get_stream_because(mail, &hdr_size, &body_size, + "mail body", &input) < 0) return -1; result_r->size_field = MAIL_FETCH_MESSAGE_PARTS; break; diff --git a/src/lib-storage/index/index-mail-binary.c b/src/lib-storage/index/index-mail-binary.c index 7f604dbe2e..852b9c7a06 100644 --- a/src/lib-storage/index/index-mail-binary.c +++ b/src/lib-storage/index/index-mail-binary.c @@ -361,8 +361,8 @@ blocks_count_lines(struct binary_ctx *ctx, struct istream *full_input) static int index_mail_read_binary_to_cache(struct mail *_mail, const struct message_part *part, - bool include_hdr, bool *binary_r, - bool *converted_r) + bool include_hdr, const char *reason, + bool *binary_r, bool *converted_r) { struct index_mail *mail = (struct index_mail *)_mail; struct mail_binary_cache *cache = &_mail->box->storage->binary_cache; @@ -373,7 +373,7 @@ index_mail_read_binary_to_cache(struct mail *_mail, t_array_init(&ctx.blocks, 8); mail_storage_free_binary_cache(_mail->box->storage); - if (mail_get_stream(_mail, NULL, NULL, &ctx.input) < 0) + if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0) return -1; if (add_binary_part(&ctx, part, include_hdr) < 0) { @@ -490,7 +490,7 @@ index_mail_get_binary_size(struct mail *_mail, if (!get_cached_binary_parts(mail)) { /* not found. parse the whole message */ if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE, - &binary, &converted) < 0) + "binary.size", &binary, &converted) < 0) return -1; } @@ -567,7 +567,7 @@ int index_mail_get_binary_stream(struct mail *_mail, converted = TRUE; } else { if (index_mail_read_binary_to_cache(_mail, part, include_hdr, - &binary, &converted) < 0) + "binary stream", &binary, &converted) < 0) return -1; mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY; } diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c index 0fb13b16d1..8f6a3548fb 100644 --- a/src/lib-storage/index/index-mail-headers.c +++ b/src/lib-storage/index/index-mail-headers.c @@ -425,7 +425,8 @@ static void index_mail_init_parser(struct index_mail *mail) } int index_mail_parse_headers(struct index_mail *mail, - struct mailbox_header_lookup_ctx *headers) + struct mailbox_header_lookup_ctx *headers, + const char *reason) { struct index_mail_data *data = &mail->data; struct istream *input; @@ -433,7 +434,7 @@ int index_mail_parse_headers(struct index_mail *mail, old_offset = data->stream == NULL ? 0 : data->stream->v_offset; - if (mail_get_hdr_stream(&mail->mail.mail, NULL, &input) < 0) + if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0) return -1; index_mail_parse_header_init(mail, headers); @@ -635,10 +636,12 @@ index_mail_get_raw_headers(struct index_mail *mail, const char *field, if (mail->header_seq != mail->data.seq || index_mail_header_is_parsed(mail, field_idx) < 0) { /* parse */ + const char *reason = index_mail_cache_reason(_mail, + t_strdup_printf("header %s", field)); headers[0] = field; headers[1] = NULL; headers_ctx = mailbox_header_lookup_init(_mail->box, headers); - ret = index_mail_parse_headers(mail, headers_ctx); + ret = index_mail_parse_headers(mail, headers_ctx, reason); mailbox_header_lookup_unref(&headers_ctx); if (ret < 0) return -1; @@ -857,11 +860,14 @@ int index_mail_get_header_stream(struct mail *_mail, struct istream *input; string_t *dest; + i_assert(headers->count > 0); i_assert(headers->box == _mail->box); if (mail->data.save_bodystructure_header) { /* we have to parse the header. */ - if (index_mail_parse_headers(mail, headers) < 0) + const char *reason = + index_mail_cache_reason(_mail, "bodystructure"); + if (index_mail_parse_headers(mail, headers, reason) < 0) return -1; } @@ -882,7 +888,25 @@ int index_mail_get_header_stream(struct mail *_mail, /* not in cache / error */ p_free(mail->mail.data_pool, dest); - if (mail_get_hdr_stream(_mail, NULL, &input) < 0) + unsigned int first_not_found = UINT_MAX, not_found_count = 0; + for (unsigned int i = 0; i < headers->count; i++) { + if (mail_cache_field_exists(_mail->transaction->cache_view, + _mail->seq, headers->idx[i]) <= 0) { + if (not_found_count++ == 0) + first_not_found = i; + } + } + + const char *reason; + if (not_found_count == 0) + reason = "BUG: all headers seem to exist in cache"; + else { + i_assert(first_not_found != UINT_MAX); + reason = index_mail_cache_reason(_mail, t_strdup_printf( + "%u/%u headers not cached (first=%s)", + not_found_count, headers->count, headers->name[first_not_found])); + } + if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0) return -1; if (mail->data.filter_stream != NULL) diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index ffb6e32496..bdbf6759de 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -321,7 +321,9 @@ int index_mail_get_parts(struct mail *_mail, struct message_part **parts_r) } if (data->parser_ctx == NULL) { - if (index_mail_parse_headers(mail, NULL) < 0) + const char *reason = + index_mail_cache_reason(_mail, "mime parts"); + if (index_mail_parse_headers(mail, NULL, reason) < 0) return -1; } @@ -510,7 +512,8 @@ int index_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r) return 0; old_offset = data->stream == NULL ? 0 : data->stream->v_offset; - if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0) + if (mail_get_stream_because(_mail, &hdr_size, &body_size, + index_mail_cache_reason(_mail, "virtual size"), &input) < 0) return -1; i_stream_seek(data->stream, old_offset); @@ -945,7 +948,8 @@ static int index_mail_write_body_snippet(struct index_mail *mail) } old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset; - if (mail_get_stream(&mail->mail.mail, NULL, NULL, &input) < 0) + const char *reason = index_mail_cache_reason(&mail->mail.mail, "snippet"); + if (mail_get_stream_because(&mail->mail.mail, NULL, NULL, reason, &input) < 0) return -1; i_assert(mail->data.stream != NULL); @@ -1177,7 +1181,7 @@ int index_mail_init_stream(struct index_mail *mail, if (!data->hdr_size_set) { if ((data->access_part & PARSE_HDR) != 0) { (void)get_cached_parts(mail); - if (index_mail_parse_headers(mail, NULL) < 0) + if (index_mail_parse_headers(mail, NULL, "parse header") < 0) return -1; } else { if (message_get_header_size(data->stream, @@ -1247,10 +1251,12 @@ static int index_mail_parse_bodystructure(struct index_mail *mail, !data->save_bodystructure_body || field == MAIL_CACHE_BODY_SNIPPET) { /* we haven't parsed the header yet */ + const char *reason = + index_mail_cache_reason(&mail->mail.mail, "bodystructure"); data->save_bodystructure_header = TRUE; data->save_bodystructure_body = TRUE; (void)get_cached_parts(mail); - if (index_mail_parse_headers(mail, NULL) < 0) { + if (index_mail_parse_headers(mail, NULL, reason) < 0) { data->save_bodystructure_header = TRUE; return -1; } @@ -1745,7 +1751,7 @@ void index_mail_update_access_parts_post(struct mail *_mail) hdr = mail_index_get_header(_mail->transaction->view); if (!_mail->saving && _mail->uid < hdr->next_uid) { if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0) - (void)mail_get_stream(_mail, NULL, NULL, &input); + (void)mail_get_stream_because(_mail, NULL, NULL, "access", &input); else (void)mail_get_hdr_stream(_mail, NULL, &input); } @@ -1808,7 +1814,7 @@ bool index_mail_prefetch(struct mail *_mail) } if (mail->data.stream == NULL) { - (void)mail_get_stream(_mail, NULL, NULL, &input); + (void)mail_get_stream_because(_mail, NULL, NULL, "prefetch", &input); if (mail->data.stream == NULL) return TRUE; } @@ -2107,7 +2113,7 @@ static void index_mail_parse(struct mail *mail, bool parse_body) struct index_mail *imail = (struct index_mail *)mail; imail->data.access_part |= PARSE_HDR; - if (index_mail_parse_headers(imail, NULL) == 0) { + if (index_mail_parse_headers(imail, NULL, "precache") == 0) { if (parse_body) { imail->data.access_part |= PARSE_BODY; (void)index_mail_parse_body(imail, 0); @@ -2231,3 +2237,10 @@ void index_mail_save_finish(struct mail_save_context *ctx) p_strdup(imail->mail.data_pool, ctx->data.from_envelope); } } + +const char *index_mail_cache_reason(struct mail *mail, const char *reason) +{ + const char *cache_reason = + mail_cache_get_missing_reason(mail->transaction->cache_view, mail->seq); + return t_strdup_printf("%s (%s)", reason, cache_reason); +} diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h index e7b1ed6b65..1dd4bd7c01 100644 --- a/src/lib-storage/index/index-mail.h +++ b/src/lib-storage/index/index-mail.h @@ -188,7 +188,8 @@ void index_mail_parse_header(struct message_part *part, struct message_header_line *hdr, struct index_mail *mail) ATTR_NULL(1); int index_mail_parse_headers(struct index_mail *mail, - struct mailbox_header_lookup_ctx *headers) + struct mailbox_header_lookup_ctx *headers, + const char *reason) ATTR_NULL(2); int index_mail_headers_get_envelope(struct index_mail *mail); @@ -266,4 +267,6 @@ int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, unsigned int field_idx); void index_mail_save_finish(struct mail_save_context *ctx); +const char *index_mail_cache_reason(struct mail *mail, const char *reason); + #endif diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c index a117d17be6..325dd5b4e7 100644 --- a/src/lib-storage/index/index-search.c +++ b/src/lib-storage/index/index-search.c @@ -713,8 +713,8 @@ static int search_arg_match_text(struct mail_search_arg *args, } else if (have_headers) { /* we need to read the entire header */ ret = have_body ? - mail_get_stream(ctx->cur_mail, NULL, NULL, &input) : - mail_get_hdr_stream(ctx->cur_mail, NULL, &input); + mail_get_stream_because(ctx->cur_mail, NULL, NULL, "search", &input) : + mail_get_hdr_stream_because(ctx->cur_mail, NULL, "search", &input); if (ret < 0) failed = TRUE; else { @@ -766,7 +766,7 @@ static int search_arg_match_text(struct mail_search_arg *args, /* we didn't search headers. */ struct message_size hdr_size; - if (mail_get_stream(ctx->cur_mail, &hdr_size, NULL, &input) < 0) + if (mail_get_stream_because(ctx->cur_mail, &hdr_size, NULL, "search", &input) < 0) return -1; i_stream_seek(input, hdr_size.physical_size); } diff --git a/src/lib-storage/mail-copy.c b/src/lib-storage/mail-copy.c index db71052f1a..db499724d3 100644 --- a/src/lib-storage/mail-copy.c +++ b/src/lib-storage/mail-copy.c @@ -66,7 +66,7 @@ mail_storage_try_copy(struct mail_save_context **_ctx, struct mail *mail) to help anything. */ pmail->v.set_uid_cache_updates(mail, TRUE); - if (mail_get_stream(mail, NULL, NULL, &input) < 0) { + if (mail_get_stream_because(mail, NULL, NULL, "copying", &input) < 0) { mail_copy_set_failed(ctx, mail, "stream"); return -1; } diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c index 2a334afa17..abac1d8f59 100644 --- a/src/plugins/fts/fts-build-mail.c +++ b/src/plugins/fts/fts-build-mail.c @@ -468,7 +468,7 @@ fts_build_mail_real(struct fts_backend_update_context *update_ctx, const char *error; int ret; - if (mail_get_stream(mail, NULL, NULL, &input) < 0) { + if (mail_get_stream_because(mail, NULL, NULL, "fts indexing", &input) < 0) { if (mail->expunged) return 0; i_error("Failed to read mailbox %s mail UID=%u stream: %s", diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c index 1321aab817..33241b05f4 100644 --- a/src/plugins/pop3-migration/pop3-migration-plugin.c +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c @@ -273,7 +273,7 @@ get_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN So we'll try to avoid this by falling back to full FETCH BODY[] (and/or RETR) and we'll parse the header ourself from it. This should work around any similar bugs in all IMAP/POP3 servers. */ - if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) { + if (mail_get_stream_because(mail, &hdr_size, NULL, "pop3-migration", &input) < 0) { errstr = mailbox_get_last_error(mail->box, &error); i_error("pop3_migration: Failed to get body for msg %u: %s", mail->seq, errstr); diff --git a/src/pop3/pop3-commands.c b/src/pop3/pop3-commands.c index 27e596045e..7b48ca0493 100644 --- a/src/pop3/pop3-commands.c +++ b/src/pop3/pop3-commands.c @@ -438,7 +438,7 @@ static int client_reply_msg_expunged(struct client *client, unsigned int msgnum) } static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines, - uoff_t *byte_counter) + const char *reason, uoff_t *byte_counter) { struct fetch_context *ctx; int ret; @@ -451,7 +451,7 @@ static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines, MAIL_FETCH_STREAM_BODY, NULL); mail_set_seq(ctx->mail, msgnum_to_seq(client, msgnum)); - if (mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) { + if (mail_get_stream_because(ctx->mail, NULL, NULL, reason, &ctx->stream) < 0) { ret = client_reply_msg_expunged(client, msgnum); fetch_deinit(ctx); return ret; @@ -495,7 +495,7 @@ static int cmd_retr(struct client *client, const char *args) client->last_seen_pop3_msn = msgnum+1; client->retr_count++; - return fetch(client, msgnum, (uoff_t)-1, &client->retr_bytes); + return fetch(client, msgnum, (uoff_t)-1, "RETR", &client->retr_bytes); } static int cmd_rset(struct client *client, const char *args ATTR_UNUSED) @@ -556,7 +556,7 @@ static int cmd_top(struct client *client, const char *args) return -1; client->top_count++; - return fetch(client, msgnum, max_lines, &client->top_bytes); + return fetch(client, msgnum, max_lines, "TOP", &client->top_bytes); } struct cmd_uidl_context {