From: Timo Sirainen Date: Sat, 16 Nov 2013 17:36:40 +0000 (+0200) Subject: imapc: Added imapc_feature fetch-header. X-Git-Tag: 2.2.8~29 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b8eb60a9ba83e2f3f6d969e810553eb937be2128;p=thirdparty%2Fdovecot%2Fcore.git imapc: Added imapc_feature fetch-header. It uses FETCH BODY.PEEK[HEADER.FIELDS (...)] whenever possible instead of fetching the entire header. --- diff --git a/TODO b/TODO index 948fddd3ab..7bab378408 100644 --- a/TODO +++ b/TODO @@ -111,10 +111,6 @@ - doveadm search savedbefore 7d could be optimized in large mailboxes.. - mdbox: storage rebuilding could log about changes it does - mdbox: broken extrefs header keeps causing index rebuilds - - cache: mail_cache_lookup() should be able to return mail_cache_add()ed - fields even before they've been flushed to disk. this is useful when copying - messages and some plugin (e.g. mail_log) fetches some fields that are - already added to cache (to avoid opening and parsing the message) - doveadm -A <storage->client->client); } +static bool +headers_have_subset(const char *const *superset, const char *const *subset) +{ + unsigned int i; + + if (superset == NULL) + return FALSE; + if (subset != NULL) { + for (i = 0; subset[i] != NULL; i++) { + if (!str_array_icase_find(superset, subset[i])) + return FALSE; + } + } + return TRUE; +} + +static const char *const * +headers_merge(pool_t pool, const char *const *h1, const char *const *h2) +{ + ARRAY_TYPE(const_string) headers; + const char *value; + unsigned int i; + + p_array_init(&headers, pool, 16); + if (h1 != NULL) { + for (i = 0; h1[i] != NULL; i++) { + value = p_strdup(pool, h1[i]); + array_append(&headers, &value, 1); + } + } + if (h2 != NULL) { + for (i = 0; h2[i] != NULL; i++) { + if (h1 == NULL || !str_array_icase_find(h1, h2[i])) { + value = p_strdup(pool, h2[i]); + array_append(&headers, &value, 1); + } + } + } + array_append_zero(&headers); + return array_idx(&headers, 0); +} + static int -imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields) +imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) { struct imapc_mail *mail = (struct imapc_mail *)_mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; @@ -60,13 +105,16 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields) struct mail_index_view *view; string_t *str; uint32_t seq; + unsigned int i; if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER) return -1; /* drop any fields that we may already be fetching currently */ fields &= ~mail->fetching_fields; - if (fields == 0) + if (headers_have_subset(mail->fetching_headers, headers)) + headers = NULL; + if (fields == 0 && headers == NULL) return 0; if (!_mail->saving) { @@ -106,6 +154,19 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields) str_append(str, "BODY.PEEK[] "); else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0) str_append(str, "BODY.PEEK[HEADER] "); + else if (headers != NULL) { + mail->fetching_headers = + headers_merge(mail->imail.mail.pool, headers, + mail->fetching_headers); + str_append(str, "BODY.PEEK[HEADER.FIELDS ("); + for (i = 0; mail->fetching_headers[i] != NULL; i++) { + if (i > 0) + str_append_c(str, ' '); + imap_append_astring(str, mail->fetching_headers[i]); + } + str_append(str, ")] "); + mail->header_list_fetched = FALSE; + } str_truncate(str, str_len(str)-1); str_append_c(str, ')'); @@ -174,7 +235,8 @@ bool imapc_mail_prefetch(struct mail *_mail) fields |= MAIL_FETCH_STREAM_HEADER; } if (fields != 0) T_BEGIN { - (void)imapc_mail_send_fetch(_mail, fields); + (void)imapc_mail_send_fetch(_mail, fields, + data->wanted_headers->name); } T_END; return !mail->imail.data.prefetch_sent; } @@ -207,7 +269,8 @@ imapc_mail_have_fields(struct imapc_mail *imail, enum mail_fetch_field fields) return TRUE; } -int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields) +int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields, + const char *const *headers) { struct imapc_mail *imail = (struct imapc_mail *)_mail; struct imapc_mailbox *mbox = @@ -223,7 +286,7 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields) } T_BEGIN { - ret = imapc_mail_send_fetch(_mail, fields); + ret = imapc_mail_send_fetch(_mail, fields, headers); } T_END; if (ret < 0) return -1; @@ -231,7 +294,9 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields) /* we'll continue waiting until we've got all the fields we wanted, or until all FETCH replies have been received (i.e. some FETCHes failed) */ - while (!imapc_mail_have_fields(imail, fields) && imail->fetch_count > 0) + while (imail->fetch_count > 0 && + (!imapc_mail_have_fields(imail, fields) || + !imail->header_list_fetched)) imapc_storage_run(mbox->storage); return 0; } @@ -360,6 +425,72 @@ imapc_fetch_stream(struct imapc_mail *mail, imapc_mail_init_stream(mail, body); } +static void +imapc_fetch_header_stream(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args) +{ + const enum message_header_parser_flags hdr_parser_flags = + MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP | + MESSAGE_HEADER_PARSER_FLAG_DROP_CR; + const struct imap_arg *hdr_list; + struct mailbox_header_lookup_ctx *headers_ctx; + struct message_header_parser_ctx *parser; + struct message_header_line *hdr; + struct istream *input; + ARRAY_TYPE(const_string) hdr_arr; + const char *value; + int ret, fd; + + if (!imap_arg_get_list(args, &hdr_list)) + return; + if (!imap_arg_atom_equals(args+1, "]")) + return; + args += 2; + + /* see if this is reply to the latest headers list request + (parse it even if it's not) */ + t_array_init(&hdr_arr, 16); + while (imap_arg_get_astring(hdr_list, &value)) { + array_append(&hdr_arr, &value, 1); + hdr_list++; + } + if (hdr_list->type != IMAP_ARG_EOL) + return; + array_append_zero(&hdr_arr); + + if (headers_have_subset(array_idx(&hdr_arr, 0), mail->fetching_headers)) + mail->header_list_fetched = TRUE; + + if (args->type == IMAP_ARG_LITERAL_SIZE) { + if (!imapc_find_lfile_arg(reply, args, &fd)) + return; + input = i_stream_create_fd(fd, 0, FALSE); + } else { + if (!imap_arg_get_nstring(args, &value)) + return; + if (value == NULL) { + mail_set_expunged(&mail->imail.mail.mail); + return; + } + input = i_stream_create_from_data(value, args->str_len); + } + + headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box, + array_idx(&hdr_arr, 0)); + index_mail_parse_header_init(&mail->imail, headers_ctx); + + parser = message_parse_header_init(input, NULL, hdr_parser_flags); + while ((ret = message_parse_header_next(parser, &hdr)) > 0) + index_mail_parse_header(NULL, hdr, &mail->imail); + i_assert(ret != 0); + index_mail_parse_header(NULL, NULL, &mail->imail); + message_parse_header_deinit(&parser); + + mailbox_header_lookup_unref(&headers_ctx); + i_stream_destroy(&input); +} + void imapc_mail_fetch_update(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, const struct imap_arg *args) @@ -384,6 +515,9 @@ void imapc_mail_fetch_update(struct imapc_mail *mail, } else if (strcasecmp(key, "BODY[HEADER]") == 0) { imapc_fetch_stream(mail, reply, &args[i+1], FALSE); match = TRUE; + } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) { + imapc_fetch_header_stream(mail, reply, &args[i+1]); + match = TRUE; } else if (strcasecmp(key, "INTERNALDATE") == 0) { if (imap_arg_get_astring(&args[i+1], &value) && imap_parse_datetime(value, &t, &tz)) diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c index dfa6ae95c2..4b32d7ccf5 100644 --- a/src/lib-storage/index/imapc/imapc-mail.c +++ b/src/lib-storage/index/imapc/imapc-mail.c @@ -80,7 +80,7 @@ static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) return 0; if (data->received_date == (time_t)-1) { - if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0) + if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0) return -1; if (data->received_date == (time_t)-1) { if (imapc_mail_failed(_mail, "INTERNALDATE") < 0) @@ -127,7 +127,7 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && data->stream == NULL) { /* trust RFC822.SIZE to be correct */ - if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE) < 0) + if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0) return -1; if (data->physical_size == (uoff_t)-1) { if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0) @@ -158,6 +158,70 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) return 0; } +static int +imapc_mail_get_header_stream(struct mail *_mail, + struct mailbox_header_lookup_ctx *headers, + struct istream **stream_r) +{ + struct imapc_mail *mail = (struct imapc_mail *)_mail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + enum mail_lookup_abort old_abort = _mail->lookup_abort; + int ret; + + if (mail->imail.data.access_part != 0 || + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) { + /* we're going to be reading the header/body anyway */ + return index_mail_get_header_stream(_mail, headers, stream_r); + } + + /* see if the wanted headers are already in cache */ + _mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + ret = index_mail_get_header_stream(_mail, headers, stream_r); + _mail->lookup_abort = old_abort; + if (ret == 0) + return 0; + + /* fetch only the wanted headers */ + if (imapc_mail_fetch(_mail, 0, headers->name) < 0) + return -1; + /* the headers should cached now. */ + return index_mail_get_header_stream(_mail, headers, stream_r); +} + +static int +imapc_mail_get_headers(struct mail *_mail, const char *field, + bool decode_to_utf8, const char *const **value_r) +{ + struct mailbox_header_lookup_ctx *headers; + const char *header_names[2]; + struct istream *input; + int ret; + + header_names[0] = field; + header_names[1] = NULL; + headers = mailbox_header_lookup_init(_mail->box, header_names); + ret = mail_get_header_stream(_mail, headers, &input); + mailbox_header_lookup_unref(&headers); + if (ret < 0) + return -1; + /* the header should cached now. */ + return index_mail_get_headers(_mail, field, decode_to_utf8, value_r); +} + +static int +imapc_mail_get_first_header(struct mail *_mail, const char *field, + bool decode_to_utf8, const char **value_r) +{ + const char *const *values; + int ret; + + ret = imapc_mail_get_headers(_mail, field, decode_to_utf8, &values); + if (ret <= 0) + return ret; + *value_r = values[0]; + return 1; +} + static int imapc_mail_get_stream(struct mail *_mail, bool get_body, struct message_size *hdr_size, @@ -182,7 +246,7 @@ imapc_mail_get_stream(struct mail *_mail, bool get_body, fetch_field = get_body || (data->access_part & READ_BODY) != 0 ? MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER; - if (imapc_mail_fetch(_mail, fetch_field) < 0) + if (imapc_mail_fetch(_mail, fetch_field, NULL) < 0) return -1; if (data->stream == NULL) { @@ -359,7 +423,7 @@ static int imapc_mail_get_guid(struct mail *_mail, const char **value_r) /* GUID not in cache, fetch it */ if (mbox->guid_fetch_field_name != NULL) { - if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID) < 0) + if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0) return -1; if (imail->data.guid == NULL) { (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name); @@ -420,9 +484,9 @@ struct mail_vfuncs imapc_mail_vfuncs = { imapc_mail_get_save_date, index_mail_get_virtual_size, imapc_mail_get_physical_size, - index_mail_get_first_header, - index_mail_get_headers, - index_mail_get_header_stream, + imapc_mail_get_first_header, + imapc_mail_get_headers, + imapc_mail_get_header_stream, imapc_mail_get_stream, index_mail_get_binary_stream, imapc_mail_get_special, diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h index 7493c027b0..ba94f89dd2 100644 --- a/src/lib-storage/index/imapc/imapc-mail.h +++ b/src/lib-storage/index/imapc/imapc-mail.h @@ -10,11 +10,13 @@ struct imapc_mail { struct index_mail imail; enum mail_fetch_field fetching_fields; + const char *const *fetching_headers; unsigned int fetch_count; int fd; buffer_t *body; bool body_fetched; + bool header_list_fetched; }; extern struct mail_vfuncs imapc_mail_vfuncs; @@ -23,7 +25,8 @@ struct mail * imapc_mail_alloc(struct mailbox_transaction_context *t, enum mail_fetch_field wanted_fields, struct mailbox_header_lookup_ctx *wanted_headers); -int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields); +int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields, + const char *const *headers); bool imapc_mail_prefetch(struct mail *mail); void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body); diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c index feabe023a7..5a89ade59d 100644 --- a/src/lib-storage/index/imapc/imapc-settings.c +++ b/src/lib-storage/index/imapc/imapc-settings.c @@ -77,6 +77,7 @@ struct imapc_feature_list { static const struct imapc_feature_list imapc_feature_list[] = { { "rfc822.size", IMAPC_FEATURE_RFC822_SIZE }, { "guid-forced", IMAPC_FEATURE_GUID_FORCED }, + { "fetch-headers", IMAPC_FEATURE_FETCH_HEADERS }, { NULL, 0 } }; diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h index 939f3bbf1a..05f2b7476f 100644 --- a/src/lib-storage/index/imapc/imapc-settings.h +++ b/src/lib-storage/index/imapc/imapc-settings.h @@ -4,7 +4,8 @@ /* */ enum imapc_features { IMAPC_FEATURE_RFC822_SIZE = 0x01, - IMAPC_FEATURE_GUID_FORCED = 0x02 + IMAPC_FEATURE_GUID_FORCED = 0x02, + IMAPC_FEATURE_FETCH_HEADERS = 0x04 }; /* */