]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
imapc: Added imapc_feature fetch-header.
authorTimo Sirainen <tss@iki.fi>
Sat, 16 Nov 2013 17:36:40 +0000 (19:36 +0200)
committerTimo Sirainen <tss@iki.fi>
Sat, 16 Nov 2013 17:36:40 +0000 (19:36 +0200)
It uses FETCH BODY.PEEK[HEADER.FIELDS (...)] whenever possible instead of
fetching the entire header.

TODO
src/lib-storage/index/imapc/imapc-mail-fetch.c
src/lib-storage/index/imapc/imapc-mail.c
src/lib-storage/index/imapc/imapc-mail.h
src/lib-storage/index/imapc/imapc-settings.c
src/lib-storage/index/imapc/imapc-settings.h

diff --git a/TODO b/TODO
index 948fddd3abd7caedb80cdf1bb855b7d003eb5b51..7bab3784087cab6dc4473423e99cea97125ce779 100644 (file)
--- a/TODO
+++ b/TODO
  - doveadm search savedbefore 7d could be optimized in large mailboxes..
  - mdbox: storage rebuilding could log about changes it does
  - mdbox: broken extrefs header keeps causing index rebuilds
- - cache: mail_cache_lookup() should be able to return mail_cache_add()ed
-   fields even before they've been flushed to disk. this is useful when copying
-   messages and some plugin (e.g. mail_log) fetches some fields that are
-   already added to cache (to avoid opening and parsing the message)
  - doveadm -A <<EOF expunge + purge + EOF
  - sent, drafts: .Sent/dovecot.index: modseq_hdr.log_offset too large
  - mail_max_lock_timeout error could be reported more nicely, also ones coming
index 13b17fb743fb6d56f6fb83f2001457714b705430..5d6594ff723f78d4f8d161f4c482397913bc90cd 100644 (file)
@@ -4,8 +4,10 @@
 #include "str.h"
 #include "istream.h"
 #include "istream-header-filter.h"
+#include "message-header-parser.h"
 #include "imap-arg.h"
 #include "imap-date.h"
+#include "imap-quote.h"
 #include "imapc-client.h"
 #include "imapc-mail.h"
 #include "imapc-storage.h"
@@ -51,8 +53,51 @@ imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
        imapc_client_stop(mbox->storage->client->client);
 }
 
+static bool
+headers_have_subset(const char *const *superset, const char *const *subset)
+{
+       unsigned int i;
+
+       if (superset == NULL)
+               return FALSE;
+       if (subset != NULL) {
+               for (i = 0; subset[i] != NULL; i++) {
+                       if (!str_array_icase_find(superset, subset[i]))
+                               return FALSE;
+               }
+       }
+       return TRUE;
+}
+
+static const char *const *
+headers_merge(pool_t pool, const char *const *h1, const char *const *h2)
+{
+       ARRAY_TYPE(const_string) headers;
+       const char *value;
+       unsigned int i;
+
+       p_array_init(&headers, pool, 16);
+       if (h1 != NULL) {
+               for (i = 0; h1[i] != NULL; i++) {
+                       value = p_strdup(pool, h1[i]);
+                       array_append(&headers, &value, 1);
+               }
+       }
+       if (h2 != NULL) {
+               for (i = 0; h2[i] != NULL; i++) {
+                       if (h1 == NULL || !str_array_icase_find(h1, h2[i])) {
+                               value = p_strdup(pool, h2[i]);
+                               array_append(&headers, &value, 1);
+                       }
+               }
+       }
+       array_append_zero(&headers);
+       return array_idx(&headers, 0);
+}
+
 static int
-imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields)
+imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
+                     const char *const *headers)
 {
        struct imapc_mail *mail = (struct imapc_mail *)_mail;
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
@@ -60,13 +105,16 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields)
        struct mail_index_view *view;
        string_t *str;
        uint32_t seq;
+       unsigned int i;
 
        if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)
                return -1;
 
        /* drop any fields that we may already be fetching currently */
        fields &= ~mail->fetching_fields;
-       if (fields == 0)
+       if (headers_have_subset(mail->fetching_headers, headers))
+               headers = NULL;
+       if (fields == 0 && headers == NULL)
                return 0;
 
        if (!_mail->saving) {
@@ -106,6 +154,19 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields)
                str_append(str, "BODY.PEEK[] ");
        else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0)
                str_append(str, "BODY.PEEK[HEADER] ");
+       else if (headers != NULL) {
+               mail->fetching_headers =
+                       headers_merge(mail->imail.mail.pool, headers,
+                                     mail->fetching_headers);
+               str_append(str, "BODY.PEEK[HEADER.FIELDS (");
+               for (i = 0; mail->fetching_headers[i] != NULL; i++) {
+                       if (i > 0)
+                               str_append_c(str, ' ');
+                       imap_append_astring(str, mail->fetching_headers[i]);
+               }
+               str_append(str, ")] ");
+               mail->header_list_fetched = FALSE;
+       }
        str_truncate(str, str_len(str)-1);
        str_append_c(str, ')');
 
@@ -174,7 +235,8 @@ bool imapc_mail_prefetch(struct mail *_mail)
                        fields |= MAIL_FETCH_STREAM_HEADER;
        }
        if (fields != 0) T_BEGIN {
-               (void)imapc_mail_send_fetch(_mail, fields);
+               (void)imapc_mail_send_fetch(_mail, fields,
+                                           data->wanted_headers->name);
        } T_END;
        return !mail->imail.data.prefetch_sent;
 }
@@ -207,7 +269,8 @@ imapc_mail_have_fields(struct imapc_mail *imail, enum mail_fetch_field fields)
        return TRUE;
 }
 
-int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields)
+int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields,
+                    const char *const *headers)
 {
        struct imapc_mail *imail = (struct imapc_mail *)_mail;
        struct imapc_mailbox *mbox =
@@ -223,7 +286,7 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields)
        }
 
        T_BEGIN {
-               ret = imapc_mail_send_fetch(_mail, fields);
+               ret = imapc_mail_send_fetch(_mail, fields, headers);
        } T_END;
        if (ret < 0)
                return -1;
@@ -231,7 +294,9 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields)
        /* we'll continue waiting until we've got all the fields we wanted,
           or until all FETCH replies have been received (i.e. some FETCHes
           failed) */
-       while (!imapc_mail_have_fields(imail, fields) && imail->fetch_count > 0)
+       while (imail->fetch_count > 0 &&
+              (!imapc_mail_have_fields(imail, fields) ||
+               !imail->header_list_fetched))
                imapc_storage_run(mbox->storage);
        return 0;
 }
@@ -360,6 +425,72 @@ imapc_fetch_stream(struct imapc_mail *mail,
        imapc_mail_init_stream(mail, body);
 }
 
+static void
+imapc_fetch_header_stream(struct imapc_mail *mail,
+                         const struct imapc_untagged_reply *reply,
+                         const struct imap_arg *args)
+{
+       const enum message_header_parser_flags hdr_parser_flags =
+               MESSAGE_HEADER_PARSER_FLAG_SKIP_INITIAL_LWSP |
+               MESSAGE_HEADER_PARSER_FLAG_DROP_CR;
+       const struct imap_arg *hdr_list;
+       struct mailbox_header_lookup_ctx *headers_ctx;
+       struct message_header_parser_ctx *parser;
+       struct message_header_line *hdr;
+       struct istream *input;
+       ARRAY_TYPE(const_string) hdr_arr;
+       const char *value;
+       int ret, fd;
+
+       if (!imap_arg_get_list(args, &hdr_list))
+               return;
+       if (!imap_arg_atom_equals(args+1, "]"))
+               return;
+       args += 2;
+
+       /* see if this is reply to the latest headers list request
+          (parse it even if it's not) */
+       t_array_init(&hdr_arr, 16);
+       while (imap_arg_get_astring(hdr_list, &value)) {
+               array_append(&hdr_arr, &value, 1);
+               hdr_list++;
+       }
+       if (hdr_list->type != IMAP_ARG_EOL)
+               return;
+       array_append_zero(&hdr_arr);
+
+       if (headers_have_subset(array_idx(&hdr_arr, 0), mail->fetching_headers))
+               mail->header_list_fetched = TRUE;
+
+       if (args->type == IMAP_ARG_LITERAL_SIZE) {
+               if (!imapc_find_lfile_arg(reply, args, &fd))
+                       return;
+               input = i_stream_create_fd(fd, 0, FALSE);
+       } else {
+               if (!imap_arg_get_nstring(args, &value))
+                       return;
+               if (value == NULL) {
+                       mail_set_expunged(&mail->imail.mail.mail);
+                       return;
+               }
+               input = i_stream_create_from_data(value, args->str_len);
+       }
+
+       headers_ctx = mailbox_header_lookup_init(mail->imail.mail.mail.box,
+                                                array_idx(&hdr_arr, 0));
+       index_mail_parse_header_init(&mail->imail, headers_ctx);
+
+       parser = message_parse_header_init(input, NULL, hdr_parser_flags);
+       while ((ret = message_parse_header_next(parser, &hdr)) > 0)
+               index_mail_parse_header(NULL, hdr, &mail->imail);
+       i_assert(ret != 0);
+       index_mail_parse_header(NULL, NULL, &mail->imail);
+       message_parse_header_deinit(&parser);
+
+       mailbox_header_lookup_unref(&headers_ctx);
+       i_stream_destroy(&input);
+}
+
 void imapc_mail_fetch_update(struct imapc_mail *mail,
                             const struct imapc_untagged_reply *reply,
                             const struct imap_arg *args)
@@ -384,6 +515,9 @@ void imapc_mail_fetch_update(struct imapc_mail *mail,
                } else if (strcasecmp(key, "BODY[HEADER]") == 0) {
                        imapc_fetch_stream(mail, reply, &args[i+1], FALSE);
                        match = TRUE;
+               } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) {
+                       imapc_fetch_header_stream(mail, reply, &args[i+1]);
+                       match = TRUE;
                } else if (strcasecmp(key, "INTERNALDATE") == 0) {
                        if (imap_arg_get_astring(&args[i+1], &value) &&
                            imap_parse_datetime(value, &t, &tz))
index dfa6ae95c2823c9f84ee000ca013d053dbf71f94..4b32d7ccf5481e9ebd29579daea9761bf0ed0b2f 100644 (file)
@@ -80,7 +80,7 @@ static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
                return 0;
 
        if (data->received_date == (time_t)-1) {
-               if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0)
+               if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0)
                        return -1;
                if (data->received_date == (time_t)-1) {
                        if (imapc_mail_failed(_mail, "INTERNALDATE") < 0)
@@ -127,7 +127,7 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
        if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) &&
            data->stream == NULL) {
                /* trust RFC822.SIZE to be correct */
-               if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE) < 0)
+               if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
                        return -1;
                if (data->physical_size == (uoff_t)-1) {
                        if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0)
@@ -158,6 +158,70 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
        return 0;
 }
 
+static int
+imapc_mail_get_header_stream(struct mail *_mail,
+                            struct mailbox_header_lookup_ctx *headers,
+                            struct istream **stream_r)
+{
+       struct imapc_mail *mail = (struct imapc_mail *)_mail;
+       struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+       enum mail_lookup_abort old_abort = _mail->lookup_abort;
+       int ret;
+
+       if (mail->imail.data.access_part != 0 ||
+           !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_HEADERS)) {
+               /* we're going to be reading the header/body anyway */
+               return index_mail_get_header_stream(_mail, headers, stream_r);
+       }
+
+       /* see if the wanted headers are already in cache */
+       _mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL;
+       ret = index_mail_get_header_stream(_mail, headers, stream_r);
+       _mail->lookup_abort = old_abort;
+       if (ret == 0)
+               return 0;
+
+       /* fetch only the wanted headers */
+       if (imapc_mail_fetch(_mail, 0, headers->name) < 0)
+               return -1;
+       /* the headers should cached now. */
+       return index_mail_get_header_stream(_mail, headers, stream_r);
+}
+
+static int
+imapc_mail_get_headers(struct mail *_mail, const char *field,
+                      bool decode_to_utf8, const char *const **value_r)
+{
+       struct mailbox_header_lookup_ctx *headers;
+       const char *header_names[2];
+       struct istream *input;
+       int ret;
+
+       header_names[0] = field;
+       header_names[1] = NULL;
+       headers = mailbox_header_lookup_init(_mail->box, header_names);
+       ret = mail_get_header_stream(_mail, headers, &input);
+       mailbox_header_lookup_unref(&headers);
+       if (ret < 0)
+               return -1;
+       /* the header should cached now. */
+       return index_mail_get_headers(_mail, field, decode_to_utf8, value_r);
+}
+
+static int
+imapc_mail_get_first_header(struct mail *_mail, const char *field,
+                           bool decode_to_utf8, const char **value_r)
+{
+       const char *const *values;
+       int ret;
+
+       ret = imapc_mail_get_headers(_mail, field, decode_to_utf8, &values);
+       if (ret <= 0)
+               return ret;
+       *value_r = values[0];
+       return 1;
+}
+
 static int
 imapc_mail_get_stream(struct mail *_mail, bool get_body,
                      struct message_size *hdr_size,
@@ -182,7 +246,7 @@ imapc_mail_get_stream(struct mail *_mail, bool get_body,
                fetch_field = get_body ||
                        (data->access_part & READ_BODY) != 0 ?
                        MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER;
-               if (imapc_mail_fetch(_mail, fetch_field) < 0)
+               if (imapc_mail_fetch(_mail, fetch_field, NULL) < 0)
                        return -1;
 
                if (data->stream == NULL) {
@@ -359,7 +423,7 @@ static int imapc_mail_get_guid(struct mail *_mail, const char **value_r)
 
        /* GUID not in cache, fetch it */
        if (mbox->guid_fetch_field_name != NULL) {
-               if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID) < 0)
+               if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0)
                        return -1;
                if (imail->data.guid == NULL) {
                        (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
@@ -420,9 +484,9 @@ struct mail_vfuncs imapc_mail_vfuncs = {
        imapc_mail_get_save_date,
        index_mail_get_virtual_size,
        imapc_mail_get_physical_size,
-       index_mail_get_first_header,
-       index_mail_get_headers,
-       index_mail_get_header_stream,
+       imapc_mail_get_first_header,
+       imapc_mail_get_headers,
+       imapc_mail_get_header_stream,
        imapc_mail_get_stream,
        index_mail_get_binary_stream,
        imapc_mail_get_special,
index 7493c027b08b859992dcf00e3af00df91b8e6a40..ba94f89dd2fc7d391f1e2814b7a48291e860c98d 100644 (file)
@@ -10,11 +10,13 @@ struct imapc_mail {
        struct index_mail imail;
 
        enum mail_fetch_field fetching_fields;
+       const char *const *fetching_headers;
        unsigned int fetch_count;
 
        int fd;
        buffer_t *body;
        bool body_fetched;
+       bool header_list_fetched;
 };
 
 extern struct mail_vfuncs imapc_mail_vfuncs;
@@ -23,7 +25,8 @@ struct mail *
 imapc_mail_alloc(struct mailbox_transaction_context *t,
                 enum mail_fetch_field wanted_fields,
                 struct mailbox_header_lookup_ctx *wanted_headers);
-int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields);
+int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields,
+                    const char *const *headers);
 bool imapc_mail_prefetch(struct mail *mail);
 void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body);
 
index feabe023a79e1df02407ff9c0e637c8749f4257c..5a89ade59ddefefc72df21edadf79de97f6cfba5 100644 (file)
@@ -77,6 +77,7 @@ struct imapc_feature_list {
 static const struct imapc_feature_list imapc_feature_list[] = {
        { "rfc822.size", IMAPC_FEATURE_RFC822_SIZE },
        { "guid-forced", IMAPC_FEATURE_GUID_FORCED },
+       { "fetch-headers", IMAPC_FEATURE_FETCH_HEADERS },
        { NULL, 0 }
 };
 
index 939f3bbf1a69f5c0a779c3000fcc1ae8fa8aa652..05f2b7476fb608f8e16451fc69e2aae3502b7883 100644 (file)
@@ -4,7 +4,8 @@
 /* <settings checks> */
 enum imapc_features {
        IMAPC_FEATURE_RFC822_SIZE       = 0x01,
-       IMAPC_FEATURE_GUID_FORCED       = 0x02
+       IMAPC_FEATURE_GUID_FORCED       = 0x02,
+       IMAPC_FEATURE_FETCH_HEADERS     = 0x04
 };
 /* </settings checks> */