imapc: Merge identical FETCH commands together (only updating UID range)

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index 91a9aa02844cc3ad561a9f3b885151dcf074dc6f..26f88242e6a673eb6da777c32f3cb5b43096fb31 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "str.h"
+#include "ioloop.h"
 #include "istream.h"
 #include "istream-header-filter.h"
 #include "message-header-parser.h"
@@ -13,29 +14,37 @@
 #include "imapc-storage.h"
 
 static void
-imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
-                            void *context)
+imapc_mail_fetch_callback(const struct imapc_command_reply *reply,
+                         void *context)
 {
-       struct imapc_mail *mail = context;
-       struct imapc_mailbox *mbox =
-               (struct imapc_mailbox *)mail->imail.mail.mail.box;
+       struct imapc_fetch_request *request = context;
+       struct imapc_fetch_request *const *requests;
+       struct imapc_mail *const *mailp;
+       struct imapc_mailbox *mbox = NULL;
+       unsigned int i, count;
 
-       i_assert(mail->fetch_count > 0);
+       array_foreach(&request->mails, mailp) {
+               struct imapc_mail *mail = *mailp;
 
-       if (--mail->fetch_count == 0) {
-               struct imapc_mail *const *fetch_mails;
-               unsigned int i, count;
+               i_assert(mail->fetch_count > 0);
+               if (--mail->fetch_count == 0)
+                       mail->fetching_fields = 0;
+               pool_unref(&mail->imail.mail.pool);
+               mbox = (struct imapc_mailbox *)mail->imail.mail.mail.box;
+       }
+       i_assert(mbox != NULL);
 
-               fetch_mails = array_get(&mbox->fetch_mails, &count);
-               for (i = 0; i < count; i++) {
-                       if (fetch_mails[i] == mail) {
-                               array_delete(&mbox->fetch_mails, i, 1);
-                               break;
-                       }
+       requests = array_get(&mbox->fetch_requests, &count);
+       for (i = 0; i < count; i++) {
+               if (requests[i] == request) {
+                       array_delete(&mbox->fetch_requests, i, 1);
+                       break;
                }
-               i_assert(i != count);
-               mail->fetching_fields = 0;
        }
+       i_assert(i < count);
+
+       array_free(&request->mails);
+       i_free(request);
 
        if (reply->state == IMAPC_COMMAND_STATE_OK)
                ;
@@ -47,9 +56,8 @@ imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
                mail_storage_set_internal_error(&mbox->storage->storage);
        } else {
                mail_storage_set_critical(&mbox->storage->storage,
-                       "imapc: Mail prefetch failed: %s", reply->text_full);
+                       "imapc: Mail FETCH failed: %s", reply->text_full);
        }
-       pool_unref(&mail->imail.mail.pool);
        imapc_client_stop(mbox->storage->client->client);
 }
 
@@ -95,13 +103,61 @@ headers_merge(pool_t pool, const char *const *h1, const char *const *h2)
        return array_idx(&headers, 0);
 }
 
+static bool
+imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str)
+{
+       const char *s1 = str_c(str);
+       const char *s2 = str_c(mbox->pending_fetch_cmd);
+       const char *p1, *p2;
+
+       i_assert(strncmp(s1, "UID FETCH ", 10) == 0);
+       i_assert(strncmp(s2, "UID FETCH ", 10) == 0);
+
+       /* skip over UID range */
+       p1 = strchr(s1+10, ' ');
+       p2 = strchr(s2+10, ' ');
+
+       if (null_strcmp(p1, p2) != 0)
+               return FALSE;
+       /* append the new UID to the pending FETCH UID range */
+       str_truncate(str, p1-s1);
+       str_insert(mbox->pending_fetch_cmd, p2-s2, ",");
+       str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10);
+       return TRUE;
+}
+
+static void
+imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str)
+{
+       struct imapc_mailbox *mbox =
+               (struct imapc_mailbox *)mail->imail.mail.mail.box;
+
+       if (mbox->pending_fetch_request != NULL &&
+           !imapc_mail_try_merge_fetch(mbox, str)) {
+               /* send the previous FETCH and create a new one */
+               imapc_mail_fetch_flush(mbox);
+       }
+       if (mbox->pending_fetch_request == NULL) {
+               mbox->pending_fetch_request =
+                       i_new(struct imapc_fetch_request, 1);
+               i_array_init(&mbox->pending_fetch_request->mails, 4);
+               i_assert(mbox->pending_fetch_cmd->used == 0);
+               str_append_str(mbox->pending_fetch_cmd, str);
+       }
+       array_append(&mbox->pending_fetch_request->mails, &mail, 1);
+
+       if (mbox->to_pending_fetch_send == NULL) {
+               mbox->to_pending_fetch_send =
+                       timeout_add_short(0, imapc_mail_fetch_flush, mbox);
+       }
+}
+
 static int
 imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
                      const char *const *headers)
 {
        struct imapc_mail *mail = (struct imapc_mail *)_mail;
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
-       struct imapc_command *cmd;
        struct mail_index_view *view;
        string_t *str;
        uint32_t seq;
@@ -172,15 +228,10 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
 
        pool_ref(mail->imail.mail.pool);
        mail->fetching_fields |= fields;
-       if (mail->fetch_count++ == 0)
-               array_append(&mbox->fetch_mails, &mail, 1);
+       mail->fetch_count++;
 
-       cmd = imapc_client_mailbox_cmd(mbox->client_box,
-                                      imapc_mail_prefetch_callback, mail);
-       imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
-       imapc_command_send(cmd, str_c(str));
-       mail->imail.data.prefetch_sent = TRUE;
-       return 0;
+       imapc_mail_delayed_send_or_merge(mail, str);
+       return 1;
 }
 
 static void imapc_mail_cache_get(struct imapc_mail *mail,
@@ -252,9 +303,10 @@ bool imapc_mail_prefetch(struct mail *_mail)
 
        fields = imapc_mail_get_wanted_fetch_fields(mail);
        if (fields != 0) T_BEGIN {
-               (void)imapc_mail_send_fetch(_mail, fields,
-                                           data->wanted_headers == NULL ? NULL :
-                                           data->wanted_headers->name);
+               if (imapc_mail_send_fetch(_mail, fields,
+                                         data->wanted_headers == NULL ? NULL :
+                                         data->wanted_headers->name) > 0)
+                       mail->imail.data.prefetch_sent = TRUE;
        } T_END;
        return !mail->imail.data.prefetch_sent;
 }
@@ -316,10 +368,31 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields,
        while (imail->fetch_count > 0 &&
               (!imapc_mail_have_fields(imail, fields) ||
                !imail->header_list_fetched))
-               imapc_storage_run(mbox->storage);
+               imapc_mailbox_run(mbox);
        return 0;
 }
 
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox)
+{
+       struct imapc_command *cmd;
+
+       if (mbox->pending_fetch_request == NULL) {
+               i_assert(mbox->to_pending_fetch_send == NULL);
+               return;
+       }
+
+       cmd = imapc_client_mailbox_cmd(mbox->client_box,
+                                      imapc_mail_fetch_callback,
+                                      mbox->pending_fetch_request);
+       imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+       imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd));
+
+       array_append(&mbox->fetch_requests, &mbox->pending_fetch_request, 1);
+       mbox->pending_fetch_request = NULL;
+       timeout_remove(&mbox->to_pending_fetch_send);
+       str_truncate(mbox->pending_fetch_cmd, 0);
+}
+
 static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
                                 const struct imap_arg *arg, int *fd_r)
 {

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 2984240c0571c8a636e9ffe899ab554717b5d4c4..a6f3fb7918ee65fb365083b374df0c64714f132d 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -358,7 +358,7 @@ static void imapc_mail_close(struct mail *_mail)
        struct imapc_mail_cache *cache = &mbox->prev_mail_cache;
 
        while (mail->fetch_count > 0)
-               imapc_storage_run(mbox->storage);
+               imapc_mailbox_run(mbox);
 
        index_mail_close(_mail);
 

diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h
index f3c61fa12ca7f9998a7b7bca12b7de4661bd6720..888781f50c449cbe9376375190cf620aaeffab1f 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.h
+++ b/src/lib-storage/index/imapc/imapc-mail.h
@@ -5,6 +5,7 @@
 
 struct imap_arg;
 struct imapc_untagged_reply;
+struct imapc_mailbox;
 
 struct imapc_mail {
        struct index_mail imail;
@@ -28,11 +29,13 @@ imapc_mail_alloc(struct mailbox_transaction_context *t,
 int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields,
                     const char *const *headers);
 bool imapc_mail_prefetch(struct mail *mail);
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox);
 void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body);
 
 void imapc_mail_fetch_update(struct imapc_mail *mail,
                             const struct imapc_untagged_reply *reply,
                             const struct imap_arg *args);
 void imapc_mail_update_access_parts(struct index_mail *mail);
+void imapc_mail_command_flush(struct imapc_mailbox *mbox);
 
 #endif

diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index f62d4ea641ab7105e26f6b718008413bb3f49986..06c112ff216e075d6cbff161c62fd45e562b920e 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -254,6 +254,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                                 struct imapc_mailbox *mbox)
 {
        uint32_t lseq, rseq = reply->num;
+       struct imapc_fetch_request *const *fetch_requestp;
        struct imapc_mail *const *mailp;
        const struct imap_arg *list, *flags_list;
        const char *atom;
@@ -303,11 +304,13 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                return;
 
        /* if this is a reply to some FETCH request, update the mail's fields */
-       array_foreach(&mbox->fetch_mails, mailp) {
-               struct imapc_mail *mail = *mailp;
+       array_foreach(&mbox->fetch_requests, fetch_requestp) {
+               array_foreach(&(*fetch_requestp)->mails, mailp) {
+                       struct imapc_mail *mail = *mailp;
 
-               if (mail->imail.mail.mail.uid == uid)
-                       imapc_mail_fetch_update(mail, reply, list);
+                       if (mail->imail.mail.mail.uid == uid)
+                               imapc_mail_fetch_update(mail, reply, list);
+               }
        }
 
        if (lseq == 0) {

diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c
index dec39b2c063952b04096c3d1008299a3df14cd03..2abecdc127d77c9d960cccf2290f54bc13b465ae 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-save.c
+++ b/src/lib-storage/index/imapc/imapc-save.c
@@ -246,7 +246,7 @@ static int imapc_save_append(struct imapc_save_context *ctx)
                            ctx->mbox->box.name, flags, internaldate, input);
        i_stream_unref(&input);
        while (sctx.ret == -2)
-               imapc_storage_run(ctx->mbox->storage);
+               imapc_mailbox_run(ctx->mbox);
 
        if (sctx.ret == 0 && ctx->mbox->selected &&
            !ctx->mbox->exists_received) {
@@ -259,7 +259,7 @@ static int imapc_save_append(struct imapc_save_context *ctx)
                                       imapc_save_noop_callback, &sctx);
                imapc_command_send(cmd, "NOOP");
                while (sctx.ret == -2)
-                       imapc_storage_run(ctx->mbox->storage);
+                       imapc_mailbox_run(ctx->mbox);
        }
        return sctx.ret;
 }
@@ -421,7 +421,7 @@ int imapc_copy(struct mail_save_context *_ctx, struct mail *mail)
                imapc_command_sendf(cmd, "UID COPY %u %s",
                                    mail->uid, _t->box->name);
                while (sctx.ret == -2)
-                       imapc_storage_run(src_mbox->storage);
+                       imapc_mailbox_run(src_mbox);
                ctx->finished = TRUE;
                index_save_context_free(_ctx);
                return sctx.ret;

diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index cffe06564beb31e1d0dbbe608f8200f977478d75..f596e0c007277e4b2fbfae33097cbb1316660957 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -113,11 +113,12 @@ void imapc_simple_run(struct imapc_simple_context *sctx)
                imapc_client_run(sctx->client->client);
 }
 
-void imapc_storage_run(struct imapc_storage *storage)
+void imapc_mailbox_run(struct imapc_mailbox *mbox)
 {
+       imapc_mail_fetch_flush(mbox);
        do {
-               imapc_client_run(storage->client->client);
-       } while (storage->reopen_count > 0);
+               imapc_client_run(mbox->storage->client->client);
+       } while (mbox->storage->reopen_count > 0);
 }
 
 void imapc_simple_callback(const struct imapc_command_reply *reply,
@@ -343,8 +344,9 @@ imapc_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
 
        p_array_init(&mbox->untagged_callbacks, pool, 16);
        p_array_init(&mbox->resp_text_callbacks, pool, 16);
-       p_array_init(&mbox->fetch_mails, pool, 16);
+       p_array_init(&mbox->fetch_requests, pool, 16);
        p_array_init(&mbox->delayed_expunged_uids, pool, 16);
+       mbox->pending_fetch_cmd = str_new(pool, 128);
        mbox->prev_mail_cache.fd = -1;
        imapc_mailbox_register_callbacks(mbox);
        return &mbox->box;
@@ -480,7 +482,7 @@ int imapc_mailbox_select(struct imapc_mailbox *mbox)
                imapc_command_sendf(cmd, "SELECT %s", mbox->box.name);
 
        while (ctx.ret == -2)
-               imapc_storage_run(mbox->storage);
+               imapc_mailbox_run(mbox);
        return ctx.ret;
 }
 
@@ -529,6 +531,7 @@ static void imapc_mailbox_close(struct mailbox *box)
 {
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
 
+       imapc_mail_fetch_flush(mbox);
        if (mbox->client_box != NULL)
                imapc_client_mailbox_close(&mbox->client_box);
        if (mbox->delayed_sync_view != NULL)

diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index e811f85dbb4e72dc3fb75fafa0c3b28fbfb17e7c..a91c483a9b73c1faffaf0ca0d9544b86429f7e3d 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -76,6 +76,10 @@ struct imapc_mail_cache {
        buffer_t *buf;
 };
 
+struct imapc_fetch_request {
+       ARRAY(struct imapc_mail *) mails;
+};
+
 struct imapc_mailbox {
        struct mailbox box;
        struct imapc_storage *storage;
@@ -85,7 +89,13 @@ struct imapc_mailbox {
        struct mail_index_view *sync_view, *delayed_sync_view;
        struct timeout *to_idle_check, *to_idle_delay;
 
-       ARRAY(struct imapc_mail *) fetch_mails;
+       ARRAY(struct imapc_fetch_request *) fetch_requests;
+       /* if non-empty, contains the latest FETCH command we're going to be
+          sending soon (but still waiting to see if we can increase its
+          UID range) */
+       string_t *pending_fetch_cmd;
+       struct imapc_fetch_request *pending_fetch_request;
+       struct timeout *to_pending_fetch_send;
 
        ARRAY(struct imapc_mailbox_event_callback) untagged_callbacks;
        ARRAY(struct imapc_mailbox_event_callback) resp_text_callbacks;
@@ -142,7 +152,7 @@ void imapc_transaction_save_commit_post(struct mail_save_context *ctx,
                                        struct mail_index_transaction_commit_result *result);
 void imapc_transaction_save_rollback(struct mail_save_context *ctx);
 
-void imapc_storage_run(struct imapc_storage *storage);
+void imapc_mailbox_run(struct imapc_mailbox *mbox);
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 

diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index e83cf4e025ccd4111143ce2a00ce597f81861b41..16fe0b2a20fed7bb72fb92e9cd5f6b308c4f9e3a 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -322,7 +322,7 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
 
        imapc_sync_expunge_finish(ctx);
        while (ctx->sync_command_count > 0)
-               imapc_storage_run(mbox->storage);
+               imapc_mailbox_run(mbox);
        array_free(&ctx->expunged_uids);
 
        /* add uidnext after all appends */