From: Timo Sirainen <tss@iki.fi>
Date: Sun, 17 Nov 2013 14:59:33 +0000 (+0200)
Subject: imapc: Merge identical FETCH commands together (only updating UID range)
X-Git-Tag: 2.2.8~12
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1a878b9d2a823abc6b1c8b1631e50a15d534665f;p=thirdparty%2Fdovecot%2Fcore.git

imapc: Merge identical FETCH commands together (only updating UID range)
---

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index 91a9aa0284..26f88242e6 100644
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "str.h"
+#include "ioloop.h"
 #include "istream.h"
 #include "istream-header-filter.h"
 #include "message-header-parser.h"
@@ -13,29 +14,37 @@
 #include "imapc-storage.h"
 
 static void
-imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
-			     void *context)
+imapc_mail_fetch_callback(const struct imapc_command_reply *reply,
+			  void *context)
 {
-	struct imapc_mail *mail = context;
-	struct imapc_mailbox *mbox =
-		(struct imapc_mailbox *)mail->imail.mail.mail.box;
+	struct imapc_fetch_request *request = context;
+	struct imapc_fetch_request *const *requests;
+	struct imapc_mail *const *mailp;
+	struct imapc_mailbox *mbox = NULL;
+	unsigned int i, count;
 
-	i_assert(mail->fetch_count > 0);
+	array_foreach(&request->mails, mailp) {
+		struct imapc_mail *mail = *mailp;
 
-	if (--mail->fetch_count == 0) {
-		struct imapc_mail *const *fetch_mails;
-		unsigned int i, count;
+		i_assert(mail->fetch_count > 0);
+		if (--mail->fetch_count == 0)
+			mail->fetching_fields = 0;
+		pool_unref(&mail->imail.mail.pool);
+		mbox = (struct imapc_mailbox *)mail->imail.mail.mail.box;
+	}
+	i_assert(mbox != NULL);
 
-		fetch_mails = array_get(&mbox->fetch_mails, &count);
-		for (i = 0; i < count; i++) {
-			if (fetch_mails[i] == mail) {
-				array_delete(&mbox->fetch_mails, i, 1);
-				break;
-			}
+	requests = array_get(&mbox->fetch_requests, &count);
+	for (i = 0; i < count; i++) {
+		if (requests[i] == request) {
+			array_delete(&mbox->fetch_requests, i, 1);
+			break;
 		}
-		i_assert(i != count);
-		mail->fetching_fields = 0;
 	}
+	i_assert(i < count);
+
+	array_free(&request->mails);
+	i_free(request);
 
 	if (reply->state == IMAPC_COMMAND_STATE_OK)
 		;
@@ -47,9 +56,8 @@ imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
 		mail_storage_set_internal_error(&mbox->storage->storage);
 	} else {
 		mail_storage_set_critical(&mbox->storage->storage,
-			"imapc: Mail prefetch failed: %s", reply->text_full);
+			"imapc: Mail FETCH failed: %s", reply->text_full);
 	}
-	pool_unref(&mail->imail.mail.pool);
 	imapc_client_stop(mbox->storage->client->client);
 }
 
@@ -95,13 +103,61 @@ headers_merge(pool_t pool, const char *const *h1, const char *const *h2)
 	return array_idx(&headers, 0);
 }
 
+static bool
+imapc_mail_try_merge_fetch(struct imapc_mailbox *mbox, string_t *str)
+{
+	const char *s1 = str_c(str);
+	const char *s2 = str_c(mbox->pending_fetch_cmd);
+	const char *p1, *p2;
+
+	i_assert(strncmp(s1, "UID FETCH ", 10) == 0);
+	i_assert(strncmp(s2, "UID FETCH ", 10) == 0);
+
+	/* skip over UID range */
+	p1 = strchr(s1+10, ' ');
+	p2 = strchr(s2+10, ' ');
+
+	if (null_strcmp(p1, p2) != 0)
+		return FALSE;
+	/* append the new UID to the pending FETCH UID range */
+	str_truncate(str, p1-s1);
+	str_insert(mbox->pending_fetch_cmd, p2-s2, ",");
+	str_insert(mbox->pending_fetch_cmd, p2-s2+1, str_c(str) + 10);
+	return TRUE;
+}
+
+static void
+imapc_mail_delayed_send_or_merge(struct imapc_mail *mail, string_t *str)
+{
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)mail->imail.mail.mail.box;
+
+	if (mbox->pending_fetch_request != NULL &&
+	    !imapc_mail_try_merge_fetch(mbox, str)) {
+		/* send the previous FETCH and create a new one */
+		imapc_mail_fetch_flush(mbox);
+	}
+	if (mbox->pending_fetch_request == NULL) {
+		mbox->pending_fetch_request =
+			i_new(struct imapc_fetch_request, 1);
+		i_array_init(&mbox->pending_fetch_request->mails, 4);
+		i_assert(mbox->pending_fetch_cmd->used == 0);
+		str_append_str(mbox->pending_fetch_cmd, str);
+	}
+	array_append(&mbox->pending_fetch_request->mails, &mail, 1);
+
+	if (mbox->to_pending_fetch_send == NULL) {
+		mbox->to_pending_fetch_send =
+			timeout_add_short(0, imapc_mail_fetch_flush, mbox);
+	}
+}
+
 static int
 imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
 		      const char *const *headers)
 {
 	struct imapc_mail *mail = (struct imapc_mail *)_mail;
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
-	struct imapc_command *cmd;
 	struct mail_index_view *view;
 	string_t *str;
 	uint32_t seq;
@@ -172,15 +228,10 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields,
 
 	pool_ref(mail->imail.mail.pool);
 	mail->fetching_fields |= fields;
-	if (mail->fetch_count++ == 0)
-		array_append(&mbox->fetch_mails, &mail, 1);
+	mail->fetch_count++;
 
-	cmd = imapc_client_mailbox_cmd(mbox->client_box,
-				       imapc_mail_prefetch_callback, mail);
-	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
-	imapc_command_send(cmd, str_c(str));
-	mail->imail.data.prefetch_sent = TRUE;
-	return 0;
+	imapc_mail_delayed_send_or_merge(mail, str);
+	return 1;
 }
 
 static void imapc_mail_cache_get(struct imapc_mail *mail,
@@ -252,9 +303,10 @@ bool imapc_mail_prefetch(struct mail *_mail)
 
 	fields = imapc_mail_get_wanted_fetch_fields(mail);
 	if (fields != 0) T_BEGIN {
-		(void)imapc_mail_send_fetch(_mail, fields,
-					    data->wanted_headers == NULL ? NULL :
-					    data->wanted_headers->name);
+		if (imapc_mail_send_fetch(_mail, fields,
+					  data->wanted_headers == NULL ? NULL :
+					  data->wanted_headers->name) > 0)
+			mail->imail.data.prefetch_sent = TRUE;
 	} T_END;
 	return !mail->imail.data.prefetch_sent;
 }
@@ -316,10 +368,31 @@ int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields,
 	while (imail->fetch_count > 0 &&
 	       (!imapc_mail_have_fields(imail, fields) ||
 		!imail->header_list_fetched))
-		imapc_storage_run(mbox->storage);
+		imapc_mailbox_run(mbox);
 	return 0;
 }
 
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox)
+{
+	struct imapc_command *cmd;
+
+	if (mbox->pending_fetch_request == NULL) {
+		i_assert(mbox->to_pending_fetch_send == NULL);
+		return;
+	}
+
+	cmd = imapc_client_mailbox_cmd(mbox->client_box,
+				       imapc_mail_fetch_callback,
+				       mbox->pending_fetch_request);
+	imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
+	imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd));
+
+	array_append(&mbox->fetch_requests, &mbox->pending_fetch_request, 1);
+	mbox->pending_fetch_request = NULL;
+	timeout_remove(&mbox->to_pending_fetch_send);
+	str_truncate(mbox->pending_fetch_cmd, 0);
+}
+
 static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
 				 const struct imap_arg *arg, int *fd_r)
 {
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 2984240c05..a6f3fb7918 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -358,7 +358,7 @@ static void imapc_mail_close(struct mail *_mail)
 	struct imapc_mail_cache *cache = &mbox->prev_mail_cache;
 
 	while (mail->fetch_count > 0)
-		imapc_storage_run(mbox->storage);
+		imapc_mailbox_run(mbox);
 
 	index_mail_close(_mail);
 
diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h
index f3c61fa12c..888781f50c 100644
--- a/src/lib-storage/index/imapc/imapc-mail.h
+++ b/src/lib-storage/index/imapc/imapc-mail.h
@@ -5,6 +5,7 @@
 
 struct imap_arg;
 struct imapc_untagged_reply;
+struct imapc_mailbox;
 
 struct imapc_mail {
 	struct index_mail imail;
@@ -28,11 +29,13 @@ imapc_mail_alloc(struct mailbox_transaction_context *t,
 int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields,
 		     const char *const *headers);
 bool imapc_mail_prefetch(struct mail *mail);
+void imapc_mail_fetch_flush(struct imapc_mailbox *mbox);
 void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body);
 
 void imapc_mail_fetch_update(struct imapc_mail *mail,
 			     const struct imapc_untagged_reply *reply,
 			     const struct imap_arg *args);
 void imapc_mail_update_access_parts(struct index_mail *mail);
+void imapc_mail_command_flush(struct imapc_mailbox *mbox);
 
 #endif
diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index f62d4ea641..06c112ff21 100644
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -254,6 +254,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 				 struct imapc_mailbox *mbox)
 {
 	uint32_t lseq, rseq = reply->num;
+	struct imapc_fetch_request *const *fetch_requestp;
 	struct imapc_mail *const *mailp;
 	const struct imap_arg *list, *flags_list;
 	const char *atom;
@@ -303,11 +304,13 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 		return;
 
 	/* if this is a reply to some FETCH request, update the mail's fields */
-	array_foreach(&mbox->fetch_mails, mailp) {
-		struct imapc_mail *mail = *mailp;
+	array_foreach(&mbox->fetch_requests, fetch_requestp) {
+		array_foreach(&(*fetch_requestp)->mails, mailp) {
+			struct imapc_mail *mail = *mailp;
 
-		if (mail->imail.mail.mail.uid == uid)
-			imapc_mail_fetch_update(mail, reply, list);
+			if (mail->imail.mail.mail.uid == uid)
+				imapc_mail_fetch_update(mail, reply, list);
+		}
 	}
 
 	if (lseq == 0) {
diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c
index dec39b2c06..2abecdc127 100644
--- a/src/lib-storage/index/imapc/imapc-save.c
+++ b/src/lib-storage/index/imapc/imapc-save.c
@@ -246,7 +246,7 @@ static int imapc_save_append(struct imapc_save_context *ctx)
 			    ctx->mbox->box.name, flags, internaldate, input);
 	i_stream_unref(&input);
 	while (sctx.ret == -2)
-		imapc_storage_run(ctx->mbox->storage);
+		imapc_mailbox_run(ctx->mbox);
 
 	if (sctx.ret == 0 && ctx->mbox->selected &&
 	    !ctx->mbox->exists_received) {
@@ -259,7 +259,7 @@ static int imapc_save_append(struct imapc_save_context *ctx)
 				       imapc_save_noop_callback, &sctx);
 		imapc_command_send(cmd, "NOOP");
 		while (sctx.ret == -2)
-			imapc_storage_run(ctx->mbox->storage);
+			imapc_mailbox_run(ctx->mbox);
 	}
 	return sctx.ret;
 }
@@ -421,7 +421,7 @@ int imapc_copy(struct mail_save_context *_ctx, struct mail *mail)
 		imapc_command_sendf(cmd, "UID COPY %u %s",
 				    mail->uid, _t->box->name);
 		while (sctx.ret == -2)
-			imapc_storage_run(src_mbox->storage);
+			imapc_mailbox_run(src_mbox);
 		ctx->finished = TRUE;
 		index_save_context_free(_ctx);
 		return sctx.ret;
diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index cffe06564b..f596e0c007 100644
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -113,11 +113,12 @@ void imapc_simple_run(struct imapc_simple_context *sctx)
 		imapc_client_run(sctx->client->client);
 }
 
-void imapc_storage_run(struct imapc_storage *storage)
+void imapc_mailbox_run(struct imapc_mailbox *mbox)
 {
+	imapc_mail_fetch_flush(mbox);
 	do {
-		imapc_client_run(storage->client->client);
-	} while (storage->reopen_count > 0);
+		imapc_client_run(mbox->storage->client->client);
+	} while (mbox->storage->reopen_count > 0);
 }
 
 void imapc_simple_callback(const struct imapc_command_reply *reply,
@@ -343,8 +344,9 @@ imapc_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list,
 
 	p_array_init(&mbox->untagged_callbacks, pool, 16);
 	p_array_init(&mbox->resp_text_callbacks, pool, 16);
-	p_array_init(&mbox->fetch_mails, pool, 16);
+	p_array_init(&mbox->fetch_requests, pool, 16);
 	p_array_init(&mbox->delayed_expunged_uids, pool, 16);
+	mbox->pending_fetch_cmd = str_new(pool, 128);
 	mbox->prev_mail_cache.fd = -1;
 	imapc_mailbox_register_callbacks(mbox);
 	return &mbox->box;
@@ -480,7 +482,7 @@ int imapc_mailbox_select(struct imapc_mailbox *mbox)
 		imapc_command_sendf(cmd, "SELECT %s", mbox->box.name);
 
 	while (ctx.ret == -2)
-		imapc_storage_run(mbox->storage);
+		imapc_mailbox_run(mbox);
 	return ctx.ret;
 }
 
@@ -529,6 +531,7 @@ static void imapc_mailbox_close(struct mailbox *box)
 {
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
 
+	imapc_mail_fetch_flush(mbox);
 	if (mbox->client_box != NULL)
 		imapc_client_mailbox_close(&mbox->client_box);
 	if (mbox->delayed_sync_view != NULL)
diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index e811f85dbb..a91c483a9b 100644
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -76,6 +76,10 @@ struct imapc_mail_cache {
 	buffer_t *buf;
 };
 
+struct imapc_fetch_request {
+	ARRAY(struct imapc_mail *) mails;
+};
+
 struct imapc_mailbox {
 	struct mailbox box;
 	struct imapc_storage *storage;
@@ -85,7 +89,13 @@ struct imapc_mailbox {
 	struct mail_index_view *sync_view, *delayed_sync_view;
 	struct timeout *to_idle_check, *to_idle_delay;
 
-	ARRAY(struct imapc_mail *) fetch_mails;
+	ARRAY(struct imapc_fetch_request *) fetch_requests;
+	/* if non-empty, contains the latest FETCH command we're going to be
+	   sending soon (but still waiting to see if we can increase its
+	   UID range) */
+	string_t *pending_fetch_cmd;
+	struct imapc_fetch_request *pending_fetch_request;
+	struct timeout *to_pending_fetch_send;
 
 	ARRAY(struct imapc_mailbox_event_callback) untagged_callbacks;
 	ARRAY(struct imapc_mailbox_event_callback) resp_text_callbacks;
@@ -142,7 +152,7 @@ void imapc_transaction_save_commit_post(struct mail_save_context *ctx,
 					struct mail_index_transaction_commit_result *result);
 void imapc_transaction_save_rollback(struct mail_save_context *ctx);
 
-void imapc_storage_run(struct imapc_storage *storage);
+void imapc_mailbox_run(struct imapc_mailbox *mbox);
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 
diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index e83cf4e025..16fe0b2a20 100644
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -322,7 +322,7 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
 
 	imapc_sync_expunge_finish(ctx);
 	while (ctx->sync_command_count > 0)
-		imapc_storage_run(mbox->storage);
+		imapc_mailbox_run(mbox);
 	array_free(&ctx->expunged_uids);
 
 	/* add uidnext after all appends */