From: Timo Sirainen <tss@iki.fi>
Date: Sun, 30 Jan 2011 18:47:43 +0000 (+0200)
Subject: imapc: Send UID FETCH commands with larger uidset parameter if possible.
X-Git-Tag: 2.1.alpha1~396
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=94a77d6e5d7600859e4c5e4a7ea19dc3e91998b9;p=thirdparty%2Fdovecot%2Fcore.git

imapc: Send UID FETCH commands with larger uidset parameter if possible.
This code assumes that server doesn't reorganize FETCH replies when it's
given an increasing UID range, which in theory IMAP server would be allowed
to do, but I doubt there are any.
---

diff --git a/src/lib-storage/index/imapc/Makefile.am b/src/lib-storage/index/imapc/Makefile.am
index c4e23bb8b0..4f47d890d0 100644
--- a/src/lib-storage/index/imapc/Makefile.am
+++ b/src/lib-storage/index/imapc/Makefile.am
@@ -25,6 +25,7 @@ libstorage_imapc_la_SOURCES = \
 headers = \
 	imapc-connection.h \
 	imapc-list.h \
+	imapc-mail.h \
 	imapc-seqmap.h \
 	imapc-storage.h \
 	imapc-sync.h
diff --git a/src/lib-storage/index/imapc/imapc-client-private.h b/src/lib-storage/index/imapc/imapc-client-private.h
index 0862cf4ab2..ff8e27ef85 100644
--- a/src/lib-storage/index/imapc/imapc-client-private.h
+++ b/src/lib-storage/index/imapc/imapc-client-private.h
@@ -18,6 +18,8 @@ struct imapc_client {
 	ARRAY_DEFINE(conns, struct imapc_client_connection *);
 
 	struct ioloop *ioloop;
+
+	unsigned int stop_now:1;
 };
 
 struct imapc_client_mailbox {
diff --git a/src/lib-storage/index/imapc/imapc-client.c b/src/lib-storage/index/imapc/imapc-client.c
index 23425e9e06..4871686595 100644
--- a/src/lib-storage/index/imapc/imapc-client.c
+++ b/src/lib-storage/index/imapc/imapc-client.c
@@ -70,15 +70,24 @@ void imapc_client_run(struct imapc_client *client)
 {
 	struct imapc_client_connection *const *connp;
 	struct ioloop *prev_ioloop = current_ioloop;
+	bool handle_pending = client->stop_now;
 
 	i_assert(client->ioloop == NULL);
 
+	client->stop_now = FALSE;
+
 	client->ioloop = io_loop_create();
+	io_loop_set_running(client->ioloop);
+
 	array_foreach(&client->conns, connp) {
 		imapc_connection_ioloop_changed((*connp)->conn);
 		imapc_connection_connect((*connp)->conn);
+		if (handle_pending)
+			imapc_connection_input_pending((*connp)->conn);
 	}
-	io_loop_run(client->ioloop);
+
+	if (io_loop_is_running(client->ioloop))
+		io_loop_run(client->ioloop);
 
 	current_ioloop = prev_ioloop;
 	array_foreach(&client->conns, connp)
@@ -94,6 +103,12 @@ void imapc_client_stop(struct imapc_client *client)
 		io_loop_stop(client->ioloop);
 }
 
+void imapc_client_stop_now(struct imapc_client *client)
+{
+	client->stop_now = TRUE;
+	imapc_client_stop(client);
+}
+
 static struct imapc_client_connection *
 imapc_client_add_connection(struct imapc_client *client)
 {
diff --git a/src/lib-storage/index/imapc/imapc-client.h b/src/lib-storage/index/imapc/imapc-client.h
index dd6ab9d99b..d9762a19e5 100644
--- a/src/lib-storage/index/imapc/imapc-client.h
+++ b/src/lib-storage/index/imapc/imapc-client.h
@@ -86,6 +86,9 @@ void imapc_client_register_untagged(struct imapc_client *client,
 
 void imapc_client_run(struct imapc_client *client);
 void imapc_client_stop(struct imapc_client *client);
+/* Stop immediately, don't finish even any already read pending replies.
+   They'll be finished when imapc_client_run() is again called. */
+void imapc_client_stop_now(struct imapc_client *client);
 
 struct imapc_client_mailbox *
 imapc_client_mailbox_open(struct imapc_client *client, const char *name,
diff --git a/src/lib-storage/index/imapc/imapc-connection.c b/src/lib-storage/index/imapc/imapc-connection.c
index 3e2451457e..364566ec56 100644
--- a/src/lib-storage/index/imapc/imapc-connection.c
+++ b/src/lib-storage/index/imapc/imapc-connection.c
@@ -677,8 +677,6 @@ static int imapc_connection_input_one(struct imapc_connection *conn)
 
 static void imapc_connection_input(struct imapc_connection *conn)
 {
-	int ret;
-
 	if (i_stream_read(conn->input) == -1) {
 		/* disconnected */
 		i_error("imapc(%s): Server disconnected unexpectedly",
@@ -687,15 +685,7 @@ static void imapc_connection_input(struct imapc_connection *conn)
 		return;
 	}
 
-	o_stream_cork(conn->output);
-	do {
-		T_BEGIN {
-			ret = imapc_connection_input_one(conn);
-		} T_END;
-	} while (ret > 0);
-
-	if (conn->output != NULL)
-		o_stream_uncork(conn->output);
+	imapc_connection_input_pending(conn);
 }
 
 static void imapc_connection_connected(struct imapc_connection *conn)
@@ -800,6 +790,24 @@ void imapc_connection_connect(struct imapc_connection *conn)
 	}
 }
 
+void imapc_connection_input_pending(struct imapc_connection *conn)
+{
+	int ret = 1;
+
+	if (conn->input == NULL)
+		return;
+
+	o_stream_cork(conn->output);
+	while (ret > 0 && !conn->client->stop_now) {
+		T_BEGIN {
+			ret = imapc_connection_input_one(conn);
+		} T_END;
+	}
+
+	if (conn->output != NULL)
+		o_stream_uncork(conn->output);
+}
+
 static struct imapc_command *
 imapc_command_begin(imapc_command_callback_t *callback, void *context)
 {
diff --git a/src/lib-storage/index/imapc/imapc-connection.h b/src/lib-storage/index/imapc/imapc-connection.h
index 75c28a91a8..842b014ed7 100644
--- a/src/lib-storage/index/imapc/imapc-connection.h
+++ b/src/lib-storage/index/imapc/imapc-connection.h
@@ -23,6 +23,7 @@ void imapc_connection_deinit(struct imapc_connection **conn);
 
 void imapc_connection_connect(struct imapc_connection *conn);
 void imapc_connection_ioloop_changed(struct imapc_connection *conn);
+void imapc_connection_input_pending(struct imapc_connection *conn);
 
 void imapc_connection_cmd(struct imapc_connection *conn, const char *cmdline,
 			  imapc_command_callback_t *callback, void *context);
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index cb3e3deccd..1df5168727 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -3,22 +3,24 @@
 #include "lib.h"
 #include "str.h"
 #include "istream.h"
-#include "index-mail.h"
+#include "imapc-mail.h"
 #include "imapc-client.h"
 #include "imapc-storage.h"
 
-static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq)
+struct mail *
+imapc_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers)
 {
-	index_mail_set_seq(_mail, seq);
-	imapc_mail_fetch(_mail);
-}
+	struct imapc_mail *mail;
+	pool_t pool;
 
-static bool imapc_mail_set_uid(struct mail *_mail, uint32_t uid)
-{
-	if (!index_mail_set_uid(_mail, uid))
-		return FALSE;
-	imapc_mail_fetch(_mail);
-	return TRUE;
+	pool = pool_alloconly_create("mail", 2048);
+	mail = p_new(pool, struct imapc_mail, 1);
+	mail->imail.mail.pool = pool;
+
+	index_mail_init(&mail->imail, t, wanted_fields, wanted_headers);
+	return &mail->imail.mail.mail;
 }
 
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
@@ -26,8 +28,15 @@ static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct index_mail_data *data = &mail->data;
 
-	if (data->received_date == (time_t)-1)
-		return -1;
+	if (data->received_date == (time_t)-1) {
+		if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0)
+			return -1;
+		if (data->received_date == (time_t)-1) {
+			mail_storage_set_critical(_mail->box->storage,
+				"imapc: Remote server didn't send INTERNALDATE");
+			return -1;
+		}
+	}
 	*date_r = data->received_date;
 	return 0;
 }
@@ -37,8 +46,10 @@ static int imapc_mail_get_save_date(struct mail *_mail, time_t *date_r)
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct index_mail_data *data = &mail->data;
 
-	if (data->save_date == (time_t)-1)
+	if (data->save_date == (time_t)-1) {
+		/* FIXME */
 		return -1;
+	}
 	*date_r = data->save_date;
 	return 0;
 }
@@ -90,9 +101,25 @@ imapc_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct index_mail_data *data = &mail->data;
+	enum mail_fetch_field fetch_field;
 
-	if (data->stream == NULL)
-		return -1;
+	if (data->stream == NULL) {
+		if (!mail->data.initialized) {
+			/* coming here from mail_set_seq() */
+			return -1;
+		}
+		fetch_field = body_size != NULL ||
+			(mail->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0 ?
+			MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER;
+		if (imapc_mail_fetch(_mail, fetch_field) < 0)
+			return -1;
+
+		if (data->stream == NULL) {
+			mail_storage_set_critical(_mail->box->storage,
+				"imapc: Remote server didn't send BODY[]");
+			return -1;
+		}
+	}
 
 	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
 }
@@ -100,8 +127,8 @@ imapc_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
 struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_close,
 	index_mail_free,
-	imapc_mail_set_seq,
-	imapc_mail_set_uid,
+	index_mail_set_seq,
+	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 
 	index_mail_get_flags,
diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h
new file mode 100644
index 0000000000..28c8720f32
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-mail.h
@@ -0,0 +1,19 @@
+#ifndef IMAPC_MAIL_H
+#define IMAPC_MAIL_H
+
+#include "index-mail.h"
+
+struct imapc_mail {
+	struct index_mail imail;
+	unsigned int searching:1;
+};
+
+extern struct mail_vfuncs imapc_mail_vfuncs;
+
+struct mail *
+imapc_mail_alloc(struct mailbox_transaction_context *t,
+		 enum mail_fetch_field wanted_fields,
+		 struct mailbox_header_lookup_ctx *wanted_headers);
+int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index 609fd0fea4..28758b2a7c 100644
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -92,7 +92,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 	enum mail_flags flags;
 	uint32_t uid, old_count;
 	unsigned int i, j;
-	ARRAY_TYPE(const_string) keywords;
+	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE;
 
 	if (mbox == NULL || seq == 0 || !imap_arg_get_list(reply->args, &list))
diff --git a/src/lib-storage/index/imapc/imapc-search.c b/src/lib-storage/index/imapc/imapc-search.c
index 5553fec875..663dc6f085 100644
--- a/src/lib-storage/index/imapc/imapc-search.c
+++ b/src/lib-storage/index/imapc/imapc-search.c
@@ -6,41 +6,94 @@
 #include "str.h"
 #include "imap-arg.h"
 #include "imap-date.h"
+#include "imap-util.h"
 #include "mail-user.h"
-#include "index-mail.h"
+#include "mail-search.h"
+#include "index-search-private.h"
+#include "imapc-mail.h"
 #include "imapc-client.h"
 #include "imapc-storage.h"
 
-void imapc_mail_fetch(struct mail *mail)
+struct imapc_search_context {
+	struct index_search_context ictx;
+
+	/* non-NULL during _search_next_nonblock() */
+	struct mail *cur_mail;
+
+	/* sequences of messages we're next wanting to fetch. */
+	ARRAY_TYPE(seq_range) next_seqs;
+	uint32_t next_pending_seq, saved_seq;
+
+	unsigned int fetching:1;
+};
+
+static void imapc_search_fetch_callback(const struct imapc_command_reply *reply,
+					void *context)
 {
-	struct mail_private *pmail = (struct mail_private *)mail;
-	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
-	string_t *str;
-	unsigned int orig_len;
+	struct imapc_search_context *ctx = context;
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)ctx->ictx.mail_ctx.transaction->box;
 
-	str = t_str_new(64);
-	str_printfa(str, "UID FETCH %u (", mail->uid);
-	orig_len = str_len(str);
-
-	if ((pmail->wanted_fields & (MAIL_FETCH_STREAM_HEADER |
-				     MAIL_FETCH_STREAM_BODY |
-				     MAIL_FETCH_MESSAGE_PARTS |
-				     MAIL_FETCH_NUL_STATE |
-				     MAIL_FETCH_IMAP_BODY |
-				     MAIL_FETCH_IMAP_BODYSTRUCTURE |
-				     MAIL_FETCH_PHYSICAL_SIZE |
-				     MAIL_FETCH_VIRTUAL_SIZE)) != 0)
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
+					    reply);
+	} else {
+		mail_storage_set_critical(&mbox->storage->storage,
+			"imapc: Command failed: %s", reply->text_full);
+	}
+	ctx->fetching = FALSE;
+	imapc_client_stop(mbox->storage->client);
+}
+
+static bool
+imapc_append_wanted_fields(string_t *str, enum mail_fetch_field fields,
+			   bool want_headers)
+{
+	bool ret = FALSE;
+
+	if ((fields & (MAIL_FETCH_STREAM_BODY |
+		       MAIL_FETCH_MESSAGE_PARTS |
+		       MAIL_FETCH_NUL_STATE |
+		       MAIL_FETCH_IMAP_BODY |
+		       MAIL_FETCH_IMAP_BODYSTRUCTURE |
+		       MAIL_FETCH_PHYSICAL_SIZE |
+		       MAIL_FETCH_VIRTUAL_SIZE)) != 0) {
 		str_append(str, "BODY.PEEK[] ");
-	else if ((pmail->wanted_fields & (MAIL_FETCH_IMAP_ENVELOPE |
-					  MAIL_FETCH_HEADER_MD5 |
-					  MAIL_FETCH_DATE)) != 0 ||
-		 pmail->wanted_headers != NULL)
+		ret = TRUE;
+	} else if (want_headers ||
+		   (fields & (MAIL_FETCH_STREAM_HEADER |
+			      MAIL_FETCH_IMAP_ENVELOPE |
+			      MAIL_FETCH_HEADER_MD5 |
+			      MAIL_FETCH_DATE)) != 0) {
 		str_append(str, "BODY.PEEK[HEADER] ");
+		ret = TRUE;
+	}
 
-	if ((pmail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0)
+	if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) {
 		str_append(str, "INTERNALDATE ");
+		ret = TRUE;
+	}
+	return ret;
+}
+
+static void
+imapc_search_send_fetch(struct imapc_search_context *ctx,
+			const ARRAY_TYPE(seq_range) *uids)
+{
+	struct mail *mail = ctx->cur_mail;
+	struct mail_private *pmail = (struct mail_private *)mail;
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
+	string_t *str;
+
+	str = t_str_new(64);
+	str_append(str, "UID FETCH ");
+	imap_write_seq_range(str, uids);
+	str_append(str, " (");
 
-	if (str_len(str) == orig_len) {
+	if (!imapc_append_wanted_fields(str, pmail->wanted_fields,
+					pmail->wanted_headers != NULL)) {
 		/* we don't need to fetch anything */
 		return;
 	}
@@ -48,11 +101,9 @@ void imapc_mail_fetch(struct mail *mail)
 	str_truncate(str, str_len(str) - 1);
 	str_append_c(str, ')');
 
-	mbox->cur_fetch_mail = mail;
-	imapc_client_mailbox_cmdf(mbox->client_box, imapc_async_stop_callback,
-				  mbox->storage, "%1s", str_c(str));
-	imapc_client_run(mbox->storage->client);
-	mbox->cur_fetch_mail = NULL;
+	ctx->fetching = TRUE;
+	imapc_client_mailbox_cmdf(mbox->client_box, imapc_search_fetch_callback,
+				  ctx, "%1s", str_c(str));
 }
 
 struct mail_search_context *
@@ -60,16 +111,122 @@ imapc_search_init(struct mailbox_transaction_context *t,
 		  struct mail_search_args *args,
 		  const enum mail_sort_type *sort_program)
 {
-	return index_storage_search_init(t, args, sort_program);
+	struct imapc_search_context *ctx;
+
+	ctx = i_new(struct imapc_search_context, 1);
+	index_storage_search_init_context(&ctx->ictx, t, args, sort_program);
+	i_array_init(&ctx->next_seqs, 64);
+	ctx->ictx.recheck_index_args = TRUE;
+	return &ctx->ictx.mail_ctx;
+}
+
+int imapc_search_deinit(struct mail_search_context *_ctx)
+{
+	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)_ctx->transaction->box;
+
+	while (ctx->fetching)
+		imapc_client_run(mbox->storage->client);
+
+	array_free(&ctx->next_seqs);
+	return index_storage_search_deinit(_ctx);
 }
 
 bool imapc_search_next_nonblock(struct mail_search_context *_ctx,
 				struct mail *mail, bool *tryagain_r)
 {
-	if (!index_storage_search_next_nonblock(_ctx, mail, tryagain_r))
+	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)_ctx->transaction->box;
+	struct imapc_mail *imail = (struct imapc_mail *)mail;
+	bool ret;
+
+	imail->searching = TRUE;
+	ctx->cur_mail = mail;
+	ret = index_storage_search_next_nonblock(_ctx, mail, tryagain_r);
+	ctx->cur_mail = NULL;
+	imail->searching = FALSE;
+	if (!ret)
 		return FALSE;
 
-	imapc_mail_fetch(mail);
+	if (ctx->fetching) {
+		mbox->cur_fetch_mail = mail;
+		imapc_client_run(mbox->storage->client);
+		mbox->cur_fetch_mail = NULL;
+	}
+	return TRUE;
+}
+
+static void imapc_get_short_uid_range(struct mailbox *box,
+				      const ARRAY_TYPE(seq_range) *seqs,
+				      ARRAY_TYPE(seq_range) *uids)
+{
+	const struct seq_range *range;
+	unsigned int i, count;
+	uint32_t uid1, uid2;
+
+	range = array_get(seqs, &count);
+	for (i = 0; i < count; i++) {
+		mail_index_lookup_uid(box->view, range[i].seq1, &uid1);
+		mail_index_lookup_uid(box->view, range[i].seq2, &uid2);
+		seq_range_array_add_range(uids, uid1, uid2);
+	}
+}
+
+static void imapc_search_update_next_seqs(struct imapc_search_context *ctx)
+{
+	struct mail_search_context *_ctx = &ctx->ictx.mail_ctx;
+	uint32_t prev_seq;
+
+	/* add messages to the next_seqs list as long as the sequences
+	   are incrementing */
+	if (ctx->next_pending_seq == 0)
+		prev_seq = 0;
+	else {
+		prev_seq = ctx->next_pending_seq;
+		seq_range_array_add(&ctx->next_seqs, 0, prev_seq);
+	}
+	if (ctx->saved_seq != 0)
+		_ctx->seq = ctx->saved_seq;
+	while (index_storage_search_next_update_seq(_ctx)) {
+		mail_search_args_reset(_ctx->args->args, FALSE);
+		if (_ctx->seq < prev_seq) {
+			ctx->next_pending_seq = _ctx->seq;
+			break;
+		}
+		seq_range_array_add(&ctx->next_seqs, 0, _ctx->seq);
+	}
+	ctx->saved_seq = _ctx->seq;
+	if (array_count(&ctx->next_seqs) > 0) T_BEGIN {
+		ARRAY_TYPE(seq_range) uids;
+
+		t_array_init(&uids, array_count(&ctx->next_seqs)*2);
+		imapc_get_short_uid_range(_ctx->transaction->box,
+					  &ctx->next_seqs, &uids);
+		imapc_search_send_fetch(ctx, &uids);
+	} T_END;
+}
+
+bool imapc_search_next_update_seq(struct mail_search_context *_ctx)
+{
+	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
+	struct seq_range *seqs;
+	unsigned int count;
+
+	seqs = array_get_modifiable(&ctx->next_seqs, &count);
+	if (count == 0) {
+		imapc_search_update_next_seqs(ctx);
+		seqs = array_get_modifiable(&ctx->next_seqs, &count);
+		if (count == 0)
+			return FALSE;
+	}
+
+	_ctx->seq = seqs[0].seq1;
+	if (seqs[0].seq1 < seqs[0].seq2)
+		seqs[0].seq1++;
+	else
+		array_delete(&ctx->next_seqs, 0, 1);
 	return TRUE;
 }
 
@@ -123,6 +280,8 @@ imapc_fetch_stream(struct index_mail *imail, const char *value, bool body)
 
 void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args)
 {
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)mail->transaction->box;
 	struct index_mail *imail = (struct index_mail *)mail;
 	const char *key, *value;
 	unsigned int i;
@@ -151,4 +310,31 @@ void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args)
 			imail->data.received_date = t;
 		}
 	}
+	imapc_client_stop_now(mbox->storage->client);
+}
+
+int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields)
+{
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)mail->transaction->box;
+	struct imapc_simple_context sctx;
+	string_t *str;
+
+	str = t_str_new(64);
+	str_printfa(str, "UID FETCH %u (", mail->uid);
+
+	if (!imapc_append_wanted_fields(str, fields, FALSE))
+		return 0;
+
+	str_truncate(str, str_len(str) - 1);
+	str_append_c(str, ')');
+
+	sctx.storage = mbox->storage;
+	imapc_client_mailbox_cmdf(mbox->client_box, imapc_async_stop_callback,
+				  mbox->storage, "%1s", str_c(str));
+
+	mbox->cur_fetch_mail = mail;
+	imapc_client_run(mbox->storage->client);
+	mbox->cur_fetch_mail = NULL;
+	return sctx.ret;
 }
diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index 70845329ef..96beba6ce9 100644
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -6,7 +6,7 @@
 #include "safe-mkstemp.h"
 #include "imap-arg.h"
 #include "imap-resp-code.h"
-#include "index-mail.h"
+#include "imapc-mail.h"
 #include "imapc-client.h"
 #include "imapc-list.h"
 #include "imapc-sync.h"
@@ -544,11 +544,11 @@ struct mailbox imapc_mailbox = {
 		index_transaction_commit,
 		index_transaction_rollback,
 		NULL,
-		index_mail_alloc,
+		imapc_mail_alloc,
 		imapc_search_init,
-		index_storage_search_deinit,
+		imapc_search_deinit,
 		imapc_search_next_nonblock,
-		index_storage_search_next_update_seq,
+		imapc_search_next_update_seq,
 		imapc_save_alloc,
 		imapc_save_begin,
 		imapc_save_continue,
diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 77be532612..a4b9f97864 100644
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -60,8 +60,6 @@ struct imapc_simple_context {
 	int ret;
 };
 
-extern struct mail_vfuncs imapc_mail_vfuncs;
-
 struct mail_save_context *
 imapc_save_alloc(struct mailbox_transaction_context *_t);
 int imapc_save_begin(struct mail_save_context *ctx, struct istream *input);
@@ -75,13 +73,14 @@ void imapc_transaction_save_commit_post(struct mail_save_context *ctx,
 					struct mail_index_transaction_commit_result *result);
 void imapc_transaction_save_rollback(struct mail_save_context *ctx);
 
-void imapc_mail_fetch(struct mail *mail);
 struct mail_search_context *
 imapc_search_init(struct mailbox_transaction_context *t,
 		  struct mail_search_args *args,
 		  const enum mail_sort_type *sort_program);
+int imapc_search_deinit(struct mail_search_context *_ctx);
 bool imapc_search_next_nonblock(struct mail_search_context *_ctx,
 				struct mail *mail, bool *tryagain_r);
+bool imapc_search_next_update_seq(struct mail_search_context *_ctx);
 void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args);
 
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 5a411e771f..3e6ea1e1e7 100644
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -1340,6 +1340,7 @@ void index_mail_set_seq(struct mail *_mail, uint32_t seq)
 		if (!_mail->saving && _mail->uid < hdr->next_uid)
 			(void)mail_get_stream(_mail, NULL, NULL, &input);
 	}
+	mail->data.initialized = TRUE;
 }
 
 bool index_mail_set_uid(struct mail *_mail, uint32_t uid)
diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 6c4f2bb0af..18eb89e13d 100644
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -96,6 +96,7 @@ struct index_mail_data {
 	ARRAY_TYPE(keywords) keywords;
 	ARRAY_TYPE(keyword_indexes) keyword_indexes;
 
+	unsigned int initialized:1;
 	unsigned int save_sent_date:1;
 	unsigned int sent_date_parsed:1;
 	unsigned int save_envelope:1;