From: Timo Sirainen <tss@iki.fi>
Date: Sun, 4 Sep 2011 14:59:58 +0000 (+0300)
Subject: imapc: Redesigned remote sequence <-> index file record mapping is done.
X-Git-Tag: 2.1.alpha2~85
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1af95e0fea59835842dafe207e86ebc53691c596;p=thirdparty%2Fdovecot%2Fcore.git

imapc: Redesigned remote sequence <-> index file record mapping is done.
The previous code didn't work when multiple connections modified the same
index files.
---

diff --git a/src/lib-storage/index/imapc/Makefile.am b/src/lib-storage/index/imapc/Makefile.am
index d1b228ab78..baed6dfecb 100644
--- a/src/lib-storage/index/imapc/Makefile.am
+++ b/src/lib-storage/index/imapc/Makefile.am
@@ -20,8 +20,8 @@ libstorage_imapc_la_SOURCES = \
 	imapc-mail.c \
 	imapc-mail-fetch.c \
 	imapc-mailbox.c \
+	imapc-msgmap.c \
 	imapc-save.c \
-	imapc-seqmap.c \
 	imapc-settings.c \
 	imapc-sync.c \
 	imapc-storage.c
@@ -32,29 +32,10 @@ headers = \
 	imapc-connection.h \
 	imapc-list.h \
 	imapc-mail.h \
-	imapc-seqmap.h \
+	imapc-msgmap.h \
 	imapc-settings.h \
 	imapc-storage.h \
 	imapc-sync.h
 
 pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
-
-test_programs = \
-	test-imapc-seqmap
-
-noinst_PROGRAMS = $(test_programs)
-
-test_libs = \
-	../../../lib-test/libtest.la \
-	../../../lib/liblib.la
-
-test_imapc_seqmap_SOURCES = test-imapc-seqmap.c
-test_imapc_seqmap_LDADD = imapc-seqmap.lo $(test_libs)
-test_imapc_seqmap_DEPENDENCIES = imapc-seqmap.lo $(test_libs)
-
-check: check-am check-test
-check-test: all-am
-	for bin in $(test_programs); do \
-	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
-	done
diff --git a/src/lib-storage/index/imapc/imapc-client-private.h b/src/lib-storage/index/imapc/imapc-client-private.h
index d8442ab058..a8ac691f52 100644
--- a/src/lib-storage/index/imapc/imapc-client-private.h
+++ b/src/lib-storage/index/imapc/imapc-client-private.h
@@ -26,7 +26,7 @@ struct imapc_client {
 struct imapc_client_mailbox {
 	struct imapc_client *client;
 	struct imapc_connection *conn;
-	struct imapc_seqmap *seqmap;
+	struct imapc_msgmap *msgmap;
 
 	void *untagged_box_context;
 	unsigned int pending_box_command_count;
diff --git a/src/lib-storage/index/imapc/imapc-client.c b/src/lib-storage/index/imapc/imapc-client.c
index 254abd1d07..faa1dc5e25 100644
--- a/src/lib-storage/index/imapc/imapc-client.c
+++ b/src/lib-storage/index/imapc/imapc-client.c
@@ -6,7 +6,7 @@
 #include "ioloop.h"
 #include "safe-mkstemp.h"
 #include "iostream-ssl.h"
-#include "imapc-seqmap.h"
+#include "imapc-msgmap.h"
 #include "imapc-connection.h"
 #include "imapc-client-private.h"
 
@@ -219,7 +219,7 @@ imapc_client_mailbox_open(struct imapc_client *client,
 	conn = imapc_client_get_unboxed_connection(client);
 	conn->box = box;
 	box->conn = conn->conn;
-	box->seqmap = imapc_seqmap_init();
+	box->msgmap = imapc_msgmap_init();
 
 	imapc_connection_select(box, name, examine, callback, context);
 	return box;
@@ -247,7 +247,7 @@ void imapc_client_mailbox_close(struct imapc_client_mailbox **_box)
 
 	if (box->conn != NULL)
 		imapc_connection_unselect(box);
-	imapc_seqmap_deinit(&box->seqmap);
+	imapc_msgmap_deinit(&box->msgmap);
 	i_free(box);
 }
 
@@ -340,10 +340,10 @@ void imapc_client_mailbox_cmdf(struct imapc_client_mailbox *box,
 	va_end(args);
 }
 
-struct imapc_seqmap *
-imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box)
+struct imapc_msgmap *
+imapc_client_mailbox_get_msgmap(struct imapc_client_mailbox *box)
 {
-	return box->seqmap;
+	return box->msgmap;
 }
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box)
diff --git a/src/lib-storage/index/imapc/imapc-client.h b/src/lib-storage/index/imapc/imapc-client.h
index 052527e7bd..c11b21e02f 100644
--- a/src/lib-storage/index/imapc/imapc-client.h
+++ b/src/lib-storage/index/imapc/imapc-client.h
@@ -135,8 +135,8 @@ void imapc_client_mailbox_cmdf(struct imapc_client_mailbox *box,
 			       imapc_command_callback_t *callback,
 			       void *context, const char *cmd_fmt, ...)
 	ATTR_FORMAT(4, 5);
-struct imapc_seqmap *
-imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box);
+struct imapc_msgmap *
+imapc_client_mailbox_get_msgmap(struct imapc_client_mailbox *box);
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box);
 bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box);
diff --git a/src/lib-storage/index/imapc/imapc-connection.c b/src/lib-storage/index/imapc/imapc-connection.c
index 6801815178..f73229950c 100644
--- a/src/lib-storage/index/imapc/imapc-connection.c
+++ b/src/lib-storage/index/imapc/imapc-connection.c
@@ -14,7 +14,6 @@
 #include "imap-util.h"
 #include "imap-parser.h"
 #include "imapc-client-private.h"
-#include "imapc-seqmap.h"
 #include "imapc-connection.h"
 
 #include <unistd.h>
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index c26269bd27..cb669ed701 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -4,7 +4,7 @@
 #include "str.h"
 #include "istream.h"
 #include "imap-envelope.h"
-#include "imapc-seqmap.h"
+#include "imapc-msgmap.h"
 #include "imapc-mail.h"
 #include "imapc-client.h"
 #include "imapc-storage.h"
@@ -101,8 +101,8 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 static bool imapc_mail_is_expunged(struct mail *_mail)
 {
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
-	struct imapc_seqmap *seqmap;
-	uint32_t lseq;
+	struct imapc_msgmap *msgmap;
+	uint32_t lseq, rseq;
 
 	/* first we'll need to convert the mail's sequence to sync_view's
 	   sequence. if there's no sync_view, then no mails have been
@@ -113,8 +113,8 @@ static bool imapc_mail_is_expunged(struct mail *_mail)
 	if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq))
 		return TRUE;
 
-	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	return imapc_seqmap_lseq_to_rseq(seqmap, lseq) == 0;
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
 }
 
 static int
diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index 6ad5f62f4a..e0c9d7009b 100644
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -5,8 +5,8 @@
 #include "imap-arg.h"
 #include "imap-util.h"
 #include "imapc-client.h"
-#include "imapc-seqmap.h"
 #include "imapc-mail.h"
+#include "imapc-msgmap.h"
 #include "imapc-sync.h"
 #include "imapc-storage.h"
 
@@ -22,6 +22,8 @@ static void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
 		mbox->box.name, t_strdup_vprintf(reason, va));
 	va_end(va);
 
+	sleep(3600);
+
 	mail_index_mark_corrupted(mbox->box.index);
 	imapc_client_mailbox_disconnect(mbox->client_box);
 }
@@ -44,8 +46,6 @@ static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
 					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
 	mbox->delayed_sync_view =
 		mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
-	mbox->min_append_uid =
-		mail_index_get_header(mbox->delayed_sync_view)->next_uid;
 }
 
 int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
@@ -73,10 +73,7 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
 				  struct imapc_mailbox *mbox)
 {
 	struct mail_index_view *view = mbox->delayed_sync_view;
-	uint32_t rcount = reply->num;
 	const struct mail_index_header *hdr;
-	struct imapc_seqmap *seqmap;
-	uint32_t next_lseq, next_rseq;
 
 	if (mbox == NULL)
 		return;
@@ -84,26 +81,10 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
 	if (view == NULL)
 		view = imapc_mailbox_get_sync_view(mbox);
 
-	if (rcount == 0) {
-		/* nothing in this mailbox */
-		return;
-	}
 	if (mbox->opening) {
 		/* We don't know the latest flags, refresh them. */
 		mbox->sync_fetch_first_uid = 1;
-	} else {
-		seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-		next_lseq = mail_index_view_get_messages_count(view) + 1;
-		next_rseq = imapc_seqmap_lseq_to_rseq(seqmap, next_lseq);
-		if (rcount < next_rseq) {
-			if (rcount == next_rseq-1) {
-				/* duplicate EXISTS - ignore */
-				return;
-			}
-			imapc_mailbox_set_corrupted(mbox,
-				"EXISTS reply shrank mailbox size");
-			return;
-		}
+	} else if (mbox->sync_fetch_first_uid != 1) {
 		hdr = mail_index_get_header(view);
 		mbox->sync_fetch_first_uid = hdr->next_uid;
 	}
@@ -152,13 +133,13 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 				 struct imapc_mailbox *mbox)
 {
 	uint32_t lseq, rseq = reply->num;
-	struct imapc_seqmap *seqmap;
 	struct imapc_mail *const *mailp;
 	const struct imap_arg *list, *flags_list;
 	const char *atom;
 	const struct mail_index_record *rec = NULL;
+	struct imapc_msgmap *msgmap;
 	enum mail_flags flags;
-	uint32_t uid, cur_count;
+	uint32_t fetch_uid, uid, msg_count;
 	unsigned int i, j;
 	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE;
@@ -166,14 +147,14 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 	if (mbox == NULL || rseq == 0 || !imap_arg_get_list(reply->args, &list))
 		return;
 
-	uid = 0; flags = 0;
+	fetch_uid = 0; flags = 0;
 	for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) {
 		if (!imap_arg_get_atom(&list[i], &atom))
 			return;
 
 		if (strcasecmp(atom, "UID") == 0) {
 			if (!imap_arg_get_atom(&list[i+1], &atom) ||
-			    str_to_uint32(atom, &uid) < 0)
+			    str_to_uint32(atom, &fetch_uid) < 0)
 				return;
 		} else if (strcasecmp(atom, "FLAGS") == 0) {
 			if (!imap_arg_get_list(&list[i+1], &flags_list))
@@ -196,12 +177,52 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 	/* FIXME: need to do something about recent flags */
 	flags &= ~MAIL_RECENT;
 
-	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
+	imapc_mailbox_init_delayed_trans(mbox);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	msg_count = imapc_msgmap_count(msgmap);
+	if (rseq > msg_count) {
+		/* newly seen message */
+		if (!mbox->syncing || fetch_uid == 0 || rseq != msg_count+1)
+			return;
+		uid = fetch_uid;
+
+		if (uid < imapc_msgmap_uidnext(msgmap)) {
+			imapc_mailbox_set_corrupted(mbox,
+				"Expunged message reappeared "
+				"(uid=%u < next_uid=%u)",
+				uid, imapc_msgmap_uidnext(msgmap));
+			return;
+		}
+
+		imapc_msgmap_append(msgmap, rseq, uid);
+		if (uid < mbox->min_append_uid) {
+			/* message is already added to index */
+			lseq = 0;
+		} else {
+			mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
+			mbox->min_append_uid = uid + 1;
+		}
+	} else {
+		uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+		if (uid != fetch_uid && fetch_uid != 0) {
+			imapc_mailbox_set_corrupted(mbox,
+				"FETCH UID mismatch (%u != %u)",
+				fetch_uid, uid);
+			return;
+		}
+		lseq = 0;
+	}
+	if (lseq == 0) {
+		if (!mail_index_lookup_seq(mbox->delayed_sync_view,
+					   uid, &lseq)) {
+			/* already expunged by another session */
+			return;
+		}
+		rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
+	}
 
-	/* fetch_mails' view is different from sync_view, so we can't compare
-	   their sequences directly. that is why this code supports only
-	   UID FETCH commands which are guaranteed to have UID in the reply. */
+	/* if this is a reply to some FETCH request, update the mail's fields */
 	array_foreach(&mbox->fetch_mails, mailp) {
 		struct imapc_mail *mail = *mailp;
 
@@ -209,42 +230,19 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 			imapc_mail_fetch_update(mail, reply, list);
 	}
 
-	imapc_mailbox_init_delayed_trans(mbox);
-	cur_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
-	while (lseq <= cur_count) {
-		rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
-		if (rec->uid == uid || uid == 0)
-			break;
+	if (rseq == mbox->sync_next_rseq) {
+		/* we're doing the initial full sync of mails. expunge any
+		   mails that no longer exist. */
+		i_assert(mbox->syncing);
 
-		if (!mbox->opening || uid < rec->uid) {
-			imapc_mailbox_set_corrupted(mbox,
-				"Message UID changed %u -> %u", rec->uid, uid);
-			return;
+		while (mbox->sync_next_lseq < lseq) {
+			mail_index_expunge(mbox->delayed_sync_trans,
+					   mbox->sync_next_lseq);
+			mbox->sync_next_lseq++;
 		}
-		/* we're opening the mailbox. this message was expunged
-		   externally, so expunge it ourself too. this code assumes
-		   that FETCH responses come in ascending order when opening
-		   mailbox. */
-		imapc_seqmap_expunge(seqmap, rseq);
-		mail_index_expunge(mbox->delayed_sync_trans, lseq);
-		lseq++;
-	}
-	if (lseq > cur_count) {
-		if (!mbox->syncing)
-			return;
-		if (uid == 0 || lseq != cur_count + 1)
-			return;
-		if (uid < mbox->min_append_uid) {
-			imapc_mailbox_set_corrupted(mbox,
-				"Expunged message reappeared "
-				"(uid=%u < next_uid=%u)",
-				uid, mbox->min_append_uid);
-			return;
-		}
-		i_assert(lseq == cur_count + 1);
-		mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
-		mbox->min_append_uid = uid + 1;
-		rec = NULL;
+		i_assert(lseq == mbox->sync_next_lseq);
+		mbox->sync_next_rseq++;
+		mbox->sync_next_lseq++;
 	}
 	if (seen_flags && (rec == NULL || rec->flags != flags)) {
 		mail_index_update_flags(mbox->delayed_sync_trans, lseq,
@@ -267,38 +265,32 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 		}
 		mail_index_keywords_unref(&kw);
 	} T_END;
-	if (mbox->highest_seen_uid < uid)
-		mbox->highest_seen_uid = uid;
 	imapc_mailbox_idle_notify(mbox);
 }
 
 static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
 				   struct imapc_mailbox *mbox)
 {
-	struct imapc_seqmap *seqmap;
-	uint32_t lseq, rseq = reply->num;
+	struct imapc_msgmap *msgmap;
+	uint32_t lseq, uid, rseq = reply->num;
 	
 	if (mbox == NULL || rseq == 0)
 		return;
 
-	imapc_mailbox_init_delayed_trans(mbox);
-
-	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (rseq > imapc_msgmap_count(msgmap)) {
+		/* we haven't even seen this message yet */
+		return;
+	}
+	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+	imapc_msgmap_expunge(msgmap, rseq);
 
-	if (lseq <= mail_index_view_get_messages_count(mbox->sync_view)) {
-		/* expunging a message in index */
-		imapc_seqmap_expunge(seqmap, rseq);
-		mail_index_expunge(mbox->delayed_sync_trans, lseq);
-	} else if (lseq <= mail_index_view_get_messages_count(mbox->delayed_sync_view)) {
-		/* expunging a message that was added to transaction,
-		   but not yet committed. expunging it here takes
-		   effect immediately. */
-		mail_index_expunge(mbox->delayed_sync_trans, lseq);
+	imapc_mailbox_init_delayed_trans(mbox);
+	if (!mail_index_lookup_seq(mbox->delayed_sync_view, uid, &lseq)) {
+		/* already expunged by another session */
 	} else {
-		/* expunging a message whose UID wasn't known yet */
+		mail_index_expunge(mbox->delayed_sync_trans, lseq);
 	}
-
 	imapc_mailbox_idle_notify(mbox);
 }
 
diff --git a/src/lib-storage/index/imapc/imapc-msgmap.c b/src/lib-storage/index/imapc/imapc-msgmap.c
new file mode 100644
index 0000000000..712d477c64
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-msgmap.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "imapc-msgmap.h"
+
+struct imapc_msgmap {
+	ARRAY_TYPE(uint32_t) uids;
+};
+
+struct imapc_msgmap *imapc_msgmap_init(void)
+{
+	struct imapc_msgmap *msgmap;
+
+	msgmap = i_new(struct imapc_msgmap, 1);
+	i_array_init(&msgmap->uids, 128);
+	return msgmap;
+}
+
+void imapc_msgmap_deinit(struct imapc_msgmap **_msgmap)
+{
+	struct imapc_msgmap *msgmap = *_msgmap;
+
+	*_msgmap = NULL;
+
+	array_free(&msgmap->uids);
+	i_free(msgmap);
+}
+
+uint32_t imapc_msgmap_count(struct imapc_msgmap *msgmap)
+{
+	return array_count(&msgmap->uids);
+}
+
+uint32_t imapc_msgmap_uidnext(struct imapc_msgmap *msgmap)
+{
+	return imapc_msgmap_count(msgmap) == 0 ? 1 :
+		imapc_msgmap_rseq_to_uid(msgmap, 1) + 1;
+}
+
+uint32_t imapc_msgmap_rseq_to_uid(struct imapc_msgmap *msgmap, uint32_t rseq)
+{
+	const uint32_t *uidp;
+
+	uidp = array_idx(&msgmap->uids, rseq-1);
+	return *uidp;
+}
+
+static int uint32_cmp(const uint32_t *p1, const uint32_t *p2)
+{
+	return *p1 < *p2 ? -1 :
+		(*p1 > *p2 ? 1 : 0);
+}
+
+bool imapc_msgmap_uid_to_rseq(struct imapc_msgmap *msgmap,
+			      uint32_t uid, uint32_t *rseq_r)
+{
+	const uint32_t *p, *first;
+
+	p = array_bsearch(&msgmap->uids, &uid, uint32_cmp);
+	if (p == NULL) {
+		*rseq_r = 0;
+		return FALSE;
+	}
+
+	first = array_idx(&msgmap->uids, 0);
+	*rseq_r = (p - first) + 1;
+	return TRUE;
+}
+
+void imapc_msgmap_append(struct imapc_msgmap *msgmap,
+			 uint32_t rseq, uint32_t uid)
+{
+	i_assert(rseq == imapc_msgmap_count(msgmap) + 1);
+
+	array_append(&msgmap->uids, &uid, 1);
+}
+
+void imapc_msgmap_expunge(struct imapc_msgmap *msgmap, uint32_t rseq)
+{
+	i_assert(rseq > 0);
+	i_assert(rseq <= imapc_msgmap_count(msgmap));
+
+	array_delete(&msgmap->uids, rseq-1, 1);
+}
diff --git a/src/lib-storage/index/imapc/imapc-msgmap.h b/src/lib-storage/index/imapc/imapc-msgmap.h
new file mode 100644
index 0000000000..51424fda4b
--- /dev/null
+++ b/src/lib-storage/index/imapc/imapc-msgmap.h
@@ -0,0 +1,17 @@
+#ifndef IMAPC_MSGMAP_H
+#define IMAPC_MSGMAP_H
+
+struct imapc_msgmap *imapc_msgmap_init(void);
+void imapc_msgmap_deinit(struct imapc_msgmap **msgmap);
+
+uint32_t imapc_msgmap_count(struct imapc_msgmap *msgmap);
+uint32_t imapc_msgmap_uidnext(struct imapc_msgmap *msgmap);
+uint32_t imapc_msgmap_rseq_to_uid(struct imapc_msgmap *msgmap, uint32_t rseq);
+bool imapc_msgmap_uid_to_rseq(struct imapc_msgmap *msgmap,
+			      uint32_t uid, uint32_t *rseq_r);
+
+void imapc_msgmap_append(struct imapc_msgmap *msgmap,
+			 uint32_t rseq, uint32_t uid);
+void imapc_msgmap_expunge(struct imapc_msgmap *msgmap, uint32_t rseq);
+
+#endif
diff --git a/src/lib-storage/index/imapc/imapc-seqmap.c b/src/lib-storage/index/imapc/imapc-seqmap.c
deleted file mode 100644
index 3713cdc9bd..0000000000
--- a/src/lib-storage/index/imapc/imapc-seqmap.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "array.h"
-#include "bsearch-insert-pos.h"
-#include "imapc-seqmap.h"
-
-struct imapc_seqmap {
-	ARRAY_TYPE(uint32_t) queue;
-	ARRAY_TYPE(uint32_t) expunges;
-};
-
-struct imapc_seqmap *imapc_seqmap_init(void)
-{
-	struct imapc_seqmap *seqmap;
-
-	seqmap = i_new(struct imapc_seqmap, 1);
-	i_array_init(&seqmap->queue, 64);
-	i_array_init(&seqmap->expunges, 64);
-	return seqmap;
-}
-
-void imapc_seqmap_deinit(struct imapc_seqmap **_seqmap)
-{
-	struct imapc_seqmap *seqmap = *_seqmap;
-
-	*_seqmap = NULL;
-	array_free(&seqmap->expunges);
-	array_free(&seqmap->queue);
-	i_free(seqmap);
-}
-
-void imapc_seqmap_reset(struct imapc_seqmap *seqmap)
-{
-	array_clear(&seqmap->queue);
-	array_clear(&seqmap->expunges);
-}
-
-bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap)
-{
-	return array_count(&seqmap->queue) == 0 &&
-		array_count(&seqmap->expunges) == 0;
-}
-
-void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq)
-{
-	i_assert(rseq > 0);
-
-	array_append(&seqmap->queue, &rseq, 1);
-}
-
-static int uint32_cmp_p(const uint32_t *p1, const uint32_t *p2)
-{
-	if (*p1 < *p2)
-		return -1;
-	else if (*p1 > *p2)
-		return 1;
-	else
-		return 0;
-}
-
-static uint32_t
-imapc_seqmap_rseq_idx_lookup(struct imapc_seqmap *seqmap, uint32_t rseq,
-			     unsigned int *idx_r)
-{
-	const uint32_t *seqs;
-	unsigned int idx, count;
-	uint32_t lseq = rseq;
-
-	seqs = array_get(&seqmap->expunges, &count);
-	for (;;) {
-		array_bsearch_insert_pos(&seqmap->expunges, &lseq, uint32_cmp_p, &idx);
-		lseq = rseq + idx;
-		if (idx == count || seqs[idx] > lseq) {
-			*idx_r = idx;
-			return lseq;
-		}
-		if (seqs[idx] == lseq)
-			lseq++;
-	}
-}
-
-static void
-imapc_seqmap_dequeue_rseq(struct imapc_seqmap *seqmap, uint32_t rseq)
-{
-	unsigned int idx;
-	uint32_t lseq;
-
-	lseq = imapc_seqmap_rseq_idx_lookup(seqmap, rseq, &idx);
-	array_insert(&seqmap->expunges, idx, &lseq, 1);
-}
-
-static void imapc_seqmap_dequeue(struct imapc_seqmap *seqmap)
-{
-	const uint32_t *seqp;
-
-	array_foreach(&seqmap->queue, seqp)
-		imapc_seqmap_dequeue_rseq(seqmap, *seqp);
-	array_clear(&seqmap->queue);
-}
-
-uint32_t imapc_seqmap_rseq_to_lseq(struct imapc_seqmap *seqmap, uint32_t rseq)
-{
-	unsigned int idx;
-
-	i_assert(rseq > 0);
-
-	imapc_seqmap_dequeue(seqmap);
-	return imapc_seqmap_rseq_idx_lookup(seqmap, rseq, &idx);
-}
-
-uint32_t imapc_seqmap_lseq_to_rseq(struct imapc_seqmap *seqmap, uint32_t lseq)
-{
-	unsigned int idx;
-
-	i_assert(lseq > 0);
-
-	imapc_seqmap_dequeue(seqmap);
-	if (array_bsearch_insert_pos(&seqmap->expunges, &lseq,
-				     uint32_cmp_p, &idx))
-		return 0;
-
-	return lseq - idx;
-}
diff --git a/src/lib-storage/index/imapc/imapc-seqmap.h b/src/lib-storage/index/imapc/imapc-seqmap.h
deleted file mode 100644
index 7e8d4a49e5..0000000000
--- a/src/lib-storage/index/imapc/imapc-seqmap.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef IMAPC_SEQMAP_H
-#define IMAPC_SEQMAP_H
-
-/* Defines a mapping between remote and local sequence numbers.
-   Initially they start the same, but remote sequences can be marked as
-   expunged, which alters the mapping until the seqmap is reset (i.e. when the
-   mailbox is synced and local sequences are expunged too).
-
-   So for example calling imapc_seqmap_expunge(seqmap, 1) twice expunges the
-   first and the second local sequence. imapc_seqmap_rseq_to_lseq(seqmap, 1)
-   will afterward return 3. */
-
-struct imapc_seqmap *imapc_seqmap_init(void);
-void imapc_seqmap_deinit(struct imapc_seqmap **seqmap);
-
-/* Reset local and remote sequences to be equal. */
-void imapc_seqmap_reset(struct imapc_seqmap *seqmap);
-bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap);
-
-/* Mark given remote sequence expunged. */
-void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq);
-/* Convert remote sequence to local sequence. */
-uint32_t imapc_seqmap_rseq_to_lseq(struct imapc_seqmap *seqmap, uint32_t rseq);
-/* Convert local sequence to remote sequence. */
-uint32_t imapc_seqmap_lseq_to_rseq(struct imapc_seqmap *seqmap, uint32_t lseq);
-
-#endif
diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 7c0d4fa991..8c5093574a 100644
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -54,15 +54,16 @@ struct imapc_mailbox {
 	ARRAY_DEFINE(untagged_callbacks, struct imapc_mailbox_event_callback);
 	ARRAY_DEFINE(resp_text_callbacks, struct imapc_mailbox_event_callback);
 
-	uint32_t min_append_uid;
-	uint32_t highest_seen_uid;
-
 	uint32_t sync_uid_validity;
 	uint32_t sync_uid_next;
 	uint32_t sync_fetch_first_uid;
+	uint32_t sync_next_lseq;
+	uint32_t sync_next_rseq;
+	uint32_t min_append_uid;
 
 	unsigned int opening:1;
 	unsigned int syncing:1;
+	unsigned int initial_sync_done:1;
 };
 
 struct imapc_simple_context {
diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index 6beb41ba27..346725d0bb 100644
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -7,7 +7,6 @@
 #include "index-sync-private.h"
 #include "imapc-storage.h"
 #include "imapc-client.h"
-#include "imapc-seqmap.h"
 #include "imapc-sync.h"
 
 static void imapc_sync_callback(const struct imapc_command_reply *reply,
@@ -168,19 +167,27 @@ static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
 
 static void imapc_sync_expunge_eom(struct imapc_sync_context *ctx)
 {
-	const struct mail_index_record *rec;
-	uint32_t lseq, cur_count;
+	struct imapc_mailbox *mbox = ctx->mbox;
+	uint32_t lseq, uid, msg_count;
+
+	if (mbox->sync_next_lseq == 0)
+		return;
 
 	/* if we haven't seen FETCH reply for some messages at the end of
 	   mailbox they've been externally expunged. */
-	cur_count = mail_index_view_get_messages_count(ctx->sync_view);
-	for (lseq = cur_count; lseq > 0; lseq--) {
-		rec = mail_index_lookup(ctx->sync_view, lseq);
-		if (rec->uid <= ctx->mbox->highest_seen_uid)
+	msg_count = mail_index_view_get_messages_count(ctx->sync_view);
+	for (lseq = mbox->sync_next_lseq; lseq <= msg_count; lseq++) {
+		mail_index_lookup_uid(ctx->sync_view, lseq, &uid);
+		if (uid >= mbox->sync_uid_next) {
+			/* another process already added new messages to index
+			   that our IMAP connection hasn't seen yet */
 			break;
-
+		}
 		mail_index_expunge(ctx->trans, lseq);
 	}
+
+	mbox->sync_next_lseq = 0;
+	mbox->sync_next_rseq = 0;
 }
 
 static void imapc_sync_index_header(struct imapc_sync_context *ctx)
@@ -211,12 +218,12 @@ static void imapc_sync_index_header(struct imapc_sync_context *ctx)
 
 static void imapc_sync_index(struct imapc_sync_context *ctx)
 {
-	struct mailbox *box = &ctx->mbox->box;
+	struct imapc_mailbox *mbox = ctx->mbox;
 	struct mail_index_sync_rec sync_rec;
 	uint32_t seq1, seq2;
 
 	i_array_init(&ctx->expunged_uids, 64);
-	ctx->keywords = mail_index_get_keywords(box->index);
+	ctx->keywords = mail_index_get_keywords(mbox->box.index);
 
 	imapc_sync_index_header(ctx);
 	while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
@@ -246,24 +253,32 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
 		}
 	} T_END;
 
-	if (ctx->mbox->sync_fetch_first_uid != 0) {
+	if (!mbox->initial_sync_done) {
+		/* with initial syncing we're fetching all messages' flags and
+		   expunge mails from local index that no longer exist on
+		   remote server */
+		i_assert(mbox->sync_fetch_first_uid == 1);
+		mbox->sync_next_lseq = 1;
+		mbox->sync_next_rseq = 1;
+	}
+	if (mbox->sync_fetch_first_uid != 0) {
 		/* we'll resync existing messages' flags and add new messages.
 		   adding new messages requires sync locking to avoid
 		   duplicates. */
 		imapc_sync_cmd(ctx, t_strdup_printf(
-			"UID FETCH %u:* FLAGS",
-			ctx->mbox->sync_fetch_first_uid));
-		ctx->mbox->sync_fetch_first_uid = 0;
+			"UID FETCH %u:* FLAGS", mbox->sync_fetch_first_uid));
+		mbox->sync_fetch_first_uid = 0;
 	}
 
 	imapc_sync_expunge_finish(ctx);
 	while (ctx->sync_command_count > 0)
-		imapc_storage_run(ctx->mbox->storage);
+		imapc_storage_run(mbox->storage);
 	array_free(&ctx->expunged_uids);
 
 	imapc_sync_expunge_eom(ctx);
-	if (box->v.sync_notify != NULL)
-		box->v.sync_notify(box, 0, 0);
+	if (mbox->box.v.sync_notify != NULL)
+		mbox->box.v.sync_notify(&mbox->box, 0, 0);
+	mbox->initial_sync_done = TRUE;
 }
 
 static int
@@ -300,6 +315,7 @@ imapc_sync_begin(struct imapc_mailbox *mbox,
 	mbox->delayed_sync_view =
 		mail_index_transaction_open_updated_view(ctx->trans);
 	mbox->delayed_sync_trans = ctx->trans;
+	mbox->min_append_uid = mail_index_get_header(ctx->sync_view)->next_uid;
 
 	mbox->syncing = TRUE;
 	imapc_sync_index(ctx);
@@ -334,25 +350,14 @@ static int imapc_sync_finish(struct imapc_sync_context **_ctx)
 static int imapc_sync(struct imapc_mailbox *mbox)
 {
 	struct imapc_sync_context *sync_ctx;
-	struct imapc_seqmap *seqmap;
 	bool force = mbox->sync_fetch_first_uid != 0;
 
-	/* if there are any pending expunges, they're now committed. syncing
-	   will return a view where they no longer exist, so reset the seqmap
-	   before syncing. */
-	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	imapc_seqmap_reset(seqmap);
-
 	if (imapc_sync_begin(mbox, &sync_ctx, force) < 0)
 		return -1;
 	if (sync_ctx == NULL)
 		return 0;
 	if (imapc_sync_finish(&sync_ctx) < 0)
 		return -1;
-
-	/* syncing itself may have also seen new expunges, which are also now
-	   committed and synced. reset the seqmap again. */
-	imapc_seqmap_reset(seqmap);
 	return 0;
 }
 
diff --git a/src/lib-storage/index/imapc/test-imapc-seqmap.c b/src/lib-storage/index/imapc/test-imapc-seqmap.c
deleted file mode 100644
index 78b1e3b69b..0000000000
--- a/src/lib-storage/index/imapc/test-imapc-seqmap.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "array.h"
-#include "imapc-seqmap.h"
-#include "test-common.h"
-
-#include <stdlib.h>
-
-static void test_imapc_seqmap(void)
-{
-	struct imapc_seqmap *seqmap;
-
-	test_begin("imapc seqmap");
-	seqmap = imapc_seqmap_init();
-
-	imapc_seqmap_expunge(seqmap, 4);
-	imapc_seqmap_expunge(seqmap, 3);
-	imapc_seqmap_expunge(seqmap, 2);
-
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 1) == 1);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 2) == 5);
-
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 1) == 1);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 2) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 3) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 4) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 5) == 2);
-
-	imapc_seqmap_reset(seqmap);
-	imapc_seqmap_expunge(seqmap, 3);
-	imapc_seqmap_expunge(seqmap, 3);
-	imapc_seqmap_expunge(seqmap, 3);
-
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 1) == 1);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 2) == 2);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 3) == 6);
-
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 1) == 1);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 2) == 2);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 3) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 4) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 5) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 6) == 3);
-
-	imapc_seqmap_reset(seqmap);
-	/* 9,8,5,4,2,1 */
-	imapc_seqmap_expunge(seqmap, 4);
-	imapc_seqmap_expunge(seqmap, 4);
-	imapc_seqmap_expunge(seqmap, 1);
-	imapc_seqmap_expunge(seqmap, 1);
-	imapc_seqmap_expunge(seqmap, 4);
-	imapc_seqmap_expunge(seqmap, 4);
-
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 1) == 3);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 2) == 6);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 3) == 7);
-	test_assert(imapc_seqmap_rseq_to_lseq(seqmap, 4) == 10);
-
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 1) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 2) == 0);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 3) == 1);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 6) == 2);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 7) == 3);
-	test_assert(imapc_seqmap_lseq_to_rseq(seqmap, 10) == 4);
-
-	imapc_seqmap_deinit(&seqmap);
-	test_end();
-}
-
-static void test_imapc_seqmap_random(void)
-{
-#define UIDMAP_SIZE 1000
-	struct imapc_seqmap *seqmap;
-	ARRAY_TYPE(uint32_t) uidmap;
-	const uint32_t *uids;
-	unsigned int i, count;
-	uint32_t seq, uid;
-
-	test_begin("imapc seqmap random");
-	seqmap = imapc_seqmap_init();
-
-	t_array_init(&uidmap, UIDMAP_SIZE);
-	for (uid = 1; uid <= UIDMAP_SIZE; uid++)
-		array_append(&uidmap, &uid, 1);
-
-	for (i = 0; i < 100; i++) {
-		seq = (rand() % array_count(&uidmap)) + 1;
-		array_delete(&uidmap, seq-1, 1);
-		imapc_seqmap_expunge(seqmap, seq);
-	}
-
-	uids = array_get(&uidmap, &count);
-	for (i = 0; i < 100; i++) {
-		seq = i + 1;
-		test_assert(imapc_seqmap_rseq_to_lseq(seqmap, seq) == uids[i]);
-		test_assert(imapc_seqmap_lseq_to_rseq(seqmap, uids[i]) == seq);
-	}
-	imapc_seqmap_deinit(&seqmap);
-	test_end();
-}
-
-int main(void)
-{
-	static void (*test_functions[])(void) = {
-		test_imapc_seqmap,
-		test_imapc_seqmap_random,
-		NULL
-	};
-	return test_run(test_functions);
-}