From: Timo Sirainen <tss@iki.fi>
Date: Wed, 31 Aug 2011 08:39:50 +0000 (+0300)
Subject: Redesigned mail precaching APIs.
X-Git-Tag: 2.1.alpha1~27
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=421d30619384e72a27e2a5d13ff6525aff4d17fe;p=thirdparty%2Fdovecot%2Fcore.git

Redesigned mail precaching APIs.
There's no longer a one monolithic mailbox_sync(MAILBOX_SYNC_FLAG_PRECACHE)
call, but rather one mail_precache() call for each mail to be precached.
This allows the callers to show the progress and in general is cleaner.
---

diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c
index 6daaf2be16..f376ec0cbd 100644
--- a/src/doveadm/doveadm-mail-index.c
+++ b/src/doveadm/doveadm-mail-index.c
@@ -11,6 +11,8 @@
 #include "doveadm-settings.h"
 #include "doveadm-mail.h"
 
+#include <stdio.h>
+
 #define INDEXER_SOCKET_NAME "indexer"
 #define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
 
@@ -23,6 +25,61 @@ struct index_cmd_context {
 	unsigned int have_wildcards:1;
 };
 
+static int cmd_index_box_precache(struct mailbox *box)
+{
+	struct mailbox_status status;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_args *search_args;
+	struct mail_search_context *ctx;
+	struct mail *mail;
+	struct mailbox_metadata metadata;
+	uint32_t seq;
+	unsigned int counter = 0, max;
+	int ret = 0;
+
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
+				 &metadata) < 0)
+		return -1;
+	mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
+				&status);
+
+	seq = status.last_cached_seq + 1;
+	if (seq > status.messages) {
+		if (doveadm_verbose) {
+			i_info("%s: Cache is already up to date",
+			       mailbox_get_vname(box));
+		}
+		return 0;
+	}
+	if (doveadm_verbose) {
+		i_info("%s: Caching mails seq=%u..%u",
+		       mailbox_get_vname(box), seq, status.messages);
+	}
+
+	trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq, status.messages);
+	ctx = mailbox_search_init(trans, search_args, NULL,
+				  metadata.precache_fields, NULL);
+	mail_search_args_unref(&search_args);
+
+	max = status.messages - seq + 1;
+	while (mailbox_search_next(ctx, &mail)) {
+		mail_precache(mail);
+		if (doveadm_verbose && ++counter % 100 == 0) {
+			printf("\r%u/%u", counter, max);
+			fflush(stdout);
+		}
+	}
+	if (doveadm_verbose)
+		printf("\r%u/%u\n", counter, max);
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = -1;
+	if (mailbox_transaction_commit(&trans) < 0)
+		ret = -1;
+	return ret;
+}
+
 static int
 cmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info)
 {
@@ -49,13 +106,13 @@ cmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info)
 		}
 	}
 
-	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ |
-			 MAILBOX_SYNC_FLAG_PRECACHE) < 0) {
+	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
 		i_error("Syncing mailbox %s failed: %s", info->name,
 			mail_storage_get_last_error(mailbox_get_storage(box), NULL));
 		ret = -1;
+	} else {
+		ret = cmd_index_box_precache(box);
 	}
-
 	mailbox_free(&box);
 	return ret;
 }
diff --git a/src/indexer/master-connection.c b/src/indexer/master-connection.c
index b3b0349b1b..5feecd9236 100644
--- a/src/indexer/master-connection.c
+++ b/src/indexer/master-connection.c
@@ -11,6 +11,7 @@
 #include "mail-namespace.h"
 #include "mail-storage.h"
 #include "mail-storage-service.h"
+#include "mail-search-build.h"
 #include "master-connection.h"
 
 #include <unistd.h>
@@ -33,15 +34,69 @@ struct master_connection {
 };
 
 static void
-indexer_worker_refresh_proctitle(const char *username, const char *mailbox)
+indexer_worker_refresh_proctitle(const char *username, const char *mailbox,
+				 uint32_t seq1, uint32_t seq2)
 {
 	if (!master_service_settings_get(master_service)->verbose_proctitle)
 		return;
 
-	if (username != NULL)
-		process_title_set(t_strdup_printf("[%s %s]", username, mailbox));
-	else
+	if (username == NULL)
 		process_title_set("[idling]");
+	else if (seq1 == 0)
+		process_title_set(t_strdup_printf("[%s %s]", username, mailbox));
+	else {
+		process_title_set(t_strdup_printf("[%s %s - %u/%u]",
+						  username, mailbox, seq1, seq2));
+	}
+}
+
+static int index_mailbox_precache(struct mailbox *box)
+{
+	struct mail_storage *storage = mailbox_get_storage(box);
+	const char *username = mail_storage_get_user(storage)->username;
+	const char *box_vname = mailbox_get_vname(box);
+	struct mailbox_status status;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_args *search_args;
+	struct mail_search_context *ctx;
+	struct mail *mail;
+	struct mailbox_metadata metadata;
+	uint32_t seq;
+	unsigned int counter = 0, max;
+	int ret = 0;
+
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
+				 &metadata) < 0)
+		return -1;
+
+	mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
+				&status);
+	seq = status.last_cached_seq + 1;
+
+	trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq, status.messages);
+	ctx = mailbox_search_init(trans, search_args, NULL,
+				  metadata.precache_fields, NULL);
+	mail_search_args_unref(&search_args);
+
+	max = status.messages - seq + 1;
+	while (mailbox_search_next(ctx, &mail)) {
+		mail_precache(mail);
+		if (++counter % 100 == 0) {
+			indexer_worker_refresh_proctitle(username, box_vname,
+							 counter, max);
+		}
+	}
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = -1;
+	if (mailbox_transaction_commit(&trans) < 0)
+		ret = -1;
+	if (ret == 0) {
+		i_info("Indexed %u messages in %s",
+		       counter, mailbox_get_vname(box));
+	}
+	return ret;
 }
 
 static int index_mailbox(struct mail_user *user, const char *mailbox,
@@ -81,8 +136,6 @@ static int index_mailbox(struct mail_user *user, const char *mailbox,
 		}
 	}
 
-	if (strchr(what, 'i') != NULL)
-		sync_flags |= MAILBOX_SYNC_FLAG_PRECACHE;
 	if (strchr(what, 'o') != NULL)
 		sync_flags |= MAILBOX_SYNC_FLAG_OPTIMIZE;
 
@@ -97,6 +150,8 @@ static int index_mailbox(struct mail_user *user, const char *mailbox,
 				mailbox, errstr);
 		}
 		ret = -1;
+	} else if (strchr(what, 'i') != NULL) {
+		index_mailbox_precache(box);
 	}
 	mailbox_free(&box);
 	return ret;
@@ -130,9 +185,9 @@ master_connection_input_line(struct master_connection *conn, const char *line)
 		i_error("User %s lookup failed: %s", args[0], error);
 		ret = -1;
 	} else {
-		indexer_worker_refresh_proctitle(user->username, args[1]);
+		indexer_worker_refresh_proctitle(user->username, args[1], 0, 0);
 		ret = index_mailbox(user, args[1], max_recent_msgs, args[3]);
-		indexer_worker_refresh_proctitle(NULL, NULL);
+		indexer_worker_refresh_proctitle(NULL, NULL, 0, 0);
 		mail_user_unref(&user);
 		mail_storage_service_user_free(&service_user);
 	}
diff --git a/src/lib-storage/index/cydir/cydir-mail.c b/src/lib-storage/index/cydir/cydir-mail.c
index 112d797ce7..92a8cd8784 100644
--- a/src/lib-storage/index/cydir/cydir-mail.c
+++ b/src/lib-storage/index/cydir/cydir-mail.c
@@ -130,6 +130,7 @@ struct mail_vfuncs cydir_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -152,7 +153,6 @@ struct mail_vfuncs cydir_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/dbox-multi/mdbox-mail.c b/src/lib-storage/index/dbox-multi/mdbox-mail.c
index b596c0fb37..8d5afebabc 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-mail.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c
@@ -191,6 +191,7 @@ struct mail_vfuncs mdbox_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -213,7 +214,6 @@ struct mail_vfuncs mdbox_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/dbox-single/sdbox-mail.c b/src/lib-storage/index/dbox-single/sdbox-mail.c
index 03e33e1165..73a3f356cb 100644
--- a/src/lib-storage/index/dbox-single/sdbox-mail.c
+++ b/src/lib-storage/index/dbox-single/sdbox-mail.c
@@ -99,6 +99,7 @@ struct mail_vfuncs sdbox_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -121,7 +122,6 @@ struct mail_vfuncs sdbox_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 7e08ad4400..1cd0dcf72e 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -221,6 +221,7 @@ struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	imapc_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -243,7 +244,6 @@ struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 26074bce1e..7888bb6886 100644
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -1557,7 +1557,7 @@ void index_mail_expunge(struct mail *mail)
 	}
 }
 
-void index_mail_parse(struct mail *mail, bool parse_body)
+static void index_mail_parse(struct mail *mail, bool parse_body)
 {
 	struct index_mail *imail = (struct index_mail *)mail;
 
@@ -1570,6 +1570,38 @@ void index_mail_parse(struct mail *mail, bool parse_body)
 	}
 }
 
+void index_mail_precache(struct mail *mail)
+{
+	struct index_mail *imail = (struct index_mail *)mail;
+	enum mail_fetch_field cache;
+	time_t date;
+	uoff_t size;
+	const char *str;
+
+	if (mail_cache_field_exists_any(mail->transaction->cache_view,
+					mail->seq)) {
+		/* already cached this mail (we should get here only if FTS
+		   plugin decreased the first precached seq) */
+		return;
+	}
+
+	cache = imail->wanted_fields;
+	if ((cache & (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY)) != 0)
+		index_mail_parse(mail, (cache & MAIL_FETCH_STREAM_BODY) != 0);
+	if ((cache & MAIL_FETCH_RECEIVED_DATE) != 0)
+		(void)mail_get_received_date(mail, &date);
+	if ((cache & MAIL_FETCH_SAVE_DATE) != 0)
+		(void)mail_get_save_date(mail, &date);
+	if ((cache & MAIL_FETCH_VIRTUAL_SIZE) != 0)
+		(void)mail_get_virtual_size(mail, &size);
+	if ((cache & MAIL_FETCH_PHYSICAL_SIZE) != 0)
+		(void)mail_get_physical_size(mail, &size);
+	if ((cache & MAIL_FETCH_UIDL_BACKEND) != 0)
+		(void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &str);
+	if ((cache & MAIL_FETCH_GUID) != 0)
+		(void)mail_get_special(mail, MAIL_FETCH_GUID, &str);
+}
+
 void index_mail_set_cache_corrupted(struct mail *mail,
 				    enum mail_fetch_field field)
 {
diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 49c6ad86f2..802375ac6c 100644
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -207,7 +207,7 @@ void index_mail_update_keywords(struct mail *mail, enum modify_type modify_type,
 				struct mail_keywords *keywords);
 void index_mail_update_modseq(struct mail *mail, uint64_t min_modseq);
 void index_mail_expunge(struct mail *mail);
-void index_mail_parse(struct mail *mail, bool parse_body);
+void index_mail_precache(struct mail *mail);
 void index_mail_set_cache_corrupted(struct mail *mail,
 				    enum mail_fetch_field field);
 int index_mail_opened(struct mail *mail, struct istream **stream);
diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c
index 1c1682acd0..4bfe873d65 100644
--- a/src/lib-storage/index/index-status.c
+++ b/src/lib-storage/index/index-status.c
@@ -7,6 +7,28 @@
 #include "index-storage.h"
 #include "mail-index-modseq.h"
 
+static void
+get_last_cached_seq(struct mailbox *box, uint32_t *last_cached_seq_r)
+{
+	const struct mail_index_header *hdr;
+	struct mail_cache_view *cache_view;
+	uint32_t seq;
+
+	*last_cached_seq_r = 0;
+	if (!mail_cache_exists(box->cache))
+		return;
+
+	cache_view = mail_cache_view_open(box->cache, box->view);
+	hdr = mail_index_get_header(box->view);
+	for (seq = hdr->messages_count; seq > 0; seq--) {
+		if (mail_cache_field_exists_any(cache_view, seq)) {
+			*last_cached_seq_r = seq;
+			break;
+		}
+	}
+	mail_cache_view_close(cache_view);
+}
+
 int index_storage_get_status(struct mailbox *box,
 			     enum mailbox_status_items items,
 			     struct mailbox_status *status_r)
@@ -50,6 +72,8 @@ int index_storage_get_status(struct mailbox *box,
 		mail_index_lookup_first(box->view, 0, MAIL_SEEN,
 					&status_r->first_unseen_seq);
 	}
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0)
+		get_last_cached_seq(box, &status_r->last_cached_seq);
 
 	if ((items & STATUS_KEYWORDS) != 0)
 		status_r->keywords = mail_index_get_keywords(box->index);
@@ -86,6 +110,44 @@ get_metadata_cache_fields(struct mailbox *box,
 	metadata_r->cache_fields = cache_fields;
 }
 
+static void get_metadata_precache_fields(struct mailbox *box,
+					 struct mailbox_metadata *metadata_r)
+{
+	const struct mail_cache_field *fields;
+	unsigned int i, count;
+	enum mail_fetch_field cache = 0;
+
+	fields = mail_cache_register_get_list(box->cache,
+					      pool_datastack_create(), &count);
+	for (i = 0; i < count; i++) {
+		const char *name = fields[i].name;
+
+		if (strncmp(name, "hdr.", 4) == 0 ||
+		    strcmp(name, "date.sent") == 0 ||
+		    strcmp(name, "imap.envelope") == 0)
+			cache |= MAIL_FETCH_STREAM_HEADER;
+		else if (strcmp(name, "mime.parts") == 0 ||
+			 strcmp(name, "imap.body") == 0 ||
+			 strcmp(name, "imap.bodystructure") == 0)
+			cache |= MAIL_FETCH_STREAM_BODY;
+		else if (strcmp(name, "date.received") == 0)
+			cache |= MAIL_FETCH_RECEIVED_DATE;
+		else if (strcmp(name, "date.save") == 0)
+			cache |= MAIL_FETCH_SAVE_DATE;
+		else if (strcmp(name, "size.virtual") == 0)
+			cache |= MAIL_FETCH_VIRTUAL_SIZE;
+		else if (strcmp(name, "size.physical") == 0)
+			cache |= MAIL_FETCH_PHYSICAL_SIZE;
+		else if (strcmp(name, "pop3.uidl") == 0)
+			cache |= MAIL_FETCH_UIDL_BACKEND;
+		else if (strcmp(name, "guid") == 0)
+			cache |= MAIL_FETCH_GUID;
+		else if (box->storage->set->mail_debug)
+			i_debug("Ignoring unknown cache field: %s", name);
+	}
+	metadata_r->precache_fields = cache;
+}
+
 static int
 virtual_size_add_new(struct mailbox *box,
 		     struct index_vsize_header *vsize_hdr)
@@ -199,11 +261,13 @@ int index_mailbox_get_metadata(struct mailbox *box,
 			       enum mailbox_metadata_items items,
 			       struct mailbox_metadata *metadata_r)
 {
-	if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0)
-		get_metadata_cache_fields(box, metadata_r);
 	if ((items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) {
 		if (get_metadata_virtual_size(box, metadata_r) < 0)
 			return -1;
 	}
+	if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0)
+		get_metadata_cache_fields(box, metadata_r);
+	if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0)
+		get_metadata_precache_fields(box, metadata_r);
 	return 0;
 }
diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
index 7d3082f31e..f2fb90c12c 100644
--- a/src/lib-storage/index/index-sync.c
+++ b/src/lib-storage/index/index-sync.c
@@ -11,17 +11,6 @@ struct index_storage_list_index_record {
 	uint32_t mtime;
 };
 
-enum cache_mask {
-	CACHE_HDR		= 0x01,
-	CACHE_BODY		= 0x02,
-	CACHE_RECEIVED_DATE	= 0x04,
-	CACHE_SAVE_DATE		= 0x08,
-	CACHE_VIRTUAL_SIZE	= 0x10,
-	CACHE_PHYSICAL_SIZE	= 0x20,
-	CACHE_POP3_UIDL		= 0x40,
-	CACHE_GUID		= 0x80
-};
-
 enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box)
 {
 	enum mail_index_sync_flags sync_flags = 0;
@@ -341,128 +330,6 @@ index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx)
 #endif
 }
 
-static enum cache_mask
-cache_fields_get(const struct mailbox_metadata *metadata, bool debug)
-{
-	const char *const *cache_fields;
-	unsigned int i, count;
-	enum cache_mask cache = 0;
-
-	cache_fields = array_get(metadata->cache_fields, &count);
-	for (i = 0; i < count; i++) {
-		if (strncmp(cache_fields[i], "hdr.", 4) == 0 ||
-		    strcmp(cache_fields[i], "date.sent") == 0 ||
-		    strcmp(cache_fields[i], "imap.envelope") == 0)
-			cache |= CACHE_HDR;
-		else if (strcmp(cache_fields[i], "mime.parts") == 0 ||
-			 strcmp(cache_fields[i], "imap.body") == 0 ||
-			 strcmp(cache_fields[i], "imap.bodystructure") == 0)
-			cache |= CACHE_BODY;
-		else if (strcmp(cache_fields[i], "date.received") == 0)
-			cache |= CACHE_RECEIVED_DATE;
-		else if (strcmp(cache_fields[i], "date.save") == 0)
-			cache |= CACHE_SAVE_DATE;
-		else if (strcmp(cache_fields[i], "size.virtual") == 0)
-			cache |= CACHE_VIRTUAL_SIZE;
-		else if (strcmp(cache_fields[i], "size.physical") == 0)
-			cache |= CACHE_PHYSICAL_SIZE;
-		else if (strcmp(cache_fields[i], "pop3.uidl") == 0)
-			cache |= CACHE_POP3_UIDL;
-		else if (strcmp(cache_fields[i], "guid") == 0)
-			cache |= CACHE_GUID;
-		else if (debug) {
-			i_debug("Ignoring unknown cache field: %s",
-				cache_fields[i]);
-		}
-	}
-	return cache;
-}
-
-static int cache_add(struct mailbox *box, enum cache_mask cache)
-{
-	struct mailbox_status status;
-	struct mailbox_transaction_context *trans;
-	struct mail *mail;
-	uint32_t seq;
-	time_t date;
-	uoff_t size;
-	const char *str;
-
-	if (cache == 0) {
-		if (box->storage->set->mail_debug) {
-			i_debug("%s: Nothing in mailbox cache, skipping",
-				box->vname);
-		}
-		return 0;
-	}
-
-	/* find the first message we need to index */
-	mailbox_get_open_status(box, STATUS_MESSAGES, &status);
-	trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
-	mail = mail_alloc(trans, 0, NULL);
-	for (seq = status.messages; seq > 0; seq--) {
-		mail_set_seq(mail, seq);
-		if (mail_is_cached(mail))
-			break;
-	}
-	seq++;
-
-	if (box->storage->set->mail_debug) {
-		if (seq > status.messages) {
-			i_debug("%s: Cache is already up to date", box->vname);
-		} else {
-			i_debug("%s: Caching mails seq=%u..%u cache=0x%x",
-				box->vname, seq, status.messages, cache);
-		}
-	}
-
-	for (; seq <= status.messages; seq++) {
-		mail_set_seq(mail, seq);
-
-		if ((cache & (CACHE_HDR | CACHE_BODY)) != 0)
-			mail_parse(mail, (cache & CACHE_BODY) != 0);
-		if ((cache & CACHE_RECEIVED_DATE) != 0)
-			(void)mail_get_received_date(mail, &date);
-		if ((cache & CACHE_SAVE_DATE) != 0)
-			(void)mail_get_save_date(mail, &date);
-		if ((cache & CACHE_VIRTUAL_SIZE) != 0)
-			(void)mail_get_virtual_size(mail, &size);
-		if ((cache & CACHE_PHYSICAL_SIZE) != 0)
-			(void)mail_get_physical_size(mail, &size);
-		if ((cache & CACHE_POP3_UIDL) != 0) {
-			(void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND,
-					       &str);
-		}
-		if ((cache & CACHE_GUID) != 0)
-			(void)mail_get_special(mail, MAIL_FETCH_GUID, &str);
-	}
-	mail_free(&mail);
-	if (mailbox_transaction_commit(&trans) < 0) {
-		mail_storage_set_critical(box->storage,
-			"Commiting mailbox %s failed: %s", box->vname,
-			mailbox_get_last_error(box, NULL));
-		return -1;
-	}
-	return 0;
-}
-
-static int index_sync_precache(struct mailbox *box)
-{
-	struct mailbox_metadata metadata;
-	enum cache_mask cache;
-
-	if (mailbox_get_metadata(box, MAILBOX_METADATA_CACHE_FIELDS,
-				 &metadata) < 0) {
-		mail_storage_set_critical(box->storage,
-			"Metadata lookup from mailbox %s failed: %s", box->vname,
-			mailbox_get_last_error(box, NULL));
-		return -1;
-	}
-
-	cache = cache_fields_get(&metadata, box->storage->set->mail_debug);
-	return cache_add(box, cache);
-}
-
 void index_sync_update_recent_count(struct mailbox *box)
 {
 	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
@@ -525,10 +392,6 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
 	if (array_is_created(&ctx->all_flag_update_uids))
 		array_free(&ctx->all_flag_update_uids);
 
-	if ((_ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0 && ret == 0) {
-		if (index_sync_precache(_ctx->box) < 0)
-			ret = -1;
-	}
 	i_free(ctx);
 	return ret;
 }
diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c
index 7b83adb8c2..a7bcc77651 100644
--- a/src/lib-storage/index/maildir/maildir-mail.c
+++ b/src/lib-storage/index/maildir/maildir-mail.c
@@ -633,6 +633,7 @@ struct mail_vfuncs maildir_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -655,7 +656,6 @@ struct mail_vfuncs maildir_mail_vfuncs = {
 	index_mail_update_modseq,
 	maildir_update_pop3_uidl,
 	index_mail_expunge,
-	index_mail_parse,
 	maildir_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c
index 0513e1d2de..031fbded11 100644
--- a/src/lib-storage/index/mbox/mbox-mail.c
+++ b/src/lib-storage/index/mbox/mbox-mail.c
@@ -397,6 +397,7 @@ struct mail_vfuncs mbox_mail_vfuncs = {
 	mbox_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -419,7 +420,6 @@ struct mail_vfuncs mbox_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/index/raw/raw-mail.c b/src/lib-storage/index/raw/raw-mail.c
index 29d4039177..2b941180d4 100644
--- a/src/lib-storage/index/raw/raw-mail.c
+++ b/src/lib-storage/index/raw/raw-mail.c
@@ -118,6 +118,7 @@ struct mail_vfuncs raw_mail_vfuncs = {
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
 	index_mail_prefetch,
+	index_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -140,7 +141,6 @@ struct mail_vfuncs raw_mail_vfuncs = {
 	index_mail_update_modseq,
 	NULL,
 	index_mail_expunge,
-	index_mail_parse,
 	index_mail_set_cache_corrupted,
 	index_mail_opened
 };
diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 75551b5cf2..9d4fba572e 100644
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -276,6 +276,7 @@ struct mail_vfuncs {
 	bool (*set_uid)(struct mail *mail, uint32_t uid);
 	void (*set_uid_cache_updates)(struct mail *mail, bool set);
 	bool (*prefetch)(struct mail *mail);
+	void (*precache)(struct mail *mail);
 
 	enum mail_flags (*get_flags)(struct mail *mail);
 	const char *const *(*get_keywords)(struct mail *mail);
@@ -313,7 +314,6 @@ struct mail_vfuncs {
 	void (*update_modseq)(struct mail *mail, uint64_t min_modseq);
 	void (*update_pop3_uidl)(struct mail *mail, const char *uidl);
 	void (*expunge)(struct mail *mail);
-	void (*parse)(struct mail *mail, bool parse_body);
 	void (*set_cache_corrupted)(struct mail *mail,
 				    enum mail_fetch_field field);
 	int (*istream_opened)(struct mail *mail, struct istream **input);
diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h
index 5d8197f6aa..d666900e90 100644
--- a/src/lib-storage/mail-storage.h
+++ b/src/lib-storage/mail-storage.h
@@ -73,13 +73,15 @@ enum mailbox_status_items {
 	STATUS_KEYWORDS		= 0x40,
 	STATUS_HIGHESTMODSEQ	= 0x80,
 	STATUS_PERMANENT_FLAGS	= 0x200,
-	STATUS_FIRST_RECENT_UID	= 0x400
+	STATUS_FIRST_RECENT_UID	= 0x400,
+	STATUS_LAST_CACHED_SEQ	= 0x800
 };
 
 enum mailbox_metadata_items {
-	MAILBOX_METADATA_GUID		= 0x01,
-	MAILBOX_METADATA_VIRTUAL_SIZE	= 0x02,
-	MAILBOX_METADATA_CACHE_FIELDS	= 0x04
+	MAILBOX_METADATA_GUID			= 0x01,
+	MAILBOX_METADATA_VIRTUAL_SIZE		= 0x02,
+	MAILBOX_METADATA_CACHE_FIELDS		= 0x04,
+	MAILBOX_METADATA_PRECACHE_FIELDS	= 0x08
 };
 
 enum mailbox_search_result_flags {
@@ -175,8 +177,6 @@ enum mailbox_sync_flags {
 	MAILBOX_SYNC_FLAG_EXPUNGE		= 0x80,
 	/* Force doing a full resync of indexes. */
 	MAILBOX_SYNC_FLAG_FORCE_RESYNC		= 0x100,
-	/* Add all missing data to cache and fts index ("doveadm index") */
-	MAILBOX_SYNC_FLAG_PRECACHE		= 0x200,
 	/* FIXME: kludge until something better comes along:
 	   Request full text search index optimization */
 	MAILBOX_SYNC_FLAG_OPTIMIZE		= 0x400
@@ -208,6 +208,7 @@ struct mailbox_status {
 
 	uint32_t first_unseen_seq;
 	uint32_t first_recent_uid;
+	uint32_t last_cached_seq;
 	uint64_t highest_modseq;
 
 	/* NULL-terminated array of keywords */
@@ -230,6 +231,8 @@ struct mailbox_metadata {
 	uint64_t virtual_size;
 	/* Fields that have "temp" or "yes" caching decision. */
 	const ARRAY_TYPE(const_string) *cache_fields;
+	/* Fields that should be precached */
+	enum mail_fetch_field precache_fields;
 };
 
 struct mailbox_update {
@@ -741,11 +744,8 @@ void mail_update_pop3_uidl(struct mail *mail, const char *uidl);
 /* Expunge this message. Sequence numbers don't change until commit. */
 void mail_expunge(struct mail *mail);
 
-/* Returns TRUE if anything is cached for the mail, FALSE if not. */
-bool mail_is_cached(struct mail *mail);
-/* Parse mail's header and optionally body so that fields using them get
-   cached. */
-void mail_parse(struct mail *mail, bool parse_body);
+/* Add missing fields to cache. */
+void mail_precache(struct mail *mail);
 /* Mark a cached field corrupted and have it recalculated. */
 void mail_set_cache_corrupted(struct mail *mail, enum mail_fetch_field field);
 
diff --git a/src/lib-storage/mail.c b/src/lib-storage/mail.c
index 342a791d74..0f7f666408 100644
--- a/src/lib-storage/mail.c
+++ b/src/lib-storage/mail.c
@@ -255,17 +255,11 @@ void mail_set_expunged(struct mail *mail)
 	mail->expunged = TRUE;
 }
 
-bool mail_is_cached(struct mail *mail)
-{
-	return mail_cache_field_exists_any(mail->transaction->cache_view,
-					   mail->seq);
-}
-
-void mail_parse(struct mail *mail, bool parse_body)
+void mail_precache(struct mail *mail)
 {
 	struct mail_private *p = (struct mail_private *)mail;
 
-	p->v.parse(mail, parse_body);
+	p->v.precache(mail);
 }
 
 void mail_set_cache_corrupted(struct mail *mail, enum mail_fetch_field field)
diff --git a/src/lib-storage/test-mail.c b/src/lib-storage/test-mail.c
index 58f2ba47a2..6b610cb399 100644
--- a/src/lib-storage/test-mail.c
+++ b/src/lib-storage/test-mail.c
@@ -59,6 +59,10 @@ static bool test_mail_prefetch(struct mail *mail ATTR_UNUSED)
 	return TRUE;
 }
 
+static void test_mail_precache(struct mail *mail ATTR_UNUSED)
+{
+}
+
 static enum mail_flags test_mail_get_flags(struct mail *mail ATTR_UNUSED)
 {
 	return 0;
@@ -201,11 +205,6 @@ static void test_mail_expunge(struct mail *mail ATTR_UNUSED)
 {
 }
 
-static void test_mail_parse(struct mail *mail ATTR_UNUSED,
-			    bool parse_body ATTR_UNUSED)
-{
-}
-
 static void
 test_mail_set_cache_corrupted(struct mail *mail ATTR_UNUSED,
 			      enum mail_fetch_field field ATTR_UNUSED)
@@ -219,6 +218,7 @@ struct mail_vfuncs test_mail_vfuncs = {
 	test_mail_set_uid,
 	test_mail_set_uid_cache_updates,
 	test_mail_prefetch,
+	test_mail_precache,
 
 	test_mail_get_flags,
 	test_mail_get_keywords,
@@ -241,7 +241,6 @@ struct mail_vfuncs test_mail_vfuncs = {
 	test_mail_update_modseq,
 	NULL,
 	test_mail_expunge,
-	test_mail_parse,
 	test_mail_set_cache_corrupted,
 	NULL
 };
diff --git a/src/plugins/fts-lucene/fts-backend-lucene.c b/src/plugins/fts-lucene/fts-backend-lucene.c
index dea0b532a4..fdf63bd534 100644
--- a/src/plugins/fts-lucene/fts-backend-lucene.c
+++ b/src/plugins/fts-lucene/fts-backend-lucene.c
@@ -9,6 +9,7 @@
 #include "mail-storage-private.h"
 #include "fts-expunge-log.h"
 #include "lucene-wrapper.h"
+#include "fts-indexer.h"
 #include "fts-lucene-plugin.h"
 
 #include <wchar.h>
@@ -85,6 +86,8 @@ fts_backend_select(struct lucene_fts_backend *backend, struct mailbox *box)
 	buffer_t buf;
 	unsigned int i;
 
+	i_assert(box != NULL);
+
 	if (backend->selected_box == box &&
 	    backend->selected_box_generation == box->generation_sequence)
 		return 0;
diff --git a/src/plugins/fts/Makefile.am b/src/plugins/fts/Makefile.am
index 7195c38b18..a96ec7fdb2 100644
--- a/src/plugins/fts/Makefile.am
+++ b/src/plugins/fts/Makefile.am
@@ -18,11 +18,9 @@ module_LTLIBRARIES = \
 
 lib20_fts_plugin_la_SOURCES = \
 	fts-api.c \
-	fts-build.c \
-	fts-build-indexer.c \
-	fts-build-mailbox.c \
-	fts-build-virtual.c \
+	fts-build-mail.c \
 	fts-expunge-log.c \
+	fts-indexer.c \
 	fts-parser.c \
 	fts-parser-html.c \
 	fts-parser-script.c \
@@ -36,9 +34,9 @@ noinst_HEADERS = \
 	html-entities.h \
 	fts-api.h \
 	fts-api-private.h \
-	fts-build.h \
-	fts-build-private.h \
+	fts-build-mail.h \
 	fts-expunge-log.h \
+	fts-indexer.h \
 	fts-parser.h \
 	fts-plugin.h \
 	fts-search-serialize.h \
diff --git a/src/plugins/fts/fts-api-private.h b/src/plugins/fts/fts-api-private.h
index 3479d7715e..1125d31fda 100644
--- a/src/plugins/fts/fts-api-private.h
+++ b/src/plugins/fts/fts-api-private.h
@@ -112,8 +112,4 @@ bool fts_header_want_indexed(const char *hdr_name);
 
 int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r);
 
-/* Returns fd, which you can either read from or close. */
-int fts_indexer_cmd(struct mail_user *user, const char *cmd,
-		    const char **path_r);
-
 #endif
diff --git a/src/plugins/fts/fts-build.c b/src/plugins/fts/fts-build-mail.c
similarity index 58%
rename from src/plugins/fts/fts-build.c
rename to src/plugins/fts/fts-build-mail.c
index d85580d4ba..7ddcbbe2c0 100644
--- a/src/plugins/fts/fts-build.c
+++ b/src/plugins/fts/fts-build-mail.c
@@ -1,22 +1,26 @@
 /* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
-#include "ioloop.h"
 #include "istream.h"
 #include "str.h"
-#include "time-util.h"
 #include "rfc822-parser.h"
 #include "message-address.h"
 #include "message-parser.h"
 #include "message-decoder.h"
-#include "../virtual/virtual-storage.h"
-#include "fts-api-private.h"
+#include "mail-storage.h"
 #include "fts-parser.h"
-#include "fts-build-private.h"
+#include "fts-api-private.h"
+#include "fts-build-mail.h"
+
+struct fts_mail_build_context {
+	struct mail *mail;
+	struct fts_backend_update_context *update_ctx;
 
-#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10
+	char *content_type, *content_disposition;
+	struct fts_parser *body_parser;
+};
 
-static void fts_build_parse_content_type(struct fts_storage_build_context *ctx,
+static void fts_build_parse_content_type(struct fts_mail_build_context *ctx,
 					 const struct message_header_line *hdr)
 {
 	struct rfc822_parser_context parser;
@@ -36,7 +40,7 @@ static void fts_build_parse_content_type(struct fts_storage_build_context *ctx,
 }
 
 static void
-fts_build_parse_content_disposition(struct fts_storage_build_context *ctx,
+fts_build_parse_content_disposition(struct fts_mail_build_context *ctx,
 				    const struct message_header_line *hdr)
 {
 	/* just pass it as-is to backend. */
@@ -45,7 +49,7 @@ fts_build_parse_content_disposition(struct fts_storage_build_context *ctx,
 		i_strndup(hdr->full_value, hdr->full_value_len);
 }
 
-static void fts_parse_mail_header(struct fts_storage_build_context *ctx,
+static void fts_parse_mail_header(struct fts_mail_build_context *ctx,
 				  const struct message_block *raw_block)
 {
 	const struct message_header_line *hdr = raw_block->hdr;
@@ -57,7 +61,7 @@ static void fts_parse_mail_header(struct fts_storage_build_context *ctx,
 }
 
 static void
-fts_build_unstructured_header(struct fts_storage_build_context *ctx,
+fts_build_unstructured_header(struct fts_mail_build_context *ctx,
 			      const struct message_header_line *hdr)
 {
 	const unsigned char *data = hdr->full_value;
@@ -82,7 +86,7 @@ fts_build_unstructured_header(struct fts_storage_build_context *ctx,
 	i_free(buf);
 }
 
-static void fts_build_mail_header(struct fts_storage_build_context *ctx,
+static void fts_build_mail_header(struct fts_mail_build_context *ctx,
 				  const struct message_block *block)
 {
 	const struct message_header_line *hdr = block->hdr;
@@ -94,7 +98,7 @@ static void fts_build_mail_header(struct fts_storage_build_context *ctx,
 	/* hdr->full_value is always set because we get the block from
 	   message_decoder */
 	memset(&key, 0, sizeof(key));
-	key.uid = ctx->uid;
+	key.uid = ctx->mail->uid;
 	key.type = block->part->physical_pos == 0 ?
 		FTS_BACKEND_BUILD_KEY_HDR : FTS_BACKEND_BUILD_KEY_MIME_HDR;
 	key.hdr_name = hdr->name;
@@ -125,8 +129,9 @@ static void fts_build_mail_header(struct fts_storage_build_context *ctx,
 }
 
 static bool
-fts_build_body_begin(struct fts_storage_build_context *ctx, bool *binary_body_r)
+fts_build_body_begin(struct fts_mail_build_context *ctx, bool *binary_body_r)
 {
+	struct mail_storage *storage;
 	const char *content_type;
 	struct fts_backend_build_key key;
 
@@ -134,7 +139,7 @@ fts_build_body_begin(struct fts_storage_build_context *ctx, bool *binary_body_r)
 
 	*binary_body_r = FALSE;
 	memset(&key, 0, sizeof(key));
-	key.uid = ctx->uid;
+	key.uid = ctx->mail->uid;
 
 	content_type = ctx->content_type != NULL ?
 		ctx->content_type : "text/plain";
@@ -143,7 +148,9 @@ fts_build_body_begin(struct fts_storage_build_context *ctx, bool *binary_body_r)
 		return FALSE;
 	}
 
-	if (fts_parser_init(ctx->box->storage->user,
+	
+	storage = mailbox_get_storage(ctx->mail->box);
+	if (fts_parser_init(mail_storage_get_user(storage),
 			    content_type, ctx->content_disposition,
 			    &ctx->body_parser)) {
 		/* extract text using the the returned parser */
@@ -155,7 +162,8 @@ fts_build_body_begin(struct fts_storage_build_context *ctx, bool *binary_body_r)
 		key.type = FTS_BACKEND_BUILD_KEY_BODY_PART;
 	} else {
 		/* possibly binary */
-		if (!ctx->binary_mime_parts)
+		if ((ctx->update_ctx->backend->flags &
+		     FTS_BACKEND_FLAG_BINARY_MIME_PARTS) == 0)
 			return FALSE;
 		*binary_body_r = TRUE;
 		key.type = FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY;
@@ -167,7 +175,7 @@ fts_build_body_begin(struct fts_storage_build_context *ctx, bool *binary_body_r)
 	return fts_backend_update_set_build_key(ctx->update_ctx, &key);
 }
 
-static int fts_body_parser_finish(struct fts_storage_build_context *ctx)
+static int fts_body_parser_finish(struct fts_mail_build_context *ctx)
 {
 	struct message_block block;
 	int ret = 0;
@@ -187,8 +195,11 @@ static int fts_body_parser_finish(struct fts_storage_build_context *ctx)
 	return ret;
 }
 
-int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
+static int
+fts_build_mail_real(struct fts_backend_update_context *update_ctx,
+		    struct mail *mail)
 {
+	struct fts_mail_build_context ctx;
 	enum message_decoder_flags decoder_flags = 0;
 	struct istream *input;
 	struct message_parser_ctx *parser;
@@ -199,17 +210,19 @@ int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
 	bool binary_body;
 	int ret;
 
-	ctx->uid = mail->uid;
-
 	if (mail_get_stream(mail, NULL, NULL, &input) < 0)
 		return mail->expunged ? 0 : -1;
 
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.update_ctx = update_ctx;
+	ctx.mail = mail;
+
 	prev_part = NULL;
 	parser = message_parser_init(pool_datastack_create(), input,
 				     MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
 				     0);
 
-	if (ctx->dtcase)
+	if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_BUILD_DTCASE) != 0)
 		decoder_flags |= MESSAGE_DECODER_FLAG_DTCASE;
 	decoder = message_decoder_init(decoder_flags);
 	for (;;) {
@@ -224,17 +237,17 @@ int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
 		if (raw_block.part != prev_part) {
 			/* body part changed. we're now parsing the end of
 			   boundary, possibly followed by message epilogue */
-			if (ctx->body_parser != NULL) {
-				if (fts_body_parser_finish(ctx) < 0) {
+			if (ctx.body_parser != NULL) {
+				if (fts_body_parser_finish(&ctx) < 0) {
 					ret = -1;
 					break;
 				}
 			}
 			message_decoder_set_return_binary(decoder, FALSE);
-			fts_backend_update_unset_build_key(ctx->update_ctx);
+			fts_backend_update_unset_build_key(update_ctx);
 			prev_part = raw_block.part;
-			i_free_and_null(ctx->content_type);
-			i_free_and_null(ctx->content_disposition);
+			i_free_and_null(ctx.content_type);
+			i_free_and_null(ctx.content_disposition);
 
 			if (raw_block.size != 0) {
 				/* multipart. skip until beginning of next
@@ -247,7 +260,7 @@ int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
 			/* always handle headers */
 		} else if (raw_block.size == 0) {
 			/* end of headers */
-			skip_body = !fts_build_body_begin(ctx, &binary_body);
+			skip_body = !fts_build_body_begin(&ctx, &binary_body);
 			if (binary_body)
 				message_decoder_set_return_binary(decoder, TRUE);
 			body_part = TRUE;
@@ -261,15 +274,15 @@ int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
 			continue;
 
 		if (block.hdr != NULL) {
-			fts_parse_mail_header(ctx, &raw_block);
-			fts_build_mail_header(ctx, &block);
+			fts_parse_mail_header(&ctx, &raw_block);
+			fts_build_mail_header(&ctx, &block);
 		} else if (block.size == 0) {
 			/* end of headers */
 		} else {
 			i_assert(body_part);
-			if (ctx->body_parser != NULL)
-				fts_parser_more(ctx->body_parser, &block);
-			if (fts_backend_update_build_more(ctx->update_ctx,
+			if (ctx.body_parser != NULL)
+				fts_parser_more(ctx.body_parser, &block);
+			if (fts_backend_update_build_more(update_ctx,
 							  block.data,
 							  block.size) < 0) {
 				ret = -1;
@@ -278,132 +291,27 @@ int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
 			body_added = TRUE;
 		}
 	}
-	if (ret == 0 && ctx->body_parser != NULL)
-		ret = fts_body_parser_finish(ctx);
+	if (ret == 0 && ctx.body_parser != NULL)
+		ret = fts_body_parser_finish(&ctx);
 	if (ret == 0 && body_part && !skip_body && !body_added) {
 		/* make sure body is added even when it doesn't exist */
-		ret = fts_backend_update_build_more(ctx->update_ctx, NULL, 0);
+		ret = fts_backend_update_build_more(update_ctx, NULL, 0);
 	}
-	if (ret == 0)
-		ctx->indexed_msg_count++;
 	if (message_parser_deinit(&parser, &parts) < 0)
 		mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS);
 	message_decoder_deinit(&decoder);
-	return ret;
+	i_free(ctx.content_type);
+	i_free(ctx.content_disposition);
+	return ret < 0 ? -1 : 1;
 }
 
-static void fts_build_notify(struct fts_storage_build_context *ctx)
+int fts_build_mail(struct fts_backend_update_context *update_ctx,
+		   struct mail *mail)
 {
-	double completed_frac;
-	unsigned int eta_secs;
-
-	if (ioloop_time - ctx->last_notify.tv_sec < FTS_BUILD_NOTIFY_INTERVAL_SECS)
-		return;
-	ctx->last_notify = ioloop_timeval;
-
-	if (ctx->box->storage->callbacks.notify_ok == NULL ||
-	    ctx->mail_idx == 0)
-		return;
-
-	/* mail_count is counted before indexing actually begins.
-	   by the time the mailbox is actually indexed it may already
-	   have more (or less) mails. so mail_idx can be higher than
-	   mail_count. */
-	completed_frac = ctx->mail_idx >= ctx->mail_count ? 1 :
-		(double)ctx->mail_idx / ctx->mail_count;
-
-	if (completed_frac >= 0.000001) {
-		unsigned int elapsed_msecs, est_total_msecs;
-
-		elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
-						   &ctx->search_start_time);
-		est_total_msecs = elapsed_msecs / completed_frac;
-		eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
-	} else {
-		eta_secs = 0;
-	}
+	int ret;
 
 	T_BEGIN {
-		const char *text;
-
-		text = t_strdup_printf("Indexed %d%% of the mailbox, "
-				       "ETA %d:%02d", (int)(completed_frac * 100.0),
-				       eta_secs/60, eta_secs%60);
-		ctx->box->storage->callbacks.
-			notify_ok(ctx->box, text,
-				  ctx->box->storage->callback_context);
-		ctx->notified = TRUE;
+		ret = fts_build_mail_real(update_ctx, mail);
 	} T_END;
-}
-
-int fts_build_init(struct fts_backend *backend, struct mailbox *box,
-		   bool precache,
-		   struct fts_storage_build_context **build_ctx_r)
-{
-	const struct fts_storage_build_vfuncs *v;
-	int ret;
-
-	*build_ctx_r = NULL;
-
-	/* unless we're precaching (i.e. indexer service, doveadm index)
-	   use the indexer service */
-	if (!precache)
-		v = &fts_storage_build_indexer_vfuncs;
-	else if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0)
-		v = &fts_storage_build_virtual_vfuncs;
-	else
-		v = &fts_storage_build_mailbox_vfuncs;
-
-	if ((ret = v->init(backend, box, build_ctx_r)) <= 0)
-		return ret;
-
-	(*build_ctx_r)->box = box;
-	(*build_ctx_r)->v = *v;
-	(*build_ctx_r)->dtcase =
-		(backend->flags & FTS_BACKEND_FLAG_BUILD_DTCASE) != 0;
-	(*build_ctx_r)->binary_mime_parts =
-		(backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0;
-	return 1;
-}
-
-int fts_build_deinit(struct fts_storage_build_context **_ctx)
-{
-	struct fts_storage_build_context *ctx = *_ctx;
-	int ret = ctx->failed ? -1 : 0;
-
-	*_ctx = NULL;
-
-	if (ctx->v.deinit(ctx) < 0)
-		ret = -1;
-	if (ctx->indexed_msg_count > 0) {
-		i_info("Indexed %u messages in %s", ctx->indexed_msg_count,
-		       mailbox_get_vname(ctx->box));
-	}
-	if (ctx->update_ctx != NULL) {
-		if (fts_backend_update_deinit(&ctx->update_ctx) < 0)
-			ret = -1;
-	}
-	if (ctx->notified) {
-		/* we notified at least once */
-		ctx->box->storage->callbacks.
-			notify_ok(ctx->box, "Mailbox indexing finished",
-				  ctx->box->storage->callback_context);
-	}
-	i_free(ctx->content_type);
-	i_free(ctx->content_disposition);
-	i_free(ctx);
-	return ret;
-}
-
-int fts_build_more(struct fts_storage_build_context *ctx)
-{
-	int ret;
-
-	if ((ret = ctx->v.more(ctx)) < 0) {
-		ctx->failed = TRUE;
-		return -1;
-	}
-
-	fts_build_notify(ctx);
 	return ret;
 }
diff --git a/src/plugins/fts/fts-build-mail.h b/src/plugins/fts/fts-build-mail.h
new file mode 100644
index 0000000000..3dffc1ba84
--- /dev/null
+++ b/src/plugins/fts/fts-build-mail.h
@@ -0,0 +1,7 @@
+#ifndef FTS_BUILD_MAIL_H
+#define FTS_BUILD_MAIL_H
+
+int fts_build_mail(struct fts_backend_update_context *update_ctx,
+		   struct mail *mail);
+
+#endif
diff --git a/src/plugins/fts/fts-build-mailbox.c b/src/plugins/fts/fts-build-mailbox.c
deleted file mode 100644
index b25bd81a89..0000000000
--- a/src/plugins/fts/fts-build-mailbox.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "mail-storage-private.h"
-#include "mail-search-build.h"
-#include "fts-api-private.h"
-#include "fts-build-private.h"
-
-#define FTS_SEARCH_NONBLOCK_COUNT 50
-
-static int
-fts_build_mailbox_init(struct fts_backend *backend, struct mailbox *box,
-		       struct fts_storage_build_context **build_ctx_r)
-{
-	struct fts_storage_build_context *ctx;
-	struct mail_search_args *search_args;
-	struct fts_backend_update_context *update_ctx;
-	struct mailbox_status status;
-	uint32_t last_uid, seq1, seq2;
-
-	if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
-		return -1;
-
-	mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
-	if (status.uidnext == last_uid+1) {
-		/* everything is already indexed */
-		return 0;
-	}
-
-	mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
-	if (seq1 == 0) {
-		/* no new messages (last messages in mailbox were expunged) */
-		return 0;
-	}
-
-	update_ctx = fts_backend_update_init(backend);
-	fts_backend_update_set_mailbox(update_ctx, box);
-
-	search_args = mail_search_build_init();
-	mail_search_build_add_seqset(search_args, seq1, seq2);
-
-	ctx = i_new(struct fts_storage_build_context, 1);
-	ctx->update_ctx = update_ctx;
-	ctx->mail_count = seq2 - seq1 + 1;
-
-	ctx->trans = mailbox_transaction_begin(box, 0);
-	ctx->search_ctx = mailbox_search_init(ctx->trans, search_args,
-					      NULL, 0, NULL);
-	ctx->search_ctx->progress_hidden = TRUE;
-	mail_search_args_unref(&search_args);
-
-	*build_ctx_r = ctx;
-	return 1;
-}
-
-static int fts_build_mailbox_deinit(struct fts_storage_build_context *ctx)
-{
-	int ret;
-
-	ret = mailbox_search_deinit(&ctx->search_ctx);
-	(void)mailbox_transaction_commit(&ctx->trans);
-	return ret;
-}
-
-static int fts_build_mailbox_more(struct fts_storage_build_context *ctx)
-{
-	struct mail *mail = NULL;
-	unsigned int count = 0;
-	int ret;
-
-	while (mailbox_search_next(ctx->search_ctx, &mail)) {
-		T_BEGIN {
-			ret = fts_build_mail(ctx, mail);
-		} T_END;
-
-		if (ret < 0)
-			return -1;
-
-		ctx->mail_idx++;
-		if (++count == FTS_SEARCH_NONBLOCK_COUNT)
-			return 0;
-	}
-	return 1;
-}
-
-const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs = {
-	fts_build_mailbox_init,
-	fts_build_mailbox_deinit,
-	fts_build_mailbox_more
-};
diff --git a/src/plugins/fts/fts-build-private.h b/src/plugins/fts/fts-build-private.h
index 29d3f27a85..8cdbd02b49 100644
--- a/src/plugins/fts/fts-build-private.h
+++ b/src/plugins/fts/fts-build-private.h
@@ -5,42 +5,5 @@
 
 struct fts_storage_build_context;
 
-#define FTS_SEARCH_NONBLOCK_COUNT 50
-
-struct fts_storage_build_vfuncs {
-	int (*init)(struct fts_backend *backend, struct mailbox *box,
-		    struct fts_storage_build_context **build_ctx_r);
-	int (*deinit)(struct fts_storage_build_context *ctx);
-	int (*more)(struct fts_storage_build_context *ctx);
-};
-
-struct fts_storage_build_context {
-	struct mailbox *box;
-	struct fts_backend_update_context *update_ctx;
-	struct fts_storage_build_vfuncs v;
-
-	struct timeval search_start_time, last_notify;
-	unsigned int mail_idx, mail_count;
-
-	struct mailbox_transaction_context *trans;
-	struct mail_search_context *search_ctx;
-
-	uint32_t uid;
-	char *content_type, *content_disposition;
-	struct fts_parser *body_parser;
-
-	unsigned int indexed_msg_count;
-
-	unsigned int binary_mime_parts:1;
-	unsigned int dtcase:1;
-	unsigned int notified:1;
-	unsigned int failed:1;
-};
-
-extern const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs;
-extern const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs;
-extern const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs;
-
-int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail);
 
 #endif
diff --git a/src/plugins/fts/fts-build-virtual.c b/src/plugins/fts/fts-build-virtual.c
deleted file mode 100644
index 8026bf16f1..0000000000
--- a/src/plugins/fts/fts-build-virtual.c
+++ /dev/null
@@ -1,203 +0,0 @@
-/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "mail-storage-private.h"
-#include "mail-search-build.h"
-#include "../virtual/virtual-storage.h"
-#include "fts-api-private.h"
-#include "fts-storage.h"
-#include "fts-build-private.h"
-
-#define FTS_SEARCH_NONBLOCK_COUNT 50
-
-struct virtual_fts_storage_build_context {
-	struct fts_storage_build_context ctx;
-
-	struct fts_backend *update_backend;
-	uint32_t virtual_last_uid;
-
-	ARRAY_TYPE(mailboxes) mailboxes;
-	unsigned int mailbox_idx;
-};
-
-static int
-fts_mailbox_get_seqs(struct mailbox *box, uint32_t *seq1_r, uint32_t *seq2_r)
-{
-	struct mailbox_status status;
-	uint32_t last_uid;
-
-	if (fts_backend_get_last_uid(fts_mailbox_backend(box),
-				     box, &last_uid) < 0)
-		return -1;
-
-	mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
-	if (status.uidnext <= last_uid+1)
-		*seq1_r = *seq2_r = 0;
-	else {
-		mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1,
-				      seq1_r, seq2_r);
-	}
-	return 0;
-}
-
-static int
-fts_build_virtual_mailboxes_get(struct virtual_fts_storage_build_context *ctx)
-{
-	struct virtual_mailbox *vbox =
-		(struct virtual_mailbox *)ctx->ctx.box;
-	struct mailbox *const *boxp;
-	ARRAY_TYPE(mailboxes) all_mailboxes;
-	uint32_t seq1, seq2;
-
-	t_array_init(&all_mailboxes, 64);
-	i_array_init(&ctx->mailboxes, 64);
-	vbox->vfuncs.get_virtual_backend_boxes(ctx->ctx.box,
-					       &all_mailboxes, TRUE);
-
-	array_foreach(&all_mailboxes, boxp) {
-		if (fts_mailbox_get_seqs(*boxp, &seq1, &seq2) < 0) {
-			array_free(&ctx->mailboxes);
-			return -1;
-		}
-		if (seq1 != 0) {
-			ctx->ctx.mail_count += seq2 - seq1 + 1;
-			array_append(&ctx->mailboxes, boxp, 1);
-		}
-	}
-	return 0;
-}
-
-static void
-fts_build_virtual_mailbox_close(struct virtual_fts_storage_build_context *ctx)
-{
-	if (mailbox_search_deinit(&ctx->ctx.search_ctx) < 0)
-		ctx->ctx.failed = TRUE;
-	(void)mailbox_transaction_commit(&ctx->ctx.trans);
-}
-
-static bool
-fts_build_virtual_mailbox_next(struct virtual_fts_storage_build_context *ctx)
-{
-	struct mail_search_args *search_args;
-	struct mailbox *const *boxes, *box;
-	struct fts_backend *backend;
-	unsigned int count;
-	uint32_t seq1, seq2;
-
-	boxes = array_get(&ctx->mailboxes, &count);
-	if (ctx->mailbox_idx == count)
-		return FALSE;
-	box = boxes[ctx->mailbox_idx++];
-
-	if (ctx->ctx.trans != NULL)
-		fts_build_virtual_mailbox_close(ctx);
-
-	if (fts_mailbox_get_seqs(box, &seq1, &seq2) < 0) {
-		ctx->ctx.failed = TRUE;
-		return fts_build_virtual_mailbox_next(ctx);
-	}
-
-	backend = fts_mailbox_backend(box);
-	if (ctx->update_backend != backend) {
-		if (ctx->ctx.update_ctx != NULL) {
-			if (fts_backend_update_deinit(&ctx->ctx.update_ctx) < 0)
-				ctx->ctx.failed = TRUE;
-		}
-		ctx->update_backend = backend;
-		ctx->ctx.update_ctx = fts_backend_update_init(backend);
-	}
-
-
-	fts_backend_update_set_mailbox(ctx->ctx.update_ctx, box);
-	search_args = mail_search_build_init();
-	mail_search_build_add_seqset(search_args, seq1, seq2);
-
-	ctx->ctx.trans = mailbox_transaction_begin(box, 0);
-	ctx->ctx.search_ctx = mailbox_search_init(ctx->ctx.trans, search_args,
-						  NULL, 0, NULL);
-	ctx->ctx.search_ctx->progress_hidden = TRUE;
-	mail_search_args_unref(&search_args);
-	return TRUE;
-}
-
-static int
-fts_build_virtual_init(struct fts_backend *backend, struct mailbox *box,
-		       struct fts_storage_build_context **build_ctx_r)
-{
-	struct virtual_fts_storage_build_context *ctx;
-	struct mailbox_status status;
-	uint32_t last_uid;
-
-	/* first do a quick check: is the virtual mailbox's last indexed
-	   UID up to date? */
-	if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
-		return -1;
-
-	mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
-	if (status.uidnext == last_uid+1)
-		return 0;
-
-	/* nope. we'll need to go through its mailboxes and check their
-	   indexes. FIXME: we could optimize by going through only those
-	   mailboxes that exist in >last_uid mails */
-	ctx = i_new(struct virtual_fts_storage_build_context, 1);
-	ctx->ctx.box = box;
-	ctx->virtual_last_uid = status.uidnext - 1;
-
-	if (fts_build_virtual_mailboxes_get(ctx) < 0) {
-		i_free(ctx);
-		return -1;
-	}
-	fts_build_virtual_mailbox_next(ctx);
-
-	*build_ctx_r = &ctx->ctx;
-	return 1;
-}
-
-static int fts_build_virtual_deinit(struct fts_storage_build_context *_ctx)
-{
-	struct virtual_fts_storage_build_context *ctx =
-		(struct virtual_fts_storage_build_context *)_ctx;
-
-	if (!_ctx->failed) {
-		(void)fts_index_set_last_uid(ctx->ctx.box,
-					     ctx->virtual_last_uid);
-	}
-
-	fts_build_virtual_mailbox_close(ctx);
-	array_free(&ctx->mailboxes);
-	return 0;
-}
-
-static int fts_build_virtual_more(struct fts_storage_build_context *_ctx)
-{
-	struct virtual_fts_storage_build_context *ctx =
-		(struct virtual_fts_storage_build_context *)_ctx;
-	struct mail *mail;
-	unsigned int count = 0;
-	int ret;
-
-	while (mailbox_search_next(_ctx->search_ctx, &mail)) {
-		T_BEGIN {
-			ret = fts_build_mail(_ctx, mail);
-		} T_END;
-
-		if (ret < 0)
-			return -1;
-
-		_ctx->mail_idx++;
-		if (++count == FTS_SEARCH_NONBLOCK_COUNT)
-			return 0;
-	}
-
-	if (fts_build_virtual_mailbox_next(ctx))
-		return fts_build_virtual_more(_ctx);
-	return 1;
-}
-
-const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs = {
-	fts_build_virtual_init,
-	fts_build_virtual_deinit,
-	fts_build_virtual_more
-};
diff --git a/src/plugins/fts/fts-build.h b/src/plugins/fts/fts-build.h
deleted file mode 100644
index c2209693ac..0000000000
--- a/src/plugins/fts/fts-build.h
+++ /dev/null
@@ -1,18 +0,0 @@
-#ifndef FTS_BUILD_H
-#define FTS_BUILD_H
-
-struct fts_storage_build_context;
-
-/* Initialize building. Returns 1 if we need to build (build_ctx set),
-   0 if not (build_ctx NULL) or -1 if error. */
-int fts_build_init(struct fts_backend *backend, struct mailbox *box,
-		   bool precache,
-		   struct fts_storage_build_context **build_ctx_r);
-/* Returns 0 if ok, -1 if error. */
-int fts_build_deinit(struct fts_storage_build_context **ctx);
-
-/* Build more. Returns 1 if finished, 0 if this function needs to be called
-   again, -1 if error. */
-int fts_build_more(struct fts_storage_build_context *ctx);
-
-#endif
diff --git a/src/plugins/fts/fts-build-indexer.c b/src/plugins/fts/fts-indexer.c
similarity index 57%
rename from src/plugins/fts/fts-build-indexer.c
rename to src/plugins/fts/fts-indexer.c
index 7eb86101b9..7f254dec7b 100644
--- a/src/plugins/fts/fts-build-indexer.c
+++ b/src/plugins/fts/fts-indexer.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
 #include "ioloop.h"
@@ -6,21 +6,30 @@
 #include "istream.h"
 #include "write-full.h"
 #include "strescape.h"
+#include "time-util.h"
 #include "mail-user.h"
 #include "mail-storage-private.h"
-#include "fts-api-private.h"
-#include "fts-build-private.h"
+#include "fts-api.h"
+#include "fts-indexer.h"
+
+#define INDEXER_NOTIFY_INTERVAL_SECS 10
 
 #define INDEXER_SOCKET_NAME "indexer"
 #define INDEXER_WAIT_MSECS 250
 #define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
 
-struct indexer_fts_storage_build_context {
-	struct fts_storage_build_context ctx;
+struct fts_indexer_context {
+	struct mailbox *box;
+
+	struct timeval search_start_time, last_notify;
+	unsigned int percentage;
 
 	char *path;
 	int fd;
 	struct istream *input;
+
+	unsigned int notified:1;
+	unsigned int failed:1;
 };
 
 int fts_indexer_cmd(struct mail_user *user, const char *cmd,
@@ -47,11 +56,41 @@ int fts_indexer_cmd(struct mail_user *user, const char *cmd,
 	return fd;
 }
 
-static int
-fts_build_indexer_init(struct fts_backend *backend, struct mailbox *box,
-		       struct fts_storage_build_context **build_ctx_r)
+static void fts_indexer_notify(struct fts_indexer_context *ctx)
+{
+	unsigned long long elapsed_msecs, est_total_msecs;
+	unsigned int eta_secs;
+
+	if (ioloop_time - ctx->last_notify.tv_sec < INDEXER_NOTIFY_INTERVAL_SECS)
+		return;
+	ctx->last_notify = ioloop_timeval;
+
+	if (ctx->box->storage->callbacks.notify_ok == NULL ||
+	    ctx->percentage == 0)
+		return;
+
+	elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
+					   &ctx->search_start_time);
+	est_total_msecs = elapsed_msecs * 100 / ctx->percentage;
+	eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
+
+	T_BEGIN {
+		const char *text;
+
+		text = t_strdup_printf("Indexed %d%% of the mailbox, "
+				       "ETA %d:%02d", ctx->percentage,
+				       eta_secs/60, eta_secs%60);
+		ctx->box->storage->callbacks.
+			notify_ok(ctx->box, text,
+				  ctx->box->storage->callback_context);
+		ctx->notified = TRUE;
+	} T_END;
+}
+
+int fts_indexer_init(struct fts_backend *backend, struct mailbox *box,
+		     struct fts_indexer_context **ctx_r)
 {
-	struct indexer_fts_storage_build_context *ctx;
+	struct fts_indexer_context *ctx;
 	struct mailbox_status status;
 	uint32_t last_uid, seq1, seq2;
 	const char *path, *cmd;
@@ -80,31 +119,39 @@ fts_build_indexer_init(struct fts_backend *backend, struct mailbox *box,
 		return -1;
 
 	/* connect to indexer and request immediate indexing of the mailbox */
-	ctx = i_new(struct indexer_fts_storage_build_context, 1);
-	ctx->ctx.mail_count = 100;
+	ctx = i_new(struct fts_indexer_context, 1);
+	ctx->box = box;
 	ctx->path = i_strdup(path);
 	ctx->fd = fd;
 	ctx->input = i_stream_create_fd(fd, 128, FALSE);
+	ctx->search_start_time = ioloop_timeval;
 
-	*build_ctx_r = &ctx->ctx;
+	*ctx_r = ctx;
 	return 1;
 }
 
-static int
-fts_build_indexer_deinit(struct fts_storage_build_context *_ctx)
+int fts_indexer_deinit(struct fts_indexer_context **_ctx)
 {
-	struct indexer_fts_storage_build_context *ctx =
-		(struct indexer_fts_storage_build_context *)_ctx;
+	struct fts_indexer_context *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : 0;
+
+	*_ctx = NULL;
 
 	i_stream_destroy(&ctx->input);
 	if (close(ctx->fd) < 0)
 		i_error("close(%s) failed: %m", ctx->path);
+	if (ctx->notified) {
+		/* we notified at least once */
+		ctx->box->storage->callbacks.
+			notify_ok(ctx->box, "Mailbox indexing finished",
+				  ctx->box->storage->callback_context);
+	}
 	i_free(ctx->path);
-	return 0;
+	i_free(ctx);
+	return ret;
 }
 
-static int
-fts_build_indexer_input(struct indexer_fts_storage_build_context *ctx)
+static int fts_indexer_input(struct fts_indexer_context *ctx)
 {
 	const char *line;
 	int percentage;
@@ -126,10 +173,10 @@ fts_build_indexer_input(struct indexer_fts_storage_build_context *ctx)
 		if (percentage < 0) {
 			/* indexing failed */
 			i_error("indexer failed to index mailbox %s",
-				ctx->ctx.box->vname);
+				ctx->box->vname);
 			return -1;
 		}
-		ctx->ctx.mail_idx = percentage;
+		ctx->percentage = percentage;
 		if (percentage == 100) {
 			/* finished */
 			return 1;
@@ -142,16 +189,14 @@ fts_build_indexer_input(struct indexer_fts_storage_build_context *ctx)
 	return 0;
 }
 
-static int fts_build_indexer_more(struct fts_storage_build_context *_ctx)
+static int fts_indexer_more_int(struct fts_indexer_context *ctx)
 {
-	struct indexer_fts_storage_build_context *ctx =
-		(struct indexer_fts_storage_build_context *)_ctx;
 	struct ioloop *ioloop;
 	struct io *io;
 	struct timeout *to;
 	int ret;
 
-	if ((ret = fts_build_indexer_input(ctx)) != 0)
+	if ((ret = fts_indexer_input(ctx)) != 0)
 		return ret;
 
 	/* wait for a while for the reply. FIXME: once search API supports
@@ -164,11 +209,18 @@ static int fts_build_indexer_more(struct fts_storage_build_context *_ctx)
 	timeout_remove(&to);
 	io_loop_destroy(&ioloop);
 
-	return fts_build_indexer_input(ctx);
+	return fts_indexer_input(ctx);
 }
 
-const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs = {
-	fts_build_indexer_init,
-	fts_build_indexer_deinit,
-	fts_build_indexer_more
-};
+int fts_indexer_more(struct fts_indexer_context *ctx)
+{
+	int ret;
+
+	if ((ret = fts_indexer_more_int(ctx)) < 0) {
+		ctx->failed = TRUE;
+		return -1;
+	}
+	if (ret == 0)
+		fts_indexer_notify(ctx);
+	return ret;
+}
diff --git a/src/plugins/fts/fts-indexer.h b/src/plugins/fts/fts-indexer.h
new file mode 100644
index 0000000000..73901c66ca
--- /dev/null
+++ b/src/plugins/fts/fts-indexer.h
@@ -0,0 +1,21 @@
+#ifndef FTS_BUILD_H
+#define FTS_BUILD_H
+
+struct fts_indexer_context;
+
+/* Initialize indexing the given mailbox via indexer service. Returns 1 if
+   indexing started, 0 if there was no need to index or -1 if error. */
+int fts_indexer_init(struct fts_backend *backend, struct mailbox *box,
+		     struct fts_indexer_context **ctx_r);
+/* Returns 0 if ok, -1 if error. */
+int fts_indexer_deinit(struct fts_indexer_context **ctx);
+
+/* Build more. Returns 1 if finished, 0 if this function needs to be called
+   again, -1 if error. */
+int fts_indexer_more(struct fts_indexer_context *ctx);
+
+/* Returns fd, which you can either read from or close. */
+int fts_indexer_cmd(struct mail_user *user, const char *cmd,
+		    const char **path_r);
+
+#endif
diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index f7e40264c5..ff36acab9b 100644
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -2,12 +2,13 @@
 
 #include "lib.h"
 #include "array.h"
-#include "mail-search.h"
+#include "mail-search-build.h"
 #include "mail-storage-private.h"
 #include "mailbox-list-private.h"
 #include "../virtual/virtual-storage.h"
 #include "fts-api-private.h"
-#include "fts-build.h"
+#include "fts-indexer.h"
+#include "fts-build-mail.h"
 #include "fts-search-serialize.h"
 #include "fts-plugin.h"
 #include "fts-storage.h"
@@ -24,6 +25,9 @@
 struct fts_mailbox_list {
 	union mailbox_list_module_context module_ctx;
 	struct fts_backend *backend;
+
+	struct fts_backend_update_context *update_ctx;
+	unsigned int update_ctx_refcount;
 };
 
 struct fts_mailbox {
@@ -35,11 +39,18 @@ struct fts_transaction_context {
 	union mailbox_transaction_module_context module_ctx;
 
 	struct fts_scores *scores;
+	uint32_t next_index_seq;
+	uint32_t highest_virtual_uid;
+
+	unsigned int precached:1;
+	unsigned int failed:1;
 };
 
 struct fts_mail {
 	union mail_module_context module_ctx;
 	char score[30];
+
+	unsigned int virtual_mail:1;
 };
 
 static MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
@@ -48,6 +59,45 @@ static MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
 static MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
 				  &mailbox_list_module_register);
 
+static int fts_mailbox_get_last_cached_seq(struct mailbox *box, uint32_t *seq_r)
+{
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
+	uint32_t seq1, seq2, last_uid;
+
+	if (fts_backend_get_last_uid(flist->backend, box, &last_uid) < 0)
+		return -1;
+
+	if (last_uid == 0)
+		*seq_r = 0;
+	else {
+		mailbox_get_seq_range(box, 1, last_uid, &seq1, &seq2);
+		*seq_r = seq2;
+	}
+	return 0;
+}
+
+static int
+fts_mailbox_get_status(struct mailbox *box, enum mailbox_status_items items,
+		       struct mailbox_status *status_r)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT(box);
+	uint32_t seq;
+
+	if (fbox->module_ctx.super.get_status(box, items, status_r) < 0)
+		return -1;
+
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0) {
+		if (fts_mailbox_get_last_cached_seq(box, &seq) < 0)
+			return -1;
+
+		/* use whichever is smaller */
+		if (status_r->last_cached_seq > seq)
+			status_r->last_cached_seq = seq;
+	}
+	return 0;
+}
+
+
 static void fts_scores_unref(struct fts_scores **_scores)
 {
 	struct fts_scores *scores = *_scores;
@@ -59,17 +109,13 @@ static void fts_scores_unref(struct fts_scores **_scores)
 	}
 }
 
-static bool fts_try_build_init(struct mail_search_context *ctx,
+static void fts_try_build_init(struct mail_search_context *ctx,
 			       struct fts_search_context *fctx)
 {
-	if (fts_backend_is_updating(fctx->backend)) {
-		/* this process is already building the indexes */
-		return FALSE;
-	}
-	fctx->build_initialized = TRUE;
+	i_assert(!fts_backend_is_updating(fctx->backend));
 
-	switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE,
-			       &fctx->build_ctx)) {
+	switch (fts_indexer_init(fctx->backend, ctx->transaction->box,
+				 &fctx->indexer_ctx)) {
 	case -1:
 		break;
 	case 0:
@@ -81,7 +127,6 @@ static bool fts_try_build_init(struct mail_search_context *ctx,
 		ctx->progress_hidden = TRUE;
 		break;
 	}
-	return TRUE;
 }
 
 static bool fts_want_build_args(const struct mail_search_arg *args)
@@ -133,15 +178,14 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t,
 	fctx->args = args;
 	fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
 	fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
+	fctx->virtual_mailbox =
+		strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
 	i_array_init(&fctx->levels, 8);
 	fctx->scores = i_new(struct fts_scores, 1);
 	fctx->scores->refcount = 1;
 	i_array_init(&fctx->scores->score_map, 64);
 	MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
 
-	fctx->virtual_mailbox =
-		strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
-
 	/* transaction contains the last search's scores. they can be
 	   queried later with mail_get_special() */
 	if (ft->scores != NULL)
@@ -150,11 +194,9 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t,
 	ft->scores->refcount++;
 
 	if (fts_want_build_args(args->args))
-		(void)fts_try_build_init(ctx, fctx);
-	else {
-		fctx->build_initialized = TRUE;
+		fts_try_build_init(ctx, fctx);
+	else
 		fts_search_lookup(fctx);
-	}
 	return ctx;
 }
 
@@ -166,20 +208,13 @@ static bool fts_mailbox_build_continue(struct mail_search_context *ctx)
 	if (fctx == NULL)
 		return TRUE;
 
-	if (!fctx->build_initialized) {
-		/* we're still waiting for this process (but another command)
-		   to finish building the indexes */
-		if (!fts_try_build_init(ctx, fctx))
-			return FALSE;
-	}
-
-	if (fctx->build_ctx != NULL) {
+	if (fctx->indexer_ctx != NULL) {
 		/* this command is still building the indexes */
-		ret = fts_build_more(fctx->build_ctx);
+		ret = fts_indexer_more(fctx->indexer_ctx);
 		if (ret == 0)
 			return FALSE;
 		ctx->progress_hidden = FALSE;
-		if (fts_build_deinit(&fctx->build_ctx) < 0)
+		if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
 			ret = -1;
 		if (ret > 0)
 			fts_search_lookup(fctx);
@@ -258,12 +293,13 @@ static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
 static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
 {
 	struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
+	struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
 	struct fts_search_context *fctx = FTS_CONTEXT(ctx);
 
 	if (fctx != NULL) {
-		if (fctx->build_ctx != NULL) {
-			/* the search was cancelled */
-			(void)fts_build_deinit(&fctx->build_ctx);
+		if (fctx->indexer_ctx != NULL) {
+			if (fts_indexer_deinit(&fctx->indexer_ctx) < 0)
+				ft->failed = TRUE;
 		}
 
 		buffer_free(&fctx->orig_matches);
@@ -306,6 +342,102 @@ static int fts_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
 	return fmail->module_ctx.super.get_special(_mail, field, value_r);
 }
 
+static int
+fts_mail_precache_range(struct mailbox_transaction_context *trans,
+			struct fts_backend_update_context *update_ctx,
+			uint32_t seq1, uint32_t seq2)
+{
+	struct mail_search_args *search_args;
+	struct mail_search_context *ctx;
+	struct mail *mail;
+	int ret = 0;
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq1, seq2);
+	ctx = mailbox_search_init(trans, search_args, NULL,
+				  MAIL_FETCH_STREAM_HEADER |
+				  MAIL_FETCH_STREAM_BODY, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(ctx, &mail)) {
+		if (fts_build_mail(update_ctx, mail) < 0) {
+			ret = -1;
+			break;
+		}
+		mail_precache(mail);
+	}
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = -1;
+	return ret;
+}
+
+static int fts_mail_precache_init(struct mail *_mail)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(_mail->box->list);
+	uint32_t last_seq;
+
+	if (fts_mailbox_get_last_cached_seq(_mail->box, &last_seq) < 0)
+		return -1;
+
+	ft->precached = TRUE;
+	ft->next_index_seq = last_seq + 1;
+	if (flist->update_ctx == NULL)
+		flist->update_ctx = fts_backend_update_init(flist->backend);
+	flist->update_ctx_refcount++;
+	return 0;
+}
+
+static void fts_mail_index(struct mail *_mail)
+{
+	struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(_mail->box->list);
+
+	if (ft->failed)
+		return;
+
+	if (!ft->precached) {
+		if (fts_mail_precache_init(_mail) < 0) {
+			ft->failed = TRUE;
+			return;
+		}
+	}
+	if (ft->next_index_seq < _mail->seq) {
+		/* most likely a virtual mailbox. we'll first need to
+		   index all mails up to the current one. */
+		fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
+		if (fts_mail_precache_range(_mail->transaction,
+					    flist->update_ctx,
+					    ft->next_index_seq,
+					    _mail->seq-1) < 0) {
+			ft->failed = TRUE;
+			return;
+		}
+	}
+
+	if (ft->next_index_seq == _mail->seq) {
+		fts_backend_update_set_mailbox(flist->update_ctx, _mail->box);
+		if (fts_build_mail(flist->update_ctx, _mail) < 0)
+			ft->failed = TRUE;
+		ft->next_index_seq = _mail->seq + 1;
+	}
+}
+
+static void fts_mail_precache(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
+	struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
+
+	fmail->module_ctx.super.precache(_mail);
+	if (fmail->virtual_mail) {
+		if (ft->highest_virtual_uid < _mail->uid)
+			ft->highest_virtual_uid = _mail->uid;
+	} else T_BEGIN {
+		fts_mail_index(_mail);
+	} T_END;
+}
+
 void fts_mail_allocated(struct mail *_mail)
 {
 	struct mail_private *mail = (struct mail_private *)_mail;
@@ -319,8 +451,11 @@ void fts_mail_allocated(struct mail *_mail)
 	fmail = p_new(mail->pool, struct fts_mail, 1);
 	fmail->module_ctx.super = *v;
 	mail->vlast = &fmail->module_ctx.super;
+	fmail->virtual_mail =
+		strcmp(_mail->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
 
 	v->get_special = fts_mail_get_special;
+	v->precache = fts_mail_precache;
 	MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
 }
 
@@ -339,14 +474,33 @@ fts_transaction_begin(struct mailbox *box,
 	return t;
 }
 
-static void fts_transaction_rollback(struct mailbox_transaction_context *t)
+static int fts_transaction_end(struct mailbox_transaction_context *t)
 {
-	struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
 	struct fts_transaction_context *ft = FTS_CONTEXT(t);
+	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
+	int ret = ft->failed ? -1 : 0;
 
+	if (ft->precached) {
+		i_assert(flist->update_ctx_refcount > 0);
+		if (--flist->update_ctx_refcount == 0) {
+			if (fts_backend_update_deinit(&flist->update_ctx) < 0)
+				ret = -1;
+		}
+	} else if (ft->highest_virtual_uid > 0) {
+		if (fts_index_set_last_uid(t->box, ft->highest_virtual_uid) < 0)
+			ret = -1;
+	}
 	if (ft->scores != NULL)
 		fts_scores_unref(&ft->scores);
 	i_free(ft);
+	return ret;
+}
+
+static void fts_transaction_rollback(struct mailbox_transaction_context *t)
+{
+	struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
+
+	(void)fts_transaction_end(t);
 	fbox->module_ctx.super.transaction_rollback(t);
 }
 
@@ -355,12 +509,12 @@ fts_transaction_commit(struct mailbox_transaction_context *t,
 		       struct mail_transaction_commit_changes *changes_r)
 {
 	struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
-	struct fts_transaction_context *ft = FTS_CONTEXT(t);
+	int ret;
 
-	if (ft->scores != NULL)
-		fts_scores_unref(&ft->scores);
-	i_free(ft);
-	return fbox->module_ctx.super.transaction_commit(t, changes_r);
+	ret = fts_transaction_end(t);
+	if (fbox->module_ctx.super.transaction_commit(t, changes_r) < 0)
+		ret = -1;
+	return ret;
 }
 
 static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
@@ -393,39 +547,15 @@ static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 	fts_backend_update_expunge(fbox->sync_update_ctx, uid);
 }
 
-static int fts_update(struct mailbox *box)
-{
-	struct fts_storage_build_context *build_ctx;
-	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
-	int ret = 0;
-
-	if ((ret = fts_build_init(flist->backend, box,
-				  TRUE, &build_ctx)) <= 0) {
-		if (box->storage->set->mail_debug)
-			i_debug("%s: FTS index is up to date", box->vname);
-		return ret;
-	}
-
-	if (box->storage->set->mail_debug)
-		i_debug("%s: Updating FTS index", box->vname);
-
-	while ((ret = fts_build_more(build_ctx)) == 0) ;
-
-	if (fts_build_deinit(&build_ctx) < 0)
-		ret = -1;
-	return ret < 0 ? -1 : 0;
-}
-
 static int fts_sync_deinit(struct mailbox_sync_context *ctx,
 			   struct mailbox_sync_status *status_r)
 {
 	struct mailbox *box = ctx->box;
 	struct fts_mailbox *fbox = FTS_CONTEXT(box);
 	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
-	bool precache, optimize;
+	bool optimize;
 	int ret = 0;
 
-	precache = (ctx->flags & MAILBOX_SYNC_FLAG_PRECACHE) != 0;
 	optimize = (ctx->flags & (MAILBOX_SYNC_FLAG_FORCE_RESYNC |
 				  MAILBOX_SYNC_FLAG_OPTIMIZE)) != 0;
 	if (fbox->module_ctx.super.sync_deinit(ctx, status_r) < 0)
@@ -441,14 +571,6 @@ static int fts_sync_deinit(struct mailbox_sync_context *ctx,
 			ret = -1;
 		}
 	}
-	if (precache) {
-		if (fts_update(box) < 0) {
-			mail_storage_set_critical(box->storage,
-				"FTS index update for mailbox %s failed",
-				box->vname);
-			ret = -1;
-		}
-	}
 	flist->backend->syncing = FALSE;
 	return ret;
 }
@@ -466,6 +588,7 @@ void fts_mailbox_allocated(struct mailbox *box)
 	fbox->module_ctx.super = *v;
 	box->vlast = &fbox->module_ctx.super;
 
+	v->get_status = fts_mailbox_get_status;
 	v->search_init = fts_mailbox_search_init;
 	v->search_next_nonblock = fts_mailbox_search_next_nonblock;
 	v->search_next_update_seq = fts_mailbox_search_next_update_seq;
diff --git a/src/plugins/fts/fts-storage.h b/src/plugins/fts/fts-storage.h
index 98f8cf365f..a2e03c8926 100644
--- a/src/plugins/fts/fts-storage.h
+++ b/src/plugins/fts/fts-storage.h
@@ -32,10 +32,9 @@ struct fts_search_context {
 	/* final scores, combined from all levels */
 	struct fts_scores *scores;
 
-	struct fts_storage_build_context *build_ctx;
+	struct fts_indexer_context *indexer_ctx;
 
 	unsigned int virtual_mailbox:1;
-	unsigned int build_initialized:1;
 	unsigned int fts_lookup_success:1;
 };
 
diff --git a/src/plugins/virtual/virtual-mail.c b/src/plugins/virtual/virtual-mail.c
index 151e088171..bf5cd753f8 100644
--- a/src/plugins/virtual/virtual-mail.c
+++ b/src/plugins/virtual/virtual-mail.c
@@ -171,6 +171,14 @@ static bool virtual_mail_prefetch(struct mail *mail)
 	return p->v.prefetch(vmail->backend_mail);
 }
 
+static void virtual_mail_precache(struct mail *mail)
+{
+	struct virtual_mail *vmail = (struct virtual_mail *)mail;
+	struct mail_private *p = (struct mail_private *)vmail->backend_mail;
+
+	p->v.precache(vmail->backend_mail);
+}
+
 static int virtual_mail_handle_lost(struct virtual_mail *vmail)
 {
 	if (!vmail->lost)
@@ -378,15 +386,6 @@ static void virtual_mail_expunge(struct mail *mail)
 	mail_expunge(vmail->backend_mail);
 }
 
-static void virtual_mail_parse(struct mail *mail, bool parse_body)
-{
-	struct virtual_mail *vmail = (struct virtual_mail *)mail;
-
-	if (virtual_mail_handle_lost(vmail) < 0)
-		return;
-	mail_parse(vmail->backend_mail, parse_body);
-}
-
 static void
 virtual_mail_set_cache_corrupted(struct mail *mail, enum mail_fetch_field field)
 {
@@ -404,6 +403,7 @@ struct mail_vfuncs virtual_mail_vfuncs = {
 	virtual_mail_set_uid,
 	virtual_mail_set_uid_cache_updates,
 	virtual_mail_prefetch,
+	virtual_mail_precache,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
@@ -426,7 +426,6 @@ struct mail_vfuncs virtual_mail_vfuncs = {
 	index_mail_update_modseq,
 	virtual_mail_update_pop3_uidl,
 	virtual_mail_expunge,
-	virtual_mail_parse,
 	virtual_mail_set_cache_corrupted,
 	NULL
 };
diff --git a/src/plugins/virtual/virtual-storage.c b/src/plugins/virtual/virtual-storage.c
index d060bc1c93..880cea0aa9 100644
--- a/src/plugins/virtual/virtual-storage.c
+++ b/src/plugins/virtual/virtual-storage.c
@@ -335,6 +335,29 @@ virtual_mailbox_update(struct mailbox *box,
 	return -1;
 }
 
+static int
+virtual_storage_get_status(struct mailbox *box,
+			   enum mailbox_status_items items,
+			   struct mailbox_status *status_r)
+{
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0)
+		items |= STATUS_MESSAGES;
+
+	if (index_storage_get_status(box, items, status_r) < 0)
+		return -1;
+
+	if ((items & STATUS_LAST_CACHED_SEQ) != 0) {
+		/* Virtual mailboxes have no cached data of their own, so the
+		   current value is always 0. The most important use for this
+		   functionality is for "doveadm index" to do FTS indexing and
+		   it doesn't really matter there if we set this value
+		   correctly or not. So for now just assume that everything is
+		   indexed. */
+		status_r->last_cached_seq = status_r->messages;
+	}
+	return 0;
+}
+
 static int
 virtual_mailbox_get_metadata(struct mailbox *box,
 			     enum mailbox_metadata_items items,
@@ -501,7 +524,7 @@ struct mailbox virtual_mailbox = {
 		virtual_mailbox_update,
 		index_storage_mailbox_delete,
 		index_storage_mailbox_rename,
-		index_storage_get_status,
+		virtual_storage_get_status,
 		virtual_mailbox_get_metadata,
 		NULL,
 		NULL,