From: Timo Sirainen <tss@iki.fi>
Date: Sun, 9 May 2004 23:06:27 +0000 (+0300)
Subject: syncing fixes
X-Git-Tag: 1.1.alpha1~4122
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d67f54632110cfb6aafe2d7cd1f99b031c0b208a;p=thirdparty%2Fdovecot%2Fcore.git

syncing fixes

--HG--
branch : HEAD
---

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index 3fb7c3f580..70518ee2a7 100644
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -112,11 +112,6 @@ int mail_index_fix_header(struct mail_index *index, struct mail_index_map *map,
 void mail_index_view_transaction_ref(struct mail_index_view *view);
 void mail_index_view_transaction_unref(struct mail_index_view *view);
 
-int mail_index_sync_get_rec(struct mail_index_view *view,
-			    struct mail_index_sync_rec *rec,
-			    const struct mail_transaction_header *hdr,
-			    const void *data, size_t *data_offset);
-
 void mail_index_set_inconsistent(struct mail_index *index);
 
 int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h
index 3790896a2f..1cf82fdcc3 100644
--- a/src/lib-index/mail-index-sync-private.h
+++ b/src/lib-index/mail-index-sync-private.h
@@ -31,4 +31,11 @@ void mail_index_header_update_counts(struct mail_index_header *hdr,
 void mail_index_header_update_lowwaters(struct mail_index_header *hdr,
 					const struct mail_index_record *rec);
 
+void
+mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
+			    const struct mail_transaction_expunge *exp);
+void
+mail_index_sync_get_update(struct mail_index_sync_rec *rec,
+			   const struct mail_transaction_flag_update *update);
+
 #endif
diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index 4e5d41a944..3fdbbb37bf 100644
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -198,7 +198,7 @@ int mail_index_sync_begin(struct mail_index *index,
 	return 1;
 }
 
-static void
+void
 mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
 			    const struct mail_transaction_expunge *exp)
 {
@@ -207,7 +207,7 @@ mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
 	rec->seq2 = exp->seq2;
 }
 
-static void
+void
 mail_index_sync_get_update(struct mail_index_sync_rec *rec,
 			   const struct mail_transaction_flag_update *update)
 {
@@ -252,45 +252,6 @@ static int mail_index_sync_rec_check(struct mail_index_view *view,
 	return TRUE;
 }
 
-int mail_index_sync_get_rec(struct mail_index_view *view,
-			    struct mail_index_sync_rec *rec,
-			    const struct mail_transaction_header *hdr,
-			    const void *data, size_t *data_offset)
-{
-	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
-	case MAIL_TRANSACTION_APPEND: {
-		rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
-		rec->seq1 = view->index->map->records_count + 1;
-		rec->seq2 = rec->seq1 + hdr->size /
-			sizeof(struct mail_index_record) - 1;
-		rec->appends = NULL;
-
-		*data_offset += hdr->size;
-		break;
-	}
-	case MAIL_TRANSACTION_EXPUNGE: {
-		const struct mail_transaction_expunge *exp =
-			CONST_PTR_OFFSET(data, *data_offset);
-
-		*data_offset += sizeof(*exp);
-                mail_index_sync_get_expunge(rec, exp);
-		break;
-	}
-	case MAIL_TRANSACTION_FLAG_UPDATE: {
-		const struct mail_transaction_flag_update *update =
-			CONST_PTR_OFFSET(data, *data_offset);
-
-		*data_offset += sizeof(*update);
-                mail_index_sync_get_update(rec, update);
-		break;
-	}
-	default:
-		i_unreached();
-	}
-
-	return mail_index_sync_rec_check(view, rec);
-}
-
 int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
 			 struct mail_index_sync_rec *sync_rec)
 {
diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
index 542ea202e8..9f02f8dde2 100644
--- a/src/lib-index/mail-index-view-sync.c
+++ b/src/lib-index/mail-index-view-sync.c
@@ -12,6 +12,7 @@ struct mail_index_view_sync_ctx {
 	enum mail_index_sync_type sync_mask;
 	struct mail_index_map *sync_map;
 	buffer_t *expunges;
+	uint32_t messages_count;
 
 	const struct mail_transaction_header *hdr;
 	const void *data;
@@ -104,6 +105,7 @@ int mail_index_view_sync_begin(struct mail_index_view *view,
 	ctx->sync_mask = sync_mask;
 	ctx->sync_map = map;
 	ctx->expunges = expunges;
+	ctx->messages_count = mail_index_view_get_message_count(view);
 
 	*ctx_r = ctx;
 	return 0;
@@ -246,6 +248,46 @@ static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx,
 	return 1;
 }
 
+static void
+mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
+			     struct mail_index_sync_rec *rec)
+{
+	const struct mail_transaction_header *hdr = ctx->hdr;
+	const void *data = ctx->data;
+
+	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+	case MAIL_TRANSACTION_APPEND: {
+		rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
+		rec->seq1 = ctx->messages_count + 1;
+		ctx->messages_count +=
+			hdr->size / sizeof(struct mail_index_record);
+		rec->seq2 = ctx->messages_count;
+		rec->appends = NULL;
+
+		ctx->data_offset += hdr->size;
+		break;
+	}
+	case MAIL_TRANSACTION_EXPUNGE: {
+		const struct mail_transaction_expunge *exp =
+			CONST_PTR_OFFSET(data, ctx->data_offset);
+
+		ctx->data_offset += sizeof(*exp);
+                mail_index_sync_get_expunge(rec, exp);
+		break;
+	}
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		const struct mail_transaction_flag_update *update =
+			CONST_PTR_OFFSET(data, ctx->data_offset);
+
+		ctx->data_offset += sizeof(*update);
+                mail_index_sync_get_update(rec, update);
+		break;
+	}
+	default:
+		i_unreached();
+	}
+}
+
 int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
 			      struct mail_index_sync_rec *sync_rec)
 {
@@ -278,9 +320,7 @@ int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
 		}
 	}
 
-	if (!mail_index_sync_get_rec(view, sync_rec, ctx->hdr, ctx->data,
-				     &ctx->data_offset))
-		return -1;
+	mail_index_view_sync_get_rec(ctx, sync_rec);
 	return 1;
 }
 
diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index 036fc2c700..ac4a8a4f42 100644
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -277,8 +277,10 @@ int mail_index_map(struct mail_index *index, int force)
 		hdr = map->mmap_base;
                 used_size = hdr->header_size +
 			hdr->messages_count * sizeof(struct mail_index_record);
-		if (map->mmap_size >= used_size && !force)
+		if (map->mmap_size >= used_size && !force) {
+			map->records_count = hdr->messages_count;
 			return 1;
+		}
 
 		if (munmap(map->mmap_base, map->mmap_size) < 0)
 			mail_index_set_syscall_error(index, "munmap()");
diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
index 39d8bbafde..259f6c3eda 100644
--- a/src/lib-storage/index/index-sync.c
+++ b/src/lib-storage/index/index-sync.c
@@ -15,7 +15,7 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 	size_t i, expunges_count;
 	void *sc_context;
 	enum mail_index_sync_type sync_mask;
-	uint32_t seq, new_count, recent_count;
+	uint32_t seq, messages_count, recent_count;
 	int ret, appends;
 
 	sync_mask = MAIL_INDEX_SYNC_MASK_ALL;
@@ -39,6 +39,8 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 	sc_context = ibox->storage->callback_context;
 	appends = FALSE;
 
+	messages_count = mail_index_view_get_message_count(ibox->view);
+
 	memset(&full_flags, 0, sizeof(full_flags));
 	while ((ret = mail_index_view_sync_next(ctx, &sync)) > 0) {
 		switch (sync.type) {
@@ -52,6 +54,9 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 			if (sc->update_flags == NULL)
 				break;
 
+			if (sync.seq2 > messages_count)
+				sync.seq2 = messages_count;
+
 			/* FIXME: hide the flag updates for expunged messages */
 			for (seq = sync.seq1; seq <= sync.seq2; seq++) {
 				if (mail_index_lookup(ibox->view,
@@ -72,7 +77,10 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 
 	if (sc->expunge != NULL) {
 		for (i = expunges_count*2; i > 0; i -= 2) {
-			for (seq = expunges[i-1]; seq >= expunges[i-2]; seq--)
+			seq = expunges[i-1];
+			if (seq > messages_count)
+				seq = messages_count;
+			for (; seq >= expunges[i-2]; seq--)
 				sc->expunge(&ibox->box, seq, sc_context);
 		}
 	}
@@ -80,9 +88,9 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 	mail_index_view_sync_end(ctx);
 
 	if (appends) {
-		new_count = mail_index_view_get_message_count(ibox->view);
+		messages_count = mail_index_view_get_message_count(ibox->view);
 		recent_count = ibox->get_recent_count(ibox);
-		sc->new_messages(&ibox->box, new_count, recent_count,
+		sc->new_messages(&ibox->box, messages_count, recent_count,
 				 sc_context);
 	}
 
diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c
index c774586152..fb97254c38 100644
--- a/src/lib-storage/index/maildir/maildir-sync.c
+++ b/src/lib-storage/index/maildir/maildir-sync.c
@@ -392,8 +392,13 @@ static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
 
 		ret = maildir_uidlist_sync_next_pre(ctx->uidlist_sync_ctx,
 						    dp->d_name);
-		if (ret == 0)
+		if (ret == 0) {
+			if (new_dir)
+				ctx->ibox->last_new_mtime = 0;
+			else
+				ctx->ibox->dirty_cur_time = ioloop_time;
 			continue;
+		}
 		if (ret < 0)
 			break;
 
@@ -406,7 +411,7 @@ static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir)
 			if (strchr(dp->d_name, ':') == NULL)
 				str_append(dest, ":2,");
 			if (rename(str_c(src), str_c(dest)) == 0) {
-				/* we moved it - it's \Recent for use */
+				/* we moved it - it's \Recent for us */
                                 ctx->ibox->dirty_cur_time = ioloop_time;
 				flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
 					MAILDIR_UIDLIST_REC_FLAG_RECENT;
@@ -495,9 +500,11 @@ static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
 		/* cur/ changed, or delayed cur/ check */
 		*cur_changed_r = TRUE;
 		ibox->last_cur_mtime = cur_mtime;
+
+		ibox->dirty_cur_time =
+			cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
+			cur_mtime : 0;
 	}
-	ibox->dirty_cur_time =
-		cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ? cur_mtime : 0;
 
 	return 0;
 }
@@ -547,6 +554,17 @@ static int maildir_sync_index(struct maildir_sync_context *ctx)
 		}
 
 		if (seq > hdr->messages_count) {
+			if (uid < hdr->next_uid) {
+				/* message not in index, but next_uid header
+				   is updated? shouldn't really happen.. */
+				mail_storage_set_critical(ibox->box.storage,
+					"Maildir sync: UID < next_uid "
+					"(%u < %u)", uid, hdr->next_uid);
+				mail_index_mark_corrupted(ibox->index);
+				ret = -1;
+				break;
+			}
+
 			mail_index_append(trans, uid, &seq);
 			mail_index_update_flags(trans, seq, MODIFY_REPLACE,
 						flags, keywords);
@@ -570,7 +588,8 @@ static int maildir_sync_index(struct maildir_sync_context *ctx)
 			   shouldn't happen */
 			mail_storage_set_critical(ibox->box.storage,
 				"Maildir sync: UID inserted in the middle "
-				"of mailbox (%u > %u)", rec->uid, uid);
+				"of mailbox (%u > %u, file = %s)",
+				rec->uid, uid, filename);
 			mail_index_mark_corrupted(ibox->index);
 			ret = -1;
 			break;
@@ -586,6 +605,8 @@ static int maildir_sync_index(struct maildir_sync_context *ctx)
 		if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
 		    memcmp(keywords, rec->keywords,
 			   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
+			/* FIXME: this is wrong if there's syncs later.
+			   it gets fixed in next sync however.. */
 			mail_index_update_flags(trans, seq, MODIFY_REPLACE,
 						flags, keywords);
 		}
@@ -656,11 +677,14 @@ static int maildir_sync_context(struct maildir_sync_context *ctx)
 			return -1;
 	}
 
+	/* finish uidlist syncing, but keep it still locked */
+        maildir_uidlist_sync_finish(ctx->uidlist_sync_ctx);
+	if (maildir_sync_index(ctx) < 0)
+		return -1;
+
 	ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
         ctx->uidlist_sync_ctx = NULL;
 
-	if (ret == 0)
-		ret = maildir_sync_index(ctx);
 	return ret;
 }
 
diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index deb4ff09b8..f882153470 100644
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -59,6 +59,7 @@ struct maildir_uidlist_sync_ctx {
 	unsigned int partial:1;
 	unsigned int synced:1;
 	unsigned int locked:1;
+	unsigned int finished:1;
 	unsigned int failed:1;
 };
 
@@ -176,6 +177,8 @@ static int maildir_uidlist_next(struct maildir_uidlist *uidlist,
 			uidlist->fname, uid, uidlist->last_read_uid);
 		return 0;
 	}
+	uidlist->last_read_uid = uid;
+
 	if (uid >= uidlist->next_uid) {
                 mail_storage_set_critical(uidlist->ibox->box.storage,
 			"UID larger than next_uid in file %s (%u >= %u)",
@@ -636,6 +639,8 @@ int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
 	struct maildir_uidlist *uidlist = ctx->uidlist;
 	struct maildir_uidlist_rec *rec, *old_rec;
 
+	i_assert(!ctx->locked);
+
 	if (ctx->failed)
 		return -1;
 
@@ -761,10 +766,8 @@ static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx)
 	}
 }
 
-int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx)
+void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx)
 {
-	int ret = ctx->failed ? -1 : 0;
-
 	if (!ctx->partial) {
 		if (!ctx->failed && !ctx->locked)
 			maildir_uidlist_swap(ctx);
@@ -775,6 +778,15 @@ int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx)
 		}
 		maildir_uidlist_mark_all(ctx->uidlist, FALSE);
 	}
+	ctx->finished = TRUE;
+}
+
+int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx)
+{
+	int ret = ctx->failed ? -1 : 0;
+
+	if (!ctx->finished)
+		maildir_uidlist_sync_finish(ctx);
 
 	if (ctx->new_files_count != 0 && !ctx->failed && !ctx->locked)
 		ret = maildir_uidlist_rewrite(ctx->uidlist);
diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h
index b7ba6b2cab..4a3a7f174e 100644
--- a/src/lib-storage/index/maildir/maildir-uidlist.h
+++ b/src/lib-storage/index/maildir/maildir-uidlist.h
@@ -37,6 +37,7 @@ int maildir_uidlist_sync_next_pre(struct maildir_uidlist_sync_ctx *ctx,
 int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
 			      const char *filename,
 			      enum maildir_uidlist_rec_flag flags);
+void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx);
 int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx);
 
 /* List all maildir files. */