From: Timo Sirainen <tss@iki.fi>
Date: Sun, 23 May 2004 01:58:32 +0000 (+0300)
Subject: mbox syncing fixes
X-Git-Tag: 1.1.alpha1~4065
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3cfff0ca01961d885bdbd6ef08d761880116af07;p=thirdparty%2Fdovecot%2Fcore.git

mbox syncing fixes

--HG--
branch : HEAD
---

diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
index dcb1aa73ea..4accf4ac17 100644
--- a/src/lib-storage/index/index-sync.c
+++ b/src/lib-storage/index/index-sync.c
@@ -15,8 +15,9 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 	size_t i, expunges_count;
 	void *sc_context;
 	enum mail_index_sync_type sync_mask;
-	uint32_t seq, seq1, seq2, messages_count, recent_count;
-	int ret, appends;
+	uint32_t seq, seq1, seq2;
+	uint32_t messages_count, new_messages_count, recent_count;
+	int ret;
 
 	sync_mask = MAIL_INDEX_SYNC_MASK_ALL;
 	if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0)
@@ -37,15 +38,11 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 
 	sc = ibox->storage->callbacks;
 	sc_context = ibox->storage->callback_context;
-	appends = FALSE;
-
-	messages_count = mail_index_view_get_message_count(ibox->view);
 
 	memset(&full_flags, 0, sizeof(full_flags));
 	while ((ret = mail_index_view_sync_next(ctx, &sync)) > 0) {
 		switch (sync.type) {
 		case MAIL_INDEX_SYNC_TYPE_APPEND:
-			appends = TRUE;
 			break;
 		case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
 			/* later */
@@ -82,20 +79,25 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)
 	if (ret < 0)
 		mail_storage_set_index_error(ibox);
 
+	messages_count = new_messages_count =
+		mail_index_view_get_message_count(ibox->view);
+
 	if (sc->expunge != NULL) {
 		for (i = expunges_count*2; i > 0; i -= 2) {
 			seq = expunges[i-1];
 			if (seq > messages_count)
 				seq = messages_count;
-			for (; seq >= expunges[i-2]; seq--)
+			for (; seq >= expunges[i-2]; seq--) {
 				sc->expunge(&ibox->box, seq, sc_context);
+				new_messages_count--;
+			}
 		}
 	}
 
 	mail_index_view_sync_end(ctx);
 
-	if (appends) {
-		messages_count = mail_index_view_get_message_count(ibox->view);
+	messages_count = mail_index_view_get_message_count(ibox->view);
+	if (messages_count != new_messages_count) {
 		recent_count = ibox->get_recent_count(ibox);
 		sc->new_messages(&ibox->box, messages_count, recent_count,
 				 sc_context);
diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c
index 941e2a1933..0eebf230c1 100644
--- a/src/lib-storage/index/mbox/istream-raw-mbox.c
+++ b/src/lib-storage/index/mbox/istream-raw-mbox.c
@@ -105,7 +105,7 @@ static ssize_t _read(struct _istream *stream)
 	const char *fromp;
 	char *sender, eoh_char;
 	time_t received_time;
-	size_t i, pos;
+	size_t i, pos, new_pos;
 	ssize_t ret;
 
 	i_stream_seek(rstream->input, stream->istream.v_offset);
@@ -137,6 +137,8 @@ static ssize_t _read(struct _istream *stream)
 		return _read(stream);
 	}
 
+	i = 0;
+
 	if (pos >= 31) {
 		if (memcmp(buf, "\nFrom ", 6) == 0 &&
 		    mbox_from_parse(buf+6, pos-6,
@@ -150,6 +152,9 @@ static ssize_t _read(struct _istream *stream)
 			i_assert(stream->pos == 0);
 			return -1;
 		}
+
+		/* we don't want to get stuck at invalid From-line */
+		i += 6;
 	} else if (ret == -1) {
 		/* last few bytes, can't contain From-line */
 		if (buf[pos-1] == '\n') {
@@ -172,7 +177,7 @@ static ssize_t _read(struct _istream *stream)
 	/* See if we have From-line here - note that it works right only
 	   because all characters are different in mbox_from. */
 	eoh_char = rstream->body_offset == (uoff_t)-1 ? '\n' : '\0';
-	for (i = 0, fromp = mbox_from; i < pos; i++) {
+	for (fromp = mbox_from; i < pos; i++) {
 		if (buf[i] == eoh_char && i > 0 && buf[i-1] == '\n') {
 			rstream->body_offset = stream->istream.v_offset + i + 1;
 			eoh_char = '\0';
@@ -189,12 +194,18 @@ static ssize_t _read(struct _istream *stream)
 				fromp++;
 		}
 	}
-	pos = i - (fromp - mbox_from);
+	new_pos = i - (fromp - mbox_from);
 
-	ret = pos <= stream->pos ? -1 :
+	ret = new_pos <= stream->pos ? -1 :
 		(ssize_t) (pos - stream->pos);
 	stream->buffer = buf;
-	stream->pos = pos;
+	stream->pos = new_pos;
+
+	if (i < pos && new_pos == stream->pos) {
+		/* beginning from From-line, try again */
+		ret = 0;
+	}
+
 	return ret;
 }
 
diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c
index 90c620ccc1..0a8299806d 100644
--- a/src/lib-storage/index/mbox/mbox-sync-parse.c
+++ b/src/lib-storage/index/mbox/mbox-sync-parse.c
@@ -130,8 +130,7 @@ static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
 	}
 
 	if (space > ctx->mail.space) {
-		ctx->mail.offset = ctx->hdr_offset +
-			hdr->full_value_offset + i;
+		ctx->mail.offset = hdr->full_value_offset + i;
 		ctx->mail.space = space;
 	}
 
@@ -179,8 +178,7 @@ static int parse_x_uid(struct mbox_sync_mail_context *ctx,
 	if (extra_space != 0 && ctx->mail.space == 0) {
 		/* set it only if X-Keywords hasn't been seen. spaces in X-UID
 		   should be removed when writing X-Keywords. */
-		ctx->mail.offset = ctx->hdr_offset +
-			hdr->full_value_offset + space_pos;
+		ctx->mail.offset = hdr->full_value_offset + space_pos;
 		ctx->mail.space = extra_space;
 	}
 	return TRUE;
@@ -236,7 +234,8 @@ static struct header_func *header_func_find(const char *header)
 }
 
 void mbox_sync_parse_next_mail(struct istream *input,
-			       struct mbox_sync_mail_context *ctx)
+			       struct mbox_sync_mail_context *ctx,
+			       int rewriting)
 {
 	struct message_header_parser_ctx *hdr_ctx;
 	struct message_header_line *hdr;
@@ -283,10 +282,12 @@ void mbox_sync_parse_next_mail(struct istream *input,
 				}
 				continue;
 			}
+			buffer_append(ctx->header, hdr->full_value,
+				      hdr->full_value_len);
+		} else {
+			buffer_append(ctx->header, hdr->value,
+				      hdr->value_len);
 		}
-
-		buffer_append(ctx->header, hdr->full_value,
-			      hdr->full_value_len);
 		if (!hdr->no_newline)
 			str_append_c(ctx->header, '\n');
 	}
@@ -296,10 +297,11 @@ void mbox_sync_parse_next_mail(struct istream *input,
 		/* missing X-IMAPbase */
 		ctx->need_rewrite = TRUE;
 	}
-	if (ctx->mail.uid == 0) {
+	if (ctx->mail.uid == 0 && !rewriting) {
 		/* missing X-UID */
 		ctx->need_rewrite = TRUE;
 		ctx->mail.uid = ctx->sync_ctx->next_uid++;
+		ctx->sync_ctx->prev_msg_uid = ctx->mail.uid;
 	}
 
 	ctx->body_offset = input->v_offset;
diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h
index c6cce72093..180b7cc3c0 100644
--- a/src/lib-storage/index/mbox/mbox-sync-private.h
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h
@@ -66,7 +66,8 @@ struct mbox_sync_context {
 
 int mbox_sync(struct index_mailbox *ibox, int last_commit);
 void mbox_sync_parse_next_mail(struct istream *input,
-			       struct mbox_sync_mail_context *ctx);
+			       struct mbox_sync_mail_context *ctx,
+			       int rewriting);
 void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
 			     buffer_t *syncs_buf);
 void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
index 3299aca62a..cf170e6714 100644
--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c
+++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
@@ -204,7 +204,7 @@ static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
 	mail_ctx.mail.offset = mails[idx].offset;
 	mail_ctx.mail.body_size = mails[idx].body_size;
 
-	mbox_sync_parse_next_mail(sync_ctx->file_input, &mail_ctx);
+	mbox_sync_parse_next_mail(sync_ctx->file_input, &mail_ctx, TRUE);
 	mbox_sync_update_header_from(&mail_ctx, &mails[idx]);
 
 	i_assert(mail_ctx.mail.space == mails[idx].space);
@@ -249,7 +249,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
 {
 	struct mbox_sync_mail *mails;
 	size_t size;
-	uoff_t offset, end_offset;
+	uoff_t offset, end_offset, dest_offset;
 	uint32_t idx, extra_per_mail;
 	int ret = 0;
 
@@ -264,7 +264,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
 	extra_per_mail = (extra_space / (last_seq - first_seq + 1));
 
 	mails[last_seq-1].space -= extra_per_mail;
-	i_assert(mails[last_seq-1].space > 0);
+	i_assert(mails[last_seq-1].space >= 0);
 	end_offset = mails[last_seq-1].offset + mails[last_seq-1].space;
 
 	/* start moving backwards */
@@ -288,8 +288,9 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
 			   move data forward so mails before us gets the extra
 			   space (ie. we temporarily get more space to us) */
 			offset = mails[idx].offset + mails[idx].space;
-			if (mbox_move(sync_ctx, offset + mails[idx+1].space,
-				      offset, end_offset - offset)) {
+			dest_offset = offset + mails[idx+1].space;
+			if (mbox_move(sync_ctx, dest_offset, offset,
+				      end_offset - dest_offset)) {
 				// FIXME: error handling
 				ret = -1;
 				break;
diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c
index 5b56e20201..a697fdff07 100644
--- a/src/lib-storage/index/mbox/mbox-sync.c
+++ b/src/lib-storage/index/mbox/mbox-sync.c
@@ -116,7 +116,7 @@ mbox_sync_next_mail(struct mbox_sync_context *sync_ctx,
 	mail_ctx->mail.offset =
 		istream_raw_mbox_get_header_offset(sync_ctx->input);
 
-	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
+	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
 	i_assert(sync_ctx->input->v_offset != from_offset);
 
 	mail_ctx->mail.body_size =
@@ -182,6 +182,7 @@ static int mbox_sync_do(struct index_mailbox *ibox,
 	sync_ctx.input = ibox->mbox_stream;
 	sync_ctx.fd = ibox->mbox_fd;
 	sync_ctx.header = str_new(default_pool, 4096);
+	sync_ctx.next_uid = 1;
 
 	input = sync_ctx.input;
 	istream_raw_mbox_seek(input, 0);
@@ -193,11 +194,12 @@ static int mbox_sync_do(struct index_mailbox *ibox,
 	messages_count = mail_index_view_get_message_count(sync_view);
 
 	space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
-	for (seq = 1;; seq++) {
+	for (seq = 0;;) {
 		/* set input->eof */
 		(void)istream_raw_mbox_get_header_offset(input);
 		if (input->eof)
 			break;
+		seq++;
 
 		mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
 
@@ -325,7 +327,6 @@ static int mbox_sync_do(struct index_mailbox *ibox,
 					ret = -1;
 					break;
 				}
-
 				/* mail_ctx may contain wrong data after
 				   rewrite, so make sure we don't try to access
 				   it */
@@ -339,13 +340,14 @@ static int mbox_sync_do(struct index_mailbox *ibox,
 	if (need_space_seq != 0) {
 		i_assert(space_diff < 0);
 		extra_space = MBOX_HEADER_EXTRA_SPACE *
-			((seq-1) - need_space_seq);
+			(seq - need_space_seq + 1);
+
 		if (mbox_sync_grow_file(&sync_ctx, &mail_ctx,
 					-space_diff + extra_space) < 0)
 			ret = -1;
 		else if (mbox_sync_try_rewrite(&mail_ctx) < 0)
 			ret = -1;
-		else if (--seq != need_space_seq) {
+		else if (seq != need_space_seq) {
 			buffer_set_used_size(mails,
 					     (seq-1) * sizeof(mail_ctx.mail));
 			buffer_append(mails, &mail_ctx.mail,