From: Timo Sirainen <tss@iki.fi>
Date: Mon, 23 Mar 2009 19:50:49 +0000 (-0400)
Subject: dbox fixes.
X-Git-Tag: 2.0.alpha1~1038^2~29
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b038288b43caf8945b11071dfe34d38271cdcb68;p=thirdparty%2Fdovecot%2Fcore.git

dbox fixes.

--HG--
branch : HEAD
---

diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c
index 7e5f02e1de..ce0f3c8ed7 100644
--- a/src/lib-storage/index/dbox/dbox-file.c
+++ b/src/lib-storage/index/dbox/dbox-file.c
@@ -634,8 +634,9 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
 	int ret;
 
 	if (file->cur_offset == (uoff_t)-1) {
-		/* first mail */
-		offset = file->file_header_size;
+		/* first mail. we may not have read the file at all yet,
+		   so set the offset afterwards. */
+		offset = 0;
 	} else {
 		offset = file->cur_offset + file->msg_header_size +
 			file->cur_physical_size;
@@ -650,7 +651,13 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r)
 	}
 	*last_r = FALSE;
 
-	return dbox_file_get_mail_stream(file, offset, &size, NULL, &expunged);
+	ret = dbox_file_get_mail_stream(file, offset, &size, NULL, &expunged);
+	if (ret <= 0)
+		return ret;
+
+	if (*offset_r == 0)
+		*offset_r = file->file_header_size;
+	return 1;
 }
 
 static int
diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c
index 252fe2aa79..9e3af8ca91 100644
--- a/src/lib-storage/index/dbox/dbox-mail.c
+++ b/src/lib-storage/index/dbox/dbox-mail.c
@@ -44,16 +44,50 @@ static void dbox_mail_close(struct mail *_mail)
 	index_mail_close(_mail);
 }
 
-uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
-			  struct mail_index_view *view, uint32_t seq)
+int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+		     uint32_t seq, uint32_t *map_uid_r)
 {
 	const struct dbox_mail_index_record *dbox_rec;
+	const struct dbox_index_header *hdr;
 	const void *data;
+	size_t data_size;
+	uint32_t cur_map_uid_validity;
 	bool expunged;
 
 	mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged);
 	dbox_rec = data;
-	return dbox_rec == NULL ? 0 : dbox_rec->map_uid;
+	if (dbox_rec == NULL || dbox_rec->map_uid == 0) {
+		*map_uid_r = 0;
+		return 0;
+	}
+
+	if (mbox->map_uid_validity == 0) {
+		mail_index_get_header_ext(mbox->ibox.view,
+					  mbox->dbox_hdr_ext_id,
+					  &data, &data_size);
+		if (data_size != sizeof(*hdr)) {
+			mail_storage_set_critical(&mbox->storage->storage,
+				"dbox %s: Invalid dbox header size", mbox->path);
+			mbox->storage->sync_rebuild = TRUE;
+			return -1;
+		}
+		hdr = data;
+		mbox->map_uid_validity = hdr->map_uid_validity;
+	}
+	if (dbox_map_open(mbox->storage->map) < 0)
+		return -1;
+
+	cur_map_uid_validity = dbox_map_get_uid_validity(mbox->storage->map);
+	if (cur_map_uid_validity != mbox->map_uid_validity) {
+		mail_storage_set_critical(&mbox->storage->storage,
+			"dbox %s: map uidvalidity mismatch (%u vs %u)",
+			mbox->path, mbox->map_uid_validity,
+			cur_map_uid_validity);
+		mbox->storage->sync_rebuild = TRUE;
+		return -1;
+	}
+	*map_uid_r = dbox_rec->map_uid;
+	return 0;
 }
 
 static int dbox_mail_open(struct dbox_mail *mail,
@@ -65,7 +99,9 @@ static int dbox_mail_open(struct dbox_mail *mail,
 	int ret;
 
 	if (mail->open_file == NULL) {
-		map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq);
+		if (dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq,
+				     &map_uid) < 0)
+			return -1;
 		if (map_uid == 0) {
 			mail->open_file =
 				dbox_file_init_single(mbox, _mail->uid);
diff --git a/src/lib-storage/index/dbox/dbox-map-private.h b/src/lib-storage/index/dbox/dbox-map-private.h
index f95f3cc0da..398f4ba5ee 100644
--- a/src/lib-storage/index/dbox/dbox-map-private.h
+++ b/src/lib-storage/index/dbox/dbox-map-private.h
@@ -13,6 +13,7 @@ struct dbox_map {
 	struct dbox_storage *storage;
 	struct mail_index *index;
 	struct mail_index_view *view;
+	uint32_t created_uid_validity;
 
 	uint32_t map_ext_id, ref_ext_id;
 	ARRAY_TYPE(seq_range) ref0_file_ids;
@@ -43,7 +44,6 @@ struct dbox_map_append_context {
 	unsigned int committed:1;
 };
 
-int dbox_map_open(struct dbox_map *map);
 int dbox_map_refresh(struct dbox_map *map);
 int dbox_map_view_lookup_rec(struct dbox_map *map, struct mail_index_view *view,
 			     uint32_t seq, struct dbox_mail_lookup_rec *rec_r);
diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c
index af7d7a8434..add25240cb 100644
--- a/src/lib-storage/index/dbox/dbox-map.c
+++ b/src/lib-storage/index/dbox/dbox-map.c
@@ -41,6 +41,7 @@ struct dbox_map *dbox_map_init(struct dbox_storage *storage)
 				sizeof(uint32_t));
 	map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0,
 				sizeof(uint16_t), sizeof(uint16_t));
+	map->created_uid_validity = ioloop_time;
 	return map;
 }
 
@@ -888,7 +889,7 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx,
 	while (seq_range_array_iter_nth(&iter, i++, &uid)) {
 		if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq))
 			i_unreached();
-		mail_index_expunge(ctx->trans, seq);
+		mail_index_expunge(ctx->sync_trans, seq);
 	}
 	return 0;
 }
@@ -976,3 +977,14 @@ void dbox_map_append_free(struct dbox_map_append_context **_ctx)
 	array_free(&ctx->files);
 	i_free(ctx);
 }
+
+uint32_t dbox_map_get_uid_validity(struct dbox_map *map)
+{
+	const struct mail_index_header *hdr;
+
+	i_assert(map->view != NULL);
+
+	hdr = mail_index_get_header(map->view);
+	return hdr->uid_validity != 0 ? hdr->uid_validity :
+		map->created_uid_validity;
+}
diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h
index 0d9369ac18..8147ac117e 100644
--- a/src/lib-storage/index/dbox/dbox-map.h
+++ b/src/lib-storage/index/dbox/dbox-map.h
@@ -16,7 +16,7 @@ struct dbox_mail_index_map_header {
 struct dbox_mail_index_map_record {
 	uint32_t file_id;
 	uint32_t offset;
-	uint32_t size;
+	uint32_t size; /* including pre/post metadata */
 };
 
 struct dbox_map_file_msg {
@@ -29,6 +29,10 @@ ARRAY_DEFINE_TYPE(dbox_map_file_msg, struct dbox_map_file_msg);
 struct dbox_map *dbox_map_init(struct dbox_storage *storage);
 void dbox_map_deinit(struct dbox_map **map);
 
+/* Open the map. This is done automatically for most operations.
+   Returns 0 if ok, -1 if error. */
+int dbox_map_open(struct dbox_map *map);
+
 /* Look up file_id and offset for given map UID. Returns 1 if ok, 0 if UID
    is already expunged, -1 if error. */
 int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
@@ -77,6 +81,10 @@ int dbox_map_append_move(struct dbox_map_append_context *ctx,
 int dbox_map_append_commit(struct dbox_map_append_context *ctx);
 void dbox_map_append_free(struct dbox_map_append_context **ctx);
 
+/* Get either existing uidvalidity or create a new one if map was
+   just created. */
+uint32_t dbox_map_get_uid_validity(struct dbox_map *map);
+
 void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
 	ATTR_FORMAT(2, 3);
 
diff --git a/src/lib-storage/index/dbox/dbox-storage-rebuild.c b/src/lib-storage/index/dbox/dbox-storage-rebuild.c
index 207d8ed589..5c4a45e431 100644
--- a/src/lib-storage/index/dbox/dbox-storage-rebuild.c
+++ b/src/lib-storage/index/dbox/dbox-storage-rebuild.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "array.h"
+#include "istream.h"
 #include "hash.h"
 #include "hex-binary.h"
 #include "str.h"
@@ -180,13 +181,14 @@ static int rebuild_add_file(struct dbox_storage_rebuild_context *ctx,
 		rec = p_new(ctx->pool, struct dbox_rebuild_msg, 1);
 		rec->file_id = file_id;
 		rec->offset = offset;
-		rec->size = file->cur_physical_size;
+		rec->size = file->input->v_offset - offset;
 		memcpy(rec->guid_128, guid_buf->data, sizeof(rec->guid_128));
 		array_append(&ctx->msgs, &rec, 1);
 
 		if (hash_table_lookup(ctx->guid_hash, guid_buf->data) != NULL) {
 			/* duplicate. save this as a refcount=0 to map,
 			   so it will eventually be deleted. */
+			rec->seen_zero_ref_in_map = TRUE;
 		} else {
 			hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
 		}
@@ -255,7 +257,8 @@ static int rebuild_apply_map(struct dbox_storage_rebuild_context *ctx)
 			mail_index_expunge(ctx->trans, seq);
 		} else {
 			(*pos)->map_uid = rec.map_uid;
-			(*pos)->seen_zero_ref_in_map = rec.refcount == 0;
+			if (rec.refcount == 0)
+				(*pos)->seen_zero_ref_in_map = TRUE;
 		}
 	}
 	rebuild_add_missing_map_uids(ctx, hdr->next_uid);
@@ -386,7 +389,7 @@ rebuild_mailbox(struct dbox_storage_rebuild_context *ctx, const char *name)
 		return -1;
 	}
 
-	rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+	rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans, TRUE);
 	ret = dbox_sync_index_rebuild_singles(rebuild_ctx);
 	if (ret == 0)
 		rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
@@ -456,7 +459,7 @@ static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx,
 	file = dbox_file_init_multi(ctx->storage, msg->file_id);
 	ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL,
 					&expunged);
-	if (ret > 0 && !expunged && dbox_file_metadata_read(file) == 0) {
+	if (ret > 0 && !expunged && dbox_file_metadata_read(file) > 0) {
 		mailbox = dbox_file_metadata_get(file,
 						 DBOX_METADATA_ORIG_MAILBOX);
 	}
@@ -533,6 +536,8 @@ static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx,
 	mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
 	mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->dbox_ext_id,
 			      &dbox_rec, NULL);
+	mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id,
+			      msg->guid_128, NULL);
 
 	msg->refcount++;
 	return 0;
@@ -619,10 +624,12 @@ static int rebuild_finish(struct dbox_storage_rebuild_context *ctx)
 
 static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx)
 {
+	const struct mail_index_header *hdr;
 	DIR *dir;
 	struct dirent *d;
 	string_t *path;
 	unsigned int dir_len;
+	uint32_t uid_validity;
 	int ret = 0;
 
 	if (dbox_map_open(ctx->storage->map) < 0)
@@ -639,6 +646,14 @@ static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx)
 		return -1;
 	}
 
+	uid_validity = dbox_map_get_uid_validity(ctx->storage->map);
+	hdr = mail_index_get_header(ctx->sync_view);
+	if (hdr->uid_validity != uid_validity) {
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&uid_validity, sizeof(uid_validity), TRUE);
+	}
+
 	dir = opendir(ctx->storage->storage_dir);
 	if (dir == NULL) {
 		mail_storage_set_critical(&ctx->storage->storage,
@@ -700,6 +715,7 @@ int dbox_storage_rebuild(struct dbox_storage *storage)
 			"stat(%s) failed: %m", storage->storage_dir);
 		return -1;
 	}
+	storage->have_multi_msgs = TRUE;
 
 	if (storage->sync_rebuild)
 		i_warning("dbox %s: rebuilding indexes", storage->storage_dir);
diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h
index 87e9699c91..17d62856f3 100644
--- a/src/lib-storage/index/dbox/dbox-storage.h
+++ b/src/lib-storage/index/dbox/dbox-storage.h
@@ -33,7 +33,7 @@
 #define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
 
 struct dbox_index_header {
-	uint32_t unused; /* for backwards compatibility */
+	uint32_t map_uid_validity;
 	uint32_t highest_maildir_uid;
 };
 
@@ -57,6 +57,7 @@ struct dbox_storage {
 	ARRAY_DEFINE(open_files, struct dbox_file *);
 
 	unsigned int sync_rebuild:1;
+	unsigned int have_multi_msgs:1;
 };
 
 struct dbox_mail_index_record {
@@ -69,6 +70,7 @@ struct dbox_mailbox {
 
 	struct maildir_uidlist *maildir_uidlist;
 	uint32_t highest_maildir_uid;
+	uint32_t map_uid_validity;
 
 	uint32_t dbox_ext_id, dbox_hdr_ext_id, guid_ext_id;
 
@@ -98,8 +100,8 @@ dbox_mail_alloc(struct mailbox_transaction_context *t,
 		struct mailbox_header_lookup_ctx *wanted_headers);
 
 /* Get map_uid for wanted message. */
-uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
-			  struct mail_index_view *view, uint32_t seq);
+int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+		     uint32_t seq, uint32_t *map_uid_r);
 
 struct mail_save_context *
 dbox_save_alloc(struct mailbox_transaction_context *_t);
diff --git a/src/lib-storage/index/dbox/dbox-sync-rebuild.c b/src/lib-storage/index/dbox/dbox-sync-rebuild.c
index e39d306eba..9d8e23b0b5 100644
--- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c
+++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c
@@ -7,6 +7,7 @@
 #include "maildir/maildir-uidlist.h"
 #include "maildir/maildir-keywords.h"
 #include "maildir/maildir-filename.h"
+#include "dbox-map.h"
 #include "dbox-file.h"
 #include "dbox-sync.h"
 
@@ -31,6 +32,7 @@ struct dbox_sync_rebuild_context {
 	uint32_t highest_uid;
 
 	unsigned int cache_used:1;
+	unsigned int storage_rebuild:1;
 };
 
 static uint32_t dbox_get_uidvalidity_next(struct mail_storage *storage)
@@ -365,16 +367,14 @@ static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx)
 				  ctx->mbox->dbox_hdr_ext_id,
 				  &data, &data_size);
 	hdr = data;
-	if (data_size == sizeof(*hdr)) {
-		if (hdr->highest_maildir_uid >= ctx->mbox->highest_maildir_uid) {
-			/* nothing to change */
-			return;
-		}
+	if (data_size == sizeof(*hdr))
 		new_hdr = *hdr;
-	} else {
+	else
 		memset(&new_hdr, 0, sizeof(new_hdr));
-	}
-	new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid;
+	if (new_hdr.highest_maildir_uid < ctx->mbox->highest_maildir_uid)
+		new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid;
+	new_hdr.map_uid_validity = !ctx->storage_rebuild ? 0 :
+		dbox_map_get_uid_validity(ctx->mbox->storage->map);
 	mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0,
 				     &new_hdr, sizeof(new_hdr));
 }
@@ -382,7 +382,8 @@ static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx)
 struct dbox_sync_rebuild_context *
 dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
 			     struct mail_index_view *view,
-			     struct mail_index_transaction *trans)
+			     struct mail_index_transaction *trans,
+			     bool storage_rebuild)
 {
 	struct mailbox *box = &mbox->ibox.box;
 	struct dbox_sync_rebuild_context *ctx;
@@ -393,6 +394,7 @@ dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
 	ctx->mbox = mbox;
 	ctx->view = view;
 	ctx->trans = trans;
+	ctx->storage_rebuild = storage_rebuild;
 	mail_index_reset(ctx->trans);
 	index_mailbox_reset_uidvalidity(&mbox->ibox);
 	mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx->cache_ext_id);
@@ -468,7 +470,7 @@ int dbox_sync_index_rebuild(struct dbox_mailbox *mbox)
 	trans = mail_index_transaction_begin(view,
 					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
 
-	ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+	ctx = dbox_sync_index_rebuild_init(mbox, view, trans, FALSE);
 	ret = dbox_sync_index_rebuild_singles(ctx);
 	dbox_sync_index_rebuild_deinit(&ctx);
 
diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c
index 3433152eb0..e4d6ea8c4d 100644
--- a/src/lib-storage/index/dbox/dbox-sync.c
+++ b/src/lib-storage/index/dbox/dbox-sync.c
@@ -48,7 +48,8 @@ static int dbox_sync_add_seq(struct dbox_sync_context *ctx,
 		 sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
 
 	memset(&lookup_entry, 0, sizeof(lookup_entry));
-	map_uid = dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq);
+	if (dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0)
+		return 0;
 	if (map_uid == 0)
 		mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid);
 	else {
@@ -136,7 +137,6 @@ static int dbox_sync_index(struct dbox_sync_context *ctx)
 	hdr = mail_index_get_header(ctx->sync_view);
 	if (hdr->uid_validity == 0) {
 		/* newly created index file */
-		i_warning("FIXME: newly created");
 		return 0;
 	}
 
@@ -197,8 +197,7 @@ static int dbox_refresh_header(struct dbox_mailbox *mbox)
 		if (data_size != 0 && data_size != 4) {
 			i_warning("dbox %s: Invalid dbox header size",
 				  mbox->path);
-		} else
-			i_warning("FIXME: initial sync");
+		}
 		return -1;
 	}
 	hdr = data;
@@ -252,17 +251,9 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags,
 		}
 
 		/* now that we're locked, check again if we want to rebuild */
-		if (dbox_refresh_header(mbox) < 0) {
-			if (!storage_rebuilt) {
-				/* we'll need to rebuild storage too.
-				   try again from the beginning. */
-				mail_index_sync_rollback(&ctx->index_sync_ctx);
-				i_free(ctx);
-				return dbox_sync_begin(mbox, flags, ctx_r);
-			}
+		if (dbox_refresh_header(mbox) < 0)
 			ret = 0;
-			i_warning("FIXME: poop");
-		} else {
+		else {
 			if ((ret = dbox_sync_index(ctx)) > 0)
 				break;
 		}
@@ -270,7 +261,19 @@ int dbox_sync_begin(struct dbox_mailbox *mbox, enum dbox_sync_flags flags,
 		/* failure. keep the index locked while we're doing a
 		   rebuild. */
 		if (ret == 0) {
-			if (i >= DBOX_REBUILD_COUNT) {
+			if (!storage_rebuilt) {
+				/* we'll need to rebuild storage too.
+				   try again from the beginning. */
+				mail_index_sync_rollback(&ctx->index_sync_ctx);
+				i_free(ctx);
+				return dbox_sync_begin(mbox, flags, ctx_r);
+			}
+			if (mbox->storage->have_multi_msgs) {
+				mail_storage_set_critical(storage,
+					"dbox %s: Storage keeps breaking",
+					ctx->mbox->path);
+				ret = -1;
+			} else if (i >= DBOX_REBUILD_COUNT) {
 				mail_storage_set_critical(storage,
 					"dbox %s: Index keeps breaking",
 					ctx->mbox->path);
diff --git a/src/lib-storage/index/dbox/dbox-sync.h b/src/lib-storage/index/dbox/dbox-sync.h
index e19d8d8d38..febfa4bc35 100644
--- a/src/lib-storage/index/dbox/dbox-sync.h
+++ b/src/lib-storage/index/dbox/dbox-sync.h
@@ -45,7 +45,8 @@ int dbox_sync_file_cleanup(struct dbox_file *file);
 struct dbox_sync_rebuild_context *
 dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
 			     struct mail_index_view *view,
-			     struct mail_index_transaction *trans);
+			     struct mail_index_transaction *trans,
+			     bool storage_rebuild);
 int dbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx);
 void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx,
 				      struct dbox_file *file,