From: Markus Valentin <markus.valentin@open-xchange.com>
Date: Mon, 11 Sep 2023 06:58:00 +0000 (+0200)
Subject: lib-storage: Allow rediscovering lost mailboxes when using ITERINDEX and LAYOUT=fs
X-Git-Tag: 2.4.0~1937
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=063083b4bf1b9df7fa101d2c4ff99f03d870b2f3;p=thirdparty%2Fdovecot%2Fcore.git

lib-storage: Allow rediscovering lost mailboxes when using ITERINDEX and LAYOUT=fs

This is meant to prevent losing mailboxes if the index directory of a
mailbox got lost. A force-resync allows to rediscover and recreate
missing directories.
---

diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
index 3c8e205593..345a89ccff 100644
--- a/src/doveadm/doveadm-mail.c
+++ b/src/doveadm/doveadm-mail.c
@@ -349,7 +349,8 @@ static int cmd_force_resync_run(struct doveadm_mail_cmd_context *_ctx,
 		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
 		MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
 		MAILBOX_LIST_ITER_STAR_WITHIN_NS |
-		MAILBOX_LIST_ITER_RAW_LIST;
+		MAILBOX_LIST_ITER_RAW_LIST |
+		MAILBOX_LIST_ITER_FORCE_RESYNC;
 	const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
 	struct mailbox_list_iterate_context *iter;
 	const struct mailbox_info *info;
diff --git a/src/lib-storage/list/mailbox-list-fs-iter.c b/src/lib-storage/list/mailbox-list-fs-iter.c
index 126c061d4a..1b83d43f0d 100644
--- a/src/lib-storage/list/mailbox-list-fs-iter.c
+++ b/src/lib-storage/list/mailbox-list-fs-iter.c
@@ -2,11 +2,13 @@
 
 #include "lib.h"
 #include "array.h"
+#include "hash.h"
 #include "str.h"
 #include "unichar.h"
 #include "imap-match.h"
 #include "imap-utf7.h"
 #include "mail-storage.h"
+#include "mail-storage-private.h"
 #include "mailbox-tree.h"
 #include "mailbox-list-subscriptions.h"
 #include "mailbox-list-iter-private.h"
@@ -52,6 +54,7 @@ struct fs_list_iterate_context {
 	bool inbox_found:1;
 	bool inbox_has_children:1;
 	bool listed_prefix_inbox:1;
+	bool reverse_index_iteration:1;
 };
 
 static int
@@ -531,6 +534,11 @@ fs_list_iter_init(struct mailbox_list *_list, const char *const *patterns,
 	ctx->info.ns = _list->ns;
 	ctx->ctx.iter_from_index_dir = ctx->ctx.list->set.iter_from_index_dir;
 
+	if ((flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0) {
+		i_assert(!hash_table_is_created(ctx->ctx.found_mailboxes));
+		hash_table_create(&ctx->ctx.found_mailboxes, ctx->ctx.pool, 8,
+				  str_hash, strcmp);
+	}
 	if (!fs_list_get_valid_patterns(ctx, patterns)) {
 		/* we've only invalid patterns (or INBOX). create a glob
 		   anyway to avoid any crashes due to glob being accessed
@@ -561,6 +569,7 @@ int fs_list_iter_deinit(struct mailbox_list_iterate_context *_ctx)
 		pool_unref(&dir->pool);
 	}
 
+	hash_table_destroy(&_ctx->found_mailboxes);
 	pool_unref(&ctx->info_pool);
 	pool_unref(&_ctx->pool);
 	return ret;
@@ -795,6 +804,73 @@ fs_list_next(struct fs_list_iterate_context *ctx)
 	return 0;
 }
 
+static void
+fs_list_iter_reset(struct fs_list_iterate_context *ctx)
+{
+	i_assert(ctx->dir == NULL);
+	ctx->inbox_found = FALSE;
+	ctx->inbox_has_children = FALSE;
+	ctx->listed_prefix_inbox = FALSE;
+	/* reset root */
+	ctx->root_idx = 0;
+	fs_list_next_root(ctx);
+}
+
+static bool fs_list_iter_try_insert_unseen(struct fs_list_iterate_context *_ctx,
+					   const struct mailbox_info *info)
+{
+	struct mailbox_list_iterate_context *ctx = &_ctx->ctx;
+
+	if (hash_table_lookup(ctx->found_mailboxes, info->vname) == NULL) {
+		/* Insert unseen mailbox */
+		const char *vname = p_strdup(ctx->pool, info->vname);
+		hash_table_insert(ctx->found_mailboxes, vname, POINTER_CAST(1));
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static bool fs_list_iter_mkdir_missing(struct fs_list_iterate_context *ctx)
+{
+	struct mailbox_list_iterate_context *_ctx = &ctx->ctx;
+	bool unseen = fs_list_iter_try_insert_unseen(ctx, &ctx->info);
+
+	if (unseen && ctx->reverse_index_iteration) {
+		/* This mailbox was not found in the first iteration,
+		   recreate the directory for it */
+	} else if (!ctx->reverse_index_iteration) {
+		/* This is the first iteration of this mailbox, no directory
+		   must be recreated now */
+		return FALSE;
+	} else {
+		/* This mailbox was found in both iterations */
+		return TRUE;
+	}
+
+	/* This mailbox was listed the first time on the reverse index
+	   iteration but not on the second iteration.
+	   Create the missing directory for the first iteration. */
+	enum mailbox_list_path_type type = !_ctx->iter_from_index_dir ?
+					   MAILBOX_LIST_PATH_TYPE_INDEX :
+					   MAILBOX_LIST_PATH_TYPE_MAILBOX;
+	const char *path;
+	if (mailbox_list_get_path(ctx->info.ns->list, ctx->info.vname, type,
+				  &path) == 1) {
+		struct mailbox *box = mailbox_alloc(ctx->info.ns->list,
+						    ctx->info.vname, 0);
+		if (mailbox_mkdir(box, path, type) < 0)
+			e_error(box->event,
+				"Failed to create path: %s for mailbox %s: %s",
+				path, ctx->info.vname,
+				mailbox_get_last_internal_error(box, NULL));
+		mailbox_free(&box);
+	} else
+		e_error(ctx->ctx.list->ns->user->event,
+			"Failed to get mailbox path for %s",
+			ctx->info.vname);
+	return FALSE;
+}
+
 const struct mailbox_info *
 fs_list_iter_next(struct mailbox_list_iterate_context *_ctx)
 {
@@ -809,9 +885,21 @@ fs_list_iter_next(struct mailbox_list_iterate_context *_ctx)
 		ret = fs_list_next(ctx);
 	} T_END;
 
-	if (ret == 0)
+	if (ret > 0 && (_ctx->flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0) {
+		if (fs_list_iter_mkdir_missing(ctx))
+			return fs_list_iter_next(_ctx);
+	}
+
+	if (ret == 0) {
+		if ((_ctx->flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0 &&
+		    !ctx->reverse_index_iteration) {
+			ctx->reverse_index_iteration = TRUE;
+			_ctx->iter_from_index_dir = !_ctx->iter_from_index_dir;
+			fs_list_iter_reset(ctx);
+			return fs_list_iter_next(_ctx);
+		}
 		return mailbox_list_iter_default_next(_ctx);
-	else if (ret < 0)
+	} else if (ret < 0)
 		return NULL;
 
 	if (_ctx->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED &&
diff --git a/src/lib-storage/mailbox-list-iter.h b/src/lib-storage/mailbox-list-iter.h
index ec60313c2d..a5c216ceb5 100644
--- a/src/lib-storage/mailbox-list-iter.h
+++ b/src/lib-storage/mailbox-list-iter.h
@@ -38,7 +38,9 @@ enum mailbox_list_iter_flags {
 	/* Return children flags */
 	MAILBOX_LIST_ITER_RETURN_CHILDREN	= 0x004000,
 	/* Return IMAP special use flags */
-	MAILBOX_LIST_ITER_RETURN_SPECIALUSE	= 0x008000
+	MAILBOX_LIST_ITER_RETURN_SPECIALUSE	= 0x008000,
+	/* This listing is done as part of a force resync */
+	MAILBOX_LIST_ITER_FORCE_RESYNC		= 0x010000,
 };
 
 struct mailbox_info {
diff --git a/src/lib-storage/mailbox-list-private.h b/src/lib-storage/mailbox-list-private.h
index b38ee10198..e652fca675 100644
--- a/src/lib-storage/mailbox-list-private.h
+++ b/src/lib-storage/mailbox-list-private.h
@@ -169,6 +169,7 @@ struct mailbox_list_iterate_context {
 	struct mailbox_info specialuse_info;
 
 	ARRAY(union mailbox_list_iterate_module_context *) module_contexts;
+	HASH_TABLE(const char *, void*) found_mailboxes;
 };
 
 struct mailbox_list_iter_update_context {