lib-storage: Allow rediscovering lost mailboxes when using ITERINDEX and LAYOUT=fs

This is meant to prevent losing mailboxes if the index directory of a
mailbox got lost. A force-resync allows to rediscover and recreate
missing directories.

diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
index 3c8e2055933b98ad9d59ecaf169a8db9f6ea8c6d..345a89ccffe6623c3e86e90e8721e1f8424b0dfd 100644 (file)
--- a/src/doveadm/doveadm-mail.c
+++ b/src/doveadm/doveadm-mail.c
@@ -349,7 +349,8 @@ static int cmd_force_resync_run(struct doveadm_mail_cmd_context *_ctx,
                MAILBOX_LIST_ITER_NO_AUTO_BOXES |
                MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
                MAILBOX_LIST_ITER_STAR_WITHIN_NS |
-               MAILBOX_LIST_ITER_RAW_LIST;
+               MAILBOX_LIST_ITER_RAW_LIST |
+               MAILBOX_LIST_ITER_FORCE_RESYNC;
        const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
        struct mailbox_list_iterate_context *iter;
        const struct mailbox_info *info;

diff --git a/src/lib-storage/list/mailbox-list-fs-iter.c b/src/lib-storage/list/mailbox-list-fs-iter.c
index 126c061d4a35a7eb58f74d2751d5b07f88120e77..1b83d43f0d9a08e2394d9aea2bc022742991f051 100644 (file)
--- a/src/lib-storage/list/mailbox-list-fs-iter.c
+++ b/src/lib-storage/list/mailbox-list-fs-iter.c
@@ -2,11 +2,13 @@
 
 #include "lib.h"
 #include "array.h"
+#include "hash.h"
 #include "str.h"
 #include "unichar.h"
 #include "imap-match.h"
 #include "imap-utf7.h"
 #include "mail-storage.h"
+#include "mail-storage-private.h"
 #include "mailbox-tree.h"
 #include "mailbox-list-subscriptions.h"
 #include "mailbox-list-iter-private.h"
@@ -52,6 +54,7 @@ struct fs_list_iterate_context {
        bool inbox_found:1;
        bool inbox_has_children:1;
        bool listed_prefix_inbox:1;
+       bool reverse_index_iteration:1;
 };
 
 static int
@@ -531,6 +534,11 @@ fs_list_iter_init(struct mailbox_list *_list, const char *const *patterns,
        ctx->info.ns = _list->ns;
        ctx->ctx.iter_from_index_dir = ctx->ctx.list->set.iter_from_index_dir;
 
+       if ((flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0) {
+               i_assert(!hash_table_is_created(ctx->ctx.found_mailboxes));
+               hash_table_create(&ctx->ctx.found_mailboxes, ctx->ctx.pool, 8,
+                                 str_hash, strcmp);
+       }
        if (!fs_list_get_valid_patterns(ctx, patterns)) {
                /* we've only invalid patterns (or INBOX). create a glob
                   anyway to avoid any crashes due to glob being accessed
@@ -561,6 +569,7 @@ int fs_list_iter_deinit(struct mailbox_list_iterate_context *_ctx)
                pool_unref(&dir->pool);
        }
 
+       hash_table_destroy(&_ctx->found_mailboxes);
        pool_unref(&ctx->info_pool);
        pool_unref(&_ctx->pool);
        return ret;
@@ -795,6 +804,73 @@ fs_list_next(struct fs_list_iterate_context *ctx)
        return 0;
 }
 
+static void
+fs_list_iter_reset(struct fs_list_iterate_context *ctx)
+{
+       i_assert(ctx->dir == NULL);
+       ctx->inbox_found = FALSE;
+       ctx->inbox_has_children = FALSE;
+       ctx->listed_prefix_inbox = FALSE;
+       /* reset root */
+       ctx->root_idx = 0;
+       fs_list_next_root(ctx);
+}
+
+static bool fs_list_iter_try_insert_unseen(struct fs_list_iterate_context *_ctx,
+                                          const struct mailbox_info *info)
+{
+       struct mailbox_list_iterate_context *ctx = &_ctx->ctx;
+
+       if (hash_table_lookup(ctx->found_mailboxes, info->vname) == NULL) {
+               /* Insert unseen mailbox */
+               const char *vname = p_strdup(ctx->pool, info->vname);
+               hash_table_insert(ctx->found_mailboxes, vname, POINTER_CAST(1));
+               return TRUE;
+       }
+       return FALSE;
+}
+
+static bool fs_list_iter_mkdir_missing(struct fs_list_iterate_context *ctx)
+{
+       struct mailbox_list_iterate_context *_ctx = &ctx->ctx;
+       bool unseen = fs_list_iter_try_insert_unseen(ctx, &ctx->info);
+
+       if (unseen && ctx->reverse_index_iteration) {
+               /* This mailbox was not found in the first iteration,
+                  recreate the directory for it */
+       } else if (!ctx->reverse_index_iteration) {
+               /* This is the first iteration of this mailbox, no directory
+                  must be recreated now */
+               return FALSE;
+       } else {
+               /* This mailbox was found in both iterations */
+               return TRUE;
+       }
+
+       /* This mailbox was listed the first time on the reverse index
+          iteration but not on the second iteration.
+          Create the missing directory for the first iteration. */
+       enum mailbox_list_path_type type = !_ctx->iter_from_index_dir ?
+                                          MAILBOX_LIST_PATH_TYPE_INDEX :
+                                          MAILBOX_LIST_PATH_TYPE_MAILBOX;
+       const char *path;
+       if (mailbox_list_get_path(ctx->info.ns->list, ctx->info.vname, type,
+                                 &path) == 1) {
+               struct mailbox *box = mailbox_alloc(ctx->info.ns->list,
+                                                   ctx->info.vname, 0);
+               if (mailbox_mkdir(box, path, type) < 0)
+                       e_error(box->event,
+                               "Failed to create path: %s for mailbox %s: %s",
+                               path, ctx->info.vname,
+                               mailbox_get_last_internal_error(box, NULL));
+               mailbox_free(&box);
+       } else
+               e_error(ctx->ctx.list->ns->user->event,
+                       "Failed to get mailbox path for %s",
+                       ctx->info.vname);
+       return FALSE;
+}
+
 const struct mailbox_info *
 fs_list_iter_next(struct mailbox_list_iterate_context *_ctx)
 {
@@ -809,9 +885,21 @@ fs_list_iter_next(struct mailbox_list_iterate_context *_ctx)
                ret = fs_list_next(ctx);
        } T_END;
 
-       if (ret == 0)
+       if (ret > 0 && (_ctx->flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0) {
+               if (fs_list_iter_mkdir_missing(ctx))
+                       return fs_list_iter_next(_ctx);
+       }
+
+       if (ret == 0) {
+               if ((_ctx->flags & MAILBOX_LIST_ITER_FORCE_RESYNC) != 0 &&
+                   !ctx->reverse_index_iteration) {
+                       ctx->reverse_index_iteration = TRUE;
+                       _ctx->iter_from_index_dir = !_ctx->iter_from_index_dir;
+                       fs_list_iter_reset(ctx);
+                       return fs_list_iter_next(_ctx);
+               }
                return mailbox_list_iter_default_next(_ctx);
-       else if (ret < 0)
+       } else if (ret < 0)
                return NULL;
 
        if (_ctx->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED &&

diff --git a/src/lib-storage/mailbox-list-iter.h b/src/lib-storage/mailbox-list-iter.h
index ec60313c2d0b9f60ffc6c3e58f727801621cd2f6..a5c216ceb548f861b9f1a80ddf53594570cafe2a 100644 (file)
--- a/src/lib-storage/mailbox-list-iter.h
+++ b/src/lib-storage/mailbox-list-iter.h
@@ -38,7 +38,9 @@ enum mailbox_list_iter_flags {
        /* Return children flags */
        MAILBOX_LIST_ITER_RETURN_CHILDREN       = 0x004000,
        /* Return IMAP special use flags */
-       MAILBOX_LIST_ITER_RETURN_SPECIALUSE     = 0x008000
+       MAILBOX_LIST_ITER_RETURN_SPECIALUSE     = 0x008000,
+       /* This listing is done as part of a force resync */
+       MAILBOX_LIST_ITER_FORCE_RESYNC          = 0x010000,
 };
 
 struct mailbox_info {

diff --git a/src/lib-storage/mailbox-list-private.h b/src/lib-storage/mailbox-list-private.h
index b38ee10198256adb3e66b03b96dbc5cdf39818cf..e652fca67532ff4e6d9af1ab1e286c3cabc7080c 100644 (file)
--- a/src/lib-storage/mailbox-list-private.h
+++ b/src/lib-storage/mailbox-list-private.h
@@ -169,6 +169,7 @@ struct mailbox_list_iterate_context {
        struct mailbox_info specialuse_info;
 
        ARRAY(union mailbox_list_iterate_module_context *) module_contexts;
+       HASH_TABLE(const char *, void*) found_mailboxes;
 };
 
 struct mailbox_list_iter_update_context {