lib-storage: Rebuild list-index using MAILBOX_LIST_ITER_FORCE_RESYNC

In case the list index file was just created use
MAILBOX_LIST_ITER_FORCE_RESYNC to rediscover mailboxes that might have
been hidden.

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index 55055c6de0c5d63cb22eb4ba4c544e5e857f71a0..d922f01d6e0b85141dcf31a62fa3d05f1ef50c9e 100644 (file)
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -318,6 +318,9 @@ struct mail_index {
        /* Index has been fsck'd, but mail_index_reset_fscked() hasn't been
           called yet. */
        bool fscked:1;
+       /* mail_index_open() has created new index files */
+       bool initial_created:1;
+
 };
 
 extern struct mail_index_module_register mail_index_module_register;

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index d2dd40c0e664c3d771e67d5e7bc4f186f3da411c..e6be908bbfa664fba527fc87c4f4c271bfa647f3 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -610,6 +610,7 @@ static int mail_index_open_files(struct mail_index *index,
                        index->map->hdr.indexid = index->indexid;
                }
                index->initial_create = FALSE;
+               index->initial_created = TRUE;
        }
        if (ret >= 0) {
                ret = index->map != NULL ? 1 : mail_index_try_open(index);
@@ -725,7 +726,7 @@ int mail_index_open_or_create(struct mail_index *index,
        flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
        ret = mail_index_open(index, flags);
        i_assert(ret != 0);
-       return ret < 0 ? -1 : 0;
+       return ret < 0 ? -1 : (index->initial_created ? 1 : 0);
 }
 
 void mail_index_close_file(struct mail_index *index)

diff --git a/src/lib-index/test-mail-cache-common.c b/src/lib-index/test-mail-cache-common.c
index b6620b728d223f89a32a7edb93af3c7f03473e76..55571b76f0d3984eb67fe75f59d3a838e798b5a5 100644 (file)
--- a/src/lib-index/test-mail-cache-common.c
+++ b/src/lib-index/test-mail-cache-common.c
@@ -95,7 +95,7 @@ void test_mail_cache_purge(void)
 {
        struct test_mail_cache_ctx ctx;
 
-       test_mail_cache_init(test_mail_index_open(), &ctx);
+       test_mail_cache_init(test_mail_index_open(FALSE), &ctx);
        test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
        test_mail_cache_deinit(&ctx);
 }

diff --git a/src/lib-index/test-mail-cache-fields.c b/src/lib-index/test-mail-cache-fields.c
index 1b76dc5c8132a700e1504a6310b3e1b23276e64a..1a99c774fdfb55edaaf000002f0af17cc8f706e3 100644 (file)
--- a/src/lib-index/test-mail-cache-fields.c
+++ b/src/lib-index/test-mail-cache-fields.c
@@ -22,7 +22,7 @@ static void test_mail_cache_fields_read_write(void)
 
        test_begin("mail cache fields read-write");
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_cache_register_fields(ctx.cache, &cache_field, 1,
                                   unsafe_data_stack_pool);
        test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);

diff --git a/src/lib-index/test-mail-cache-purge.c b/src/lib-index/test-mail-cache-purge.c
index ee0b33803a05c2247a42c8c3a356bdd1c0c5b880..24dd3bb74dafa26d4b7d28d6a678b6acfcdbfcaa 100644 (file)
--- a/src/lib-index/test-mail-cache-purge.c
+++ b/src/lib-index/test-mail-cache-purge.c
@@ -18,7 +18,7 @@ static void test_mail_cache_read_during_purge2(void)
        i_set_failure_prefix("index2: ");
 
        /* read from cache via 2nd index */
-       test_mail_cache_init(test_mail_index_open(), &ctx);
+       test_mail_cache_init(test_mail_index_open(FALSE), &ctx);
 
        cache_view = mail_cache_view_open(ctx.cache, ctx.view);
        test_assert(mail_cache_lookup_field(cache_view, str, 1,
@@ -36,7 +36,7 @@ static void test_mail_cache_read_during_purge(void)
        int status;
 
        test_begin("mail cache read during purge");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        /* lock the index for cache purge */
@@ -88,7 +88,7 @@ static void test_mail_cache_write_during_purge2(void)
        i_set_failure_prefix("index2: ");
 
        /* add to cache via 2nd index */
-       test_mail_cache_init(test_mail_index_open(), &ctx);
+       test_mail_cache_init(test_mail_index_open(FALSE), &ctx);
        test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar2");
        test_mail_cache_deinit(&ctx);
 }
@@ -103,7 +103,7 @@ static void test_mail_cache_write_during_purge(void)
        int status;
 
        test_begin("mail cache write during purge");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        /* lock the index for cache purge */
@@ -171,7 +171,7 @@ static void test_mail_cache_purge_while_cache_locked(void)
        int status;
 
        test_begin("mail cache purge while cache locked");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        /* lock the cache */
@@ -250,7 +250,7 @@ static void test_mail_cache_purge_during_write_n(unsigned int num_mails,
        struct mail_index_transaction *trans;
        uint32_t seq;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        /* Add mails */
@@ -379,7 +379,7 @@ static void test_mail_cache_delete_too_large_int(bool exceed_on_first_write)
        struct test_mail_cache_ctx ctx;
        struct stat st;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo2");
@@ -441,7 +441,7 @@ static void test_mail_cache_purge_too_large_int(bool exceed_size)
        struct test_mail_cache_ctx ctx;
        struct stat st;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        /* add two mails with some cache field and expunge the first mail */
@@ -503,7 +503,7 @@ static void test_mail_cache_unexpectedly_lost_int(bool read_first)
        struct test_mail_cache_ctx ctx;
        struct mail_cache_view *cache_view;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        test_mail_cache_purge();
@@ -566,7 +566,7 @@ static void test_mail_cache_resetid_mismatch_int(bool read_first)
        struct mail_cache_view *cache_view;
        const char *temp_cache_path;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        /* make a copy of the first cache file */
@@ -695,7 +695,7 @@ static void test_mail_cache_purge_field_changes_int(enum test_drop drop)
        struct mail_index_transaction *trans;
        unsigned int i;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        /* add two mails with all of the cache fields */
@@ -851,7 +851,7 @@ static void test_mail_cache_purge_already_done(void)
        struct test_mail_cache_ctx ctx;
 
        test_begin("mail cache purge already done");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
 
        test_mail_cache_purge();
@@ -886,7 +886,7 @@ static void test_mail_cache_purge_bitmask(void)
        struct mail_cache_view *cache_view;
 
        test_begin("mail cache purge bitmask");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
        ioloop_time = 1000000;
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
@@ -934,7 +934,7 @@ test_mail_cache_update_need_purge_continued_records_int(bool big_min_size)
        struct test_mail_cache_ctx ctx;
        uint32_t seq;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        for (seq = 1; seq <= 100; seq++) {
@@ -989,7 +989,7 @@ test_mail_cache_update_need_purge_deleted_records_int(bool big_min_size)
        struct test_mail_cache_ctx ctx;
        uint32_t seq;
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        for (seq = 1; seq <= 100; seq++) {

diff --git a/src/lib-index/test-mail-cache.c b/src/lib-index/test-mail-cache.c
index 9c279097ff2a5159009c9b348a6704c9cb893401..b0c97b81088fc7d24b92f9afedbcba9d7b5853d6 100644 (file)
--- a/src/lib-index/test-mail-cache.c
+++ b/src/lib-index/test-mail-cache.c
@@ -106,7 +106,7 @@ static void test_mail_cache_fields(void)
        string_t *str = t_str_new(16);
 
        test_begin("mail cache uncommitted lookups");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_cache_register_fields(ctx.cache, cache_fields,
                                   N_ELEMENTS(cache_fields),
                                   unsafe_data_stack_pool);
@@ -253,7 +253,7 @@ static void test_mail_cache_record_max_size_int(unsigned int field3_size)
        struct mail_cache_transaction_ctx *cache_trans;
        string_t *str = t_str_new(16);
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        /* Add the first cache field. In a chain of cache records each one
@@ -358,7 +358,7 @@ static void test_mail_cache_record_max_size4(void)
        string_t *str = t_str_new(16);
 
        test_begin("mail cache record max size (4)");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
 
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
@@ -400,7 +400,7 @@ static void test_mail_cache_add_decisions(void)
 
        test_begin("mail cache add decisions");
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields));
        mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT,
                                   unsafe_data_stack_pool);
@@ -471,7 +471,7 @@ static void test_mail_cache_lookup_decisions_int(bool header_lookups)
        unsigned int i;
        string_t *str = t_str_new(16);
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        /* create the initial mails and the cache file */
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
@@ -657,7 +657,7 @@ static void test_mail_cache_in_memory(void)
        test_begin("mail cache add in-memory");
 
        index = mail_index_alloc(NULL, NULL, "(in-memory)");
-       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0);
+       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 1);
        test_mail_cache_init(index, &ctx);
        mail_index_set_optimization_settings(ctx.index, &optimization_set);
        cache_view = mail_cache_view_open(ctx.cache, ctx.view);
@@ -726,7 +726,7 @@ static void test_mail_cache_size_corruption(void)
 
        test_begin("mail cache size corruption");
 
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "12345678");
        cache_view = mail_cache_view_open(ctx.cache, ctx.view);
 
@@ -768,7 +768,7 @@ static void test_mail_cache_duplicate_fields(void)
        struct mail_cache_transaction_ctx *cache_trans;
 
        test_begin("mail cache duplicate fields");
-       test_mail_cache_init(test_mail_index_init(), &ctx);
+       test_mail_cache_init(test_mail_index_init(TRUE), &ctx);
        mail_cache_register_fields(ctx.cache, cache_fields,
                                   N_ELEMENTS(cache_fields),
                                   unsafe_data_stack_pool);

diff --git a/src/lib-index/test-mail-index-modseq.c b/src/lib-index/test-mail-index-modseq.c
index 4a2d52468b77470ee193da9304454b4f87188abe..c900966254fbda08e2f202a71538855633de5e2e 100644 (file)
--- a/src/lib-index/test-mail-index-modseq.c
+++ b/src/lib-index/test-mail-index-modseq.c
@@ -27,7 +27,7 @@ static void test_mail_index_modseq_get_next_log_offset(void)
        uint32_t seq, uid;
 
        test_begin("mail_transaction_log_file_get_modseq_next_offset()");
-       index = test_mail_index_init();
+       index = test_mail_index_init(TRUE);
        view = mail_index_view_open(index);
        mail_index_modseq_enable(index);
 

diff --git a/src/lib-index/test-mail-index.c b/src/lib-index/test-mail-index.c
index 3e2fd02d645d81a8061de8bbdd4fefbfe4c48dc2..84d2b43f516754e1b3fb118df2cdc3794910fd10 100644 (file)
--- a/src/lib-index/test-mail-index.c
+++ b/src/lib-index/test-mail-index.c
@@ -15,8 +15,8 @@ static void test_mail_index_rotate(void)
        const char *reason;
 
        test_begin("mail index rotate");
-       index = test_mail_index_init();
-       index2 = test_mail_index_open();
+       index = test_mail_index_init(TRUE);
+       index2 = test_mail_index_open(FALSE);
        view = mail_index_view_open(index);
 
        /* First rotation of the index. The view will point to the old index. */
@@ -92,8 +92,8 @@ static void test_mail_index_new_extension(void)
        uint32_t seq, rec_ext_id, rec_ext = 0x12345678;
 
        test_begin("mail index new extension");
-       index = test_mail_index_init();
-       index2 = test_mail_index_open();
+       index = test_mail_index_init(TRUE);
+       index2 = test_mail_index_open(FALSE);
        view = mail_index_view_open(index);
 
        rec_ext_id = mail_index_ext_register(index, "test-rec", 0,

diff --git a/src/lib-index/test-mail-index.h b/src/lib-index/test-mail-index.h
index 75b343f92b0346ad951319445d760589ddcc6f0d..e8251e974d18f51237eba21971c09c2683a88c24 100644 (file)
--- a/src/lib-index/test-mail-index.h
+++ b/src/lib-index/test-mail-index.h
@@ -7,16 +7,17 @@
 
 #define TESTDIR_NAME ".dovecot.test"
 
-static inline struct mail_index *test_mail_index_open(void)
+static inline struct mail_index *test_mail_index_open(bool idx_initial_created)
 {
        struct mail_index *index;
 
        index = mail_index_alloc(NULL, TESTDIR_NAME, "test.dovecot.index");
-       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0);
+       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) ==
+                  (idx_initial_created ? 1 : 0));
        return index;
 }
 
-static inline struct mail_index *test_mail_index_init(void)
+static inline struct mail_index *test_mail_index_init(bool idx_initial_created)
 {
        const char *error;
 
@@ -26,7 +27,7 @@ static inline struct mail_index *test_mail_index_init(void)
 
        ioloop_time = 1;
 
-       return test_mail_index_open();
+       return test_mail_index_open(idx_initial_created);
 }
 
 static inline void test_mail_index_close(struct mail_index **index)

diff --git a/src/lib-index/test-mail-transaction-log-file.c b/src/lib-index/test-mail-transaction-log-file.c
index 6f591cef6944fc603902940276bf552f5f286513..d740947e5a10c6ed1f5c9876ee35bb75b8e0c46c 100644 (file)
--- a/src/lib-index/test-mail-transaction-log-file.c
+++ b/src/lib-index/test-mail-transaction-log-file.c
@@ -244,7 +244,7 @@ static void test_mail_transaction_update_modseq(void)
 static struct mail_index *test_mail_index_open(void)
 {
        struct mail_index *index = mail_index_alloc(NULL, NULL, "test.dovecot.index");
-       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0);
+       test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 1);
        struct mail_index_view *view = mail_index_view_open(index);
 
        struct mail_index_transaction *trans =

diff --git a/src/lib-storage/list/mailbox-list-index-sync.c b/src/lib-storage/list/mailbox-list-index-sync.c
index 87411d6cb53b5338f24c76c421a5cf368e75f9a3..06afe6529db0a2cd44de364454e5ef9d0097f0cf 100644 (file)
--- a/src/lib-storage/list/mailbox-list-index-sync.c
+++ b/src/lib-storage/list/mailbox-list-index-sync.c
@@ -295,6 +295,7 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx)
        struct mailbox_list_iterate_context *iter;
        const struct mailbox_info *info;
        enum mailbox_list_index_flags flags;
+       enum mailbox_list_iter_flags iter_flags;
        const char *patterns[2];
        struct mailbox_list_index_node *node;
        uint32_t seq;
@@ -303,14 +304,18 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx)
        /* clear EXISTS-flags, so after sync we know what can be expunged */
        mailbox_list_index_node_clear_exists(sync_ctx->ilist->mailbox_tree);
 
+       iter_flags = MAILBOX_LIST_ITER_RAW_LIST |
+                    MAILBOX_LIST_ITER_NO_AUTO_BOXES;
+       if (sync_ctx->ilist->index_created)
+               iter_flags |= MAILBOX_LIST_ITER_FORCE_RESYNC;
+
        /* don't include autocreated mailboxes in index until they're
           actually created. this index may be used by multiple users, so
           we also want to ignore ACLs here. */
        patterns[0] = "*"; patterns[1] = NULL;
        iter = sync_ctx->ilist->module_ctx.super.
                iter_init(sync_ctx->list, patterns,
-                         MAILBOX_LIST_ITER_RAW_LIST |
-                         MAILBOX_LIST_ITER_NO_AUTO_BOXES);
+                         iter_flags);
 
        sync_ctx->syncing_list = TRUE;
        while ((info = sync_ctx->ilist->module_ctx.super.iter_next(iter)) != NULL) T_BEGIN {
@@ -344,6 +349,7 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx)
 
        /* successfully listed everything, expunge any unseen mailboxes */
        sync_expunge_nonexistent(sync_ctx, sync_ctx->ilist->mailbox_tree);
+       sync_ctx->ilist->index_created = FALSE;
        return 0;
 }
 

diff --git a/src/lib-storage/list/mailbox-list-index.c b/src/lib-storage/list/mailbox-list-index.c
index 183b099c2ecd7cbad9a3597a98607dc2483d927c..ea8500f2fa5cdfe80c04d96283884d02771f6c75 100644 (file)
--- a/src/lib-storage/list/mailbox-list-index.c
+++ b/src/lib-storage/list/mailbox-list-index.c
@@ -59,6 +59,7 @@ int mailbox_list_index_index_open(struct mailbox_list *list)
        const struct mail_storage_settings *set = list->mail_set;
        enum mail_index_open_flags index_flags;
        unsigned int lock_timeout;
+       int ret;
 
        if (ilist->opened)
                return 0;
@@ -98,7 +99,7 @@ int mailbox_list_index_index_open(struct mailbox_list *list)
        mail_index_set_fsync_mode(ilist->index, set->parsed_fsync_mode, 0);
        mail_index_set_lock_method(ilist->index, set->parsed_lock_method,
                                   lock_timeout);
-       if (mail_index_open_or_create(ilist->index, index_flags) < 0) {
+       if ((ret = mail_index_open_or_create(ilist->index, index_flags)) < 0) {
                if (mail_index_move_to_memory(ilist->index) < 0) {
                        /* try opening once more. it should be created
                           directly into memory now, except if it fails with
@@ -110,6 +111,8 @@ int mailbox_list_index_index_open(struct mailbox_list *list)
                        }
                }
        }
+       if (ret == 1)
+               ilist->index_created = TRUE;
        ilist->opened = TRUE;
        return 0;
 }

diff --git a/src/lib-storage/list/mailbox-list-index.h b/src/lib-storage/list/mailbox-list-index.h
index 96e0c62238bb96a9a2e0e43e9534b5facb581797..bf711ba9f2a00f7874eef1dc91ae9e8b4d82922a 100644 (file)
--- a/src/lib-storage/list/mailbox-list-index.h
+++ b/src/lib-storage/list/mailbox-list-index.h
@@ -135,6 +135,7 @@ struct mailbox_list_index {
        bool force_resynced:1;
        bool force_resync_failed:1;
        bool last_refresh_success:1;
+       bool index_created:1;
 };
 
 struct mailbox_list_index_iterate_context {