From: Markus Valentin Date: Wed, 2 Aug 2023 15:07:34 +0000 (+0200) Subject: lib-storage: Rebuild list-index using MAILBOX_LIST_ITER_FORCE_RESYNC X-Git-Tag: 2.4.0~1934 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b7a747ceca87e4e929e9b7c1ae29017d05c0da1e;p=thirdparty%2Fdovecot%2Fcore.git lib-storage: Rebuild list-index using MAILBOX_LIST_ITER_FORCE_RESYNC In case the list index file was just created use MAILBOX_LIST_ITER_FORCE_RESYNC to rediscover mailboxes that might have been hidden. --- diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h index 55055c6de0..d922f01d6e 100644 --- a/src/lib-index/mail-index-private.h +++ b/src/lib-index/mail-index-private.h @@ -318,6 +318,9 @@ struct mail_index { /* Index has been fsck'd, but mail_index_reset_fscked() hasn't been called yet. */ bool fscked:1; + /* mail_index_open() has created new index files */ + bool initial_created:1; + }; extern struct mail_index_module_register mail_index_module_register; diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c index d2dd40c0e6..e6be908bbf 100644 --- a/src/lib-index/mail-index.c +++ b/src/lib-index/mail-index.c @@ -610,6 +610,7 @@ static int mail_index_open_files(struct mail_index *index, index->map->hdr.indexid = index->indexid; } index->initial_create = FALSE; + index->initial_created = TRUE; } if (ret >= 0) { ret = index->map != NULL ? 1 : mail_index_try_open(index); @@ -725,7 +726,7 @@ int mail_index_open_or_create(struct mail_index *index, flags |= MAIL_INDEX_OPEN_FLAG_CREATE; ret = mail_index_open(index, flags); i_assert(ret != 0); - return ret < 0 ? -1 : 0; + return ret < 0 ? -1 : (index->initial_created ? 1 : 0); } void mail_index_close_file(struct mail_index *index) diff --git a/src/lib-index/test-mail-cache-common.c b/src/lib-index/test-mail-cache-common.c index b6620b728d..55571b76f0 100644 --- a/src/lib-index/test-mail-cache-common.c +++ b/src/lib-index/test-mail-cache-common.c @@ -95,7 +95,7 @@ void test_mail_cache_purge(void) { struct test_mail_cache_ctx ctx; - test_mail_cache_init(test_mail_index_open(), &ctx); + test_mail_cache_init(test_mail_index_open(FALSE), &ctx); test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); test_mail_cache_deinit(&ctx); } diff --git a/src/lib-index/test-mail-cache-fields.c b/src/lib-index/test-mail-cache-fields.c index 1b76dc5c81..1a99c774fd 100644 --- a/src/lib-index/test-mail-cache-fields.c +++ b/src/lib-index/test-mail-cache-fields.c @@ -22,7 +22,7 @@ static void test_mail_cache_fields_read_write(void) test_begin("mail cache fields read-write"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_cache_register_fields(ctx.cache, &cache_field, 1, unsafe_data_stack_pool); test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); diff --git a/src/lib-index/test-mail-cache-purge.c b/src/lib-index/test-mail-cache-purge.c index ee0b33803a..24dd3bb74d 100644 --- a/src/lib-index/test-mail-cache-purge.c +++ b/src/lib-index/test-mail-cache-purge.c @@ -18,7 +18,7 @@ static void test_mail_cache_read_during_purge2(void) i_set_failure_prefix("index2: "); /* read from cache via 2nd index */ - test_mail_cache_init(test_mail_index_open(), &ctx); + test_mail_cache_init(test_mail_index_open(FALSE), &ctx); cache_view = mail_cache_view_open(ctx.cache, ctx.view); test_assert(mail_cache_lookup_field(cache_view, str, 1, @@ -36,7 +36,7 @@ static void test_mail_cache_read_during_purge(void) int status; test_begin("mail cache read during purge"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); /* lock the index for cache purge */ @@ -88,7 +88,7 @@ static void test_mail_cache_write_during_purge2(void) i_set_failure_prefix("index2: "); /* add to cache via 2nd index */ - test_mail_cache_init(test_mail_index_open(), &ctx); + test_mail_cache_init(test_mail_index_open(FALSE), &ctx); test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar2"); test_mail_cache_deinit(&ctx); } @@ -103,7 +103,7 @@ static void test_mail_cache_write_during_purge(void) int status; test_begin("mail cache write during purge"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); /* lock the index for cache purge */ @@ -171,7 +171,7 @@ static void test_mail_cache_purge_while_cache_locked(void) int status; test_begin("mail cache purge while cache locked"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); /* lock the cache */ @@ -250,7 +250,7 @@ static void test_mail_cache_purge_during_write_n(unsigned int num_mails, struct mail_index_transaction *trans; uint32_t seq; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); /* Add mails */ @@ -379,7 +379,7 @@ static void test_mail_cache_delete_too_large_int(bool exceed_on_first_write) struct test_mail_cache_ctx ctx; struct stat st; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo2"); @@ -441,7 +441,7 @@ static void test_mail_cache_purge_too_large_int(bool exceed_size) struct test_mail_cache_ctx ctx; struct stat st; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); /* add two mails with some cache field and expunge the first mail */ @@ -503,7 +503,7 @@ static void test_mail_cache_unexpectedly_lost_int(bool read_first) struct test_mail_cache_ctx ctx; struct mail_cache_view *cache_view; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); test_mail_cache_purge(); @@ -566,7 +566,7 @@ static void test_mail_cache_resetid_mismatch_int(bool read_first) struct mail_cache_view *cache_view; const char *temp_cache_path; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); /* make a copy of the first cache file */ @@ -695,7 +695,7 @@ static void test_mail_cache_purge_field_changes_int(enum test_drop drop) struct mail_index_transaction *trans; unsigned int i; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); /* add two mails with all of the cache fields */ @@ -851,7 +851,7 @@ static void test_mail_cache_purge_already_done(void) struct test_mail_cache_ctx ctx; test_begin("mail cache purge already done"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1"); test_mail_cache_purge(); @@ -886,7 +886,7 @@ static void test_mail_cache_purge_bitmask(void) struct mail_cache_view *cache_view; test_begin("mail cache purge bitmask"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); ioloop_time = 1000000; test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); @@ -934,7 +934,7 @@ test_mail_cache_update_need_purge_continued_records_int(bool big_min_size) struct test_mail_cache_ctx ctx; uint32_t seq; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); for (seq = 1; seq <= 100; seq++) { @@ -989,7 +989,7 @@ test_mail_cache_update_need_purge_deleted_records_int(bool big_min_size) struct test_mail_cache_ctx ctx; uint32_t seq; - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); for (seq = 1; seq <= 100; seq++) { diff --git a/src/lib-index/test-mail-cache.c b/src/lib-index/test-mail-cache.c index 9c279097ff..b0c97b8108 100644 --- a/src/lib-index/test-mail-cache.c +++ b/src/lib-index/test-mail-cache.c @@ -106,7 +106,7 @@ static void test_mail_cache_fields(void) string_t *str = t_str_new(16); test_begin("mail cache uncommitted lookups"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_cache_register_fields(ctx.cache, cache_fields, N_ELEMENTS(cache_fields), unsafe_data_stack_pool); @@ -253,7 +253,7 @@ static void test_mail_cache_record_max_size_int(unsigned int field3_size) struct mail_cache_transaction_ctx *cache_trans; string_t *str = t_str_new(16); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); /* Add the first cache field. In a chain of cache records each one @@ -358,7 +358,7 @@ static void test_mail_cache_record_max_size4(void) string_t *str = t_str_new(16); test_begin("mail cache record max size (4)"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); @@ -400,7 +400,7 @@ static void test_mail_cache_add_decisions(void) test_begin("mail cache add decisions"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields)); mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT, unsafe_data_stack_pool); @@ -471,7 +471,7 @@ static void test_mail_cache_lookup_decisions_int(bool header_lookups) unsigned int i; string_t *str = t_str_new(16); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); /* create the initial mails and the cache file */ test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); @@ -657,7 +657,7 @@ static void test_mail_cache_in_memory(void) test_begin("mail cache add in-memory"); index = mail_index_alloc(NULL, NULL, "(in-memory)"); - test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 1); test_mail_cache_init(index, &ctx); mail_index_set_optimization_settings(ctx.index, &optimization_set); cache_view = mail_cache_view_open(ctx.cache, ctx.view); @@ -726,7 +726,7 @@ static void test_mail_cache_size_corruption(void) test_begin("mail cache size corruption"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "12345678"); cache_view = mail_cache_view_open(ctx.cache, ctx.view); @@ -768,7 +768,7 @@ static void test_mail_cache_duplicate_fields(void) struct mail_cache_transaction_ctx *cache_trans; test_begin("mail cache duplicate fields"); - test_mail_cache_init(test_mail_index_init(), &ctx); + test_mail_cache_init(test_mail_index_init(TRUE), &ctx); mail_cache_register_fields(ctx.cache, cache_fields, N_ELEMENTS(cache_fields), unsafe_data_stack_pool); diff --git a/src/lib-index/test-mail-index-modseq.c b/src/lib-index/test-mail-index-modseq.c index 4a2d52468b..c900966254 100644 --- a/src/lib-index/test-mail-index-modseq.c +++ b/src/lib-index/test-mail-index-modseq.c @@ -27,7 +27,7 @@ static void test_mail_index_modseq_get_next_log_offset(void) uint32_t seq, uid; test_begin("mail_transaction_log_file_get_modseq_next_offset()"); - index = test_mail_index_init(); + index = test_mail_index_init(TRUE); view = mail_index_view_open(index); mail_index_modseq_enable(index); diff --git a/src/lib-index/test-mail-index.c b/src/lib-index/test-mail-index.c index 3e2fd02d64..84d2b43f51 100644 --- a/src/lib-index/test-mail-index.c +++ b/src/lib-index/test-mail-index.c @@ -15,8 +15,8 @@ static void test_mail_index_rotate(void) const char *reason; test_begin("mail index rotate"); - index = test_mail_index_init(); - index2 = test_mail_index_open(); + index = test_mail_index_init(TRUE); + index2 = test_mail_index_open(FALSE); view = mail_index_view_open(index); /* First rotation of the index. The view will point to the old index. */ @@ -92,8 +92,8 @@ static void test_mail_index_new_extension(void) uint32_t seq, rec_ext_id, rec_ext = 0x12345678; test_begin("mail index new extension"); - index = test_mail_index_init(); - index2 = test_mail_index_open(); + index = test_mail_index_init(TRUE); + index2 = test_mail_index_open(FALSE); view = mail_index_view_open(index); rec_ext_id = mail_index_ext_register(index, "test-rec", 0, diff --git a/src/lib-index/test-mail-index.h b/src/lib-index/test-mail-index.h index 75b343f92b..e8251e974d 100644 --- a/src/lib-index/test-mail-index.h +++ b/src/lib-index/test-mail-index.h @@ -7,16 +7,17 @@ #define TESTDIR_NAME ".dovecot.test" -static inline struct mail_index *test_mail_index_open(void) +static inline struct mail_index *test_mail_index_open(bool idx_initial_created) { struct mail_index *index; index = mail_index_alloc(NULL, TESTDIR_NAME, "test.dovecot.index"); - test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == + (idx_initial_created ? 1 : 0)); return index; } -static inline struct mail_index *test_mail_index_init(void) +static inline struct mail_index *test_mail_index_init(bool idx_initial_created) { const char *error; @@ -26,7 +27,7 @@ static inline struct mail_index *test_mail_index_init(void) ioloop_time = 1; - return test_mail_index_open(); + return test_mail_index_open(idx_initial_created); } static inline void test_mail_index_close(struct mail_index **index) diff --git a/src/lib-index/test-mail-transaction-log-file.c b/src/lib-index/test-mail-transaction-log-file.c index 6f591cef69..d740947e5a 100644 --- a/src/lib-index/test-mail-transaction-log-file.c +++ b/src/lib-index/test-mail-transaction-log-file.c @@ -244,7 +244,7 @@ static void test_mail_transaction_update_modseq(void) static struct mail_index *test_mail_index_open(void) { struct mail_index *index = mail_index_alloc(NULL, NULL, "test.dovecot.index"); - test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0); + test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 1); struct mail_index_view *view = mail_index_view_open(index); struct mail_index_transaction *trans = diff --git a/src/lib-storage/list/mailbox-list-index-sync.c b/src/lib-storage/list/mailbox-list-index-sync.c index 87411d6cb5..06afe6529d 100644 --- a/src/lib-storage/list/mailbox-list-index-sync.c +++ b/src/lib-storage/list/mailbox-list-index-sync.c @@ -295,6 +295,7 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx) struct mailbox_list_iterate_context *iter; const struct mailbox_info *info; enum mailbox_list_index_flags flags; + enum mailbox_list_iter_flags iter_flags; const char *patterns[2]; struct mailbox_list_index_node *node; uint32_t seq; @@ -303,14 +304,18 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx) /* clear EXISTS-flags, so after sync we know what can be expunged */ mailbox_list_index_node_clear_exists(sync_ctx->ilist->mailbox_tree); + iter_flags = MAILBOX_LIST_ITER_RAW_LIST | + MAILBOX_LIST_ITER_NO_AUTO_BOXES; + if (sync_ctx->ilist->index_created) + iter_flags |= MAILBOX_LIST_ITER_FORCE_RESYNC; + /* don't include autocreated mailboxes in index until they're actually created. this index may be used by multiple users, so we also want to ignore ACLs here. */ patterns[0] = "*"; patterns[1] = NULL; iter = sync_ctx->ilist->module_ctx.super. iter_init(sync_ctx->list, patterns, - MAILBOX_LIST_ITER_RAW_LIST | - MAILBOX_LIST_ITER_NO_AUTO_BOXES); + iter_flags); sync_ctx->syncing_list = TRUE; while ((info = sync_ctx->ilist->module_ctx.super.iter_next(iter)) != NULL) T_BEGIN { @@ -344,6 +349,7 @@ mailbox_list_index_sync_list(struct mailbox_list_index_sync_context *sync_ctx) /* successfully listed everything, expunge any unseen mailboxes */ sync_expunge_nonexistent(sync_ctx, sync_ctx->ilist->mailbox_tree); + sync_ctx->ilist->index_created = FALSE; return 0; } diff --git a/src/lib-storage/list/mailbox-list-index.c b/src/lib-storage/list/mailbox-list-index.c index 183b099c2e..ea8500f2fa 100644 --- a/src/lib-storage/list/mailbox-list-index.c +++ b/src/lib-storage/list/mailbox-list-index.c @@ -59,6 +59,7 @@ int mailbox_list_index_index_open(struct mailbox_list *list) const struct mail_storage_settings *set = list->mail_set; enum mail_index_open_flags index_flags; unsigned int lock_timeout; + int ret; if (ilist->opened) return 0; @@ -98,7 +99,7 @@ int mailbox_list_index_index_open(struct mailbox_list *list) mail_index_set_fsync_mode(ilist->index, set->parsed_fsync_mode, 0); mail_index_set_lock_method(ilist->index, set->parsed_lock_method, lock_timeout); - if (mail_index_open_or_create(ilist->index, index_flags) < 0) { + if ((ret = mail_index_open_or_create(ilist->index, index_flags)) < 0) { if (mail_index_move_to_memory(ilist->index) < 0) { /* try opening once more. it should be created directly into memory now, except if it fails with @@ -110,6 +111,8 @@ int mailbox_list_index_index_open(struct mailbox_list *list) } } } + if (ret == 1) + ilist->index_created = TRUE; ilist->opened = TRUE; return 0; } diff --git a/src/lib-storage/list/mailbox-list-index.h b/src/lib-storage/list/mailbox-list-index.h index 96e0c62238..bf711ba9f2 100644 --- a/src/lib-storage/list/mailbox-list-index.h +++ b/src/lib-storage/list/mailbox-list-index.h @@ -135,6 +135,7 @@ struct mailbox_list_index { bool force_resynced:1; bool force_resync_failed:1; bool last_refresh_success:1; + bool index_created:1; }; struct mailbox_list_index_iterate_context {