From: Timo Sirainen Date: Tue, 27 Apr 2021 23:01:14 +0000 (+0300) Subject: lib-index: Fix "Extension introduction for unknown id" errors after map is generated X-Git-Tag: 2.3.17~214 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ddf224808ef416095cfece8bd34d15e59a8ca339;p=thirdparty%2Fdovecot%2Fcore.git lib-index: Fix "Extension introduction for unknown id" errors after map is generated This happens when: * View is opened * Messages are expunged * View is synced with NOEXPUNGES flag * A new extension is introduced * Index is rotated at least twice * View is again synced with NOEXPUNGES flag * More changes are done to index with the new extension * Once more view is synced with NOEXPUNGES flag The last sync will see changes with the new extension ID, but the view's map doesn't know its ID. --- diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c index 3eec2f1a15..15079bc47a 100644 --- a/src/lib-index/mail-index-sync-ext.c +++ b/src/lib-index/mail-index-sync-ext.c @@ -460,6 +460,16 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, if (u->ext_id != (uint32_t)-1 && (!array_is_created(&map->extensions) || u->ext_id >= array_count(&map->extensions))) { + /* The extension ID is unknown in this map. */ + if (map->hdr.log_file_seq == 0) { + /* This map was generated by + view_sync_get_log_lost_changes(). There's no need to + update any extensions, because they won't be used + anyway. Any extension lookups will be accessed via + the latest index map. */ + i_assert(map->rec_map != ctx->view->index->map->rec_map); + return 1; + } if (!mail_index_sync_ext_unknown_complain(ctx, u->ext_id)) return -1; mail_index_sync_set_corrupted(ctx, diff --git a/src/lib-index/test-mail-index.c b/src/lib-index/test-mail-index.c index f54ea39ec9..3e2fd02d64 100644 --- a/src/lib-index/test-mail-index.c +++ b/src/lib-index/test-mail-index.c @@ -40,10 +40,129 @@ static void test_mail_index_rotate(void) test_end(); } +static void +test_mail_index_new_extension_rotate_write(struct mail_index *index2, + uint32_t uid) +{ + struct mail_index_view *view2; + struct mail_index_transaction *trans; + uint32_t hdr_ext_id, rec_ext_id, file_seq, seq, rec_ext = 0x12345678; + uoff_t file_offset; + + /* Rotate the index in the index */ + test_assert(mail_transaction_log_sync_lock(index2->log, "test", + &file_seq, &file_offset) == 0); + mail_index_write(index2, TRUE, "test"); + mail_transaction_log_sync_unlock(index2->log, "test"); + + /* Write a new extension header to the 2nd index. */ + hdr_ext_id = mail_index_ext_register(index2, "test", + sizeof(hdr_ext_id), 0, 0); + rec_ext_id = mail_index_ext_register(index2, "test-rec", 0, + sizeof(uint32_t), sizeof(uint32_t)); + view2 = mail_index_view_open(index2); + trans = mail_index_transaction_begin(view2, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header_ext(trans, hdr_ext_id, 0, + &hdr_ext_id, sizeof(hdr_ext_id)); + mail_index_append(trans, uid, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view2); +} + +static void test_mail_index_new_extension_sync(struct mail_index_view *view) +{ + struct mail_index_view_sync_ctx *sync_ctx; + struct mail_index_view_sync_rec sync_rec; + bool delayed_expunges; + + test_assert(mail_index_refresh(view->index) == 0); + sync_ctx = mail_index_view_sync_begin(view, + MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES); + test_assert(!mail_index_view_sync_next(sync_ctx, &sync_rec)); + test_assert(mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) == 0); +} + +static void test_mail_index_new_extension(void) +{ + struct mail_index *index, *index2; + struct mail_index_view *view, *view2; + struct mail_index_transaction *trans; + uint32_t seq, rec_ext_id, rec_ext = 0x12345678; + + test_begin("mail index new extension"); + index = test_mail_index_init(); + index2 = test_mail_index_open(); + view = mail_index_view_open(index); + + rec_ext_id = mail_index_ext_register(index, "test-rec", 0, + sizeof(uint32_t), sizeof(uint32_t)); + + /* Save two mails */ + uint32_t uid_validity = 123456; + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_append(trans, 1, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + mail_index_append(trans, 2, &seq); + mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL); + test_assert(mail_index_transaction_commit(&trans) == 0); + + /* refresh indexes and view */ + test_assert(mail_index_refresh(index2) == 0); + mail_index_view_close(&view); + view = mail_index_view_open(index); + + /* Expunge the mail in the 2nd index */ + view2 = mail_index_view_open(index2); + trans = mail_index_transaction_begin(view2, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mail_index_expunge(trans, 1); + test_assert(mail_index_transaction_commit(&trans) == 0); + mail_index_view_close(&view2); + + /* Sync the first view without expunges */ + test_mail_index_new_extension_sync(view); + + for (unsigned int i = 0; i < 3; i++) + test_mail_index_new_extension_rotate_write(index2, 3 + i); + + /* Sync the first view. It needs to generate the missing view. */ + test_expect_error_string("generating missing logs"); + test_mail_index_new_extension_sync(view); + test_expect_no_more_errors(); + test_assert(mail_index_get_header(view)->messages_count == 5); + + /* Make sure the extensions records are still there. + Note that this works, because the extensions are looked up from the + newly refreshed index, not the old index. */ + for (seq = 1; seq <= 5; seq++) { + const void *data; + bool expunged; + mail_index_lookup_ext(view, seq, rec_ext_id, &data, &expunged); + test_assert_idx(memcmp(data, &rec_ext, sizeof(rec_ext)) == 0, seq); + } + + /* Once more rotate and write using the new extension */ + test_mail_index_new_extension_rotate_write(index2, 6); + /* Make sure the first view understands the new extension by ID */ + test_mail_index_new_extension_sync(view); + + mail_index_view_close(&view); + test_mail_index_deinit(&index); + test_mail_index_deinit(&index2); + test_end(); +} + int main(void) { static void (*const test_functions[])(void) = { test_mail_index_rotate, + test_mail_index_new_extension, NULL }; return test_run(test_functions);