lib-index: Fix "Extension introduction for unknown id" errors after map is generated

This happens when:
 * View is opened
 * Messages are expunged
 * View is synced with NOEXPUNGES flag
 * A new extension is introduced
 * Index is rotated at least twice
 * View is again synced with NOEXPUNGES flag
 * More changes are done to index with the new extension
 * Once more view is synced with NOEXPUNGES flag

The last sync will see changes with the new extension ID, but the view's map
doesn't know its ID.

diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index 3eec2f1a154678bccac78d808b9c717b1c39c9f3..15079bc47a123b05a94d81d656c9249d5975cf3e 100644 (file)
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -460,6 +460,16 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx,
        if (u->ext_id != (uint32_t)-1 &&
            (!array_is_created(&map->extensions) ||
             u->ext_id >= array_count(&map->extensions))) {
+               /* The extension ID is unknown in this map. */
+               if (map->hdr.log_file_seq == 0) {
+                       /* This map was generated by
+                          view_sync_get_log_lost_changes(). There's no need to
+                          update any extensions, because they won't be used
+                          anyway. Any extension lookups will be accessed via
+                          the latest index map. */
+                       i_assert(map->rec_map != ctx->view->index->map->rec_map);
+                       return 1;
+               }
                if (!mail_index_sync_ext_unknown_complain(ctx, u->ext_id))
                        return -1;
                mail_index_sync_set_corrupted(ctx,

diff --git a/src/lib-index/test-mail-index.c b/src/lib-index/test-mail-index.c
index f54ea39ec9213911c6dfe259c129ce0bd505a0d3..3e2fd02d645d81a8061de8bbdd4fefbfe4c48dc2 100644 (file)
--- a/src/lib-index/test-mail-index.c
+++ b/src/lib-index/test-mail-index.c
@@ -40,10 +40,129 @@ static void test_mail_index_rotate(void)
        test_end();
 }
 
+static void
+test_mail_index_new_extension_rotate_write(struct mail_index *index2,
+                                          uint32_t uid)
+{
+       struct mail_index_view *view2;
+       struct mail_index_transaction *trans;
+       uint32_t hdr_ext_id, rec_ext_id, file_seq, seq, rec_ext = 0x12345678;
+       uoff_t file_offset;
+
+       /* Rotate the index in the index */
+       test_assert(mail_transaction_log_sync_lock(index2->log, "test",
+                                                  &file_seq, &file_offset) == 0);
+       mail_index_write(index2, TRUE, "test");
+       mail_transaction_log_sync_unlock(index2->log, "test");
+
+       /* Write a new extension header to the 2nd index. */
+       hdr_ext_id = mail_index_ext_register(index2, "test",
+                                            sizeof(hdr_ext_id), 0, 0);
+       rec_ext_id = mail_index_ext_register(index2, "test-rec", 0,
+                                            sizeof(uint32_t), sizeof(uint32_t));
+       view2 = mail_index_view_open(index2);
+       trans = mail_index_transaction_begin(view2,
+                       MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+       mail_index_update_header_ext(trans, hdr_ext_id, 0,
+                                    &hdr_ext_id, sizeof(hdr_ext_id));
+       mail_index_append(trans, uid, &seq);
+       mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL);
+       test_assert(mail_index_transaction_commit(&trans) == 0);
+       mail_index_view_close(&view2);
+}
+
+static void test_mail_index_new_extension_sync(struct mail_index_view *view)
+{
+       struct mail_index_view_sync_ctx *sync_ctx;
+       struct mail_index_view_sync_rec sync_rec;
+       bool delayed_expunges;
+
+       test_assert(mail_index_refresh(view->index) == 0);
+       sync_ctx = mail_index_view_sync_begin(view,
+               MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES);
+       test_assert(!mail_index_view_sync_next(sync_ctx, &sync_rec));
+       test_assert(mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) == 0);
+}
+
+static void test_mail_index_new_extension(void)
+{
+       struct mail_index *index, *index2;
+       struct mail_index_view *view, *view2;
+       struct mail_index_transaction *trans;
+       uint32_t seq, rec_ext_id, rec_ext = 0x12345678;
+
+       test_begin("mail index new extension");
+       index = test_mail_index_init();
+       index2 = test_mail_index_open();
+       view = mail_index_view_open(index);
+
+       rec_ext_id = mail_index_ext_register(index, "test-rec", 0,
+                                            sizeof(uint32_t), sizeof(uint32_t));
+
+       /* Save two mails */
+       uint32_t uid_validity = 123456;
+       trans = mail_index_transaction_begin(view,
+                       MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+       mail_index_update_header(trans,
+               offsetof(struct mail_index_header, uid_validity),
+               &uid_validity, sizeof(uid_validity), TRUE);
+       mail_index_append(trans, 1, &seq);
+       mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL);
+       mail_index_append(trans, 2, &seq);
+       mail_index_update_ext(trans, seq, rec_ext_id, &rec_ext, NULL);
+       test_assert(mail_index_transaction_commit(&trans) == 0);
+
+       /* refresh indexes and view */
+       test_assert(mail_index_refresh(index2) == 0);
+       mail_index_view_close(&view);
+       view = mail_index_view_open(index);
+
+       /* Expunge the mail in the 2nd index */
+       view2 = mail_index_view_open(index2);
+       trans = mail_index_transaction_begin(view2,
+                       MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+       mail_index_expunge(trans, 1);
+       test_assert(mail_index_transaction_commit(&trans) == 0);
+       mail_index_view_close(&view2);
+
+       /* Sync the first view without expunges */
+       test_mail_index_new_extension_sync(view);
+
+       for (unsigned int i = 0; i < 3; i++)
+               test_mail_index_new_extension_rotate_write(index2, 3 + i);
+
+       /* Sync the first view. It needs to generate the missing view. */
+       test_expect_error_string("generating missing logs");
+       test_mail_index_new_extension_sync(view);
+       test_expect_no_more_errors();
+       test_assert(mail_index_get_header(view)->messages_count == 5);
+
+       /* Make sure the extensions records are still there.
+          Note that this works, because the extensions are looked up from the
+          newly refreshed index, not the old index. */
+       for (seq = 1; seq <= 5; seq++) {
+               const void *data;
+               bool expunged;
+               mail_index_lookup_ext(view, seq, rec_ext_id, &data, &expunged);
+               test_assert_idx(memcmp(data, &rec_ext, sizeof(rec_ext)) == 0, seq);
+       }
+
+       /* Once more rotate and write using the new extension */
+       test_mail_index_new_extension_rotate_write(index2, 6);
+       /* Make sure the first view understands the new extension by ID */
+       test_mail_index_new_extension_sync(view);
+
+       mail_index_view_close(&view);
+       test_mail_index_deinit(&index);
+       test_mail_index_deinit(&index2);
+       test_end();
+}
+
 int main(void)
 {
        static void (*const test_functions[])(void) = {
                test_mail_index_rotate,
+               test_mail_index_new_extension,
                NULL
        };
        return test_run(test_functions);