From: Josef 'Jeff' Sipek Date: Thu, 24 Jun 2021 16:58:58 +0000 (-0400) Subject: virtual: Expunge old emails if backend box guid changed X-Git-Tag: 2.3.17~215 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=710346bcb884b464c8ed128870fdc1999c13dfd3;p=thirdparty%2Fdovecot%2Fcore.git virtual: Expunge old emails if backend box guid changed This introduces a new extensible "ext2" header to make it easier to add new fields in the future. It also allows keeping backwards/forwards compatibility with the old code, so the virtual index isn't rebuilt on upgrades or downgrades. --- diff --git a/src/plugins/virtual/virtual-storage.c b/src/plugins/virtual/virtual-storage.c index 6c7261a87a..debc6b4783 100644 --- a/src/plugins/virtual/virtual-storage.c +++ b/src/plugins/virtual/virtual-storage.c @@ -275,6 +275,7 @@ virtual_mailbox_alloc(struct mail_storage *_storage, struct mailbox_list *list, mbox->storage = storage; mbox->virtual_ext_id = (uint32_t)-1; + mbox->virtual_ext2_id = (uint32_t)-1; mbox->virtual_guid_ext_id = (uint32_t)-1; return &mbox->box; } @@ -508,6 +509,8 @@ static int virtual_mailbox_open(struct mailbox *box) mail_index_ext_register(mbox->box.index, "virtual", 0, sizeof(struct virtual_mail_index_record), sizeof(uint32_t)); + mbox->virtual_ext2_id = + mail_index_ext_register(mbox->box.index, "virtual2", 0, 0, 0); mbox->virtual_guid_ext_id = mail_index_ext_register(mbox->box.index, "virtual-guid", GUID_128_SIZE, diff --git a/src/plugins/virtual/virtual-storage.h b/src/plugins/virtual/virtual-storage.h index 6307bcd92e..1f34e3930c 100644 --- a/src/plugins/virtual/virtual-storage.h +++ b/src/plugins/virtual/virtual-storage.h @@ -13,6 +13,8 @@ #define VIRTUAL_CONTEXT_REQUIRE(obj) \ MODULE_CONTEXT_REQUIRE(obj, virtual_storage_module) +#define VIRTUAL_MAIL_INDEX_EXT2_HEADER_VERSION 1 + struct virtual_save_context; struct virtual_mail_index_header { @@ -42,6 +44,26 @@ struct virtual_mail_index_mailbox_record { uint64_t highest_modseq; }; +struct virtual_mail_index_ext2_header { + /* Version compatibility number: + VIRTUAL_MAIL_INDEX_EXT2_HEADER_VERSION */ + uint8_t version; + /* Set to sizeof(struct virtual_mail_index_mailbox_ext2_record) when + writing. */ + uint8_t ext_record_size; + /* Set to sizeof(struct virtual_mail_index_ext2_header) when writing. */ + uint16_t hdr_size; + /* Must be the same as virtual_mail_index_header.change_counter. + If not, it means the header was modified by an older Dovecot version + and this extension header should be ignored/rewritten. */ + uint32_t change_counter; +}; + +struct virtual_mail_index_mailbox_ext2_record { + /* Synced GUID value */ + uint8_t guid[16]; +}; + struct virtual_mail_index_record { uint32_t mailbox_id; uint32_t real_uid; @@ -78,6 +100,7 @@ struct virtual_backend_box { uint32_t sync_uid_validity; uint32_t sync_next_uid; uint64_t sync_highest_modseq; + guid_128_t sync_guid; /* this value is either 0 or same as sync_highest_modseq. it's kept 0 when there are pending removes that have yet to be expunged */ uint64_t ondisk_highest_modseq; @@ -126,6 +149,7 @@ struct virtual_mailbox { struct virtual_storage *storage; uint32_t virtual_ext_id; + uint32_t virtual_ext2_id; uint32_t virtual_guid_ext_id; uint32_t prev_uid_validity; diff --git a/src/plugins/virtual/virtual-sync.c b/src/plugins/virtual/virtual-sync.c index ab321dc3dc..9b0340d588 100644 --- a/src/plugins/virtual/virtual-sync.c +++ b/src/plugins/virtual/virtual-sync.c @@ -146,6 +146,70 @@ virtual_sync_get_backend_box(struct virtual_mailbox *mbox, const char *name, return -1; } +static bool +virtual_mailbox_ext2_header_read(struct virtual_mailbox *mbox, + struct mail_index_view *view, + const struct virtual_mail_index_header *ext_hdr) +{ + const char *box_path = mailbox_get_path(&mbox->box); + const struct virtual_mail_index_ext2_header *ext2_hdr; + const struct virtual_mail_index_mailbox_ext2_record *ext2_rec; + const void *ext2_data; + size_t ext2_size; + struct virtual_backend_box *bbox; + + mail_index_get_header_ext(view, mbox->virtual_ext2_id, + &ext2_data, &ext2_size); + ext2_hdr = ext2_data; + if (ext2_size == 0) { + /* ext2 is missing - silently add it */ + return FALSE; + } + if (ext2_size < sizeof(*ext2_hdr)) { + i_error("virtual index %s: Invalid ext2 header size: %zu", + box_path, ext2_size); + return FALSE; + } + if (ext2_hdr->hdr_size > ext2_size) { + i_error("virtual index %s: ext2 header size too large: %u > %zu", + box_path, ext2_hdr->hdr_size, ext2_size); + return FALSE; + } + if (ext2_hdr->ext_record_size < sizeof(*ext2_rec)) { + i_error("virtual index %s: Invalid ext2 record size: %u", + box_path, ext2_hdr->ext_record_size); + return FALSE; + } + + if (ext_hdr->change_counter != ext2_hdr->change_counter) { + i_warning("virtual index %s: " + "Extension header change_counter mismatch (%u != %u) - " + "Index was modified by an older version?", + box_path, ext_hdr->change_counter, + ext2_hdr->change_counter); + return FALSE; + } + size_t mailboxes_size = ext2_size - ext2_hdr->hdr_size; + if (mailboxes_size % ext2_hdr->ext_record_size != 0 || + mailboxes_size / ext2_hdr->ext_record_size != ext_hdr->mailbox_count) { + i_error("virtual index %s: Invalid ext2 size: " + "hdr_size=%u record_size=%u total_size=%zu mailbox_count=%u", + box_path, ext2_hdr->hdr_size, ext2_hdr->ext_record_size, + ext2_size, ext_hdr->mailbox_count); + return FALSE; + } + + ext2_rec = CONST_PTR_OFFSET(ext2_data, ext2_hdr->hdr_size); + array_foreach_elem(&mbox->backend_boxes, bbox) { + if (bbox->sync_mailbox_idx1 == 0) + continue; + + guid_128_copy(bbox->sync_guid, + ext2_rec[bbox->sync_mailbox_idx1-1].guid); + } + return TRUE; +} + int virtual_mailbox_ext_header_read(struct virtual_mailbox *mbox, struct mail_index_view *view, bool *broken_r) @@ -258,6 +322,11 @@ int virtual_mailbox_ext_header_read(struct virtual_mailbox *mbox, mbox->ext_header_rewrite = TRUE; ret = 0; } + if (!*broken_r && ext_mailbox_count > 0) { + if (!virtual_mailbox_ext2_header_read(mbox, view, ext_hdr)) + mbox->ext_header_rewrite = TRUE; + } + mbox->highest_mailbox_id = ext_hdr == NULL ? 0 : ext_hdr->highest_mailbox_id; /* do not mark it initialized if it's broken */ @@ -282,8 +351,9 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) { struct virtual_mail_index_header ext_hdr; struct virtual_mail_index_mailbox_record mailbox; + struct virtual_mail_index_mailbox_ext2_record ext2_rec; struct virtual_backend_box **bboxes; - buffer_t *buf; + buffer_t *buf, *buf2; const void *ext_data; size_t ext_size; unsigned int i, mailbox_pos, name_pos, count; @@ -294,6 +364,7 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) i_zero(&ext_hdr); i_zero(&mailbox); + i_zero(&ext2_rec); ext_hdr.change_counter = ++ctx->mbox->prev_change_counter; ext_hdr.mailbox_count = count; @@ -303,6 +374,16 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) buf = buffer_create_dynamic(default_pool, name_pos + 256); buffer_append(buf, &ext_hdr, sizeof(ext_hdr)); + struct virtual_mail_index_ext2_header ext2_hdr = { + .version = VIRTUAL_MAIL_INDEX_EXT2_HEADER_VERSION, + .ext_record_size = sizeof(struct virtual_mail_index_mailbox_ext2_record), + .hdr_size = sizeof(struct virtual_mail_index_ext2_header), + .change_counter = ext_hdr.change_counter, + }; + buf2 = buffer_create_dynamic(default_pool, sizeof(ext2_hdr) + + sizeof(ext2_rec) * count); + buffer_append(buf2, &ext2_hdr, sizeof(ext2_hdr)); + for (i = 0; i < count; i++) { i_assert(i == 0 || bboxes[i]->mailbox_id > bboxes[i-1]->mailbox_id); @@ -316,6 +397,9 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) buffer_write(buf, mailbox_pos, &mailbox, sizeof(mailbox)); buffer_write(buf, name_pos, bboxes[i]->name, mailbox.name_len); + guid_128_copy(ext2_rec.guid, bboxes[i]->sync_guid); + buffer_append(buf2, &ext2_rec, sizeof(ext2_rec)); + mailbox_pos += sizeof(mailbox); name_pos += mailbox.name_len; @@ -329,6 +413,7 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) } i_assert(buf->used == name_pos); + /* update base extension */ mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext_id, &ext_data, &ext_size); if (ext_size < name_pos) { @@ -339,7 +424,18 @@ static void virtual_sync_ext_header_rewrite(struct virtual_sync_context *ctx) } mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id, 0, buf->data, name_pos); + + /* update ext2 */ + mail_index_get_header_ext(ctx->sync_view, ctx->mbox->virtual_ext2_id, + &ext_data, &ext_size); + if (ext_size != buf2->used) { + mail_index_ext_resize(ctx->trans, ctx->mbox->virtual_ext2_id, + buf2->used, 0, 0); + } + mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext2_id, + 0, buf2->data, buf2->used); buffer_free(&buf); + buffer_free(&buf2); } static void virtual_sync_ext_header_update(struct virtual_sync_context *ctx) @@ -354,6 +450,9 @@ static void virtual_sync_ext_header_update(struct virtual_sync_context *ctx) mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id, offsetof(struct virtual_mail_index_header, change_counter), &ext_hdr.change_counter, sizeof(ext_hdr.change_counter)); + mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext2_id, + offsetof(struct virtual_mail_index_ext2_header, change_counter), + &ext_hdr.change_counter, sizeof(ext_hdr.change_counter)); } static int virtual_sync_index_rec(struct virtual_sync_context *ctx, @@ -1119,8 +1218,10 @@ static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx, uid_validity); struct mailbox_status status; struct virtual_mail_index_mailbox_record mailbox; - unsigned int mailbox_offset; + struct virtual_mail_index_mailbox_ext2_record ext2; + unsigned int mailbox_offset, ext2_offset; uint64_t wanted_ondisk_highest_modseq; + struct mailbox_metadata metadata; mailbox_get_open_status(bbox->box, STATUS_UIDVALIDITY | STATUS_HIGHESTMODSEQ, &status); @@ -1128,9 +1229,15 @@ static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx, array_count(&bbox->sync_pending_removes) > 0 ? 0 : status.highest_modseq; + /* The caller already did this successfully, so we simply assert */ + if (mailbox_get_metadata(bbox->box, MAILBOX_METADATA_GUID, + &metadata) < 0) + i_unreached(); + if (bbox->sync_uid_validity == status.uidvalidity && bbox->sync_next_uid == status.uidnext && bbox->sync_highest_modseq == status.highest_modseq && + guid_128_equals(bbox->sync_guid, metadata.guid) && bbox->ondisk_highest_modseq == wanted_ondisk_highest_modseq) return; @@ -1139,6 +1246,7 @@ static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx, bbox->sync_highest_modseq = status.highest_modseq; bbox->ondisk_highest_modseq = wanted_ondisk_highest_modseq; bbox->sync_next_uid = status.uidnext; + guid_128_copy(bbox->sync_guid, metadata.guid); if (ctx->mbox->ext_header_rewrite) { /* we'll rewrite the entire header later */ @@ -1150,13 +1258,20 @@ static void virtual_sync_backend_ext_header(struct virtual_sync_context *ctx, mailbox.highest_modseq = bbox->ondisk_highest_modseq; mailbox.next_uid = bbox->sync_next_uid; + i_zero(&ext2); + guid_128_copy(ext2.guid, bbox->sync_guid); + i_assert(bbox->sync_mailbox_idx1 > 0); mailbox_offset = sizeof(struct virtual_mail_index_header) + (bbox->sync_mailbox_idx1-1) * sizeof(mailbox); + ext2_offset = sizeof(struct virtual_mail_index_ext2_header) + + (bbox->sync_mailbox_idx1-1) * sizeof(ext2); mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext_id, mailbox_offset + uidval_pos, CONST_PTR_OFFSET(&mailbox, uidval_pos), sizeof(mailbox) - uidval_pos); + mail_index_update_header_ext(ctx->trans, ctx->mbox->virtual_ext2_id, + ext2_offset, &ext2, sizeof(ext2)); ctx->ext_header_changed = TRUE; } @@ -1217,6 +1332,8 @@ static int virtual_sync_backend_box(struct virtual_sync_context *ctx, MAILBOX_SYNC_FLAG_FAST); if (bbox->search_result == NULL) { + struct mailbox_metadata metadata; + /* a) first sync in this process. b) we had auto-closed this backend mailbox. @@ -1233,18 +1350,26 @@ static int virtual_sync_backend_box(struct virtual_sync_context *ctx, bbox->open_failed = FALSE; } - if (mailbox_get_status(bbox->box, STATUS_UIDVALIDITY | + if ((mailbox_get_status(bbox->box, STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ, - &status) < 0) { + &status) < 0) || + (mailbox_get_metadata(bbox->box, MAILBOX_METADATA_GUID, + &metadata) < 0)) { if (mailbox_get_last_mail_error(bbox->box) != MAIL_ERROR_NOTFOUND) return -1; /* mailbox was deleted */ virtual_sync_backend_box_deleted(ctx, bbox); return 0; } + if (guid_128_is_empty(bbox->sync_guid)) { + /* upgrading from old virtual index */ + guid_128_copy(bbox->sync_guid, metadata.guid); + ctx->mbox->ext_header_rewrite = TRUE; + } if (status.uidvalidity == bbox->sync_uid_validity && status.uidnext == bbox->sync_next_uid && - status.highest_modseq == bbox->sync_highest_modseq) { + status.highest_modseq == bbox->sync_highest_modseq && + guid_128_equals(metadata.guid, bbox->sync_guid)) { /* mailbox hasn't changed since we last opened it, skip it for now. @@ -1261,9 +1386,10 @@ static int virtual_sync_backend_box(struct virtual_sync_context *ctx, } virtual_backend_box_sync_mail_set(bbox); - if (status.uidvalidity != bbox->sync_uid_validity) { - /* UID validity changed since last sync (or this is - the first sync), do a full search */ + if ((status.uidvalidity != bbox->sync_uid_validity) || + !guid_128_equals(metadata.guid, bbox->sync_guid)) { + /* UID validity or GUID changed since last sync (or + this is the first sync), do a full search */ bbox->first_sync = TRUE; ret = virtual_sync_backend_box_init(bbox); } else {