From: Timo Sirainen Date: Tue, 29 Oct 2013 19:08:07 +0000 (+0200) Subject: mdbox: Added "mdbox_deleted" storage, which can be used to access messages with refco... X-Git-Tag: 2.2.7~34 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b90c23a9862b91594959b918b035d73f7bc0b265;p=thirdparty%2Fdovecot%2Fcore.git mdbox: Added "mdbox_deleted" storage, which can be used to access messages with refcount=0 For example: doveadm import mdbox_deleted:~/mdbox "" mailbox inbox subject oops --- diff --git a/src/lib-storage/index/dbox-multi/Makefile.am b/src/lib-storage/index/dbox-multi/Makefile.am index 8926c854cd..810edd21d9 100644 --- a/src/lib-storage/index/dbox-multi/Makefile.am +++ b/src/lib-storage/index/dbox-multi/Makefile.am @@ -12,6 +12,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib-storage/index/dbox-common libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ mdbox-file.c \ mdbox-mail.c \ mdbox-map.c \ diff --git a/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c new file mode 100644 index 0000000000..53642c6640 --- /dev/null +++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c @@ -0,0 +1,320 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "mail-copy.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_deleted_storage; +extern struct mailbox mdbox_deleted_mailbox; +extern struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs; + +static struct mail_storage *mdbox_deleted_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox deleted storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_deleted_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static struct mailbox * +mdbox_deleted_mailbox_alloc(struct mail_storage *storage, + struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + pool_t pool; + + flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("mdbox deleted mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_deleted_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = (struct mdbox_storage *)storage; + return &mbox->box; +} + +static int +mdbox_deleted_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mail_index_transaction *new_trans = NULL; + uint32_t uid_validity = ioloop_time; + uint32_t uid_next = 1; + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + mbox->creating = TRUE; + mdbox_update_header(mbox, trans, update); + mbox->creating = FALSE; + + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +static int +mdbox_deleted_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) + guid_128_generate(metadata_r->guid); + return 0; +} + +static struct mail_save_context * +mdbox_deleted_save_alloc(struct mailbox_transaction_context *t) +{ + struct mail_save_context *ctx; + + ctx = i_new(struct mail_save_context, 1); + ctx->transaction = t; + return ctx; +} + +static int +mdbox_deleted_save_begin(struct mail_save_context *ctx, + struct istream *input ATTR_UNUSED) +{ + mail_storage_set_error(ctx->transaction->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "mdbox_deleted doesn't support saving mails"); + return -1; +} + +static int +mdbox_deleted_save_continue(struct mail_save_context *ctx ATTR_UNUSED) +{ + return -1; +} + +static int mdbox_deleted_save_finish(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); + return -1; +} + +static void +mdbox_deleted_save_cancel(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); +} + +static int mdbox_deleted_sync(struct mdbox_mailbox *mbox, + enum mdbox_sync_flags flags) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_mail_index_record rec; + struct mdbox_map_mail_index_record map_rec; + uint16_t refcount; + uint32_t map_seq, map_count, seq, uid = 0; + int ret = 0; + + if (mbox->mdbox_deleted_synced) { + /* don't bother supporting incremental syncs */ + return 0; + } + if (!mbox->box.inbox_user && mbox->box.name[0] != '\0') { + /* since mailbox list currently shows all the existing + mailboxes, we don't want all of them to list the deleted + messages. only show messages in user's INBOX or the + namespace prefix. */ + return 0; + } + + if (mdbox_map_open(mbox->storage->map) < 0) + return -1; + + if (mdbox_deleted_mailbox_create_indexes(&mbox->box, NULL, NULL) < 0) + return -1; + + memset(&rec, 0, sizeof(rec)); + rec.save_date = ioloop_time; + + if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &trans, flags) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + map_count = mdbox_map_get_messages_count(mbox->storage->map); + for (map_seq = 1; map_seq <= map_count; map_seq++) { + if (mdbox_map_lookup_seq_full(mbox->storage->map, map_seq, + &map_rec, &refcount) < 0) { + ret = -1; + break; + } + if (refcount == 0) { + rec.map_uid = mdbox_map_lookup_uid(mbox->storage->map, + map_seq); + mail_index_append(trans, ++uid, &seq); + mail_index_update_ext(trans, seq, + mbox->ext_id, &rec, NULL); + } + } + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else { + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } else { + mbox->mdbox_deleted_synced = TRUE; + } + } + return ret; +} + +static struct mailbox_sync_context * +mdbox_deleted_storage_sync_init(struct mailbox *box, + enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + ret = -1; + } + + if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted)) + ret = mdbox_deleted_sync(mbox, mdbox_sync_flags); + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +struct mail_storage mdbox_deleted_storage = { + .name = MDBOX_DELETED_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + + .v = { + mdbox_get_setting_parser_info, + mdbox_deleted_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + NULL, + mdbox_deleted_mailbox_alloc, + NULL + } +}; + +struct mailbox mdbox_deleted_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + index_storage_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + index_storage_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_deleted_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_deleted_storage_sync_init, + index_mailbox_sync_next, + index_mailbox_sync_deinit, + NULL, + dbox_notify_changes, + index_transaction_begin, + index_transaction_commit, + index_transaction_rollback, + NULL, + dbox_mail_alloc, + index_storage_search_init, + index_storage_search_deinit, + index_storage_search_next_nonblock, + index_storage_search_next_update_seq, + mdbox_deleted_save_alloc, + mdbox_deleted_save_begin, + mdbox_deleted_save_continue, + mdbox_deleted_save_finish, + mdbox_deleted_save_cancel, + mail_storage_copy, + NULL, + NULL, + NULL, + index_storage_is_inconsistent + } +}; + +struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs = { + mdbox_file_unrefed, + mdbox_file_create_fd, + mdbox_mail_open, + mdbox_deleted_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, + mdbox_set_mailbox_corrupted, + mdbox_set_file_corrupted +}; diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c index 93bbf7b30b..071e084db6 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-map.c +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c @@ -316,9 +316,6 @@ int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, struct mdbox_map_mail_index_record *rec_r, uint16_t *refcount_r) { - const struct mdbox_map_mail_index_record *rec; - const uint16_t *ref16_p; - const void *data; uint32_t seq; int ret; @@ -328,6 +325,17 @@ int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, if ((ret = mdbox_map_get_seq(map, map_uid, &seq)) <= 0) return ret; + return mdbox_map_lookup_seq_full(map, seq, rec_r, refcount_r); +} + +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r) +{ + const struct mdbox_map_mail_index_record *rec; + const uint16_t *ref16_p; + const void *data; + if (mdbox_map_lookup_seq(map, seq, &rec) < 0) return -1; *rec_r = *rec; @@ -342,6 +350,19 @@ int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, return 1; } +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq) +{ + uint32_t uid; + + mail_index_lookup_uid(map->view, seq, &uid); + return uid; +} + +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map) +{ + return mail_index_view_get_messages_count(map->view); +} + int mdbox_map_view_lookup_rec(struct mdbox_map *map, struct mail_index_view *view, uint32_t seq, struct dbox_mail_lookup_rec *rec_r) diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.h b/src/lib-storage/index/dbox-multi/mdbox-map.h index 58f570c323..8fa05ea678 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-map.h +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h @@ -53,6 +53,14 @@ int mdbox_map_lookup(struct mdbox_map *map, uint32_t map_uid, int mdbox_map_lookup_full(struct mdbox_map *map, uint32_t map_uid, struct mdbox_map_mail_index_record *rec_r, uint16_t *refcount_r); +/* Like mdbox_map_lookup_full(), but look up with sequence. */ +int mdbox_map_lookup_seq_full(struct mdbox_map *map, uint32_t seq, + struct mdbox_map_mail_index_record *rec_r, + uint16_t *refcount_r); +/* Return map UID for the map sequence. */ +uint32_t mdbox_map_lookup_uid(struct mdbox_map *map, uint32_t seq); +/* Returns the total number of messages in the map. */ +unsigned int mdbox_map_get_messages_count(struct mdbox_map *map); /* Get all messages from file */ int mdbox_map_get_file_msgs(struct mdbox_map *map, uint32_t file_id, diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.c b/src/lib-storage/index/dbox-multi/mdbox-storage.c index 841aa3669e..a80a515d71 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c @@ -19,7 +19,6 @@ extern struct mail_storage mdbox_storage; extern struct mailbox mdbox_mailbox; -extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs; static struct mail_storage *mdbox_storage_alloc(void) { @@ -34,9 +33,8 @@ static struct mail_storage *mdbox_storage_alloc(void) return &storage->storage.storage; } -static int -mdbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, - const char **error_r) +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r) { struct mdbox_storage *storage = (struct mdbox_storage *)_storage; const char *dir; @@ -64,7 +62,7 @@ mdbox_storage_create(struct mail_storage *_storage, struct mail_namespace *ns, return dbox_storage_create(_storage, ns, error_r); } -static void mdbox_storage_destroy(struct mail_storage *_storage) +void mdbox_storage_destroy(struct mail_storage *_storage) { struct mdbox_storage *storage = (struct mdbox_storage *)_storage; @@ -166,7 +164,7 @@ mdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, return &mbox->box; } -static int mdbox_mailbox_open(struct mailbox *box) +int mdbox_mailbox_open(struct mailbox *box) { struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; @@ -355,14 +353,14 @@ mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) return ""; } -static void mdbox_set_mailbox_corrupted(struct mailbox *box) +void mdbox_set_mailbox_corrupted(struct mailbox *box) { struct mdbox_storage *mstorage = (struct mdbox_storage *)box->storage; mdbox_storage_set_corrupted(mstorage); } -static void mdbox_set_file_corrupted(struct dbox_file *file) +void mdbox_set_file_corrupted(struct dbox_file *file) { struct mdbox_storage *mstorage = (struct mdbox_storage *)file->storage; diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage.h b/src/lib-storage/index/dbox-multi/mdbox-storage.h index e1092fc5ee..b77da52ae0 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-storage.h +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h @@ -6,6 +6,7 @@ #include "mdbox-settings.h" #define MDBOX_STORAGE_NAME "mdbox" +#define MDBOX_DELETED_STORAGE_NAME "mdbox_deleted" #define MDBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" #define MDBOX_GLOBAL_DIR_NAME "storage" #define MDBOX_MAIL_FILE_PREFIX "m." @@ -58,9 +59,11 @@ struct mdbox_mailbox { uint32_t map_uid_validity; uint32_t ext_id, hdr_ext_id, guid_ext_id; + unsigned int mdbox_deleted_synced:1; unsigned int creating:1; }; +extern struct dbox_storage_vfuncs mdbox_dbox_storage_vfuncs; extern struct mail_vfuncs mdbox_mail_vfuncs; int mdbox_mail_open(struct dbox_mail *mail, uoff_t *offset_r, @@ -99,6 +102,13 @@ int mdbox_copy(struct mail_save_context *ctx, struct mail *mail); void mdbox_purge_alt_flag_change(struct mail *mail, bool move_to_alt); int mdbox_purge(struct mail_storage *storage); +int mdbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, const char **error_r); +void mdbox_storage_destroy(struct mail_storage *_storage); +int mdbox_mailbox_open(struct mailbox *box); + void mdbox_storage_set_corrupted(struct mdbox_storage *storage); +void mdbox_set_mailbox_corrupted(struct mailbox *box); +void mdbox_set_file_corrupted(struct dbox_file *file); #endif