--- /dev/null
+/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "hex-binary.h"
+#include "str.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+#include "dbox-map-private.h"
+#include "dbox-sync.h"
+#include "dbox-storage-rebuild.h"
+
+#include <stdlib.h>
+#include <dirent.h>
+#include <unistd.h>
+
+struct dbox_rebuild_msg {
+ uint8_t guid_128[DBOX_GUID_BIN_LEN];
+ uint32_t file_id;
+ uint32_t offset;
+ uint32_t size;
+ uint32_t map_uid;
+
+ uint16_t refcount;
+ unsigned int seen_zero_ref_in_map:1;
+};
+
+struct rebuild_msg_mailbox {
+ struct mailbox *box;
+ struct mail_index_sync_ctx *sync_ctx;
+ struct mail_index_view *view;
+ struct mail_index_transaction *trans;
+ uint32_t next_uid;
+};
+
+struct dbox_storage_rebuild_context {
+ struct dbox_storage *storage;
+ pool_t pool;
+
+ struct hash_table *guid_hash;
+ ARRAY_DEFINE(msgs, struct dbox_rebuild_msg *);
+
+ uint32_t prev_file_id;
+ uint32_t highest_seen_map_uid;
+
+ struct mail_index_sync_ctx *sync_ctx;
+ struct mail_index_view *sync_view;
+ struct mail_index_transaction *trans;
+
+ struct rebuild_msg_mailbox prev_msg;
+
+ unsigned int msgs_unsorted:1;
+};
+
+static unsigned int guid_hash(const void *p)
+{
+ const uint8_t *s = p;
+ unsigned int i, g, h = 0;
+
+ for (i = 0; i < DBOX_GUID_BIN_LEN; i++) {
+ h = (h << 4) + s[i];
+ if ((g = h & 0xf0000000UL)) {
+ h = h ^ (g >> 24);
+ h = h ^ g;
+ }
+ }
+ return h;
+}
+
+static int guid_cmp(const void *p1, const void *p2)
+{
+ return memcmp(p1, p2, DBOX_GUID_BIN_LEN);
+}
+
+static struct dbox_storage_rebuild_context *
+dbox_storage_rebuild_init(struct dbox_storage *storage)
+{
+ struct dbox_storage_rebuild_context *ctx;
+
+ ctx = i_new(struct dbox_storage_rebuild_context, 1);
+ ctx->storage = storage;
+ ctx->pool = pool_alloconly_create("dbox map rebuild", 1024*256);
+ ctx->guid_hash = hash_table_create(default_pool, ctx->pool, 0,
+ guid_hash, guid_cmp);
+ i_array_init(&ctx->msgs, 512);
+ return ctx;
+}
+
+static void dbox_storage_rebuild_deinit(struct dbox_storage_rebuild_context *ctx)
+{
+ if (ctx->sync_ctx != NULL)
+ mail_index_sync_rollback(&ctx->sync_ctx);
+
+ hash_table_destroy(&ctx->guid_hash);
+ pool_unref(&ctx->pool);
+ array_free(&ctx->msgs);
+ i_free(ctx);
+}
+
+static int dbox_rebuild_msg_offset_cmp(const void *p1, const void *p2)
+{
+ const struct dbox_rebuild_msg *const *m1 = p1, *const *m2 = p2;
+
+ if ((*m1)->file_id < (*m2)->file_id)
+ return -1;
+ if ((*m1)->file_id > (*m2)->file_id)
+ return 1;
+
+ if ((*m1)->offset < (*m2)->offset)
+ return -1;
+ if ((*m1)->offset > (*m2)->offset)
+ return 1;
+ return 0;
+}
+
+static int dbox_rebuild_msg_uid_cmp(const void *p1, const void *p2)
+{
+ const struct dbox_rebuild_msg *const *m1 = p1, *const *m2 = p2;
+
+ if ((*m1)->map_uid < (*m2)->map_uid)
+ return -1;
+ if ((*m1)->map_uid > (*m2)->map_uid)
+ return 1;
+ return 0;
+}
+
+static int rebuild_add_file(struct dbox_storage_rebuild_context *ctx,
+ const char *path)
+{
+ struct dbox_file *file;
+ const char *fname, *guid;
+ struct dbox_rebuild_msg *rec;
+ uint32_t file_id;
+ buffer_t *guid_buf;
+ uoff_t offset;
+ bool last;
+ int ret = 0;
+
+ fname = strrchr(path, '/');
+ i_assert(fname != NULL);
+ fname += strlen(DBOX_MAIL_FILE_MULTI_PREFIX) + 1;
+
+ file_id = strtoul(fname, NULL, 10);
+ if (!is_numeric(fname, '\0') || file_id == 0) {
+ i_warning("dbox rebuild: File name is missing ID: %s", path);
+ return 0;
+ }
+
+ /* small optimization: typically files are returned sorted. in that
+ case we don't need to sort them ourself. */
+ if (file_id < ctx->prev_file_id)
+ ctx->msgs_unsorted = TRUE;
+ ctx->prev_file_id = file_id;
+
+ guid_buf = buffer_create_dynamic(pool_datastack_create(),
+ DBOX_GUID_BIN_LEN);
+
+ file = dbox_file_init_multi(ctx->storage, file_id);
+ while ((ret = dbox_file_seek_next(file, &offset, &last)) > 0) {
+ if ((ret = dbox_file_metadata_read(file)) <= 0)
+ break;
+
+ guid = dbox_file_metadata_get(file, DBOX_METADATA_GUID);
+ if (guid == NULL) {
+ dbox_file_set_corrupted(file,
+ "Message is missing GUID");
+ ret = 0;
+ break;
+ }
+ buffer_set_used_size(guid_buf, 0);
+ if (hex_to_binary(guid, guid_buf) < 0 ||
+ guid_buf->used != sizeof(rec->guid_128)) {
+ dbox_file_set_corrupted(file,
+ "Message GUID is not 128 bit hex: %s", guid);
+ ret = 0;
+ break;
+ }
+
+ rec = p_new(ctx->pool, struct dbox_rebuild_msg, 1);
+ rec->file_id = file_id;
+ rec->offset = offset;
+ rec->size = file->cur_physical_size;
+ memcpy(rec->guid_128, guid_buf->data, sizeof(rec->guid_128));
+ array_append(&ctx->msgs, &rec, 1);
+
+ if (hash_table_lookup(ctx->guid_hash, guid_buf->data) != NULL) {
+ /* duplicate. save this as a refcount=0 to map,
+ so it will eventually be deleted. */
+ } else {
+ hash_table_insert(ctx->guid_hash, rec->guid_128, rec);
+ }
+ }
+ if (ret == 0) {
+ /* FIXME: file is corrupted. should we try to fix it? */
+ }
+ dbox_file_unref(&file);
+ return ret < 0 ? -1 : 0;
+}
+
+static void
+rebuild_add_missing_map_uids(struct dbox_storage_rebuild_context *ctx,
+ uint32_t next_uid)
+{
+ struct dbox_rebuild_msg **msgs;
+ struct dbox_mail_index_map_record rec;
+ unsigned int i, count;
+ uint32_t seq;
+
+ memset(&rec, 0, sizeof(rec));
+ msgs = array_get_modifiable(&ctx->msgs, &count);
+ for (i = 0; i < count; i++) {
+ if (msgs[i]->map_uid != 0)
+ continue;
+
+ rec.file_id = msgs[i]->file_id;
+ rec.offset = msgs[i]->offset;
+ rec.size = msgs[i]->size;
+
+ msgs[i]->map_uid = next_uid++;
+ mail_index_append(ctx->trans, msgs[i]->map_uid, &seq);
+ mail_index_update_ext(ctx->trans, seq,
+ ctx->storage->map->map_ext_id,
+ &rec, NULL);
+ }
+}
+
+static int rebuild_apply_map(struct dbox_storage_rebuild_context *ctx)
+{
+ struct dbox_map *map = ctx->storage->map;
+ const struct mail_index_header *hdr;
+ struct dbox_rebuild_msg **msgs, *pos;
+ struct dbox_rebuild_msg search_msg, *search_msgp = &search_msg;
+ struct dbox_mail_lookup_rec rec;
+ uint32_t seq;
+ unsigned int count;
+
+ msgs = array_get_modifiable(&ctx->msgs, &count);
+ if (ctx->msgs_unsorted)
+ qsort(msgs, count, sizeof(*msgs), dbox_rebuild_msg_offset_cmp);
+
+ hdr = mail_index_get_header(ctx->sync_view);
+ for (seq = 1; seq <= hdr->messages_count; seq++) {
+ if (dbox_map_view_lookup_rec(map, ctx->sync_view,
+ seq, &rec) < 0)
+ return -1;
+
+ /* look up the rebuild msg record for this message */
+ search_msg.file_id = rec.rec.file_id;
+ search_msg.offset = rec.rec.offset;
+ pos = bsearch(&search_msgp, msgs, count, sizeof(*msgs),
+ dbox_rebuild_msg_offset_cmp);
+ if (pos == NULL) {
+ /* map record points to non-existing message. */
+ mail_index_expunge(ctx->trans, seq);
+ } else {
+ pos->map_uid = rec.map_uid;
+ pos->seen_zero_ref_in_map = rec.refcount == 0;
+ }
+ }
+ rebuild_add_missing_map_uids(ctx, hdr->next_uid);
+
+ /* afterwards we're interested in looking up map_uids.
+ re-sort the messages to make it easier. */
+ qsort(msgs, count, sizeof(*msgs), dbox_rebuild_msg_uid_cmp);
+ return 0;
+}
+
+static struct dbox_rebuild_msg *
+rebuild_lookup_map_uid(struct dbox_storage_rebuild_context *ctx,
+ uint32_t map_uid)
+{
+ struct dbox_rebuild_msg search_msg, *search_msgp = &search_msg;
+ struct dbox_rebuild_msg *const *msgs;
+ unsigned int count;
+
+ search_msg.map_uid = map_uid;
+ msgs = array_get(&ctx->msgs, &count);
+ return bsearch(&search_msgp, msgs, count, sizeof(*msgs),
+ dbox_rebuild_msg_uid_cmp);
+}
+
+static void
+rebuild_mailbox_multi(struct dbox_storage_rebuild_context *ctx,
+ struct dbox_sync_rebuild_context *rebuild_ctx,
+ struct dbox_mailbox *mbox,
+ struct mail_index_view *view,
+ struct mail_index_transaction *trans)
+{
+ const struct dbox_mail_index_record *dbox_rec;
+ struct dbox_mail_index_record new_dbox_rec;
+ const struct mail_index_header *hdr;
+ struct dbox_rebuild_msg *rec;
+ const void *data;
+ bool expunged;
+ uint32_t seq, uid, new_seq, map_uid;
+
+ memset(&new_dbox_rec, 0, sizeof(new_dbox_rec));
+ hdr = mail_index_get_header(view);
+ for (seq = 1; seq <= hdr->messages_count; seq++) {
+ mail_index_lookup_ext(view, seq, mbox->dbox_ext_id,
+ &data, &expunged);
+ dbox_rec = data;
+ map_uid = dbox_rec == NULL ? 0 : dbox_rec->map_uid;
+
+ mail_index_lookup_ext(view, seq, mbox->guid_ext_id,
+ &data, &expunged);
+
+ /* see if we can find this message based on
+ 1) GUID, 2) map_uid */
+ rec = data == NULL ? NULL :
+ hash_table_lookup(ctx->guid_hash, data);
+ if (rec == NULL) {
+ if (map_uid == 0) {
+ /* not a multi-dbox message, ignore. */
+ continue;
+ }
+ /* multi-dbox message that wasn't found with GUID.
+ either it's lost or GUID has been corrupted. we can
+ still try to look it up using map_uid. */
+ rec = rebuild_lookup_map_uid(ctx, map_uid);
+ if (rec != NULL) {
+ mail_index_update_ext(trans, seq,
+ mbox->guid_ext_id,
+ rec->guid_128, NULL);
+ }
+ } else if (map_uid != rec->map_uid) {
+ /* map_uid is wrong, update it */
+ i_assert(rec->map_uid != 0);
+ new_dbox_rec.map_uid = rec->map_uid;
+ mail_index_update_ext(trans, seq, mbox->dbox_ext_id,
+ &new_dbox_rec, NULL);
+ } else {
+ /* everything was ok */
+ }
+
+ if (rec != NULL) {
+ /* keep this message */
+ rec->refcount++;
+
+ mail_index_lookup_uid(view, seq, &uid);
+ mail_index_append(trans, uid, &new_seq);
+ dbox_sync_rebuild_index_metadata(rebuild_ctx,
+ NULL, new_seq, uid);
+
+ new_dbox_rec.map_uid = rec->map_uid;
+ mail_index_update_ext(ctx->trans, new_seq,
+ mbox->dbox_ext_id,
+ &new_dbox_rec, NULL);
+ }
+ }
+}
+
+static int
+rebuild_mailbox(struct dbox_storage_rebuild_context *ctx, const char *name)
+{
+ struct mailbox *box;
+ struct dbox_mailbox *mbox;
+ struct mail_index_sync_ctx *sync_ctx;
+ struct mail_index_view *view;
+ struct mail_index_transaction *trans;
+ struct dbox_sync_rebuild_context *rebuild_ctx;
+ enum mail_error error;
+ int ret;
+
+ box = dbox_mailbox_open(&ctx->storage->storage, name, NULL,
+ MAILBOX_OPEN_READONLY |
+ MAILBOX_OPEN_KEEP_RECENT |
+ MAILBOX_OPEN_IGNORE_ACLS);
+ if (box == NULL) {
+ mail_storage_get_last_error(&ctx->storage->storage, &error);
+ if (error == MAIL_ERROR_TEMP)
+ return -1;
+ /* non-temporary error, ignore */
+ return 0;
+ }
+ mbox = (struct dbox_mailbox *)box;
+
+ ret = mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans,
+ MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES);
+ if (ret <= 0) {
+ i_assert(ret != 0);
+ mail_storage_set_index_error(&mbox->ibox);
+ (void)mailbox_close(&box);
+ return -1;
+ }
+
+ rebuild_ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+ ret = dbox_sync_index_rebuild_singles(rebuild_ctx);
+ if (ret == 0)
+ rebuild_mailbox_multi(ctx, rebuild_ctx, mbox, view, trans);
+ dbox_sync_index_rebuild_deinit(&rebuild_ctx);
+
+ if (mail_index_sync_commit(&sync_ctx) < 0) {
+ mail_storage_set_index_error(&mbox->ibox);
+ ret = -1;
+ }
+
+ (void)mailbox_close(&box);
+ return ret < 0 ? -1 : 0;
+}
+
+static int rebuild_mailboxes(struct dbox_storage_rebuild_context *ctx)
+{
+ struct mailbox_list_iterate_context *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = mailbox_list_iter_init(ctx->storage->storage.list, "*",
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ if ((info->flags & (MAILBOX_NONEXISTENT |
+ MAILBOX_NOSELECT)) == 0) {
+ if (rebuild_mailbox(ctx, info->name) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static int rebuild_msg_mailbox_commit(struct rebuild_msg_mailbox *msg)
+{
+ if (mail_index_sync_commit(&msg->sync_ctx) < 0)
+ return -1;
+ (void)mailbox_close(&msg->box);
+ memset(msg, 0, sizeof(*msg));
+ return 0;
+}
+
+static int rebuild_restore_msg(struct dbox_storage_rebuild_context *ctx,
+ struct dbox_rebuild_msg *msg)
+{
+ struct mail_storage *storage = &ctx->storage->storage;
+ struct dbox_file *file;
+ const struct mail_index_header *hdr;
+ struct dbox_mail_index_record dbox_rec;
+ const char *mailbox = NULL;
+ struct mailbox *box;
+ struct dbox_mailbox *mbox;
+ enum mail_error error;
+ bool expunged, created;
+ uoff_t size;
+ int ret;
+ uint32_t seq;
+
+ /* first see if message contains the mailbox it was originally
+ saved to */
+ file = dbox_file_init_multi(ctx->storage, msg->file_id);
+ ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL,
+ &expunged);
+ if (ret > 0 && !expunged && dbox_file_metadata_read(file) == 0) {
+ mailbox = dbox_file_metadata_get(file,
+ DBOX_METADATA_ORIG_MAILBOX);
+ }
+ dbox_file_unref(&file);
+ if (ret <= 0 || expunged) {
+ if (ret < 0)
+ return -1;
+ /* we shouldn't get here, so apparently we couldn't fix
+ something. just ignore the mail.. */
+ return 0;
+ }
+
+ if (mailbox == NULL)
+ mailbox = "INBOX";
+
+ /* we have the destination mailbox. now open it and add the message
+ there. */
+ created = FALSE;
+ box = ctx->prev_msg.box != NULL &&
+ strcmp(mailbox, ctx->prev_msg.box->name) == 0 ?
+ ctx->prev_msg.box : NULL;
+ while (box == NULL) {
+ box = dbox_mailbox_open(storage, mailbox, NULL,
+ MAILBOX_OPEN_READONLY |
+ MAILBOX_OPEN_KEEP_RECENT |
+ MAILBOX_OPEN_IGNORE_ACLS);
+ if (box != NULL)
+ break;
+
+ mail_storage_get_last_error(storage, &error);
+ if (error == MAIL_ERROR_TEMP)
+ return -1;
+
+ if (error == MAIL_ERROR_NOTFOUND && !created) {
+ /* mailbox doesn't exist currently? see if creating
+ it helps. */
+ created = TRUE;
+ (void)mail_storage_mailbox_create(storage, mailbox,
+ FALSE);
+ } else if (strcmp(mailbox, "INBOX") != 0) {
+ /* see if we can save to INBOX instead. */
+ mailbox = "INBOX";
+ } else {
+ /* this shouldn't happen */
+ return -1;
+ }
+ }
+ mbox = (struct dbox_mailbox *)box;
+
+ /* switch the mailbox cache if necessary */
+ if (box != ctx->prev_msg.box && ctx->prev_msg.box != NULL) {
+ if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
+ return -1;
+ }
+ if (ctx->prev_msg.box == NULL) {
+ ret = mail_index_sync_begin(mbox->ibox.index,
+ &ctx->prev_msg.sync_ctx,
+ &ctx->prev_msg.view,
+ &ctx->prev_msg.trans, 0);
+ if (ret <= 0) {
+ i_assert(ret != 0);
+ mail_storage_set_index_error(&mbox->ibox);
+ (void)mailbox_close(&box);
+ return -1;
+ }
+ ctx->prev_msg.box = box;
+ hdr = mail_index_get_header(ctx->prev_msg.view);
+ ctx->prev_msg.next_uid = hdr->next_uid;
+ }
+
+ /* add the new message */
+ memset(&dbox_rec, 0, sizeof(dbox_rec));
+ dbox_rec.map_uid = msg->map_uid;
+ mail_index_append(ctx->prev_msg.trans, ctx->prev_msg.next_uid++, &seq);
+ mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->dbox_ext_id,
+ &dbox_rec, NULL);
+
+ msg->refcount++;
+ return 0;
+}
+
+static int rebuild_handle_zero_refs(struct dbox_storage_rebuild_context *ctx)
+{
+ struct dbox_rebuild_msg **msgs;
+ unsigned int i, count;
+
+ /* if we have messages at this point which have refcount=0, they're
+ either already expunged or they were somehow lost for some reason.
+ we'll need to figure out what to do about them. */
+ msgs = array_get_modifiable(&ctx->msgs, &count);
+ for (i = 0; i < count; i++) {
+ if (msgs[i]->refcount != 0)
+ continue;
+
+ if (msgs[i]->seen_zero_ref_in_map) {
+ /* we've seen the map record, trust it. */
+ continue;
+ }
+ /* either map record was lost for this message or the message
+ was lost from its mailbox. safest way to handle this is to
+ restore the message. */
+ if (rebuild_restore_msg(ctx, msgs[i]) < 0)
+ return -1;
+ }
+ if (ctx->prev_msg.box != NULL) {
+ if (rebuild_msg_mailbox_commit(&ctx->prev_msg) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void rebuild_update_refcounts(struct dbox_storage_rebuild_context *ctx)
+{
+ const struct mail_index_header *hdr;
+ const void *data;
+ struct dbox_rebuild_msg **msgs;
+ const uint16_t *ref16_p;
+ bool expunged;
+ uint32_t seq, map_uid;
+ unsigned int i, count;
+
+ /* update refcounts for existing map records */
+ msgs = array_get_modifiable(&ctx->msgs, &count);
+ hdr = mail_index_get_header(ctx->sync_view);
+ for (seq = 1, i = 0; seq <= hdr->messages_count && i < count; seq++) {
+ mail_index_lookup_uid(ctx->sync_view, seq, &map_uid);
+ if (map_uid != msgs[i]->map_uid) {
+ /* we've already expunged this map record */
+ i_assert(map_uid < msgs[i]->map_uid);
+ continue;
+ }
+
+ mail_index_lookup_ext(ctx->sync_view, seq,
+ ctx->storage->map->ref_ext_id,
+ &data, &expunged);
+ ref16_p = data;
+ if (ref16_p == NULL || *ref16_p != msgs[i]->refcount) {
+ mail_index_update_ext(ctx->trans, seq,
+ ctx->storage->map->ref_ext_id,
+ &msgs[i]->refcount, NULL);
+ }
+ i++;
+ }
+
+ /* update refcounts for newly created map records */
+ for (; i < count; i++, seq++) {
+ mail_index_update_ext(ctx->trans, seq,
+ ctx->storage->map->ref_ext_id,
+ &msgs[i]->refcount, NULL);
+ }
+}
+
+static int rebuild_finish(struct dbox_storage_rebuild_context *ctx)
+{
+ if (rebuild_handle_zero_refs(ctx) < 0)
+ return -1;
+ rebuild_update_refcounts(ctx);
+ return 0;
+}
+
+static int dbox_storage_rebuild_scan(struct dbox_storage_rebuild_context *ctx)
+{
+ DIR *dir;
+ struct dirent *d;
+ string_t *path;
+ unsigned int dir_len;
+ int ret = 0;
+
+ /* begin by locking the map, so that other processes can't try to
+ rebuild at the same time. */
+ ret = mail_index_sync_begin(ctx->storage->map->index, &ctx->sync_ctx,
+ &ctx->sync_view, &ctx->trans, 0);
+ if (ret <= 0) {
+ i_assert(ret != 0);
+ mail_storage_set_internal_error(&ctx->storage->storage);
+ mail_index_reset_error(ctx->storage->map->index);
+ return -1;
+ }
+
+ dir = opendir(ctx->storage->storage_dir);
+ if (dir == NULL) {
+ mail_storage_set_critical(&ctx->storage->storage,
+ "opendir(%s) failed: %m", ctx->storage->storage_dir);
+ return -1;
+ }
+ path = t_str_new(256);
+ str_append(path, ctx->storage->storage_dir);
+ str_append_c(path, '/');
+ dir_len = str_len(path);
+
+ for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+ if (strncmp(d->d_name, DBOX_MAIL_FILE_MULTI_PREFIX,
+ strlen(DBOX_MAIL_FILE_MULTI_PREFIX)) == 0) {
+ str_truncate(path, dir_len);
+ str_append(path, d->d_name);
+ T_BEGIN {
+ ret = rebuild_add_file(ctx, d->d_name);
+ } T_END;
+ if (ret < 0) {
+ ret = -1;
+ break;
+ }
+ }
+ }
+ if (ret == 0 && errno != 0) {
+ mail_storage_set_critical(&ctx->storage->storage,
+ "readdir(%s) failed: %m", ctx->storage->storage_dir);
+ ret = -1;
+ }
+ if (closedir(dir) < 0) {
+ mail_storage_set_critical(&ctx->storage->storage,
+ "closedir(%s) failed: %m", ctx->storage->storage_dir);
+ ret = -1;
+ }
+
+ if (ret < 0 ||
+ rebuild_apply_map(ctx) < 0 ||
+ rebuild_mailboxes(ctx) < 0 ||
+ rebuild_finish(ctx) < 0 ||
+ mail_index_sync_commit(&ctx->sync_ctx) < 0)
+ return -1;
+ return 0;
+}
+
+int dbox_storage_rebuild(struct dbox_storage *storage)
+{
+ struct dbox_storage_rebuild_context *ctx;
+ struct stat st;
+ int ret;
+
+ if (stat(storage->storage_dir, &st) < 0) {
+ if (errno == ENOENT) {
+ /* no multi-dbox files */
+ return 0;
+ }
+
+ mail_storage_set_critical(&storage->storage,
+ "stat(%s) failed: %m", storage->storage_dir);
+ return -1;
+ }
+
+ ctx = dbox_storage_rebuild_init(storage);
+ ret = dbox_storage_rebuild_scan(ctx);
+ dbox_storage_rebuild_deinit(ctx);
+
+ if (ret == 0)
+ storage->sync_rebuild = FALSE;
+ return ret;
+}
uint32_t cache_ext_id;
uint32_t cache_reset_id;
+ struct mail_index *backup_index;
+ struct mail_index_view *backup_view;
+
struct maildir_uidlist_sync_ctx *maildir_sync_ctx;
struct maildir_keywords *mk;
struct maildir_keywords_sync_ctx *maildir_sync_keywords;
static void
dbox_sync_index_copy_cache(struct dbox_sync_rebuild_context *ctx,
+ struct mail_index_view *view,
uint32_t old_seq, uint32_t new_seq)
{
struct mail_index_map *map;
if (ctx->cache_ext_id == (uint32_t)-1)
return;
- mail_index_lookup_ext_full(ctx->view, old_seq, ctx->cache_ext_id,
+ mail_index_lookup_ext_full(view, old_seq, ctx->cache_ext_id,
&map, &data, &expunged);
if (expunged)
return;
- if (!mail_index_ext_get_reset_id(ctx->view, map, ctx->cache_ext_id,
+ if (!mail_index_ext_get_reset_id(view, map, ctx->cache_ext_id,
&reset_id) || reset_id == 0)
return;
static void
dbox_sync_index_copy_from_old(struct dbox_sync_rebuild_context *ctx,
+ struct mail_index_view *view,
uint32_t old_seq, uint32_t new_seq)
{
- struct mail_index *index = mail_index_view_get_index(ctx->view);
+ struct mail_index *index = mail_index_view_get_index(view);
const struct mail_index_record *rec;
ARRAY_TYPE(keyword_indexes) old_keywords;
struct mail_keywords *kw;
/* copy flags */
- rec = mail_index_lookup(ctx->view, old_seq);
+ rec = mail_index_lookup(view, old_seq);
mail_index_update_flags(ctx->trans, new_seq,
MODIFY_REPLACE, rec->flags);
/* copy keywords */
t_array_init(&old_keywords, 32);
- mail_index_lookup_keywords(ctx->view, old_seq, &old_keywords);
+ mail_index_lookup_keywords(view, old_seq, &old_keywords);
kw = mail_index_keywords_create_from_indexes(index, &old_keywords);
mail_index_update_keywords(ctx->trans, new_seq, MODIFY_REPLACE, kw);
mail_index_keywords_free(&kw);
- dbox_sync_index_copy_cache(ctx, old_seq, new_seq);
+ dbox_sync_index_copy_cache(ctx, view, old_seq, new_seq);
}
static void
mail_index_keywords_free(&keywords);
}
-static void
-dbox_sync_index_metadata(struct dbox_sync_rebuild_context *ctx,
- struct dbox_file *file, uint32_t seq, uint32_t uid)
+void dbox_sync_rebuild_index_metadata(struct dbox_sync_rebuild_context *ctx,
+ struct dbox_file *file,
+ uint32_t new_seq, uint32_t uid)
{
uint32_t old_seq;
if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) {
/* the message exists in the old index.
copy the metadata from it. */
- dbox_sync_index_copy_from_old(ctx, old_seq, seq);
- } else if (file->maildir_file) {
+ dbox_sync_index_copy_from_old(ctx, ctx->view, old_seq, new_seq);
+ } else if (ctx->backup_view != NULL &&
+ mail_index_lookup_seq(ctx->backup_view, uid, &old_seq)) {
+ /* copy the metadata from backup index. */
+ dbox_sync_index_copy_from_old(ctx, ctx->backup_view,
+ old_seq, new_seq);
+ } else if (file != NULL && file->maildir_file) {
/* we're probably doing initial sync after migration from
maildir. preserve the old flags. */
- dbox_sync_index_copy_from_maildir(ctx, file, seq);
+ dbox_sync_index_copy_from_maildir(ctx, file, new_seq);
}
}
-static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx,
- struct dbox_file *file, uoff_t *offset)
+static int dbox_sync_add_file_index(struct dbox_sync_rebuild_context *ctx,
+ struct dbox_file *file)
{
uint32_t seq;
- bool last;
+ uoff_t size;
+ bool expunged;
int ret;
- ret = dbox_file_seek_next(file, offset, &last);
+ ret = dbox_file_get_mail_stream(file, 0, &size, NULL, &expunged);
if (ret <= 0) {
if (ret < 0)
return -1;
- i_warning("%s: Ignoring broken file (header)",
- file->current_path);
+ i_warning("dbox: Ignoring broken file: %s", file->current_path);
return 0;
}
-
- ret = dbox_file_metadata_read(file);
- if (ret <= 0) {
- if (ret < 0)
- return -1;
- i_warning("%s: Ignoring broken file (metadata)",
- file->current_path);
+ if (expunged) {
+ /* the file just got deleted? */
return 0;
}
- /* FIXME: file->uid doesn't work for multi files */
mail_index_append(ctx->trans, file->uid, &seq);
- dbox_sync_index_metadata(ctx, file, seq, file->uid);
- return 1;
+ dbox_sync_rebuild_index_metadata(ctx, file, seq, file->uid);
+ return 0;
}
static int
-dbox_sync_index_uid_file(struct dbox_sync_rebuild_context *ctx,
- const char *dir, const char *fname)
+dbox_sync_add_uid_file(struct dbox_sync_rebuild_context *ctx,
+ const char *dir, const char *fname)
{
struct dbox_file *file;
unsigned long uid;
char *p;
- uoff_t offset = 0;
int ret;
- fname += sizeof(DBOX_MAIL_FILE_MULTI_PREFIX)-1;
+ fname += sizeof(DBOX_MAIL_FILE_UID_PREFIX)-1;
uid = strtoul(fname, &p, 10);
if (*p != '\0' || uid == 0 || uid >= (uint32_t)-1) {
i_warning("dbox %s: Ignoring invalid filename %s",
file = dbox_file_init_single(ctx->mbox, uid);
file->current_path = i_strdup_printf("%s/%s", dir, fname);
- ret = dbox_sync_index_file_next(ctx, file, &offset) < 0 ? -1 : 0;
+ ret = dbox_sync_add_file_index(ctx, file);
dbox_file_unref(&file);
return ret;
}
static int
-dbox_sync_index_multi_file(struct dbox_sync_rebuild_context *ctx,
- const char *dir, const char *fname)
-{
- /* FIXME */
- return 0;
-}
-
-static int
-dbox_sync_index_maildir_file(struct dbox_sync_rebuild_context *ctx,
- const char *fname)
+dbox_sync_add_maildir_file(struct dbox_sync_rebuild_context *ctx,
+ const char *fname)
{
int ret;
}
static int
-dbox_sync_index_file(struct dbox_sync_rebuild_context *ctx,
- const char *path, const char *fname, bool primary)
+dbox_sync_add_file(struct dbox_sync_rebuild_context *ctx,
+ const char *path, const char *fname, bool primary)
{
if (strncmp(fname, DBOX_MAIL_FILE_UID_PREFIX,
sizeof(DBOX_MAIL_FILE_UID_PREFIX)-1) == 0)
- return dbox_sync_index_uid_file(ctx, path, fname);
-
- if (strncmp(fname, DBOX_MAIL_FILE_MULTI_PREFIX,
- sizeof(DBOX_MAIL_FILE_MULTI_PREFIX)-1) == 0)
- return dbox_sync_index_multi_file(ctx, path, fname);
+ return dbox_sync_add_uid_file(ctx, path, fname);
if (primary && strstr(fname, ":2,") != NULL)
- return dbox_sync_index_maildir_file(ctx, fname);
+ return dbox_sync_add_maildir_file(ctx, fname);
return 0;
}
break;
T_BEGIN {
- ret = dbox_sync_index_file(ctx, path, d->d_name,
- primary);
+ ret = dbox_sync_add_file(ctx, path, d->d_name, primary);
} T_END;
} while (ret >= 0);
if (errno != 0) {
const char *fname;
enum maildir_uidlist_rec_flag flags;
uint32_t uid, next_uid;
- uoff_t offset;
int ret = 0;
if (ctx->maildir_sync_ctx == NULL)
file->current_path =
i_strdup_printf("%s/%s", ctx->mbox->path, fname);
- offset = 0;
- ret = dbox_sync_index_file_next(ctx, file, &offset);
+ ret = dbox_sync_add_file_index(ctx, file);
dbox_file_unref(&file);
if (ret < 0)
break;
&new_hdr, sizeof(new_hdr));
}
-static int dbox_sync_index_rebuild_ctx(struct dbox_sync_rebuild_context *ctx)
+struct dbox_sync_rebuild_context *
+dbox_sync_index_rebuild_init(struct dbox_mailbox *mbox,
+ struct mail_index_view *view,
+ struct mail_index_transaction *trans)
{
+ struct mailbox *box = &mbox->ibox.box;
+ struct dbox_sync_rebuild_context *ctx;
+ const char *index_dir;
+ enum mail_index_open_flags open_flags = MAIL_INDEX_OPEN_FLAG_READONLY;
+
+ ctx = i_new(struct dbox_sync_rebuild_context, 1);
+ ctx->mbox = mbox;
+ ctx->view = view;
+ ctx->trans = trans;
+ mail_index_reset(ctx->trans);
+ index_mailbox_reset_uidvalidity(&mbox->ibox);
+ mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx->cache_ext_id);
+
+ /* if backup index file exists, try to use it */
+ index_dir = mailbox_list_get_path(box->storage->list, box->name,
+ MAILBOX_LIST_PATH_TYPE_INDEX);
+ ctx->backup_index =
+ mail_index_alloc(index_dir, DBOX_INDEX_PREFIX".backup");
+
+#ifndef MMAP_CONFLICTS_WRITE
+ if ((box->storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0)
+#endif
+ open_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
+ if (mail_index_open(ctx->backup_index, open_flags,
+ box->storage->lock_method) < 0)
+ mail_index_free(&ctx->backup_index);
+ else
+ ctx->backup_view = mail_index_view_open(ctx->backup_index);
+ return ctx;
+}
+
+int dbox_sync_index_rebuild_singles(struct dbox_sync_rebuild_context *ctx)
+{
+ int ret = 0;
+
dbox_sync_set_uidvalidity(ctx);
if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->path, TRUE) < 0)
- return -1;
-
- if (ctx->mbox->alt_path != NULL) {
+ ret = -1;
+ else if (ctx->mbox->alt_path != NULL) {
if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->alt_path,
FALSE) < 0)
- return -1;
+ ret = -1;
}
- if (dbox_sync_maildir_finish(ctx) < 0)
- return -1;
+ if (ret == 0) {
+ if (dbox_sync_maildir_finish(ctx) < 0)
+ ret = -1;
+ }
+
+ if (ctx->maildir_sync_ctx != NULL) {
+ if (maildir_uidlist_sync_deinit(&ctx->maildir_sync_ctx) < 0)
+ ret = -1;
+ }
+ if (ctx->maildir_sync_keywords != NULL)
+ maildir_keywords_sync_deinit(&ctx->maildir_sync_keywords);
+ if (ctx->mk != NULL)
+ maildir_keywords_deinit(&ctx->mk);
+ return ret;
+}
+
+void dbox_sync_index_rebuild_deinit(struct dbox_sync_rebuild_context **_ctx)
+{
+ struct dbox_sync_rebuild_context *ctx = *_ctx;
+
+ *_ctx = NULL;
+ if (ctx->backup_index != NULL) {
+ mail_index_view_close(&ctx->backup_view);
+ mail_index_free(&ctx->backup_index);
+ }
dbox_sync_update_header(ctx);
- return 0;
+ i_free(ctx);
}
int dbox_sync_index_rebuild(struct dbox_mailbox *mbox)
{
- struct dbox_sync_rebuild_context ctx;
+ struct dbox_sync_rebuild_context *ctx;
+ struct mail_index_view *view;
+ struct mail_index_transaction *trans;
int ret;
- memset(&ctx, 0, sizeof(ctx));
- ctx.mbox = mbox;
- ctx.view = mail_index_view_open(mbox->ibox.index);
- ctx.trans = mail_index_transaction_begin(ctx.view,
+ view = mail_index_view_open(mbox->ibox.index);
+ trans = mail_index_transaction_begin(view,
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
- mail_index_reset(ctx.trans);
- index_mailbox_reset_uidvalidity(&mbox->ibox);
- mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx.cache_ext_id);
- if ((ret = dbox_sync_index_rebuild_ctx(&ctx)) < 0)
- mail_index_transaction_rollback(&ctx.trans);
+ ctx = dbox_sync_index_rebuild_init(mbox, view, trans);
+ ret = dbox_sync_index_rebuild_singles(ctx);
+ dbox_sync_index_rebuild_deinit(&ctx);
+
+ if (ret < 0)
+ mail_index_transaction_rollback(&trans);
else
- ret = mail_index_transaction_commit(&ctx.trans);
- mail_index_view_close(&ctx.view);
+ ret = mail_index_transaction_commit(&trans);
+ mail_index_view_close(&view);
- if (ctx.maildir_sync_ctx != NULL) {
- if (maildir_uidlist_sync_deinit(&ctx.maildir_sync_ctx) < 0)
- ret = -1;
- }
- if (ctx.maildir_sync_keywords != NULL)
- maildir_keywords_sync_deinit(&ctx.maildir_sync_keywords);
- if (ctx.mk != NULL)
- maildir_keywords_deinit(&ctx.mk);
if (ret == 0)
- mbox->storage->sync_rebuild = FALSE;
+ ctx->mbox->storage->sync_rebuild = FALSE;
return ret;
}
projects / thirdparty / dovecot / core.git / commitdiff
