mdbox: Check that m.X file doesn't have garbage at end of file when saving new data...

diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c
index 1ef0c306f00241d92094ec0e634a8dcb0b8993b7..7e52324b9b5ffaf6e8b0a685c790e303aea602da 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c
@@ -796,10 +796,25 @@ static bool dbox_try_open(struct dbox_file *file, bool want_altpath)
        return TRUE;
 }
 
+static bool dbox_file_is_ok_at(struct dbox_file *file, uoff_t offset)
+{
+       bool last;
+       int ret;
+
+       if (dbox_file_seek(file, offset) == 0)
+               return FALSE;
+
+       while ((ret = dbox_file_seek_next(file, &offset, &last)) > 0);
+       if (ret == 0 && !last)
+               return FALSE;
+       return TRUE;
+}
+
 static bool
 mdbox_map_file_try_append(struct mdbox_map_append_context *ctx,
                          bool want_altpath,
-                         uint32_t file_id, time_t stamp, uoff_t mail_size,
+                         const struct mdbox_map_mail_index_record *rec,
+                         time_t stamp, uoff_t mail_size,
                          struct dbox_file_append_context **file_append_r,
                          struct ostream **output_r, bool *retry_later_r)
 {
@@ -815,7 +830,7 @@ mdbox_map_file_try_append(struct mdbox_map_append_context *ctx,
        *output_r = NULL;
        *retry_later_r = FALSE;
 
-       file = mdbox_file_init(storage, file_id);
+       file = mdbox_file_init(storage, rec->file_id);
        if (!dbox_try_open(file, want_altpath)) {
                dbox_file_unref(&file);
                return TRUE;
@@ -830,6 +845,13 @@ mdbox_map_file_try_append(struct mdbox_map_append_context *ctx,
                if (errno != ENOENT)
                        i_error("stat(%s) failed: %m", file->cur_path);
                /* the file was unlinked between opening and locking it. */
+       } else if (st.st_size != rec->offset + rec->size &&
+                  /* check if there's any garbage at the end of file.
+                     note that there may be valid messages added by another
+                     session before we locked it (but after we refreshed
+                     map index). */
+                  !dbox_file_is_ok_at(file, rec->offset + rec->size)) {
+               /* error message was already logged */
        } else {
                file_append = dbox_file_append_init(file);
                if (dbox_file_get_append_stream(file_append, output_r) <= 0) {
@@ -1037,7 +1059,7 @@ mdbox_map_find_appendable_file(struct mdbox_map_append_context *ctx,
                }
 
                mail_index_lookup_uid(map->view, seq, &uid);
-               if (!mdbox_map_file_try_append(ctx, want_altpath, rec->file_id,
+               if (!mdbox_map_file_try_append(ctx, want_altpath, rec,
                                               stamp, mail_size, file_append_r,
                                               output_r, &retry_later)) {
                        /* file is too old. the rest of the files are too. */

diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c
index 52c6aac309eee253681670cd0e8cd2ab75963694..ce273933445581b363d14f6dd34ba78e8a18dfca 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c
@@ -301,10 +301,13 @@ int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
                return -1;
        }
 
-       /* lock the mailbox after map to avoid deadlocks. */
+       /* lock the mailbox after map to avoid deadlocks. if we've noticed
+          any corruption, deal with it later, otherwise we won't have
+          up-to-date atomic->sync_view */
        if (mdbox_sync_begin(ctx->mbox, MDBOX_SYNC_FLAG_NO_PURGE |
                             MDBOX_SYNC_FLAG_FORCE |
-                            MDBOX_SYNC_FLAG_FSYNC, ctx->atomic,
+                            MDBOX_SYNC_FLAG_FSYNC |
+                            MDBOX_SYNC_FLAG_NO_REBUILD, ctx->atomic,
                             &ctx->sync_ctx) < 0) {
                mdbox_transaction_save_rollback(_ctx);
                return -1;

diff --git a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
index 6da21346e8bfad6928c5c7c365f9e1392271f2fb..4019b937ec68c5f6c4809a5c2efdd82574e845da 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c
@@ -50,7 +50,6 @@ struct mdbox_storage_rebuild_context {
        ARRAY_TYPE(seq_range) seen_file_ids;
 
        uint32_t rebuild_count;
-       uint32_t highest_seen_map_uid;
        uint32_t highest_file_id;
 
        struct mailbox_list *default_list;

diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.c b/src/lib-storage/index/dbox-multi/mdbox-sync.c
index 3ce795bce437f789dcc8c3b050f3facee0aba00d..8a1b790f7c86f67451edb9d1a4e7fdc432319414 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c
@@ -229,7 +229,7 @@ int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,
           headers until syncing has locked the mailbox */
        rebuild = mbox->storage->corrupted ||
                (flags & MDBOX_SYNC_FLAG_FORCE_REBUILD) != 0;
-       if (rebuild) {
+       if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) {
                if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0)
                        return -1;
                index_mailbox_reset_uidvalidity(&mbox->box);
@@ -330,8 +330,10 @@ mdbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
                        ret = -1;
        }
 
-       if (mail_index_reset_fscked(box->index))
-               mdbox_storage_set_corrupted(mbox->storage);
+       if (box->opened) {
+               if (mail_index_reset_fscked(box->index))
+                       mdbox_storage_set_corrupted(mbox->storage);
+       }
        if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) ||
                         mbox->storage->corrupted)) {
                if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0)

diff --git a/src/lib-storage/index/dbox-multi/mdbox-sync.h b/src/lib-storage/index/dbox-multi/mdbox-sync.h
index bacfa2afe51c1a4194e9123347a350268020eca7..3057ce680330aae866f6a80c3a24711c73b9a4d7 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.h
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h
@@ -8,7 +8,8 @@ enum mdbox_sync_flags {
        MDBOX_SYNC_FLAG_FORCE           = 0x01,
        MDBOX_SYNC_FLAG_FSYNC           = 0x02,
        MDBOX_SYNC_FLAG_FORCE_REBUILD   = 0x04,
-       MDBOX_SYNC_FLAG_NO_PURGE        = 0x08
+       MDBOX_SYNC_FLAG_NO_PURGE        = 0x08,
+       MDBOX_SYNC_FLAG_NO_REBUILD      = 0x10
 };
 
 struct mdbox_sync_context {