Added maildir_very_dirty_syncs setting.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c
index 8836e12704e033640720b29554600e491857cb0c..a1b15a0dd47dadd31de7a7a3a750967b4d1ef152 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-keywords.c
+++ b/src/lib-storage/index/maildir/maildir-keywords.c
@@ -47,6 +47,7 @@ struct maildir_keywords_sync_ctx {
        const ARRAY_TYPE(keywords) *keywords;
        ARRAY_DEFINE(idx_to_chr, char);
        unsigned int chridx_to_idx[MAILDIR_MAX_KEYWORDS];
+       bool readonly;
 };
 
 struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox)
@@ -404,6 +405,17 @@ maildir_keywords_sync_init(struct maildir_keywords *mk,
        return ctx;
 }
 
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init_readonly(struct maildir_keywords *mk,
+                                   struct mail_index *index)
+{
+       struct maildir_keywords_sync_ctx *ctx;
+
+       ctx = maildir_keywords_sync_init(mk, index);
+       ctx->readonly = TRUE;
+       return ctx;
+}
+
 void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **_ctx)
 {
        struct maildir_keywords_sync_ctx *ctx = *_ctx;
@@ -455,13 +467,17 @@ char maildir_keywords_idx_char(struct maildir_keywords_sync_ctx *ctx,
        const char *const *name_p;
        char *chr_p;
        unsigned int chridx;
+       int ret;
 
        chr_p = array_idx_modifiable(&ctx->idx_to_chr, idx);
        if (*chr_p != '\0')
                return *chr_p;
 
        name_p = array_idx(ctx->keywords, idx);
-       if (maildir_keywords_lookup_or_create(ctx->mk, *name_p, &chridx) <= 0)
+       ret = !ctx->readonly ?
+               maildir_keywords_lookup_or_create(ctx->mk, *name_p, &chridx) :
+               maildir_keywords_lookup(ctx->mk, *name_p, &chridx);
+       if (ret <= 0)
                return '\0';
 
        *chr_p = chridx + MAILDIR_KEYWORD_FIRST;

diff --git a/src/lib-storage/index/maildir/maildir-keywords.h b/src/lib-storage/index/maildir/maildir-keywords.h
index f1d78ff250f0302d04ebb843fbe6f28a3f10d34a..ea0e3d8251c2e24999f249202d28b118ddba5490 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-keywords.h
+++ b/src/lib-storage/index/maildir/maildir-keywords.h
@@ -19,6 +19,10 @@ maildir_keywords_init_readonly(struct mailbox *box);
 struct maildir_keywords_sync_ctx *
 maildir_keywords_sync_init(struct maildir_keywords *mk,
                           struct mail_index *index);
+/* Don't try to add any nonexisting keywords */
+struct maildir_keywords_sync_ctx *
+maildir_keywords_sync_init_readonly(struct maildir_keywords *mk,
+                                   struct mail_index *index);
 void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **ctx);
 
 /* Returns keyword index. */

diff --git a/src/lib-storage/index/maildir/maildir-storage.c b/src/lib-storage/index/maildir/maildir-storage.c
index 23037c77b6343bde6f87d663d77da1ef42a8570a..aa67b8aa14e0a74be9764dfd278f86c0d734d43c 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-storage.c
+++ b/src/lib-storage/index/maildir/maildir-storage.c
@@ -442,6 +442,7 @@ maildir_open(struct maildir_storage *storage, const char *name,
        index = index_storage_alloc(&storage->storage, name, flags,
                                    MAILDIR_INDEX_PREFIX);
        mbox->ibox.index = index;
+       mbox->very_dirty_syncs = getenv("MAILDIR_VERY_DIRTY_SYNCS") != NULL;
 
        /* for shared mailboxes get the create mode from the
           permissions of dovecot-shared file. */
@@ -869,6 +870,8 @@ static int maildir_storage_mailbox_close(struct mailbox *box)
                timeout_remove(&mbox->keep_lock_to);
        }
 
+       if (mbox->flags_view != NULL)
+               mail_index_view_close(&mbox->flags_view);
        if (mbox->keywords != NULL)
                maildir_keywords_deinit(&mbox->keywords);
        maildir_uidlist_deinit(&mbox->uidlist);

diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h
index 7f5c63c65d6b51e126a47830192303ff179621e7..b8692406ba71dfa2ee7a657a8f44886d5780a703 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-storage.h
+++ b/src/lib-storage/index/maildir/maildir-storage.h
@@ -82,6 +82,7 @@ struct maildir_storage {
 struct maildir_mailbox {
        struct index_mailbox ibox;
        struct maildir_storage *storage;
+       struct mail_index_view *flags_view;
 
        const char *path;
        struct timeout *keep_lock_to;
@@ -94,6 +95,7 @@ struct maildir_mailbox {
        uint32_t maildir_ext_id;
 
        unsigned int syncing_commit:1;
+       unsigned int very_dirty_syncs:1;
 };
 
 struct maildir_transaction_context {

diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c
index 099ca7b2134ca9bc695919cd17eadf805672606e..693e88310b857cc214b9add05ba123aef9b27dce 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-sync-index.c
+++ b/src/lib-storage/index/maildir/maildir-sync-index.c
@@ -387,6 +387,8 @@ int maildir_sync_index(struct maildir_index_sync_context *ctx,
        uint32_t first_uid;
        unsigned int changes = 0;
        int ret = 0;
+       time_t time_before_sync;
+       struct stat st;
        bool expunged, full_rescan = FALSE;
 
        i_assert(!mbox->syncing_commit);
@@ -412,6 +414,7 @@ int maildir_sync_index(struct maildir_index_sync_context *ctx,
        }
        hdr_next_uid = hdr->next_uid;
 
+       time_before_sync = time(NULL);
        mbox->syncing_commit = TRUE;
        seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1);
        t_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS);
@@ -560,8 +563,12 @@ int maildir_sync_index(struct maildir_index_sync_context *ctx,
        if (mbox->ibox.box.v.sync_notify != NULL)
                mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0);
 
-       if (ctx->changed)
-               mbox->maildir_hdr.cur_mtime = time(NULL);
+       if (stat(t_strconcat(mbox->path, "/cur", NULL), &st) == 0) {
+               mbox->maildir_hdr.new_check_time =
+                       I_MAX(st.st_mtime, time_before_sync);
+               mbox->maildir_hdr.cur_mtime = st.st_mtime;
+               mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st);
+       }
 
        if (uid_validity == 0) {
                uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :

diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c
index 27cda4bfbfe732faf7de50a340aa3dd7b1ccf456..756f31dedcbacde1894877d63e1fd593436d56bf 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-sync.c
+++ b/src/lib-storage/index/maildir/maildir-sync.c
@@ -500,10 +500,23 @@ static int maildir_scan_dir(struct maildir_sync_context *ctx, bool new_dir)
        }
 
        if (dir_changed) {
-               if (new_dir)
-                       ctx->mbox->maildir_hdr.new_mtime = now;
-               else
-                       ctx->mbox->maildir_hdr.cur_mtime = now;
+               /* save the exact new times. the new mtimes should be >=
+                  "now", but just in case something weird happens and mtime
+                  doesn't update, use "now". */
+               if (stat(ctx->new_dir, &st) == 0) {
+                       ctx->mbox->maildir_hdr.new_check_time =
+                               I_MAX(st.st_mtime, now);
+                       ctx->mbox->maildir_hdr.new_mtime = st.st_mtime;
+                       ctx->mbox->maildir_hdr.new_mtime_nsecs =
+                               ST_MTIME_NSEC(st);
+               }
+               if (stat(ctx->cur_dir, &st) == 0) {
+                       ctx->mbox->maildir_hdr.new_check_time =
+                               I_MAX(st.st_mtime, now);
+                       ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime;
+                       ctx->mbox->maildir_hdr.cur_mtime_nsecs =
+                               ST_MTIME_NSEC(st);
+               }
        }
 
        return ret < 0 ? -1 :
@@ -564,7 +577,7 @@ static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty,
 
        /* try to avoid stat()ing by first checking delayed changes */
        if (DIR_DELAYED_REFRESH(hdr, new) ||
-           DIR_DELAYED_REFRESH(hdr, cur)) {
+           (DIR_DELAYED_REFRESH(hdr, cur) && !mbox->very_dirty_syncs)) {
                /* refresh index and try again */
                if (maildir_sync_header_refresh(mbox) < 0)
                        return -1;
@@ -572,7 +585,7 @@ static int maildir_sync_quick_check(struct maildir_mailbox *mbox, bool undirty,
 
                if (DIR_DELAYED_REFRESH(hdr, new))
                        *new_changed_r = TRUE;
-               if (DIR_DELAYED_REFRESH(hdr, cur))
+               if (DIR_DELAYED_REFRESH(hdr, cur) && !mbox->very_dirty_syncs)
                        *cur_changed_r = TRUE;
                if (*new_changed_r && *cur_changed_r)
                        return 0;
@@ -901,6 +914,32 @@ maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
                }
        }
 
+       if (mbox->very_dirty_syncs) {
+               struct mail_index_view_sync_ctx *sync_ctx;
+               bool b;
+
+               if (mbox->flags_view == NULL) {
+                       mbox->flags_view =
+                               mail_index_view_open(mbox->ibox.index);
+               }
+               sync_ctx = mail_index_view_sync_begin(mbox->flags_view,
+                               MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
+               if (mail_index_view_sync_commit(&sync_ctx, &b) < 0) {
+                       mail_storage_set_index_error(&mbox->ibox);
+                       ret = -1;
+               }
+               /* make sure the map stays in private memory */
+               if (mbox->flags_view->map->refcount > 1) {
+                       struct mail_index_map *map;
+
+                       map = mail_index_map_clone(mbox->flags_view->map);
+                       mail_index_unmap(&mbox->flags_view->map);
+                       mbox->flags_view->map = map;
+               }
+               mail_index_record_map_move_to_private(mbox->flags_view->map);
+               mail_index_map_move_to_memory(mbox->flags_view->map);
+               maildir_uidlist_set_all_nonsynced(mbox->uidlist);
+       }
        return index_mailbox_sync_init(box, flags, ret < 0);
 }
 

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index b6d409385e7ca0d37cb2e6366d5fa1d83b636bf4..02233fa003c2710fbc904662add2456ea366747b 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -1338,6 +1338,11 @@ static int maildir_uidlist_sync_lock(struct maildir_uidlist *uidlist,
        return 1;
 }
 
+void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist)
+{
+       maildir_uidlist_mark_all(uidlist, TRUE);
+}
+
 int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
                              enum maildir_uidlist_sync_flags sync_flags,
                              struct maildir_uidlist_sync_ctx **sync_ctx_r)

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h
index 9485c3ed9a92d0ab13d6336f3e85802772436534..cca7e51dfe427bf5983a6f911ae43e261d283d59 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-uidlist.h
+++ b/src/lib-storage/index/maildir/maildir-uidlist.h
@@ -93,6 +93,7 @@ void maildir_uidlist_set_ext(struct maildir_uidlist *uidlist, uint32_t uid,
    maildir_uidlist_set_ext() */
 int maildir_uidlist_update(struct maildir_uidlist *uidlist);
 
+void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist);
 /* Sync uidlist with what's actually on maildir. Returns same as
    maildir_uidlist_lock(). */
 int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,

diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c
index 76b0e3757048a2a10d48ecaa2d8e2acd31ff9529..26793521fcdcd3e6ede0e22f9ba1b6265e8a9b75 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-util.c
+++ b/src/lib-storage/index/maildir/maildir-util.c
@@ -1,11 +1,14 @@
 /* Copyright (c) 2004-2009 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "array.h"
 #include "ioloop.h"
 #include "str.h"
 #include "mkdir-parents.h"
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
+#include "maildir-keywords.h"
+#include "maildir-filename.h"
 #include "maildir-sync.h"
 
 #include <unistd.h>
@@ -16,35 +19,69 @@
 
 #define MAILDIR_RESYNC_RETRY_COUNT 10
 
+static const char *
+maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid,
+                      const char *fname, bool *have_flags_r)
+
+{
+       struct mail_index_view *view = mbox->flags_view;
+       struct maildir_keywords_sync_ctx *kw_ctx;
+       enum mail_flags flags;
+       ARRAY_TYPE(keyword_indexes) keywords;
+       uint32_t seq;
+
+       if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) {
+               *have_flags_r = FALSE;
+               return fname;
+       }
+
+       t_array_init(&keywords, 32);
+       mail_index_lookup_view_flags(view, seq, &flags, &keywords);
+       if (array_count(&keywords) == 0) {
+               *have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0;
+               fname = maildir_filename_set_flags(NULL, fname, flags, NULL);
+       } else {
+               *have_flags_r = TRUE;
+               kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords,
+                                                            mbox->ibox.index);
+               fname = maildir_filename_set_flags(kw_ctx, fname,
+                                                  flags, &keywords);
+               maildir_keywords_sync_deinit(&kw_ctx);
+       }
+       return fname;
+}
+
 static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid,
                               maildir_file_do_func *callback, void *context)
 {
-       const char *fname;
-        enum maildir_uidlist_rec_flag flags;
+       const char *path, *fname;
+       enum maildir_uidlist_rec_flag flags;
+       bool have_flags;
        int ret;
 
        fname = maildir_uidlist_lookup(mbox->uidlist, uid, &flags);
        if (fname == NULL)
                return -2; /* expunged */
 
+       if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+               /* let's see if we can guess the filename based on index */
+               fname = maildir_filename_guess(mbox, uid, fname, &have_flags);
+               if (have_flags) {
+                       /* don't even bother looking into new/ dir */
+                       flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+               }
+       }
+
        if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
                /* probably in new/ dir */
-               T_BEGIN {
-                       const char *path;
-
-                       path = t_strconcat(mbox->path, "/new/", fname, NULL);
-                       ret = callback(mbox, path, context);
-               } T_END;
+               path = t_strconcat(mbox->path, "/new/", fname, NULL);
+               ret = callback(mbox, path, context);
                if (ret != 0)
                        return ret;
        }
 
-       T_BEGIN {
-               const char *path;
-
-               path = t_strconcat(mbox->path, "/cur/", fname, NULL);
-               ret = callback(mbox, path, context);
-       } T_END;
+       path = t_strconcat(mbox->path, "/cur/", fname, NULL);
+       ret = callback(mbox, path, context);
        return ret;
 }
 
@@ -71,7 +108,9 @@ int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid,
 {
        int i, ret;
 
-       ret = maildir_file_do_try(mbox, uid, callback, context);
+       T_BEGIN {
+               ret = maildir_file_do_try(mbox, uid, callback, context);
+       } T_END;
        for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) {
                /* file is either renamed or deleted. sync the maildir and
                   see which one. if file appears to be renamed constantly,
@@ -79,11 +118,14 @@ int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid,
                if (maildir_storage_sync_force(mbox, uid) < 0)
                        return -1;
 
-               ret = maildir_file_do_try(mbox, uid, callback, context);
+               T_BEGIN {
+                       ret = maildir_file_do_try(mbox, uid, callback, context);
+               } T_END;
        }
 
-       if (i == MAILDIR_RESYNC_RETRY_COUNT)
+       if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN {
                ret = maildir_file_do_try(mbox, uid, do_racecheck, context);
+       } T_END;
 
        return ret == -2 ? 0 : ret;
 }

diff --git a/src/master/mail-process.c b/src/master/mail-process.c
index 3c37f140118a5af9d1c078ed4ba47dd323fef409..0ae2cdbf675f73b5af6671d0f164903b1104e8d2 100644 (file)
--- a/src/master/mail-process.c
+++ b/src/master/mail-process.c
@@ -366,6 +366,8 @@ mail_process_set_environment(struct settings *set, const char *mail,
                env_put("MAILDIR_COPY_WITH_HARDLINKS=1");
        if (set->maildir_copy_preserve_filename)
                env_put("MAILDIR_COPY_PRESERVE_FILENAME=1");
+       if (set->maildir_very_dirty_syncs)
+               env_put("MAILDIR_VERY_DIRTY_SYNCS=1");
        if (set->mail_debug)
                env_put("DEBUG=1");
        if (set->mail_full_filesystem_access)

diff --git a/src/master/master-settings-defs.c b/src/master/master-settings-defs.c
index 6d7e3bdb962ede9746ba0a97362341edc97e798e..964d3c80674e60fab20daf093c0477a6f824b66f 100644 (file)
--- a/src/master/master-settings-defs.c
+++ b/src/master/master-settings-defs.c
@@ -87,6 +87,7 @@ static struct setting_def setting_defs[] = {
        DEF_BOOL(maildir_stat_dirs),
        DEF_BOOL(maildir_copy_with_hardlinks),
        DEF_BOOL(maildir_copy_preserve_filename),
+       DEF_BOOL(maildir_very_dirty_syncs),
        DEF_STR(mbox_read_locks),
        DEF_STR(mbox_write_locks),
        DEF_INT(mbox_lock_timeout),

diff --git a/src/master/master-settings.c b/src/master/master-settings.c
index 99a45c2e0237362ca0c22d44636e85367074a170..10df1fa50465160eb04d8f4daa6e087b4d0a3ccf 100644 (file)
--- a/src/master/master-settings.c
+++ b/src/master/master-settings.c
@@ -253,6 +253,7 @@ struct settings default_settings = {
        MEMBER(maildir_stat_dirs) FALSE,
        MEMBER(maildir_copy_with_hardlinks) TRUE,
        MEMBER(maildir_copy_preserve_filename) FALSE,
+       MEMBER(maildir_very_dirty_syncs) FALSE,
        MEMBER(mbox_read_locks) "fcntl",
        MEMBER(mbox_write_locks) "dotlock fcntl",
        MEMBER(mbox_lock_timeout) 300,

diff --git a/src/master/master-settings.h b/src/master/master-settings.h
index 9ffb981b1bee90e46cf520cb3f9991a4fdaa8a9b..0238a3b8217d08675935ac18c47a134f7cfef9a1 100644 (file)
--- a/src/master/master-settings.h
+++ b/src/master/master-settings.h
@@ -99,6 +99,7 @@ struct settings {
        bool maildir_stat_dirs;
        bool maildir_copy_with_hardlinks;
        bool maildir_copy_preserve_filename;
+       bool maildir_very_dirty_syncs;
        const char *mbox_read_locks;
        const char *mbox_write_locks;
        unsigned int mbox_lock_timeout;