UIDs for appended messages can now be assigned all at once. Maildir now

updates indexes immediately while saving messages.

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h
index 64808fd1da8a45562601d9c3740fac9ed63f3678..a2b26dc5481bfd225d1314b4d6f3463e22c54408 100644 (file)
--- a/src/lib-index/mail-index-transaction-private.h
+++ b/src/lib-index/mail-index-transaction-private.h
@@ -4,7 +4,8 @@
 #include "mail-transaction-log.h"
 
 struct mail_index_transaction {
-       struct mail_index_view *view, *updated_view;
+       int refcount;
+       struct mail_index_view *view;
 
         buffer_t *appends;
        uint32_t first_new_seq, last_new_seq;
@@ -30,4 +31,7 @@ struct mail_index_transaction {
 struct mail_index_record *
 mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq);
 
+void mail_index_transaction_ref(struct mail_index_transaction *t);
+void mail_index_transaction_unref(struct mail_index_transaction *t);
+
 #endif

diff --git a/src/lib-index/mail-index-transaction-view.c b/src/lib-index/mail-index-transaction-view.c
index e4a56fce16d253776f696bf216d240a235e995d5..d1b1ee413b9d37e605e57577216db648f56beb77 100644 (file)
--- a/src/lib-index/mail-index-transaction-view.c
+++ b/src/lib-index/mail-index-transaction-view.c
@@ -17,8 +17,7 @@ static void _tview_close(struct mail_index_view *view)
        struct mail_index_view_transaction *tview =
                (struct mail_index_view_transaction *)view;
 
-       tview->t->updated_view = NULL;
-
+       mail_index_transaction_unref(tview->t);
        return tview->parent->close(view);
 }
 
@@ -147,19 +146,16 @@ static struct mail_index_view_methods view_methods = {
 };
 
 struct mail_index_view *
-mail_index_transaction_get_updated_view(struct mail_index_transaction *t)
+mail_index_transaction_open_updated_view(struct mail_index_transaction *t)
 {
        struct mail_index_view_transaction *tview;
 
-       if (t->updated_view == NULL) {
-               tview = i_new(struct mail_index_view_transaction, 1);
-               mail_index_view_clone(&tview->view, t->view);
-               tview->view.methods = view_methods;
-               tview->parent = &t->view->methods;
-               tview->t = t;
-
-               t->updated_view = &tview->view;
-       }
+       tview = i_new(struct mail_index_view_transaction, 1);
+       mail_index_view_clone(&tview->view, t->view);
+       tview->view.methods = view_methods;
+       tview->parent = &t->view->methods;
+       tview->t = t;
 
-       return t->updated_view;
+       mail_index_transaction_ref(t);
+       return &tview->view;
 }

diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c
index 0084baaae9d57bf7c3dc4faa2213ae467f44de5e..4835e72620124c51913bfb6f89eba6a9d3499934 100644 (file)
--- a/src/lib-index/mail-index-transaction.c
+++ b/src/lib-index/mail-index-transaction.c
@@ -23,6 +23,7 @@ mail_index_transaction_begin(struct mail_index_view *view, int hide)
        mail_index_view_transaction_ref(view);
 
        t = i_new(struct mail_index_transaction, 1);
+       t->refcount = 1;
        t->view = view;
        t->hide_transaction = hide;
        t->first_new_seq = mail_index_view_get_message_count(t->view)+1;
@@ -48,11 +49,20 @@ static void mail_index_transaction_free(struct mail_index_transaction *t)
                buffer_free(t->updates);
        if (t->cache_updates != NULL)
                buffer_free(t->cache_updates);
-       if (t->updated_view != NULL)
-               mail_index_view_close(t->updated_view);
        i_free(t);
 }
 
+void mail_index_transaction_ref(struct mail_index_transaction *t)
+{
+       t->refcount++;
+}
+
+void mail_index_transaction_unref(struct mail_index_transaction *t)
+{
+       if (--t->refcount == 0)
+               mail_index_transaction_free(t);
+}
+
 static void
 mail_index_buffer_convert_to_uids(struct mail_index_view *view,
                                  buffer_t *buf, size_t record_size, int range)
@@ -114,7 +124,7 @@ int mail_index_transaction_commit(struct mail_index_transaction *t,
        int ret;
 
        if (mail_index_view_is_inconsistent(t->view)) {
-               mail_index_transaction_free(t);
+               mail_index_transaction_unref(t);
                return -1;
        }
 
@@ -128,13 +138,13 @@ int mail_index_transaction_commit(struct mail_index_transaction *t,
                                                  log_file_offset_r);
        }
 
-       mail_index_transaction_free(t);
+       mail_index_transaction_unref(t);
        return ret;
 }
 
 void mail_index_transaction_rollback(struct mail_index_transaction *t)
 {
-        mail_index_transaction_free(t);
+        mail_index_transaction_unref(t);
 }
 
 static void
@@ -208,6 +218,34 @@ void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
        rec->uid = uid;
 }
 
+void mail_index_append_assign_uids(struct mail_index_transaction *t,
+                                  uint32_t first_uid, uint32_t *next_uid_r)
+{
+        struct mail_index_record *rec, *end;
+       size_t size;
+
+       if (t->appends == NULL)
+               return;
+
+       rec = buffer_get_modifyable_data(t->appends, &size);
+       end = PTR_OFFSET(rec, size);
+
+       /* find the first mail with uid = 0 */
+       while (rec != end) {
+               if (rec->uid == 0)
+                       break;
+               rec = PTR_OFFSET(rec, t->append_record_size);
+       }
+
+       while (rec != end) {
+               i_assert(rec->uid == 0);
+               rec->uid = first_uid++;
+               rec = PTR_OFFSET(rec, t->append_record_size);
+       }
+
+       *next_uid_r = first_uid;
+}
+
 void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
 {
         struct mail_transaction_expunge exp, *data;

diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index 49198e8640b56783763fb8f212532d41bb604d72..e9afe9e39e5df7147b483d7cdbd3c7be9fed2d91 100644 (file)
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -179,10 +179,10 @@ int mail_index_transaction_commit(struct mail_index_transaction *t,
 void mail_index_transaction_rollback(struct mail_index_transaction *t);
 
 /* Returns a view to transaction. Currently this differs from normal view only
-   in that it contains newly appended messages in transaction. The view is
-   destroyed when the transaction is destroyed. */
+   in that it contains newly appended messages in transaction. The view can
+   still be used after transaction has been committed. */
 struct mail_index_view *
-mail_index_transaction_get_updated_view(struct mail_index_transaction *t);
+mail_index_transaction_open_updated_view(struct mail_index_transaction *t);
 
 /* Begin synchronizing mailbox with index file. This call locks the index
    exclusively against other modifications. Returns 1 if ok, -1 if error.
@@ -270,6 +270,10 @@ int mail_index_lookup_first(struct mail_index_view *view, enum mail_flags flags,
 /* Append a new record to index. */
 void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
                       uint32_t *seq_r);
+/* Assigns UIDs for appended mails all at once. UID must have been given as 0
+   for mail_index_append(). Returns the next unused UID. */
+void mail_index_append_assign_uids(struct mail_index_transaction *t,
+                                  uint32_t first_uid, uint32_t *next_uid_r);
 /* Expunge record from index. Note that this doesn't affect sequence numbers
    until transaction is committed and mailbox is synced. */
 void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq);

diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c
index b8364e73b01f4bce6ff764a675e8e43d902c937d..159be2b82e62dd540d1f31f5301d2a99f770a3f1 100644 (file)
--- a/src/lib-storage/index/index-transaction.c
+++ b/src/lib-storage/index/index-transaction.c
@@ -9,7 +9,7 @@ void index_transaction_init(struct index_transaction_context *t,
        t->mailbox_ctx.box = &ibox->box;
        t->ibox = ibox;
        t->trans = mail_index_transaction_begin(ibox->view, hide);
-       t->trans_view = mail_index_transaction_get_updated_view(t->trans);
+       t->trans_view = mail_index_transaction_open_updated_view(t->trans);
        t->cache_view = mail_cache_view_open(ibox->cache, t->trans_view);
 }
 
@@ -19,6 +19,7 @@ static void index_transaction_free(struct index_transaction_context *t)
                (void)mail_cache_transaction_end(t->cache_trans);
 
        mail_cache_view_close(t->cache_view);
+       mail_index_view_close(t->trans_view);
        mail_index_view_unlock(t->ibox->view);
 
        if (t->fetch_mail.pool != NULL)

diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c
index 7fa7fe35b2d46ba55023bf97df1f1b60e5d690c8..b3d204233ba8a50581a9ff7c002e902b97f3f0a6 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-save.c
+++ b/src/lib-storage/index/maildir/maildir-save.c
@@ -3,7 +3,9 @@
 #include "lib.h"
 #include "ioloop.h"
 #include "ostream.h"
+#include "str.h"
 #include "maildir-storage.h"
+#include "maildir-uidlist.h"
 #include "mail-save.h"
 
 #include <stdio.h>
@@ -22,6 +24,8 @@ struct maildir_save_context {
        pool_t pool;
 
        struct index_mailbox *ibox;
+       struct mail_index_transaction *trans;
+       struct index_mail mail;
 
        const char *tmpdir, *newdir;
        struct maildir_filename *files;
@@ -103,8 +107,9 @@ static int maildir_file_move(struct maildir_save_context *ctx,
 }
 
 static struct maildir_save_context *
-mailbox_save_init(struct index_mailbox *ibox)
+mailbox_save_init(struct maildir_transaction_context *t)
 {
+        struct index_mailbox *ibox = t->ictx.ibox;
        struct maildir_save_context *ctx;
        pool_t pool;
 
@@ -112,6 +117,9 @@ mailbox_save_init(struct index_mailbox *ibox)
        ctx = p_new(pool, struct maildir_save_context, 1);
        ctx->pool = pool;
        ctx->ibox = ibox;
+       ctx->trans = t->ictx.trans;
+
+       index_mail_init(&t->ictx, &ctx->mail, 0, NULL);
 
        ctx->tmpdir = p_strconcat(pool, ibox->path, "/tmp", NULL);
        ctx->newdir = p_strconcat(pool, ibox->path, "/new", NULL);
@@ -132,9 +140,12 @@ int maildir_save(struct mailbox_transaction_context *_t,
        enum mail_flags mail_flags;
         struct utimbuf buf;
        const char *fname, *dest_fname, *tmp_path;
+       enum mail_flags save_flags;
+       keywords_mask_t keywords;
+       uint32_t seq;
 
        if (t->save_ctx == NULL)
-               t->save_ctx = mailbox_save_init(ibox);
+               t->save_ctx = mailbox_save_init(t);
        ctx = t->save_ctx;
 
        mail_flags = flags->flags;
@@ -178,35 +189,91 @@ int maildir_save(struct mailbox_transaction_context *_t,
        mf->dest = p_strdup(ctx->pool, dest_fname);
        ctx->files = mf;
 
+       /* insert into index */
+       save_flags = (flags->flags & ~MAIL_RECENT) |
+               (ibox->keep_recent ? MAIL_RECENT : 0);
+       memset(keywords, 0, INDEX_KEYWORDS_BYTE_COUNT);
+       // FIXME: set keywords
+
+       mail_index_append(t->ictx.trans, 0, &seq);
+       mail_index_update_flags(t->ictx.trans, seq, MODIFY_REPLACE,
+                               save_flags, keywords);
        t_pop();
+
+       if (mail_r != NULL) {
+               const struct mail_index_record *rec;
+
+               if (mail_index_lookup(t->ictx.trans_view, seq, &rec) < 0)
+                       return -1;
+               if (index_mail_next(&ctx->mail, rec, seq, FALSE) <= 0)
+                       return -1;
+               *mail_r = &ctx->mail.mail;
+       }
+
        return 0;
 }
 
+static void maildir_save_commit_abort(struct maildir_save_context *ctx,
+                                     struct maildir_filename *pos)
+{
+       struct maildir_filename *mf;
+       string_t *str;
+
+       t_push();
+       str = t_str_new(1024);
+
+       /* try to unlink the mails already moved */
+       for (mf = ctx->files; mf != pos; mf = mf->next) {
+               str_truncate(str, 0);
+               str_printfa(str, "%s/%s", ctx->newdir, mf->dest);
+               (void)unlink(str_c(str));
+       }
+       ctx->files = pos;
+       t_pop();
+
+       maildir_save_rollback(ctx);
+}
+
 int maildir_save_commit(struct maildir_save_context *ctx)
 {
-       struct maildir_filename *mf, *mf2;
-       const char *path;
+       struct maildir_uidlist_sync_ctx *sync_ctx;
+       struct maildir_filename *mf;
+       uint32_t first_uid, last_uid;
+        enum maildir_uidlist_rec_flag flags;
        int ret = 0;
 
+       ret = maildir_uidlist_lock(ctx->ibox->uidlist);
+       if (ret <= 0) {
+               /* error or timeout - our transaction is broken */
+               maildir_save_commit_abort(ctx, ctx->files);
+               return -1;
+       }
+
+       first_uid = maildir_uidlist_get_next_uid(ctx->ibox->uidlist);
+       mail_index_append_assign_uids(ctx->trans, first_uid, &last_uid);
+
+       flags = MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+               MAILDIR_UIDLIST_REC_FLAG_RECENT;
+
        /* move them into new/ */
+       sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist, TRUE);
        for (mf = ctx->files; mf != NULL; mf = mf->next) {
-               if (maildir_file_move(ctx, mf->src, mf->dest) < 0) {
-                       ret = -1;
-                       break;
+               if (maildir_file_move(ctx, mf->src, mf->dest) < 0 ||
+                   maildir_uidlist_sync_next(sync_ctx, mf->dest, flags) < 0) {
+                       (void)maildir_uidlist_sync_deinit(sync_ctx);
+                       maildir_save_commit_abort(ctx, mf);
+                       return -1;
                }
        }
 
-       if (ret < 0) {
-               /* failed, try to unlink the mails already moved */
-               for (mf2 = ctx->files; mf2 != mf; mf2 = mf2->next) {
-                       t_push();
-                       path = t_strconcat(ctx->newdir, "/",
-                                          mf2->dest, NULL);
-                       (void)unlink(path);
-                       t_pop();
-               }
+       if (maildir_uidlist_sync_deinit(sync_ctx) < 0) {
+               maildir_save_commit_abort(ctx, NULL);
+               return -1;
        }
 
+       i_assert(maildir_uidlist_get_next_uid(ctx->ibox->uidlist) == last_uid);
+
+       index_mail_deinit(&ctx->mail);
        pool_unref(ctx->pool);
        return ret;
 }
@@ -214,15 +281,19 @@ int maildir_save_commit(struct maildir_save_context *ctx)
 void maildir_save_rollback(struct maildir_save_context *ctx)
 {
        struct maildir_filename *mf;
-       const char *path;
+       string_t *str;
+
+       t_push();
+       str = t_str_new(1024);
 
        /* clean up the temp files */
        for (mf = ctx->files; mf != NULL; mf = mf->next) {
-               t_push();
-               path = t_strconcat(ctx->tmpdir, "/", mf->dest, NULL);
-               (void)unlink(path);
-               t_pop();
+               str_truncate(str, 0);
+               str_printfa(str, "%s/%s", ctx->tmpdir, mf->dest);
+               (void)unlink(str_c(str));
        }
+       t_pop();
 
+       index_mail_deinit(&ctx->mail);
        pool_unref(ctx->pool);
 }

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index 23605219fd7eabcdfaad2809fa163353780b759c..515e1e8447629601ddbb0874ea6f4d735e870fb0 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -66,7 +66,8 @@ struct maildir_uidlist_iter_ctx {
        const struct maildir_uidlist_rec *const *next, *const *end;
 };
 
-int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
+static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist,
+                                       unsigned int timeout)
 {
        const char *path;
        mode_t old_mask;
@@ -79,7 +80,7 @@ int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
                           "/" MAILDIR_UIDLIST_NAME, NULL);
         old_mask = umask(0777 & ~uidlist->ibox->mail_create_mode);
        fd = file_dotlock_open(path, uidlist->ibox->storage->temp_prefix,
-                              NULL, 0, 0, UIDLIST_LOCK_STALE_TIMEOUT,
+                              NULL, timeout, 0, UIDLIST_LOCK_STALE_TIMEOUT,
                               NULL, NULL);
        umask(old_mask);
        if (fd == -1) {
@@ -98,6 +99,17 @@ int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
        return 1;
 }
 
+int maildir_uidlist_lock(struct maildir_uidlist *uidlist)
+{
+       return maildir_uidlist_lock_timeout(uidlist,
+                                           UIDLIST_LOCK_STALE_TIMEOUT);
+}
+
+int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist)
+{
+       return maildir_uidlist_lock_timeout(uidlist, 0);
+}
+
 void maildir_uidlist_unlock(struct maildir_uidlist *uidlist)
 {
        const char *path;

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.h b/src/lib-storage/index/maildir/maildir-uidlist.h
index 191383493dd363556775b16af37f17dbf5d210a6..8381ef09ab4da234839776110ac001950c592dcc 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-uidlist.h
+++ b/src/lib-storage/index/maildir/maildir-uidlist.h
@@ -11,6 +11,7 @@ enum maildir_uidlist_rec_flag {
        MAILDIR_UIDLIST_REC_FLAG_RACING         = 0x10
 };
 
+int maildir_uidlist_lock(struct maildir_uidlist *uidlist);
 int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist);
 void maildir_uidlist_unlock(struct maildir_uidlist *uidlist);