dbox: Implemented copying by changing refcounts.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c
index fae1673c2004f92627f1f82c8623815c1362afa0..fde2368a8574675b7bde33be5c2f253a05779f58 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-mail.c
+++ b/src/lib-storage/index/dbox/dbox-mail.c
@@ -17,6 +17,7 @@ struct dbox_mail {
 
        struct dbox_file *open_file;
        uoff_t offset;
+       uint32_t map_uid;
 };
 
 struct mail *
@@ -90,41 +91,53 @@ int dbox_mail_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
        return 0;
 }
 
+static void dbox_mail_set_expunged(struct dbox_mail *mail)
+{
+       struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox;
+       struct mail *_mail = &mail->imail.mail.mail;
+
+       (void)mail_index_refresh(mbox->ibox.index);
+       if (mail_index_is_expunged(mbox->ibox.view, _mail->seq)) {
+               mail_set_expunged(_mail);
+               return;
+       }
+
+       if (mail->map_uid != 0) {
+               dbox_map_set_corrupted(mbox->storage->map,
+                       "Unexpectedly lost uid=%u map_uid=%u",
+                       _mail->uid, mail->map_uid);
+       } else {
+               mail_storage_set_critical(&mbox->storage->storage,
+                       "Unexpectedly lost uid=%u", _mail->uid);
+       }
+       mbox->storage->sync_rebuild = TRUE;
+}
+
 static int dbox_mail_open(struct dbox_mail *mail,
                          uoff_t *offset_r, struct dbox_file **file_r)
 {
        struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox;
        struct mail *_mail = &mail->imail.mail.mail;
-       uint32_t map_uid, file_id;
+       uint32_t file_id;
        int ret;
 
        if (mail->open_file == NULL) {
                if (dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq,
-                                    &map_uid) < 0)
+                                    &mail->map_uid) < 0)
                        return -1;
-               if (map_uid == 0) {
+               if (mail->map_uid == 0) {
                        mail->open_file =
                                dbox_file_init_single(mbox, _mail->uid);
                } else if ((ret = dbox_map_lookup(mbox->storage->map,
-                                                 map_uid, &file_id,
+                                                 mail->map_uid, &file_id,
                                                  &mail->offset)) <= 0) {
                        if (ret < 0)
                                return -1;
 
                        /* map_uid doesn't exist anymore. either it
                           got just expunged or the map index is
-                          corrupted. see if the message still exists
-                          in the mailbox index. */
-                       (void)mail_index_refresh(mbox->ibox.index);
-                       if (mail_index_is_expunged(mbox->ibox.view,
-                                                  _mail->seq)) {
-                               mail_set_expunged(_mail);
-                               return -1;
-                       }
-
-                       dbox_map_set_corrupted(mbox->storage->map,
-                               "Unexpectedly lost map_uid=%u", map_uid);
-                       mbox->storage->sync_rebuild = TRUE;
+                          corrupted. */
+                       dbox_mail_set_expunged(mail);
                        return -1;
                } else {
                        mail->open_file =
@@ -140,7 +153,6 @@ static int dbox_mail_open(struct dbox_mail *mail,
 static int
 dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r)
 {
-       struct mail *_mail = &mail->imail.mail.mail;
        uoff_t offset, size;
        bool expunged;
 
@@ -151,7 +163,7 @@ dbox_mail_metadata_read(struct dbox_mail *mail, struct dbox_file **file_r)
                                      NULL, &expunged) <= 0)
                return -1;
        if (expunged) {
-               mail_set_expunged(_mail);
+               dbox_mail_set_expunged(mail);
                return -1;
        }
        if (dbox_file_metadata_read(*file_r) <= 0)
@@ -328,7 +340,7 @@ dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
                        return -1;
                }
                if (expunged) {
-                       mail_set_expunged(_mail);
+                       dbox_mail_set_expunged(mail);
                        return -1;
                }
                data->physical_size = size;

diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c
index 1f783fe954a5bac531d2e9a5e28c67a23dacab9b..6931a2ad81bebae17233209caceb3009a7bc30d4 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.c
+++ b/src/lib-storage/index/dbox/dbox-map.c
@@ -13,7 +13,10 @@
 struct dbox_map_transaction_context {
        struct dbox_map *map;
        struct mail_index_transaction *trans;
+       struct mail_index_sync_ctx *sync_ctx;
+
        unsigned int changed:1;
+       unsigned int success:1;
 };
 
 void dbox_map_set_corrupted(struct dbox_map *map, const char *format, ...)
@@ -315,72 +318,73 @@ dbox_map_sync_handle(struct dbox_map *map, struct mail_index_sync_ctx *sync_ctx)
        }
 }
 
-int dbox_map_transaction_commit(struct dbox_map_transaction_context **_ctx)
+int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx)
 {
-       struct dbox_map_transaction_context *ctx = *_ctx;
        struct dbox_map *map = ctx->map;
-       struct mail_index_sync_ctx *sync_ctx;
        struct mail_index_view *view;
-       struct mail_index_transaction *sync_trans, *trans = ctx->trans;
+       struct mail_index_transaction *sync_trans;
        int ret;
 
-       *_ctx = NULL;
-       if (!ctx->changed) {
-               mail_index_transaction_rollback(&trans);
-               i_free(ctx);
+       if (!ctx->changed)
                return 0;
-       }
-       i_free(ctx);
 
        /* use syncing to lock the transaction log, so that we always see
           log's head_offset = tail_offset */
-       ret = mail_index_sync_begin(map->index, &sync_ctx,
+       ret = mail_index_sync_begin(map->index, &ctx->sync_ctx,
                                    &view, &sync_trans, 0);
        if (ret <= 0) {
                i_assert(ret != 0);
                mail_storage_set_internal_error(&map->storage->storage);
                mail_index_reset_error(map->index);
-               mail_index_transaction_rollback(&trans);
+               mail_index_transaction_rollback(&ctx->trans);
                return -1;
        }
-       dbox_map_sync_handle(map, sync_ctx);
+       dbox_map_sync_handle(map, ctx->sync_ctx);
 
-       if (mail_index_transaction_commit(&trans) < 0 ||
-           mail_index_sync_commit(&sync_ctx) < 0) {
+       if (mail_index_transaction_commit(&ctx->trans) < 0) {
                mail_storage_set_internal_error(&map->storage->storage);
                mail_index_reset_error(map->index);
-               if (sync_ctx != NULL)
-                       mail_index_sync_rollback(&sync_ctx);
                return -1;
        }
+       ctx->success = TRUE;
        return 0;
 }
 
-void dbox_map_transaction_rollback(struct dbox_map_transaction_context **_ctx)
+void dbox_map_transaction_free(struct dbox_map_transaction_context **_ctx)
 {
        struct dbox_map_transaction_context *ctx = *_ctx;
+       struct dbox_map *map = ctx->map;
 
        *_ctx = NULL;
-       mail_index_transaction_rollback(&ctx->trans);
+       if (ctx->success) {
+               if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
+                       mail_storage_set_internal_error(&map->storage->storage);
+                       mail_index_reset_error(map->index);
+               }
+       } else if (ctx->sync_ctx != NULL) {
+               mail_index_sync_rollback(&ctx->sync_ctx);
+       }
+       if (ctx->trans != NULL)
+               mail_index_transaction_rollback(&ctx->trans);
        i_free(ctx);
 }
 
 int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
-                             const ARRAY_TYPE(seq_range) *map_uids, int diff)
+                             const ARRAY_TYPE(uint32_t) *map_uids, int diff)
 {
        struct dbox_map *map = ctx->map;
-       struct seq_range_iter iter;
-       unsigned int i;
-       uint32_t map_uid, seq;
+       const uint32_t *uids;
+       unsigned int i, count;
+       uint32_t seq;
        int ret;
 
-       seq_range_array_iter_init(&iter, map_uids); i = 0;
-       while (seq_range_array_iter_nth(&iter, i++, &map_uid)) {
-               if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) {
+       uids = array_get(map_uids, &count);
+       for (i = 0; i < count; i++) {
+               if ((ret = dbox_map_get_seq(map, uids[i], &seq)) <= 0) {
                        if (ret == 0) {
                                dbox_map_set_corrupted(map,
                                        "refcount update lost map_uid=%u",
-                                       map_uid);
+                                       uids[i]);
                        }
                        return -1;
                }
@@ -426,10 +430,9 @@ int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id)
                        mail_index_expunge(map_trans->trans, seq);
                }
        }
-       if (ret < 0)
-               dbox_map_transaction_rollback(&map_trans);
-       else
-               (void)dbox_map_transaction_commit(&map_trans);
+       if (ret == 0)
+               (void)dbox_map_transaction_commit(map_trans);
+       dbox_map_transaction_free(&map_trans);
        return ret;
 }
 
@@ -948,13 +951,16 @@ int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx,
 
 int dbox_map_append_commit(struct dbox_map_append_context *ctx)
 {
-       i_assert(ctx->sync_ctx != NULL);
+       struct dbox_map *map = ctx->map;
+
        i_assert(ctx->trans == NULL);
 
-       if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
-               mail_storage_set_internal_error(&ctx->map->storage->storage);
-               mail_index_reset_error(ctx->map->index);
-               return -1;
+       if (ctx->sync_ctx != NULL) {
+               if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
+                       mail_storage_set_internal_error(&map->storage->storage);
+                       mail_index_reset_error(map->index);
+                       return -1;
+               }
        }
        ctx->committed = TRUE;
        return 0;

diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h
index e226130c0f3fe432e7e1ed82643a82b69e4f655a..e12eef70e205818f0bbad0f71642300f3df53d20 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.h
+++ b/src/lib-storage/index/dbox/dbox-map.h
@@ -44,11 +44,13 @@ int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id,
 
 struct dbox_map_transaction_context *
 dbox_map_transaction_begin(struct dbox_map *map, bool external);
-int dbox_map_transaction_commit(struct dbox_map_transaction_context **ctx);
-void dbox_map_transaction_rollback(struct dbox_map_transaction_context **ctx);
+/* Write transaction to map and leave it locked. Call _free() to update tail
+   offset and unlock. */
+int dbox_map_transaction_commit(struct dbox_map_transaction_context *ctx);
+void dbox_map_transaction_free(struct dbox_map_transaction_context **ctx);
 
 int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx,
-                             const ARRAY_TYPE(seq_range) *map_uids, int diff);
+                             const ARRAY_TYPE(uint32_t) *map_uids, int diff);
 int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id);
 
 /* Return all files containing messages with zero refcount. */

diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c
index 8af6e21c27304b4292a4f60811d47e3f6fe7b071..45df36e195860db04c74852656b1621ff5e41289 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-save.c
+++ b/src/lib-storage/index/dbox/dbox-save.c
@@ -11,6 +11,7 @@
 #include "ostream.h"
 #include "write-full.h"
 #include "index-mail.h"
+#include "mail-copy.h"
 #include "dbox-storage.h"
 #include "dbox-map.h"
 #include "dbox-file.h"
@@ -34,6 +35,9 @@ struct dbox_save_context {
        struct dbox_map_append_context *append_ctx;
        struct dbox_sync_context *sync_ctx;
 
+       ARRAY_TYPE(uint32_t) copy_map_uids;
+       struct dbox_map_transaction_context *map_trans;
+
        /* updated for each appended mail: */
        uint32_t seq;
        struct istream *input;
@@ -59,8 +63,11 @@ dbox_save_alloc(struct mailbox_transaction_context *_t)
 
        i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
 
-       if (t->save_ctx != NULL)
+       if (t->save_ctx != NULL) {
+               /* use the existing allocated structure */
+               t->save_ctx->finished = FALSE;
                return &t->save_ctx->ctx;
+       }
 
        ctx = t->save_ctx = i_new(struct dbox_save_context, 1);
        ctx->ctx.transaction = &t->ictx.mailbox_ctx;
@@ -71,13 +78,26 @@ dbox_save_alloc(struct mailbox_transaction_context *_t)
        return &ctx->ctx;
 }
 
+static void dbox_save_add_to_index(struct dbox_save_context *ctx)
+{
+       enum mail_flags save_flags;
+
+       save_flags = ctx->ctx.flags & ~MAIL_RECENT;
+       mail_index_append(ctx->trans, 0, &ctx->seq);
+       mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
+                               save_flags);
+       if (ctx->ctx.keywords != NULL) {
+               mail_index_update_keywords(ctx->trans, ctx->seq,
+                                          MODIFY_REPLACE, ctx->ctx.keywords);
+       }
+}
+
 int dbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
 {
        struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
        struct dbox_message_header dbox_msg_hdr;
        struct dbox_save_mail *save_mail;
        struct istream *crlf_input;
-       enum mail_flags save_flags;
        uoff_t mail_size;
 
        /* get the size of the mail to be saved, if possible */
@@ -89,15 +109,7 @@ int dbox_save_begin(struct mail_save_context *_ctx, struct istream *input)
                return -1;
        }
 
-       /* add to index */
-       save_flags = _ctx->flags & ~MAIL_RECENT;
-       mail_index_append(ctx->trans, 0, &ctx->seq);
-       mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
-                               save_flags);
-       if (_ctx->keywords != NULL) {
-               mail_index_update_keywords(ctx->trans, ctx->seq,
-                                          MODIFY_REPLACE, _ctx->keywords);
-       }
+       dbox_save_add_to_index(ctx);
 
        if (_ctx->dest_mail == NULL) {
                if (ctx->mail == NULL)
@@ -395,6 +407,18 @@ int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx)
                i_assert(next_map_uid == last_map_uid + 1);
        }
 
+       /* increase map's refcount for copied mails */
+       if (array_is_created(&ctx->copy_map_uids)) {
+               ctx->map_trans =
+                       dbox_map_transaction_begin(ctx->mbox->storage->map,
+                                                  FALSE);
+               if (dbox_map_update_refcounts(ctx->map_trans,
+                                             &ctx->copy_map_uids, 1) < 0) {
+                       dbox_transaction_save_rollback(ctx);
+                       return -1;
+               }
+       }
+
        if (ctx->mail != NULL)
                mail_free(&ctx->mail);
 
@@ -410,6 +434,8 @@ void dbox_transaction_save_commit_post(struct dbox_save_context *ctx)
 
        /* finish writing the mailbox APPENDs */
        if (dbox_sync_finish(&ctx->sync_ctx, TRUE) == 0) {
+               if (ctx->map_trans != NULL)
+                       (void)dbox_map_transaction_commit(ctx->map_trans);
                /* commit only updates the sync tail offset, everything else
                   was already written at this point. */
                (void)dbox_map_append_commit(ctx->append_ctx);
@@ -431,6 +457,10 @@ void dbox_transaction_save_rollback(struct dbox_save_context *ctx)
                dbox_save_cancel(&ctx->ctx);
        if (ctx->append_ctx != NULL)
                dbox_map_append_free(&ctx->append_ctx);
+       if (ctx->map_trans != NULL)
+               dbox_map_transaction_free(&ctx->map_trans);
+       if (array_is_created(&ctx->copy_map_uids))
+               array_free(&ctx->copy_map_uids);
 
        if (ctx->sync_ctx != NULL)
                (void)dbox_sync_finish(&ctx->sync_ctx, FALSE);
@@ -440,3 +470,46 @@ void dbox_transaction_save_rollback(struct dbox_save_context *ctx)
        array_free(&ctx->mails);
        i_free(ctx);
 }
+
+int dbox_copy(struct mail_save_context *_ctx, struct mail *mail)
+{
+       struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
+       struct dbox_mailbox *src_mbox;
+       struct dbox_mail_index_record rec;
+       const void *data;
+       bool expunged;
+
+       ctx->finished = TRUE;
+
+       if (mail->box->storage != _ctx->transaction->box->storage)
+               return mail_storage_copy(_ctx, mail);
+       src_mbox = (struct dbox_mailbox *)mail->box;
+
+       memset(&rec, 0, sizeof(rec));
+       if (dbox_mail_lookup(src_mbox, src_mbox->ibox.view, mail->seq,
+                            &rec.map_uid) < 0)
+               return -1;
+
+       if (rec.map_uid == 0) {
+               /* FIXME: we could hard link */
+               return mail_storage_copy(_ctx, mail);
+       }
+
+       /* remember the map_uid so we can later increase its refcount */
+       if (!array_is_created(&ctx->copy_map_uids))
+               i_array_init(&ctx->copy_map_uids, 32);
+       array_append(&ctx->copy_map_uids, &rec.map_uid, 1);
+
+       /* add message to mailbox index */
+       dbox_save_add_to_index(ctx);
+       mail_index_update_ext(ctx->trans, ctx->seq, ctx->mbox->dbox_ext_id,
+                             &rec, NULL);
+
+       mail_index_lookup_ext(src_mbox->ibox.view, mail->seq,
+                             src_mbox->guid_ext_id, &data, &expunged);
+       if (data != NULL) {
+               mail_index_update_ext(ctx->trans, ctx->seq,
+                                     ctx->mbox->guid_ext_id, data, NULL);
+       }
+       return 0;
+}

diff --git a/src/lib-storage/index/dbox/dbox-storage.c b/src/lib-storage/index/dbox/dbox-storage.c
index 495db9baad0fbd85b4e0b0032fa29de9d2563cc8..2703a36694b379931350d0d7dea19c71c2885b8a 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage.c
+++ b/src/lib-storage/index/dbox/dbox-storage.c
@@ -807,7 +807,7 @@ struct mailbox dbox_mailbox = {
                dbox_save_continue,
                dbox_save_finish,
                dbox_save_cancel,
-               mail_storage_copy,
+               dbox_copy,
                index_storage_is_inconsistent
        }
 };

diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h
index 3b4b464091e0ecdcd3a643fd974b02fd312a30b6..0f4bac751f9a29fa7df33d83ade69153e9f05fb9 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage.h
+++ b/src/lib-storage/index/dbox/dbox-storage.h
@@ -115,4 +115,6 @@ int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx);
 void dbox_transaction_save_commit_post(struct dbox_save_context *ctx);
 void dbox_transaction_save_rollback(struct dbox_save_context *ctx);
 
+int dbox_copy(struct mail_save_context *ctx, struct mail *mail);
+
 #endif

diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c
index 4e206fcc90699d8df1652fb99be2b543a7e7672d..f0d893b1f07f25f148a031e41043d2aac19f0da8 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-sync.c
+++ b/src/lib-storage/index/dbox/dbox-sync.c
@@ -82,7 +82,7 @@ static int dbox_sync_add_seq(struct dbox_sync_context *ctx,
                                     lookup_entry.uid != 0 ? 1 : 3);
                }
                seq_range_array_add(&entry->expunge_seqs, 0, seq);
-               seq_range_array_add(&entry->expunge_map_uids, 0, map_uid);
+               array_append(&entry->expunge_map_uids, &map_uid, 1);
        } else {
                if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
                        entry->move_to_alt = TRUE;
@@ -171,8 +171,9 @@ static int dbox_sync_index(struct dbox_sync_context *ctx)
        }
 
        if (ret > 0 && ctx->map_trans != NULL) {
-               if (dbox_map_transaction_commit(&ctx->map_trans) < 0)
+               if (dbox_map_transaction_commit(ctx->map_trans) < 0)
                        ret = -1;
+               dbox_map_transaction_free(&ctx->map_trans);
        }
 
        if (box->v.sync_notify != NULL)
@@ -305,7 +306,7 @@ int dbox_sync_finish(struct dbox_sync_context **_ctx, bool success)
        *_ctx = NULL;
 
        if (ctx->map_trans != NULL)
-               dbox_map_transaction_rollback(&ctx->map_trans);
+               dbox_map_transaction_free(&ctx->map_trans);
 
        if (success) {
                if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {

diff --git a/src/lib-storage/index/dbox/dbox-sync.h b/src/lib-storage/index/dbox/dbox-sync.h
index febfa4bc35aff2b649ecc69c37e7ef915b5e7751..98f77dff5bddb2c771e04c65034338bf1345b6d0 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-sync.h
+++ b/src/lib-storage/index/dbox/dbox-sync.h
@@ -15,7 +15,7 @@ struct dbox_sync_file_entry {
        unsigned int move_from_alt:1;
        unsigned int move_to_alt:1;
        ARRAY_TYPE(seq_range) expunge_seqs;
-       ARRAY_TYPE(seq_range) expunge_map_uids;
+       ARRAY_TYPE(uint32_t) expunge_map_uids;
 };
 
 struct dbox_sync_context {