imapc: Delay handling new messages and setting uidvalidity/uidnext until sync.

diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index 72e071e3e331a50c18ce1895f6b0f344b0ef6136..6ad5f62f4a6b12e4e2c9bf7e323612f69288676e 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -44,6 +44,8 @@ static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
                                        MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
        mbox->delayed_sync_view =
                mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
+       mbox->min_append_uid =
+               mail_index_get_header(mbox->delayed_sync_view)->next_uid;
 }
 
 int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
@@ -67,53 +69,6 @@ int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
        return ret;
 }
 
-static void imapc_mailbox_open_finish(struct imapc_mailbox *mbox)
-{
-       const struct mail_index_record *rec;
-       struct imapc_seqmap *seqmap;
-       uint32_t lseq, rseq, cur_count;
-
-       /* if we haven't seen FETCH reply for some messages at the end of
-          mailbox they've been externally expunged. */
-       imapc_mailbox_init_delayed_trans(mbox);
-       seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-
-       cur_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
-       for (lseq = cur_count; lseq > 0; lseq--) {
-               rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
-               if (rec->uid <= mbox->highest_seen_uid)
-                       break;
-
-               rseq = imapc_seqmap_lseq_to_rseq(seqmap, lseq);
-               imapc_seqmap_expunge(seqmap, rseq);
-               mail_index_expunge(mbox->delayed_sync_trans, lseq);
-       }
-}
-
-static void
-imapc_newmsgs_callback(const struct imapc_command_reply *reply,
-                      void *context)
-{
-       struct imapc_mailbox *mbox = context;
-
-       if (reply->state == IMAPC_COMMAND_STATE_OK)
-               mbox->open_success = TRUE;
-       else if (reply->state == IMAPC_COMMAND_STATE_NO) {
-               imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
-                                           reply);
-       } else if (mbox->opening ||
-                  reply->state != IMAPC_COMMAND_STATE_DISCONNECTED) {
-               mail_storage_set_critical(&mbox->storage->storage,
-                       "imapc: Mailbox newmsgs fetch failed: %s",
-                       reply->text_full);
-       }
-       if (mbox->opening) {
-               if (reply->state == IMAPC_COMMAND_STATE_OK)
-                       imapc_mailbox_open_finish(mbox);
-               imapc_client_stop(mbox->storage->client);
-       }
-}
-
 static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
                                  struct imapc_mailbox *mbox)
 {
@@ -121,7 +76,7 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
        uint32_t rcount = reply->num;
        const struct mail_index_header *hdr;
        struct imapc_seqmap *seqmap;
-       uint32_t first_uid, next_lseq, next_rseq;
+       uint32_t next_lseq, next_rseq;
 
        if (mbox == NULL)
                return;
@@ -135,7 +90,7 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
        }
        if (mbox->opening) {
                /* We don't know the latest flags, refresh them. */
-               first_uid = 1;
+               mbox->sync_fetch_first_uid = 1;
        } else {
                seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
                next_lseq = mail_index_view_get_messages_count(view) + 1;
@@ -150,12 +105,8 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
                        return;
                }
                hdr = mail_index_get_header(view);
-               first_uid = hdr->next_uid;
+               mbox->sync_fetch_first_uid = hdr->next_uid;
        }
-
-       mbox->new_msgs = TRUE;
-       imapc_client_mailbox_cmdf(mbox->client_box, imapc_newmsgs_callback,
-                                 mbox, "UID FETCH %u:* FLAGS", first_uid);
 }
 
 static void imapc_mailbox_idle_timeout(struct imapc_mailbox *mbox)
@@ -205,7 +156,6 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
        struct imapc_mail *const *mailp;
        const struct imap_arg *list, *flags_list;
        const char *atom;
-       const struct mail_index_header *old_hdr;
        const struct mail_index_record *rec = NULL;
        enum mail_flags flags;
        uint32_t uid, cur_count;
@@ -260,8 +210,6 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
        }
 
        imapc_mailbox_init_delayed_trans(mbox);
-       old_hdr = mail_index_get_header(mbox->sync_view);
-
        cur_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
        while (lseq <= cur_count) {
                rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
@@ -282,17 +230,20 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                lseq++;
        }
        if (lseq > cur_count) {
+               if (!mbox->syncing)
+                       return;
                if (uid == 0 || lseq != cur_count + 1)
                        return;
-               if (uid < old_hdr->next_uid) {
+               if (uid < mbox->min_append_uid) {
                        imapc_mailbox_set_corrupted(mbox,
                                "Expunged message reappeared "
                                "(uid=%u < next_uid=%u)",
-                               uid, old_hdr->next_uid);
+                               uid, mbox->min_append_uid);
                        return;
                }
                i_assert(lseq == cur_count + 1);
                mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
+               mbox->min_append_uid = uid + 1;
                rec = NULL;
        }
        if (seen_flags && (rec == NULL || rec->flags != flags)) {
@@ -355,47 +306,26 @@ static void
 imapc_resp_text_uidvalidity(const struct imapc_untagged_reply *reply,
                            struct imapc_mailbox *mbox)
 {
-       const struct mail_index_header *hdr;
        uint32_t uid_validity;
-       bool changes;
 
        if (mbox == NULL || reply->resp_text_value == NULL ||
            str_to_uint32(reply->resp_text_value, &uid_validity) < 0)
                return;
 
-       hdr = mail_index_get_header(imapc_mailbox_get_sync_view(mbox));
-       if (hdr->uid_validity != uid_validity) {
-               imapc_mailbox_init_delayed_trans(mbox);
-               if (hdr->uid_validity != 0) {
-                       /* uidvalidity changed, reset the entire mailbox */
-                       mail_index_reset(mbox->delayed_sync_trans);
-                       (void)imapc_mailbox_commit_delayed_trans(mbox, &changes);
-                       imapc_mailbox_init_delayed_trans(mbox);
-               }
-               mail_index_update_header(mbox->delayed_sync_trans,
-                       offsetof(struct mail_index_header, uid_validity),
-                       &uid_validity, sizeof(uid_validity), TRUE);
-       }
+       mbox->sync_uid_validity = uid_validity;
 }
 
 static void
 imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
                        struct imapc_mailbox *mbox)
 {
-       const struct mail_index_header *hdr;
        uint32_t uid_next;
 
        if (mbox == NULL || reply->resp_text_value == NULL ||
            str_to_uint32(reply->resp_text_value, &uid_next) < 0)
                return;
 
-       hdr = mail_index_get_header(imapc_mailbox_get_sync_view(mbox));
-       if (hdr->next_uid != uid_next) {
-               imapc_mailbox_init_delayed_trans(mbox);
-               mail_index_update_header(mbox->delayed_sync_trans,
-                       offsetof(struct mail_index_header, next_uid),
-                       &uid_next, sizeof(uid_next), FALSE);
-       }
+       mbox->sync_uid_next = uid_next;
 }
 
 void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox,

diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index 76c25bea08f1604d8cdf118057be255a7d366338..707979eb0eca277c8ce5bb3cb8ee6d60decf2757 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -326,8 +326,7 @@ imapc_mailbox_open_callback(const struct imapc_command_reply *reply,
                        ctx->mbox->box.name, reply->text_full);
                ctx->ret = -1;
        }
-       if (!ctx->mbox->new_msgs)
-               imapc_client_stop(ctx->mbox->storage->client);
+       imapc_client_stop(ctx->mbox->storage->client);
 }
 
 static int imapc_mailbox_open(struct mailbox *box)
@@ -357,7 +356,7 @@ static int imapc_mailbox_open(struct mailbox *box)
        while (ctx.ret == -2)
                imapc_storage_run(mbox->storage);
        mbox->opening = FALSE;
-       if (!mbox->open_success) {
+       if (ctx.ret < 0) {
                mailbox_close(box);
                return -1;
        }

diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 1d248edae1fa1c12fc3afb9bb949272ea326ce86..7c0d4fa991719ef97c62505d92efd4ea4fab2db5 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -54,11 +54,15 @@ struct imapc_mailbox {
        ARRAY_DEFINE(untagged_callbacks, struct imapc_mailbox_event_callback);
        ARRAY_DEFINE(resp_text_callbacks, struct imapc_mailbox_event_callback);
 
+       uint32_t min_append_uid;
        uint32_t highest_seen_uid;
 
+       uint32_t sync_uid_validity;
+       uint32_t sync_uid_next;
+       uint32_t sync_fetch_first_uid;
+
        unsigned int opening:1;
-       unsigned int open_success:1;
-       unsigned int new_msgs:1;
+       unsigned int syncing:1;
 };
 
 struct imapc_simple_context {

diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index 0402830c33140bfa8143f9587760937b01d7c86f..6beb41ba2780e75d84cdee939564e13cdcaa6dc8 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -166,6 +166,49 @@ static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
        imapc_sync_cmd(ctx, str_c(str));
 }
 
+static void imapc_sync_expunge_eom(struct imapc_sync_context *ctx)
+{
+       const struct mail_index_record *rec;
+       uint32_t lseq, cur_count;
+
+       /* if we haven't seen FETCH reply for some messages at the end of
+          mailbox they've been externally expunged. */
+       cur_count = mail_index_view_get_messages_count(ctx->sync_view);
+       for (lseq = cur_count; lseq > 0; lseq--) {
+               rec = mail_index_lookup(ctx->sync_view, lseq);
+               if (rec->uid <= ctx->mbox->highest_seen_uid)
+                       break;
+
+               mail_index_expunge(ctx->trans, lseq);
+       }
+}
+
+static void imapc_sync_index_header(struct imapc_sync_context *ctx)
+{
+       struct imapc_mailbox *mbox = ctx->mbox;
+       const struct mail_index_header *hdr;
+
+       hdr = mail_index_get_header(ctx->sync_view);
+       if (hdr->uid_validity != mbox->sync_uid_validity &&
+           mbox->sync_uid_validity != 0) {
+               if (hdr->uid_validity != 0) {
+                       /* uidvalidity changed, reset the entire mailbox */
+                       mail_index_reset(ctx->trans);
+                       mbox->sync_fetch_first_uid = 1;
+               }
+               mail_index_update_header(ctx->trans,
+                       offsetof(struct mail_index_header, uid_validity),
+                       &mbox->sync_uid_validity,
+                       sizeof(mbox->sync_uid_validity), TRUE);
+       }
+       if (hdr->next_uid < mbox->sync_uid_next) {
+               mail_index_update_header(ctx->trans,
+                       offsetof(struct mail_index_header, next_uid),
+                       &mbox->sync_uid_next, sizeof(mbox->sync_uid_next),
+                       FALSE);
+       }
+}
+
 static void imapc_sync_index(struct imapc_sync_context *ctx)
 {
        struct mailbox *box = &ctx->mbox->box;
@@ -175,6 +218,7 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
        i_array_init(&ctx->expunged_uids, 64);
        ctx->keywords = mail_index_get_keywords(box->index);
 
+       imapc_sync_index_header(ctx);
        while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
                if (!mail_index_lookup_seq_range(ctx->sync_view,
                                                 sync_rec.uid1, sync_rec.uid2,
@@ -202,11 +246,22 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
                }
        } T_END;
 
+       if (ctx->mbox->sync_fetch_first_uid != 0) {
+               /* we'll resync existing messages' flags and add new messages.
+                  adding new messages requires sync locking to avoid
+                  duplicates. */
+               imapc_sync_cmd(ctx, t_strdup_printf(
+                       "UID FETCH %u:* FLAGS",
+                       ctx->mbox->sync_fetch_first_uid));
+               ctx->mbox->sync_fetch_first_uid = 0;
+       }
+
        imapc_sync_expunge_finish(ctx);
        while (ctx->sync_command_count > 0)
                imapc_storage_run(ctx->mbox->storage);
        array_free(&ctx->expunged_uids);
 
+       imapc_sync_expunge_eom(ctx);
        if (box->v.sync_notify != NULL)
                box->v.sync_notify(box, 0, 0);
 }
@@ -219,6 +274,8 @@ imapc_sync_begin(struct imapc_mailbox *mbox,
        enum mail_index_sync_flags sync_flags;
        int ret;
 
+       i_assert(!mbox->syncing);
+
        ctx = i_new(struct imapc_sync_context, 1);
        ctx->mbox = mbox;
 
@@ -244,6 +301,7 @@ imapc_sync_begin(struct imapc_mailbox *mbox,
                mail_index_transaction_open_updated_view(ctx->trans);
        mbox->delayed_sync_trans = ctx->trans;
 
+       mbox->syncing = TRUE;
        imapc_sync_index(ctx);
 
        mail_index_view_close(&mbox->delayed_sync_view);
@@ -268,6 +326,7 @@ static int imapc_sync_finish(struct imapc_sync_context **_ctx)
        } else {
                mail_index_sync_rollback(&ctx->index_sync_ctx);
        }
+       ctx->mbox->syncing = FALSE;
        i_free(ctx);
        return ret;
 }
@@ -276,6 +335,7 @@ static int imapc_sync(struct imapc_mailbox *mbox)
 {
        struct imapc_sync_context *sync_ctx;
        struct imapc_seqmap *seqmap;
+       bool force = mbox->sync_fetch_first_uid != 0;
 
        /* if there are any pending expunges, they're now committed. syncing
           will return a view where they no longer exist, so reset the seqmap
@@ -283,7 +343,7 @@ static int imapc_sync(struct imapc_mailbox *mbox)
        seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
        imapc_seqmap_reset(seqmap);
 
-       if (imapc_sync_begin(mbox, &sync_ctx, FALSE) < 0)
+       if (imapc_sync_begin(mbox, &sync_ctx, force) < 0)
                return -1;
        if (sync_ctx == NULL)
                return 0;
@@ -321,7 +381,8 @@ imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
 
        if (imapc_mailbox_commit_delayed_trans(mbox, &changes) < 0)
                ret = -1;
-       if ((changes || index_mailbox_want_full_sync(&mbox->box, flags)) &&
+       if ((changes || mbox->sync_fetch_first_uid != 0 ||
+            index_mailbox_want_full_sync(&mbox->box, flags)) &&
            ret == 0)
                ret = imapc_sync(mbox);