imapc: Added support for imapc_features=modseq

If the remote server supports CONDSTORE or QRESYNC extensions we'll use the
remote's MODSEQ and HIGHESTMODSEQ counts.

There are some situations where the HIGHESTMODSEQ isn't updated exactly
correctly on an open mailbox, so this feature shouldn't be fully relied on.
It was primarily implemented for dsync+imapc support - both for preserving
modseqs and also for HIGHESTMODSEQ lookups.

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 1923a8f66a2d2b55e8b2cb7361a604ac479eb5ea..45be11e3a0e6dc868d31d9f77eb56f7b51951a8b 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -96,6 +96,26 @@ static int imapc_mail_failed(struct mail *mail, const char *field)
        return fix_broken_mail ? 0 : -1;
 }
 
+static uint64_t imapc_mail_get_modseq(struct mail *_mail)
+{
+       struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+       struct imapc_msgmap *msgmap;
+       const uint64_t *modseqs;
+       unsigned int count;
+       uint32_t rseq;
+
+       if (!imapc_storage_has_modseqs(mbox->storage))
+               return index_mail_get_modseq(_mail);
+
+       msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+       if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) {
+               modseqs = array_get(&mbox->rseq_modseqs, &count);
+               if (rseq <= count)
+                       return modseqs[rseq-1];
+       }
+       return 1; /* unknown modseq */
+}
+
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
        struct index_mail *mail = (struct index_mail *)_mail;
@@ -561,7 +581,7 @@ struct mail_vfuncs imapc_mail_vfuncs = {
        index_mail_get_flags,
        index_mail_get_keywords,
        index_mail_get_keyword_indexes,
-       index_mail_get_modseq,
+       imapc_mail_get_modseq,
        index_mail_get_pvt_modseq,
        index_mail_get_parts,
        index_mail_get_date,

diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index c3e12d17021768b61473354a3c848cd6209bf17e..349339db429e301871ace559ab090ef30cc7fca9 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "ioloop.h"
+#include "mail-index-modseq.h"
 #include "imap-arg.h"
 #include "imap-seqset.h"
 #include "imap-util.h"
@@ -282,11 +283,12 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
        uint32_t lseq, rseq = reply->num;
        struct imapc_fetch_request *const *fetch_requestp;
        struct imapc_mail *const *mailp;
-       const struct imap_arg *list, *flags_list;
+       const struct imap_arg *list, *flags_list, *modseq_list;
        const char *atom, *guid = NULL;
        const struct mail_index_record *rec = NULL;
        enum mail_flags flags;
        uint32_t fetch_uid, uid;
+       uint64_t modseq = 0;
        unsigned int i, j;
        ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
        bool seen_flags = FALSE, have_labels = FALSE;
@@ -319,6 +321,15 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                                        array_append(&keywords, &atom, 1);
                                }
                        }
+               } else if (strcasecmp(atom, "MODSEQ") == 0 &&
+                          imapc_storage_has_modseqs(mbox->storage)) {
+                       /* (modseq-number) */
+                       if (!imap_arg_get_list(&list[i+1], &modseq_list))
+                               return;
+                       if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
+                           str_to_uint64(atom, &modseq) < 0 ||
+                           modseq_list[1].type != IMAP_ARG_EOL)
+                               return;
                } else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
                           !mbox->initial_sync_done) {
                        if (imap_arg_get_atom(&list[i+1], &atom))
@@ -414,6 +425,11 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                }
                mail_index_keywords_unref(&kw);
        }
+       if (modseq != 0) {
+               if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < modseq)
+                       mail_index_update_modseq(mbox->delayed_sync_trans, lseq, modseq);
+               array_idx_set(&mbox->rseq_modseqs, rseq-1, &modseq);
+       }
        if (guid != NULL) {
                struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
                const enum index_cache_field guid_cache_idx =
@@ -454,6 +470,7 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
        }
        uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
        imapc_msgmap_expunge(msgmap, rseq);
+       array_delete(&mbox->rseq_modseqs, rseq-1, 1);
 
        imapc_mailbox_init_delayed_trans(mbox);
        if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
@@ -578,6 +595,19 @@ imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
        mbox->sync_uid_next = uid_next;
 }
 
+static void
+imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
+                             struct imapc_mailbox *mbox)
+{
+       uint64_t highestmodseq;
+
+       if (mbox == NULL ||
+           str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
+               return;
+
+       mbox->sync_highestmodseq = highestmodseq;
+}
+
 static void
 imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
                               struct imapc_mailbox *mbox)
@@ -646,6 +676,8 @@ void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox)
                                         imapc_resp_text_uidvalidity);
        imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
                                         imapc_resp_text_uidnext);
+       imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
+                                        imapc_resp_text_highestmodseq);
        imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
                                         imapc_resp_text_permanentflags);
 }

diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c
index 6325aa8349f4f066228fa4e8d6dd982cffaea3c8..2bbf274fc740af8a416bf23d6a431afcca986e99 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-settings.c
+++ b/src/lib-storage/index/imapc/imapc-settings.c
@@ -93,6 +93,7 @@ static const struct imapc_feature_list imapc_feature_list[] = {
        { "proxyauth", IMAPC_FEATURE_PROXYAUTH },
        { "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
        { "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
+       { "modseq", IMAPC_FEATURE_MODSEQ },
        { NULL, 0 }
 };
 

diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h
index bb1dec1ae7350fa73238adadbae0f1189d673757..fa8be08492fa0283ff86875d11b86863a63e5961 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-settings.h
+++ b/src/lib-storage/index/imapc/imapc-settings.h
@@ -14,7 +14,8 @@ enum imapc_features {
        IMAPC_FEATURE_NO_EXAMINE                = 0x40,
        IMAPC_FEATURE_PROXYAUTH                 = 0x80,
        IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS     = 0x100,
-       IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS    = 0x200
+       IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS    = 0x200,
+       IMAPC_FEATURE_MODSEQ                    = 0x400
 };
 /* </settings checks> */
 

diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index ebe22b20aca3cbaf7929b9a7da96655383351570..4c7e1a3eae297f1086a178bbc6e0ec3f505bfd6d 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -55,6 +55,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
                                  struct imapc_storage_client *client);
 static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
                                     struct imapc_storage_client *client);
+static int imapc_mailbox_run_status(struct mailbox *box,
+                                   enum mailbox_status_items items,
+                                   struct mailbox_status *status_r);
 
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 {
@@ -72,6 +75,16 @@ bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
        return FALSE;
 }
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage)
+{
+       enum imapc_capability capa =
+               imapc_client_get_capabilities(storage->client->client);
+
+       return (capa & (IMAPC_CAPABILITY_CONDSTORE |
+                       IMAPC_CAPABILITY_QRESYNC)) != 0 &&
+               IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_MODSEQ);
+}
+
 static struct mail_storage *imapc_storage_alloc(void)
 {
        struct imapc_storage *storage;
@@ -603,6 +616,13 @@ static int imapc_mailbox_open(struct mailbox *box)
                return -1;
        }
 
+       if (imapc_storage_has_modseqs(mbox->storage)) {
+               if (!array_is_created(&mbox->rseq_modseqs))
+                       i_array_init(&mbox->rseq_modseqs, 32);
+               else
+                       array_clear(&mbox->rseq_modseqs);
+       }
+
        if (imapc_mailbox_select(mbox) < 0) {
                mailbox_close(box);
                return -1;
@@ -635,6 +655,8 @@ static void imapc_mailbox_close(struct mailbox *box)
                if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
                        mailbox_set_index_error(&mbox->box);
        }
+       if (array_is_created(&mbox->rseq_modseqs))
+               array_free(&mbox->rseq_modseqs);
        if (mbox->sync_view != NULL)
                mail_index_view_close(&mbox->sync_view);
        if (mbox->to_idle_delay != NULL)
@@ -727,6 +749,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
                        status->uidvalidity = num;
                else if (strcasecmp(key, "UNSEEN") == 0)
                        status->unseen = num;
+               else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 &&
+                        imapc_storage_has_modseqs(storage))
+                       status->highest_modseq = num;
        }
 }
 
@@ -765,15 +790,32 @@ static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
        }
 }
 
-static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
-                                             enum mailbox_status_items items,
-                                             struct mailbox_status *status_r)
+static int imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
+                                            enum mailbox_status_items items,
+                                            struct mailbox_status *status_r)
 {
+       int ret = 0;
+
        index_storage_get_open_status(&mbox->box, items, status_r);
        if ((items & STATUS_PERMANENT_FLAGS) != 0)
                status_r->permanent_flags = mbox->permanent_flags;
        if ((items & STATUS_FIRST_RECENT_UID) != 0)
                status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1;
+       if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+               /* FIXME: this doesn't work perfectly. we're now just returning
+                  the HIGHESTMODSEQ from the current index, which may or may
+                  not be correct. with QRESYNC enabled we could be returning
+                  sync_highestmodseq, but that would require implementing
+                  VANISHED replies. and without QRESYNC we'd have to issue
+                  STATUS (HIGHESTMODSEQ), which isn't efficient since we get
+                  here constantly (after every IMAP command). */
+       }
+       if (imapc_storage_has_modseqs(mbox->storage)) {
+               /* even if local indexes are only in memory, we still
+                  have modseqs on the IMAP server itself. */
+               status_r->nonpermanent_modseqs = FALSE;
+       }
+       return ret;
 }
 
 static int imapc_mailbox_delete(struct mailbox *box)
@@ -802,6 +844,9 @@ static int imapc_mailbox_run_status(struct mailbox *box,
                str_append(str, " UIDVALIDITY");
        if ((items & STATUS_UNSEEN) != 0)
                str_append(str, " UNSEEN");
+       if ((items & STATUS_HIGHESTMODSEQ) != 0 &&
+           imapc_storage_has_modseqs(mbox->storage))
+               str_append(str, " HIGHESTMODSEQ");
 
        if (str_len(str) == 0) {
                /* nothing requested */
@@ -833,14 +878,17 @@ static int imapc_mailbox_get_status(struct mailbox *box,
                status_r->have_guids = TRUE;
 
        if (box->opened) {
-               imapc_mailbox_get_selected_status(mbox, items, status_r);
+               if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) {
+                       /* can't do anything about this */
+               }
        } else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS |
                             STATUS_PERMANENT_FLAGS |
                             STATUS_FIRST_RECENT_UID)) != 0) {
                /* getting these requires opening the mailbox */
                if (mailbox_open(box) < 0)
                        return -1;
-               imapc_mailbox_get_selected_status(mbox, items, status_r);
+               if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0)
+                       return -1;
        } else {
                if (imapc_mailbox_run_status(box, items, status_r) < 0)
                        return -1;

diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 25e398f007a72b7e252663e3c249cd59ed4e38d6..f2f89aeaa05c436654cda6ded84c1106b144bb59 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -108,9 +108,11 @@ struct imapc_mailbox {
        enum mail_flags permanent_flags;
        uint32_t highest_nonrecent_uid;
 
+       ARRAY(uint64_t) rseq_modseqs;
        ARRAY_TYPE(uint32_t) delayed_expunged_uids;
        uint32_t sync_uid_validity;
        uint32_t sync_uid_next;
+       uint64_t sync_highestmodseq;
        uint32_t sync_fetch_first_uid;
        uint32_t sync_next_lseq;
        uint32_t sync_next_rseq;
@@ -165,6 +167,7 @@ void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox);
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage);
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r);
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
                                 enum mail_error default_error,

diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index d14b12ccf518f7a5bda1166c6d2e86ac30fc188c..fa3e66f6253aa7ed19309661256bbe09ac6cb68a 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "imap-util.h"
 #include "mail-cache.h"
+#include "mail-index-modseq.h"
 #include "index-sync-private.h"
 #include "imapc-client.h"
 #include "imapc-msgmap.h"
@@ -245,6 +246,13 @@ static void imapc_sync_uid_next(struct imapc_sync_context *ctx)
        }
 }
 
+static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx)
+{
+       if (imapc_storage_has_modseqs(ctx->mbox->storage) &&
+           mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq)
+               mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq);
+}
+
 static void
 imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped)
 {
@@ -311,6 +319,10 @@ imapc_sync_send_commands(struct imapc_sync_context *ctx, uint32_t first_uid)
        string_t *cmd = t_str_new(64);
 
        str_printfa(cmd, "UID FETCH %u:* (FLAGS", first_uid);
+       if (imapc_storage_has_modseqs(ctx->mbox->storage)) {
+               str_append(cmd, " MODSEQ");
+               mail_index_modseq_enable(ctx->mbox->box.index);
+       }
        if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
                enum mailbox_info_flags flags;
 
@@ -393,8 +405,9 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
                imapc_mailbox_run(mbox);
        array_free(&ctx->expunged_uids);
 
-       /* add uidnext after all appends */
+       /* add uidnext & highestmodseq after all appends */
        imapc_sync_uid_next(ctx);
+       imapc_sync_highestmodseq(ctx);
 
        if (!ctx->failed)
                imapc_sync_expunge_eom(ctx);