dbox: Optimize POP3 MAIL_FETCH_UIDL_BACKEND.

We keep track of the highest UID known to have POP3 UIDL in index's header.
If saving adds a newer message, it'll also update the header. When fetching
UIDL_BACKEND, we can need to check only mails with lower UIDs. There are
some race conditions here, but normally UIDLs are set only once during
migration so it shouldn't matter.

diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am
index 6e2fa6f9cd741817ce8de16addac1324663b5809..220ccd7958ab41166d0376a527dfebcf991563ed 100644 (file)
--- a/src/lib-storage/index/Makefile.am
+++ b/src/lib-storage/index/Makefile.am
@@ -20,6 +20,7 @@ libstorage_index_la_SOURCES = \
        index-mail-binary.c \
        index-mail-headers.c \
        index-mailbox-size.c \
+       index-pop3-uidl.c \
        index-rebuild.c \
        index-search.c \
        index-search-result.c \
@@ -41,6 +42,7 @@ headers = \
        index-attachment.h \
        index-mail.h \
        index-mailbox-size.h \
+       index-pop3-uidl.h \
        index-rebuild.h \
        index-search-private.h \
        index-search-result.h \

diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c
index ddd93bdaa61dda53672985ab0262fe45708d2d0c..6785904be13e391d9d3739d601b23281ad2b0817 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-mail.c
+++ b/src/lib-storage/index/dbox-common/dbox-mail.c
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "index-storage.h"
 #include "index-mail.h"
+#include "index-pop3-uidl.h"
 #include "dbox-attachment.h"
 #include "dbox-storage.h"
 #include "dbox-file.h"
@@ -223,15 +224,31 @@ int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
                          const char **value_r)
 {
        struct dbox_mail *mail = (struct dbox_mail *)_mail;
+       int ret;
 
        /* keep the UIDL in cache file, otherwise POP3 would open all
           mail files and read the metadata. same for GUIDs if they're
           used. */
        switch (field) {
        case MAIL_FETCH_UIDL_BACKEND:
-               return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL,
-                                               MAIL_CACHE_POP3_UIDL, value_r);
+               if (!index_pop3_uidl_can_exist(_mail)) {
+                       *value_r = "";
+                       return 0;
+               }
+               ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL,
+                                              MAIL_CACHE_POP3_UIDL, value_r);
+               if (ret == 0) {
+                       index_pop3_uidl_update_exists(&mail->imail.mail.mail,
+                                                     (*value_r)[0] != '\0');
+               }
+               return ret;
        case MAIL_FETCH_POP3_ORDER:
+               if (!index_pop3_uidl_can_exist(_mail)) {
+                       /* we're assuming that if there's a POP3 order, there's
+                          also a UIDL */
+                       *value_r = "";
+                       return 0;
+               }
                return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER,
                                                MAIL_CACHE_POP3_ORDER, value_r);
        case MAIL_FETCH_GUID:

diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c
index fe305a6df894ab5b11801e1b71afbf7f3bf2400e..456cfe339c914d6c748a736f70a3e62fd0a1a606 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-save.c
+++ b/src/lib-storage/index/dbox-common/dbox-save.c
@@ -166,11 +166,15 @@ void dbox_save_write_metadata(struct mail_save_context *_ctx,
                str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL,
                            mdata->pop3_uidl);
                ctx->have_pop3_uidls = TRUE;
+               ctx->highest_pop3_uidl_seq =
+                       I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
        }
        if (mdata->pop3_order != 0) {
                str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER,
                            mdata->pop3_order);
                ctx->have_pop3_orders = TRUE;
+               ctx->highest_pop3_uidl_seq =
+                       I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
        }
 
        guid = mdata->guid;

diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h
index 7b16d1c87ae710796739b069a45afca23d5c4154..9d5f17333c939c36b24d446ecb11a33995bc9c7c 100644 (file)
--- a/src/lib-storage/index/dbox-common/dbox-save.h
+++ b/src/lib-storage/index/dbox-common/dbox-save.h
@@ -14,6 +14,7 @@ struct dbox_save_context {
 
        struct ostream *dbox_output;
 
+       uint32_t highest_pop3_uidl_seq;
        unsigned int failed:1;
        unsigned int finished:1;
        unsigned int have_pop3_uidls:1;

diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c
index 857ff7b7b2f8b2f66dfb13d1b5f576b8b8aa4cd0..d9bdc022b6ac499ad0eec59d329615ff2759b7fa 100644 (file)
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c
@@ -11,6 +11,7 @@
 #include "ostream.h"
 #include "write-full.h"
 #include "index-mail.h"
+#include "index-pop3-uidl.h"
 #include "mail-copy.h"
 #include "dbox-save.h"
 #include "mdbox-storage.h"
@@ -326,6 +327,17 @@ int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
        mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
                                      &_t->changes->saved_uids);
 
+       if (ctx->ctx.highest_pop3_uidl_seq != 0) {
+               struct seq_range_iter iter;
+               uint32_t uid;
+
+               seq_range_array_iter_init(&iter, &_t->changes->saved_uids);
+               if (!seq_range_array_iter_nth(&iter,
+                               ctx->ctx.highest_pop3_uidl_seq-1, &uid))
+                       i_unreached();
+               index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid);
+       }
+
        /* save map UIDs to mailbox index */
        if (first_map_uid != 0)
                mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid);

diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c
index 9f0eff8852bf3c67edc4c7164c65614136843896..5560d0d7a2106b6d788f7bec39ad4001902722bd 100644 (file)
--- a/src/lib-storage/index/dbox-single/sdbox-save.c
+++ b/src/lib-storage/index/dbox-single/sdbox-save.c
@@ -12,6 +12,7 @@
 #include "write-full.h"
 #include "index-mail.h"
 #include "mail-copy.h"
+#include "index-pop3-uidl.h"
 #include "dbox-attachment.h"
 #include "dbox-save.h"
 #include "sdbox-storage.h"
@@ -248,6 +249,10 @@ static int dbox_save_assign_uids(struct sdbox_save_context *ctx,
                i_assert(ret);
                if (sdbox_file_assign_uid(sfile, uid) < 0)
                        return -1;
+               if (ctx->ctx.highest_pop3_uidl_seq == i+1) {
+                       index_pop3_uidl_set_max_uid(&ctx->mbox->box,
+                               ctx->ctx.trans, uid);
+               }
        }
        i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
        return 0;

diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c
new file mode 100644 (file)
index 0000000..5ebc707
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.c
@@ -0,0 +1,101 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-pop3-uidl.h"
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+                                struct mail_index_transaction *trans,
+                                uint32_t uid)
+{
+       struct mailbox_index_pop3_uidl uidl;
+
+       memset(&uidl, 0, sizeof(uidl));
+       uidl.max_uid_with_pop3_uidl = uid;
+
+       mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id,
+                                    0, &uidl, sizeof(uidl));
+}
+
+bool index_pop3_uidl_can_exist(struct mail *mail)
+{
+       struct mailbox_index_pop3_uidl uidl;
+       const void *data;
+       size_t size;
+
+       /* We'll assume that if the header exists, it's up-to-date. normally
+          UIDLs are set only during migration, so this value never changes.
+          Also even if it does, it becomes out-of-date only when the mailbox
+          is modified with old Dovecot versions. To fix that we'd have to
+          add and keep updating "max tracked uid" in this header for every
+          saved mail, which isn't worth it. */
+       mail_index_get_header_ext(mail->transaction->view,
+                                 mail->box->pop3_uidl_hdr_ext_id,
+                                 &data, &size);
+       if (size < sizeof(uidl)) {
+               /* this header isn't set yet */
+               return TRUE;
+       }
+       memcpy(&uidl, data, size);
+       return mail->uid <= uidl.max_uid_with_pop3_uidl;
+}
+
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists)
+{
+       struct mailbox_transaction_context *trans = mail->transaction;
+
+       if (exists) {
+               if (trans->highest_pop3_uidl_uid < mail->uid) {
+                       trans->highest_pop3_uidl_uid = mail->uid;
+                       trans->prev_pop3_uidl_tracking_seq = mail->seq;
+               }
+       } else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) {
+               trans->prev_pop3_uidl_tracking_seq++;
+       } else {
+               /* skipping mails. we don't know the state. */
+       }
+}
+
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans)
+{
+       struct mail_index_view *view;
+       struct mailbox_index_pop3_uidl uidl;
+       const void *data;
+       size_t size;
+       bool seen_all_msgs;
+
+       if (trans->highest_pop3_uidl_uid == 0)
+               return;
+
+       /* First check that we actually looked at UIDL for all messages.
+          Otherwise we can't say for sure if the newest messages had UIDLs. */
+       if (trans->prev_pop3_uidl_tracking_seq !=
+           mail_index_view_get_messages_count(trans->view))
+               return;
+
+       /* Just to be sure: Refresh the index and check again. POP3 keeps
+          transactions open for duration of the entire session. Maybe another
+          process already added new mails (and already updated this header).
+          This check is racy, but normally UIDLs aren't added after migration
+          so it's a bit questionable if it's even worth having this check in
+          there. */
+       view = mail_index_view_open(trans->box->index);
+       seen_all_msgs = mail_index_refresh(trans->box->index) == 0 &&
+               trans->prev_pop3_uidl_tracking_seq ==
+               mail_index_view_get_messages_count(view);
+       mail_index_view_close(&view);
+       if (!seen_all_msgs)
+               return;
+
+       /* check if we have already the same header */
+       mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id,
+                                 &data, &size);
+       if (size >= sizeof(uidl)) {
+               memcpy(&uidl, data, size);
+               if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl)
+                       return;
+       }
+       index_pop3_uidl_set_max_uid(trans->box, trans->itrans,
+                                   trans->highest_pop3_uidl_uid);
+}

diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h
new file mode 100644 (file)
index 0000000..956ab7d
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.h
@@ -0,0 +1,16 @@
+#ifndef INDEX_POP3_H
+#define INDEX_POP3_H
+
+struct mail_index_transaction;
+struct mail;
+struct mailbox;
+struct mailbox_transaction_context;
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+                                struct mail_index_transaction *trans,
+                                uint32_t uid);
+bool index_pop3_uidl_can_exist(struct mail *mail);
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists);
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans);
+
+#endif

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 339ebbb7d106b3427b56896686285967daedde8f..0d5a407ad5c76daf0eb5e0047a1751285abf3857 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -293,6 +293,9 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
                mail_index_ext_register(box->index, "hdr-vsize",
                                        sizeof(struct mailbox_index_vsize), 0,
                                        sizeof(uint64_t));
+       box->pop3_uidl_hdr_ext_id =
+               mail_index_ext_register(box->index, "hdr-pop3-uidl",
+                                       sizeof(struct mailbox_index_pop3_uidl), 0, 0);
 
        box->opened = TRUE;
 

diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c
index 4c7b42cc3b69fe30ae4205104a3ce59231d9ba81..db1f21914e6383e2deb8715deedb70113896c322 100644 (file)
--- a/src/lib-storage/index/index-transaction.c
+++ b/src/lib-storage/index/index-transaction.c
@@ -5,6 +5,7 @@
 #include "dict.h"
 #include "index-storage.h"
 #include "index-sync-private.h"
+#include "index-pop3-uidl.h"
 #include "index-mail.h"
 
 static void index_transaction_free(struct mailbox_transaction_context *t)
@@ -28,6 +29,7 @@ index_transaction_index_commit(struct mail_index_transaction *index_trans,
        struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL;
        int ret = 0;
 
+       index_pop3_uidl_update_exists_finish(t);
        if (t->nontransactional_changes)
                t->changes->changed = TRUE;
 

diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 0aad8f83f80cbed4a15795ae5193162d1dfade6d..4d8bba309721b82e2d6cd59661cbb218eee499cb 100644 (file)
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -292,6 +292,10 @@ struct mailbox_index_vsize {
        uint32_t message_count;
 };
 
+struct mailbox_index_pop3_uidl {
+       uint32_t max_uid_with_pop3_uidl;
+};
+
 struct mailbox_index_first_saved {
        uint32_t uid;
        uint32_t timestamp;
@@ -346,6 +350,7 @@ struct mailbox {
        enum mailbox_feature enabled_features;
        struct mail_msgpart_partial_cache partial_cache;
        uint32_t vsize_hdr_ext_id;
+       uint32_t pop3_uidl_hdr_ext_id;
 
        /* MAIL_RECENT flags handling */
        ARRAY_TYPE(seq_range) recent_flags;
@@ -551,6 +556,9 @@ struct mailbox_transaction_context {
        struct mail_transaction_commit_changes *changes;
        ARRAY(union mailbox_transaction_module_context *) module_contexts;
 
+       uint32_t prev_pop3_uidl_tracking_seq;
+       uint32_t highest_pop3_uidl_uid;
+
        struct mail_save_context *save_ctx;
        /* number of mails saved/copied within this transaction. */
        unsigned int save_count;