From: Timo Sirainen Date: Mon, 20 Jun 2016 17:06:38 +0000 (+0300) Subject: dbox: Optimize POP3 MAIL_FETCH_UIDL_BACKEND. X-Git-Tag: 2.3.0.rc1~3444 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=2b9dbb270ad82e58d5f3581436e6f143176d5819;p=thirdparty%2Fdovecot%2Fcore.git dbox: Optimize POP3 MAIL_FETCH_UIDL_BACKEND. We keep track of the highest UID known to have POP3 UIDL in index's header. If saving adds a newer message, it'll also update the header. When fetching UIDL_BACKEND, we can need to check only mails with lower UIDs. There are some race conditions here, but normally UIDLs are set only once during migration so it shouldn't matter. --- diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am index 6e2fa6f9cd..220ccd7958 100644 --- a/src/lib-storage/index/Makefile.am +++ b/src/lib-storage/index/Makefile.am @@ -20,6 +20,7 @@ libstorage_index_la_SOURCES = \ index-mail-binary.c \ index-mail-headers.c \ index-mailbox-size.c \ + index-pop3-uidl.c \ index-rebuild.c \ index-search.c \ index-search-result.c \ @@ -41,6 +42,7 @@ headers = \ index-attachment.h \ index-mail.h \ index-mailbox-size.h \ + index-pop3-uidl.h \ index-rebuild.h \ index-search-private.h \ index-search-result.h \ diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c index ddd93bdaa6..6785904be1 100644 --- a/src/lib-storage/index/dbox-common/dbox-mail.c +++ b/src/lib-storage/index/dbox-common/dbox-mail.c @@ -5,6 +5,7 @@ #include "str.h" #include "index-storage.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-storage.h" #include "dbox-file.h" @@ -223,15 +224,31 @@ int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r) { struct dbox_mail *mail = (struct dbox_mail *)_mail; + int ret; /* keep the UIDL in cache file, otherwise POP3 would open all mail files and read the metadata. same for GUIDs if they're used. */ switch (field) { case MAIL_FETCH_UIDL_BACKEND: - return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, - MAIL_CACHE_POP3_UIDL, value_r); + if (!index_pop3_uidl_can_exist(_mail)) { + *value_r = ""; + return 0; + } + ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + if (ret == 0) { + index_pop3_uidl_update_exists(&mail->imail.mail.mail, + (*value_r)[0] != '\0'); + } + return ret; case MAIL_FETCH_POP3_ORDER: + if (!index_pop3_uidl_can_exist(_mail)) { + /* we're assuming that if there's a POP3 order, there's + also a UIDL */ + *value_r = ""; + return 0; + } return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER, MAIL_CACHE_POP3_ORDER, value_r); case MAIL_FETCH_GUID: diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c index 5128c341e9..e808c2b493 100644 --- a/src/lib-storage/index/dbox-common/dbox-save.c +++ b/src/lib-storage/index/dbox-common/dbox-save.c @@ -157,11 +157,15 @@ void dbox_save_write_metadata(struct mail_save_context *_ctx, str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); ctx->have_pop3_uidls = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } if (mdata->pop3_order != 0) { str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER, mdata->pop3_order); ctx->have_pop3_orders = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } guid = mdata->guid; diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h index c822f13f88..4f5e3a5e93 100644 --- a/src/lib-storage/index/dbox-common/dbox-save.h +++ b/src/lib-storage/index/dbox-common/dbox-save.h @@ -14,6 +14,7 @@ struct dbox_save_context { struct ostream *dbox_output; + uint32_t highest_pop3_uidl_seq; bool failed:1; bool finished:1; bool have_pop3_uidls:1; diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c index 857ff7b7b2..d9bdc022b6 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-save.c +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c @@ -11,6 +11,7 @@ #include "ostream.h" #include "write-full.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "mail-copy.h" #include "dbox-save.h" #include "mdbox-storage.h" @@ -326,6 +327,17 @@ int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, &_t->changes->saved_uids); + if (ctx->ctx.highest_pop3_uidl_seq != 0) { + struct seq_range_iter iter; + uint32_t uid; + + seq_range_array_iter_init(&iter, &_t->changes->saved_uids); + if (!seq_range_array_iter_nth(&iter, + ctx->ctx.highest_pop3_uidl_seq-1, &uid)) + i_unreached(); + index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid); + } + /* save map UIDs to mailbox index */ if (first_map_uid != 0) mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid); diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c index 9f0eff8852..5560d0d7a2 100644 --- a/src/lib-storage/index/dbox-single/sdbox-save.c +++ b/src/lib-storage/index/dbox-single/sdbox-save.c @@ -12,6 +12,7 @@ #include "write-full.h" #include "index-mail.h" #include "mail-copy.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-save.h" #include "sdbox-storage.h" @@ -248,6 +249,10 @@ static int dbox_save_assign_uids(struct sdbox_save_context *ctx, i_assert(ret); if (sdbox_file_assign_uid(sfile, uid) < 0) return -1; + if (ctx->ctx.highest_pop3_uidl_seq == i+1) { + index_pop3_uidl_set_max_uid(&ctx->mbox->box, + ctx->ctx.trans, uid); + } } i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); return 0; diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c new file mode 100644 index 0000000000..5ebc7073ff --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.c @@ -0,0 +1,101 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid) +{ + struct mailbox_index_pop3_uidl uidl; + + memset(&uidl, 0, sizeof(uidl)); + uidl.max_uid_with_pop3_uidl = uid; + + mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id, + 0, &uidl, sizeof(uidl)); +} + +bool index_pop3_uidl_can_exist(struct mail *mail) +{ + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + + /* We'll assume that if the header exists, it's up-to-date. normally + UIDLs are set only during migration, so this value never changes. + Also even if it does, it becomes out-of-date only when the mailbox + is modified with old Dovecot versions. To fix that we'd have to + add and keep updating "max tracked uid" in this header for every + saved mail, which isn't worth it. */ + mail_index_get_header_ext(mail->transaction->view, + mail->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size < sizeof(uidl)) { + /* this header isn't set yet */ + return TRUE; + } + memcpy(&uidl, data, size); + return mail->uid <= uidl.max_uid_with_pop3_uidl; +} + +void index_pop3_uidl_update_exists(struct mail *mail, bool exists) +{ + struct mailbox_transaction_context *trans = mail->transaction; + + if (exists) { + if (trans->highest_pop3_uidl_uid < mail->uid) { + trans->highest_pop3_uidl_uid = mail->uid; + trans->prev_pop3_uidl_tracking_seq = mail->seq; + } + } else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) { + trans->prev_pop3_uidl_tracking_seq++; + } else { + /* skipping mails. we don't know the state. */ + } +} + +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans) +{ + struct mail_index_view *view; + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + bool seen_all_msgs; + + if (trans->highest_pop3_uidl_uid == 0) + return; + + /* First check that we actually looked at UIDL for all messages. + Otherwise we can't say for sure if the newest messages had UIDLs. */ + if (trans->prev_pop3_uidl_tracking_seq != + mail_index_view_get_messages_count(trans->view)) + return; + + /* Just to be sure: Refresh the index and check again. POP3 keeps + transactions open for duration of the entire session. Maybe another + process already added new mails (and already updated this header). + This check is racy, but normally UIDLs aren't added after migration + so it's a bit questionable if it's even worth having this check in + there. */ + view = mail_index_view_open(trans->box->index); + seen_all_msgs = mail_index_refresh(trans->box->index) == 0 && + trans->prev_pop3_uidl_tracking_seq == + mail_index_view_get_messages_count(view); + mail_index_view_close(&view); + if (!seen_all_msgs) + return; + + /* check if we have already the same header */ + mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size >= sizeof(uidl)) { + memcpy(&uidl, data, size); + if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl) + return; + } + index_pop3_uidl_set_max_uid(trans->box, trans->itrans, + trans->highest_pop3_uidl_uid); +} diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h new file mode 100644 index 0000000000..956ab7ddd5 --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.h @@ -0,0 +1,16 @@ +#ifndef INDEX_POP3_H +#define INDEX_POP3_H + +struct mail_index_transaction; +struct mail; +struct mailbox; +struct mailbox_transaction_context; + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid); +bool index_pop3_uidl_can_exist(struct mail *mail); +void index_pop3_uidl_update_exists(struct mail *mail, bool exists); +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans); + +#endif diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index 8ea8142e36..23aa81a1ed 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -294,6 +294,9 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory) mail_index_ext_register(box->index, "hdr-vsize", sizeof(struct mailbox_index_vsize), 0, sizeof(uint64_t)); + box->pop3_uidl_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-pop3-uidl", + sizeof(struct mailbox_index_pop3_uidl), 0, 0); box->opened = TRUE; diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c index d11ae410c3..22a4b0a995 100644 --- a/src/lib-storage/index/index-transaction.c +++ b/src/lib-storage/index/index-transaction.c @@ -5,6 +5,7 @@ #include "dict.h" #include "index-storage.h" #include "index-sync-private.h" +#include "index-pop3-uidl.h" #include "index-mail.h" static void index_transaction_free(struct mailbox_transaction_context *t) @@ -29,6 +30,7 @@ index_transaction_index_commit(struct mail_index_transaction *index_trans, const char *error; int ret = 0; + index_pop3_uidl_update_exists_finish(t); if (t->nontransactional_changes) t->changes->changed = TRUE; diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index 726fd7faf1..b32e722267 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -292,6 +292,10 @@ struct mailbox_index_vsize { uint32_t message_count; }; +struct mailbox_index_pop3_uidl { + uint32_t max_uid_with_pop3_uidl; +}; + struct mailbox_index_first_saved { uint32_t uid; uint32_t timestamp; @@ -346,6 +350,7 @@ struct mailbox { enum mailbox_feature enabled_features; struct mail_msgpart_partial_cache partial_cache; uint32_t vsize_hdr_ext_id; + uint32_t pop3_uidl_hdr_ext_id; /* MAIL_RECENT flags handling */ ARRAY_TYPE(seq_range) recent_flags; @@ -551,6 +556,9 @@ struct mailbox_transaction_context { struct mail_transaction_commit_changes *changes; ARRAY(union mailbox_transaction_module_context *) module_contexts; + uint32_t prev_pop3_uidl_tracking_seq; + uint32_t highest_pop3_uidl_uid; + struct mail_save_context *save_ctx; /* number of mails saved/copied within this transaction. */ unsigned int save_count;