Save MD5 sums of some headers for readonly mboxes and use them for syncing

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h
index 0500a152500c0577852acd873ea59ab8ea6046be..d2ff2328db5612bb00cd0ad6d5f37cadb7518836 100644 (file)
--- a/src/lib-storage/index/index-storage.h
+++ b/src/lib-storage/index/index-storage.h
@@ -88,7 +88,7 @@ struct index_mailbox {
        unsigned int mbox_lock_id, mbox_mail_lock_id;
        int mbox_readonly;
 
-       uint32_t mbox_extra_idx;
+       uint32_t mbox_extra_idx, md5hdr_extra_idx;
 
        /* maildir sync: */
        struct maildir_uidlist *uidlist;

diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c
index 9f61f5192d551327a67b077e13c927c4ee9c6432..9705bef060e1615c915c8351b5f1337f40affc7d 100644 (file)
--- a/src/lib-storage/index/mbox/mbox-sync-parse.c
+++ b/src/lib-storage/index/mbox/mbox-sync-parse.c
@@ -1,5 +1,8 @@
 /* Copyright (C) 2004 Timo Sirainen */
 
+/* MD5 header summing logic was pretty much copy&pasted from popa3d by
+   Solar Designer */
+
 #include "lib.h"
 #include "buffer.h"
 #include "istream.h"
@@ -266,9 +269,64 @@ static int parse_content_length(struct mbox_sync_mail_context *ctx,
        return TRUE;
 }
 
+static int parse_date(struct mbox_sync_mail_context *ctx,
+                     struct message_header_line *hdr)
+{
+       if (!ctx->seen_received_hdr) {
+               /* Received-header contains date too, and more trusted one */
+               md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+       }
+       return TRUE;
+}
+
+static int parse_delivered_to(struct mbox_sync_mail_context *ctx,
+                             struct message_header_line *hdr)
+{
+       md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+       return TRUE;
+}
+
+static int parse_message_id(struct mbox_sync_mail_context *ctx,
+                           struct message_header_line *hdr)
+{
+       if (!ctx->seen_received_hdr) {
+               /* Received-header contains unique ID too,
+                  and more trusted one */
+               md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+       }
+       return TRUE;
+}
+
+static int parse_received(struct mbox_sync_mail_context *ctx,
+                         struct message_header_line *hdr)
+{
+       if (!ctx->seen_received_hdr) {
+               /* get only the first received-header */
+               md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+               if (!hdr->continues)
+                       ctx->seen_received_hdr = TRUE;
+       }
+       return TRUE;
+}
+
+static int parse_x_delivery_id(struct mbox_sync_mail_context *ctx,
+                              struct message_header_line *hdr)
+{
+       /* Let the local delivery agent help generate unique ID's but don't
+          blindly trust this header alone as it could just as easily come from
+          the remote. */
+       md5_update(&ctx->hdr_md5_ctx, hdr->value, hdr->value_len);
+       return TRUE;
+}
+
 static struct header_func header_funcs[] = {
        { "Content-Length", parse_content_length },
+       { "Date", parse_date },
+       { "Delivered-To", parse_delivered_to },
+       { "Message-ID", parse_message_id },
+       { "Received", parse_received },
        { "Status", parse_status },
+       { "X-Delivery-ID", parse_x_delivery_id },
        { "X-IMAP", parse_x_imap },
        { "X-IMAPbase", parse_x_imap_base },
        { "X-Keywords", parse_x_keywords },
@@ -310,6 +368,8 @@ void mbox_sync_parse_next_mail(struct istream *input,
        ctx->content_length = (uoff_t)-1;
        str_truncate(ctx->header, 0);
 
+       md5_init(&ctx->hdr_md5_ctx);
+
         line_start_pos = 0;
        hdr_ctx = message_parse_header_init(input, NULL, FALSE);
        while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
@@ -361,6 +421,8 @@ void mbox_sync_parse_next_mail(struct istream *input,
        }
        message_parse_header_deinit(hdr_ctx);
 
+       md5_final(&ctx->hdr_md5_ctx, ctx->hdr_md5_sum);
+
        if ((ctx->seq == 1 && sync_ctx->base_uid_validity == 0) ||
            (ctx->seq > 1 && sync_ctx->dest_first_mail)) {
                /* missing X-IMAPbase */

diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h
index 6cd6000878e495ae0cadacb91df71a112da5963b..24306384270a0be6ace95807bd5903b87dacbfe9 100644 (file)
--- a/src/lib-storage/index/mbox/mbox-sync-private.h
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h
@@ -1,6 +1,7 @@
 #ifndef __MBOX_SYNC_PRIVATE_H
 #define __MBOX_SYNC_PRIVATE_H
 
+#include "md5.h"
 #include "mail-index.h"
 
 struct mbox_flag_type {
@@ -56,6 +57,9 @@ struct mbox_sync_mail_context {
        size_t header_first_change, header_last_change;
        string_t *header, *uidl;
 
+       struct md5_context hdr_md5_ctx;
+       unsigned char hdr_md5_sum[16];
+
        uoff_t content_length;
 
        size_t hdr_pos[MBOX_HDR_COUNT];
@@ -67,6 +71,7 @@ struct mbox_sync_mail_context {
        unsigned int seen_imapbase:1;
        unsigned int pseudo:1;
        unsigned int updated:1;
+       unsigned int seen_received_hdr:1;
 };
 
 struct mbox_sync_context {

diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c
index 241c2cf9e36f71c5e65bf19541047242dddd17a8..3f07b3d94f5f71b84559748e73eccf8980444211 100644 (file)
--- a/src/lib-storage/index/mbox/mbox-sync.c
+++ b/src/lib-storage/index/mbox/mbox-sync.c
@@ -365,6 +365,45 @@ mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx,
        return ret;
 }
 
+static int mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx,
+                                   unsigned char hdr_md5_sum[],
+                                   const struct mail_index_record **rec_r)
+{
+        const struct mail_index_record *rec = NULL;
+       uint32_t messages_count;
+       const void *data;
+       int ret;
+
+       messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
+       while (sync_ctx->idx_seq <= messages_count) {
+               ret = mail_index_lookup(sync_ctx->sync_view,
+                                       sync_ctx->idx_seq, &rec);
+               if (ret < 0) {
+                       mail_storage_set_index_error(sync_ctx->ibox);
+                       return -1;
+               }
+
+               if (mail_index_lookup_extra(sync_ctx->sync_view,
+                                           sync_ctx->idx_seq,
+                                           sync_ctx->ibox->md5hdr_extra_idx,
+                                           &data) < 0) {
+                       mail_storage_set_index_error(sync_ctx->ibox);
+                       return -1;
+               }
+
+               if (memcmp(data, hdr_md5_sum, 16) == 0)
+                       break;
+
+               /* externally expunged message, remove from index */
+               mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq);
+                sync_ctx->idx_seq++;
+               rec = NULL;
+       }
+
+       *rec_r = rec;
+       return 0;
+}
+
 static int mbox_sync_get_from_offset(struct mbox_sync_context *sync_ctx,
                                     uint32_t seq, uint64_t *offset_r)
 {
@@ -419,6 +458,14 @@ static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
                mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
                                        MODIFY_REPLACE, mbox_flags,
                                        mail->keywords);
+
+               if (sync_ctx->ibox->md5hdr_extra_idx != 0) {
+                       mail_index_update_extra_rec(sync_ctx->t,
+                               sync_ctx->idx_seq,
+                               sync_ctx->ibox->md5hdr_extra_idx,
+                               mail_ctx->hdr_md5_sum);
+               }
+
                if (str_len(mail_ctx->uidl) > 0) {
                        /*FIXME:mail_cache_add(sync_ctx->cache_trans,
                                       MAIL_CACHE_UID_STRING,
@@ -762,13 +809,33 @@ static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
                        return -1;
 
                rec = NULL;
-               if (uid != 0 && !mail_ctx->pseudo) {
+               if (uid != 0 && !mail_ctx->pseudo &&
+                   sync_ctx->ibox->md5hdr_extra_idx == 0) {
                        ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
                        if (ret < 0)
                                return -1;
                        if (ret == 0)
                                uid = 0;
                }
+
+               if (uid == 0 && sync_ctx->ibox->mbox_readonly) {
+                       /* Use MD5 sums */
+                       if (sync_ctx->ibox->md5hdr_extra_idx == 0) {
+                               sync_ctx->ibox->md5hdr_extra_idx =
+                                       mail_index_register_record_extra(
+                                               sync_ctx->ibox->index,
+                                               "header-md5", 16);
+                       }
+
+                       if (mbox_sync_find_index_md5(sync_ctx,
+                                                    mail_ctx->hdr_md5_sum,
+                                                    &rec) < 0)
+                               return -1;
+
+                       if (rec != NULL)
+                               uid = rec->uid;
+               }
+
                if (uid == 0) {
                        /* missing/broken X-UID. all the rest of the mails
                           need new UIDs. */