imapc: Added imapc_features=fetch-fix-broken-mails

This allows ignoring a missing FETCH reply. It shouldn't be used normally,
but if there's a mail that the IMAP server simply won't serve, this can be
used to finish dsync successfully.

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index b40458b080f1ff70a6c29b11c845a906c43bfce5..e59c24e7fc28614ab9b34f20998b948e6650711e 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -59,9 +59,10 @@ static bool imapc_mail_is_expunged(struct mail *_mail)
        return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
 }
 
-static void imapc_mail_failed(struct mail *mail, const char *field)
+static int imapc_mail_failed(struct mail *mail, const char *field)
 {
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
+       bool fix_broken_mail = FALSE;
 
        if (mail->expunged || imapc_mail_is_expunged(mail)) {
                mail_set_expunged(mail);
@@ -69,18 +70,24 @@ static void imapc_mail_failed(struct mail *mail, const char *field)
                /* we've already logged a disconnection error */
                mail_storage_set_internal_error(mail->box->storage);
        } else {
-               /* NOTE: earlier we didn't treat this as a failure, because
-                  old Exchange versions fail to return any data for messages
-                  in Calendars mailbox. But it's a bad idea to always assume
-                  that a missing field is intentional, because there's
-                  potential for data loss. Ideally we could detect whether
-                  this is an Exchange issue or not, but I don't have access
-                  to such an old Exchange anymore. So at least for now until
-                  someone complains, the Exchange workaround is disabled. */
+               /* By default we'll assume that this is a critical failure,
+                  because we don't want to lose any data. We can be here
+                  either because it's a temporary failure on the server or
+                  it's a permanent failure. Unfortunately we can't know
+                  which case it is, so permanent failures need to be worked
+                  around by setting imapc_features=fetch-fix-broken-mails.
+
+                  One reason for permanent failures was that earlier Exchange
+                  versions failed to return any data for messages in Calendars
+                  mailbox. This seems to be fixed in newer versions.
+                  */
+               fix_broken_mail = IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS);
                mail_storage_set_critical(mail->box->storage,
-                       "imapc: Remote server didn't send %s for UID %u in %s",
-                       field, mail->uid, mail->box->vname);
+                       "imapc: Remote server didn't send %s for UID %u in %s%s",
+                       field, mail->uid, mail->box->vname,
+                       fix_broken_mail ? " - treating it as empty" : "");
        }
+       return fix_broken_mail ? 0 : -1;
 }
 
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
@@ -95,8 +102,11 @@ static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
                if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0)
                        return -1;
                if (data->received_date == (time_t)-1) {
-                       imapc_mail_failed(_mail, "INTERNALDATE");
-                       return -1;
+                       if (imapc_mail_failed(_mail, "INTERNALDATE") < 0)
+                               return -1;
+                       /* assume that the server never returns INTERNALDATE
+                          for this mail (see BODY[] failure handling) */
+                       data->received_date = 0;
                }
        }
        *date_r = data->received_date;
@@ -138,8 +148,11 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
                if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
                        return -1;
                if (data->physical_size == (uoff_t)-1) {
-                       imapc_mail_failed(_mail, "RFC822.SIZE");
-                       return -1;
+                       if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0)
+                               return -1;
+                       /* assume that the server never returns RFC822.SIZE
+                          for this mail (see BODY[] failure handling) */
+                       data->physical_size = 0;
                }
                *size_r = data->physical_size;
                return 0;
@@ -271,8 +284,14 @@ imapc_mail_get_stream(struct mail *_mail, bool get_body,
                        return -1;
 
                if (data->stream == NULL) {
-                       imapc_mail_failed(_mail, "BODY[]");
-                       return -1;
+                       if (imapc_mail_failed(_mail, "BODY[]"))
+                               return -1;
+                       i_assert(data->stream == NULL);
+
+                       /* return the broken email as empty */
+                       mail->body_fetched = TRUE;
+                       data->stream = i_stream_create_from_data(NULL, 0);
+                       imapc_mail_init_stream(mail);
                }
        }
 
@@ -471,7 +490,7 @@ static int imapc_mail_get_guid(struct mail *_mail, const char **value_r)
                if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0)
                        return -1;
                if (imail->data.guid == NULL) {
-                       imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
+                       (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
                        return -1;
                }
        } else {

diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c
index 60d2ba2337f90f18e7b60e79a1021fd8a4db584c..6325aa8349f4f066228fa4e8d6dd982cffaea3c8 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-settings.c
+++ b/src/lib-storage/index/imapc/imapc-settings.c
@@ -92,6 +92,7 @@ static const struct imapc_feature_list imapc_feature_list[] = {
        { "no-examine", IMAPC_FEATURE_NO_EXAMINE },
        { "proxyauth", IMAPC_FEATURE_PROXYAUTH },
        { "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
+       { "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
        { NULL, 0 }
 };
 

diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h
index d4c7a62882084c2d53fa32fc98434dcda143affb..bb1dec1ae7350fa73238adadbae0f1189d673757 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-settings.h
+++ b/src/lib-storage/index/imapc/imapc-settings.h
@@ -13,7 +13,8 @@ enum imapc_features {
        IMAPC_FEATURE_ZIMBRA_WORKAROUNDS        = 0x20,
        IMAPC_FEATURE_NO_EXAMINE                = 0x40,
        IMAPC_FEATURE_PROXYAUTH                 = 0x80,
-       IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS     = 0x100
+       IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS     = 0x100,
+       IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS    = 0x200
 };
 /* </settings checks> */