lib-mail: More fixes to istream-header-filter with large input.

Don't assume that when reading the header the second time we'll get exactly
the same header blocks as the first time.

This commit also prevents the filter callback from switching the
matching-decision on non-first header line. This shouldn't be needed and it
could just cause confusion. (It also made it a bit easier to implement this
fix.)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 3235a98273cdbe92bcba1dd358050b3b3d73f6ec..2ab2dabcd55971a06d35fc693be13fa9c727480e 100644 (file)
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -39,6 +39,7 @@ struct header_filter_istream {
        unsigned int end_body_with_lf:1;
        unsigned int last_lf_added:1;
        unsigned int eoh_not_matched:1;
+       unsigned int prev_matched:1;
 };
 
 header_filter_callback *null_header_filter_callback = NULL;
@@ -154,7 +155,6 @@ static ssize_t read_header(struct header_filter_istream *mstream)
        struct message_header_line *hdr;
        uoff_t highwater_offset;
        ssize_t ret, ret2;
-       bool matched;
        int hdr_ret;
 
        if (mstream->hdr_ctx == NULL) {
@@ -188,8 +188,10 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 
        while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx,
                                                    &hdr)) > 0) {
-               mstream->cur_line++;
+               bool matched;
 
+               if (!hdr->continued)
+                       mstream->cur_line++;
                if (hdr->eoh) {
                        mstream->seen_eoh = TRUE;
                        matched = TRUE;
@@ -211,15 +213,25 @@ static ssize_t read_header(struct header_filter_istream *mstream)
                        continue;
                }
 
-               matched = mstream->headers_count == 0 ? FALSE :
-                       i_bsearch(hdr->name, mstream->headers,
-                                 mstream->headers_count,
-                                 sizeof(*mstream->headers),
-                                 bsearch_strcasecmp) != NULL;
+               if (hdr->continued) {
+                       /* Header line continued - use only the first line's
+                          matched-result. Otherwise multiline headers might
+                          end up being only partially picked, which wouldn't
+                          be very good. However, allow callbacks to modify
+                          the headers in any way they want. */
+                       matched = mstream->prev_matched;
+               } else if (mstream->headers_count == 0) {
+                       /* no include/exclude headers - default matching */
+                       matched = FALSE;
+               } else {
+                       matched = i_bsearch(hdr->name, mstream->headers,
+                                           mstream->headers_count,
+                                           sizeof(*mstream->headers),
+                                           bsearch_strcasecmp) != NULL;
+               }
                if (mstream->callback == NULL) {
                        /* nothing gets excluded */
-               } else if (mstream->cur_line > mstream->parsed_lines ||
-                          mstream->headers_edited) {
+               } else if (!mstream->header_parsed || mstream->headers_edited) {
                        /* first time in this line or we have actually modified
                           the header so we always want to call the callbacks */
                        bool orig_matched = matched;
@@ -228,18 +240,19 @@ static ssize_t read_header(struct header_filter_istream *mstream)
                        mstream->callback(mstream, hdr, &matched,
                                          mstream->context);
                        if (matched != orig_matched &&
-                           !mstream->headers_edited) {
+                           !hdr->continued && !mstream->headers_edited) {
                                if (!array_is_created(&mstream->match_change_lines))
                                        i_array_init(&mstream->match_change_lines, 8);
                                array_append(&mstream->match_change_lines,
                                             &mstream->cur_line, 1);
                        }
-               } else {
+               } else if (!hdr->continued) {
                        /* second time in this line. was it excluded by the
                           callback the first time? */
                        if (match_line_changed(mstream))
                                matched = !matched;
                }
+               mstream->prev_matched = matched;
 
                if (matched == mstream->exclude) {
                        /* ignore */
@@ -302,6 +315,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
                mstream->hdr_ctx = NULL;
 
                if (!mstream->header_parsed && mstream->callback != NULL) {
+                       bool matched = FALSE;
                        mstream->callback(mstream, NULL,
                                          &matched, mstream->context);
                        /* check if the callback added more headers.
@@ -426,6 +440,7 @@ i_stream_header_filter_seek_to_header(struct header_filter_istream *mstream,
                message_parse_header_deinit(&mstream->hdr_ctx);
        mstream->skip_count = v_offset;
        mstream->cur_line = 0;
+       mstream->prev_matched = FALSE;
        mstream->header_read = FALSE;
        mstream->seen_eoh = FALSE;
 }

diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 048e3a3a3816c2fce01745eb18c884baa31511b1..cfa544ccb59cfdecd73812020118a3dd443923d0 100644 (file)
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -172,9 +172,11 @@ static void test_istream_filter_large_buffer(void)
                i_assert(p != NULL);
                test_assert(strcmp(p+1, str_c(output) + prefix_len) == 0);
 
-               /* seek back and retry once with caching */
+               /* seek back and retry once with caching and different
+                  buffer size */
                i_stream_seek(filter, 0);
                str_truncate(output, 0);
+               test_istream_set_max_buffer_size(istream, 4096);
        }
 
        str_free(&input);