generate RFC 2919/2369 List-* and Precedence headers natively

Mailing list messages without List-Id/Precedence headers cause vacation
autoreplies to trigger bounces leading to unsubscriptions, prevent mail
clients like Delta Chat from detecting list messages, and hurt
deliverability with major providers.

Generate List-Id, List-Post, List-Help, List-Subscribe, List-Unsubscribe,
and Precedence headers by default in do_all_the_voodoo_here(). List-Owner
is included only when control/owner exists. All headers can be disabled
via control/nolistheaders.

diff --git a/include/do_all_the_voodoo_here.h b/include/do_all_the_voodoo_here.h
index e7667fd2a3dd6207a90e5953974cf3cb06dba99b..f0d6ee104111dc3b2bd64685106cb9b0470b44b5 100644 (file)
--- a/include/do_all_the_voodoo_here.h
+++ b/include/do_all_the_voodoo_here.h
@@ -27,6 +27,7 @@
 
 int do_all_the_voodoo_here(int infd, int outfd, int hdrfd, int footfd,
              const strlist *delhdrs, struct mailhdr *readhdrs,
-             strlist *allhdrs, const char *subjectprefix, int replyto);
+             strlist *allhdrs, const char *subjectprefix, int replyto,
+             struct ml *ml);
 void scan_headers(FILE *f, struct mailhdr *readhdrs, strlist *allhdrs,
        strlist *allunfoldeds);

diff --git a/src/do_all_the_voodoo_here.c b/src/do_all_the_voodoo_here.c
index c3c52886e4b33be7f6e01e1dcc0af4695a10a608..b7626a41706eb75a2a8ab34f19846ec2f5b75544 100644 (file)
--- a/src/do_all_the_voodoo_here.c
+++ b/src/do_all_the_voodoo_here.c
@@ -30,6 +30,7 @@
 #include "gethdrline.h"
 #include "strgen.h"
 #include "ctrlvalue.h"
+#include "statctrl.h"
 #include "do_all_the_voodoo_here.h"
 #include "log_error.h"
 #include "wrappers.h"
@@ -73,9 +74,36 @@ scan_headers(FILE *f, struct mailhdr *readhdrs, strlist *allhdrs, strlist *allun
        }
 }
 
+static void
+write_list_headers(FILE *outf, struct ml *ml)
+{
+       char *owner;
+
+       if (ml == NULL)
+               return;
+       if (statctrl(ml->ctrlfd, "nolistheaders"))
+               return;
+
+       fprintf(outf, "List-Id: <%s.%s>\n", ml->name, ml->fqdn);
+       fprintf(outf, "List-Post: <mailto:%s@%s>\n", ml->name, ml->fqdn);
+       fprintf(outf, "List-Help: <mailto:%s%shelp@%s>\n",
+           ml->name, ml->delim, ml->fqdn);
+       fprintf(outf, "List-Subscribe: <mailto:%s%ssubscribe@%s>\n",
+           ml->name, ml->delim, ml->fqdn);
+       fprintf(outf, "List-Unsubscribe: <mailto:%s%sunsubscribe@%s>\n",
+           ml->name, ml->delim, ml->fqdn);
+       owner = ctrlvalue(ml->ctrlfd, "owner");
+       if (owner != NULL) {
+               fprintf(outf, "List-Owner: <mailto:%s>\n", owner);
+               free(owner);
+       }
+       fprintf(outf, "Precedence: list\n");
+}
+
 int do_all_the_voodoo_here(int infd, int outfd, int hdrfd, int footfd,
                 const strlist *delhdrs, struct mailhdr *readhdrs,
-                strlist *allhdrs, const char *prefix, int replyto)
+                strlist *allhdrs, const char *prefix, int replyto,
+                struct ml *ml)
 {
        char *hdrline, *unfolded, *unqp;
        strlist allunfoldeds = vec_init();
@@ -140,6 +168,7 @@ int do_all_the_voodoo_here(int infd, int outfd, int hdrfd, int footfd,
                                        return -1;
                                }
                        }
+                       write_list_headers(outf, ml);
                        hdrsadded = true;
                }
 
@@ -184,6 +213,7 @@ int do_all_the_voodoo_here(int infd, int outfd, int hdrfd, int footfd,
                                return -1;
                        }
                }
+               write_list_headers(outf, ml);
        }
 
        if (prefix && !subject_present) {

diff --git a/src/mlmmj-process.c b/src/mlmmj-process.c
index eb7ecdb160d469fe6808da4bf2405a1171f8733c..fb7d7fbcbc7b7cd6826b807f72342847bdc75b25 100644 (file)
--- a/src/mlmmj-process.c
+++ b/src/mlmmj-process.c
@@ -398,7 +398,7 @@ int main(int argc, char **argv)
 
        if (do_all_the_voodoo_here(rawmailfd, donemailfd, hdrfd, footfd,
                                delheaders, readhdrs,
-                               &allheaders, subjectprefix, replyto) < 0) {
+                               &allheaders, subjectprefix, replyto, &ml) < 0) {
                log_error(LOG_ARGS, "Error in do_all_the_voodoo_here");
                close(donemailfd);
                unlink(donemailname);
@@ -553,7 +553,7 @@ int main(int argc, char **argv)
                        strlist ownerhdrs = vec_init();
                        if (do_all_the_voodoo_here(rawmailfd, donemailfd, -1,
                                        -1, delheaders,
-                                       NULL, &ownerhdrs, NULL, 0) < 0) {
+                                       NULL, &ownerhdrs, NULL, 0, NULL) < 0) {
                                log_error(LOG_ARGS, "do_all_the_voodoo_here");
                                vec_free_and_free(&ownerhdrs, free);
                                close(ownfd);

diff --git a/tests/mlmmj-receive.sh b/tests/mlmmj-receive.sh
index 5ba2e0733a20ee62a5e7d64197a5588a36b3e272..1996b973b37792ddacfece318cf2d2c3872ebfc2 100644 (file)
--- a/tests/mlmmj-receive.sh
+++ b/tests/mlmmj-receive.sh
@@ -103,6 +103,7 @@ simple_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
        cat > incoming-invalid << EOF
@@ -1649,6 +1650,7 @@ subscription_moderation_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -1836,6 +1838,7 @@ moderation_init_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -1978,6 +1981,7 @@ moderation_notifymod_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2141,6 +2145,7 @@ moderation_notmetoo_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2266,6 +2271,7 @@ moderation_reject_invalid_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2352,6 +2358,7 @@ maxmailsize_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2418,6 +2425,7 @@ maxmailsize0_body() {
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2485,6 +2493,7 @@ normal_email_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -2687,6 +2696,7 @@ delheaders_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
        printf "X-H1\nNope\n" > list/control/delheaders
@@ -2736,6 +2746,7 @@ delheaders_extras_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
        printf "X-k3\nx-h1\nx-L\n\n\n \n\t\nplop\nx-sym-colon:\nNope\n" > list/control/delheaders
@@ -2811,6 +2822,7 @@ customheaders_body()
        echo test@mlmmjtest > list/control/listaddress
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
+       touch list/control/nolistheaders
        printf "X-H1: test\nNope: really not\n" > list/control/customheaders
 
        printf "user@test\nuser2@test" > list/subscribers.d/u
@@ -2903,6 +2915,7 @@ customheaders_blanks_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
        printf "X-H1: test\nNope: really not\n\n   \n" > list/control/customheaders
@@ -2997,6 +3010,7 @@ customheaders_with_subst_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
        printf "X-H1: test\nNope: really not\nX-Poster-Address: \$posteraddr\$\n" > list/control/customheaders
@@ -3095,6 +3109,7 @@ verp_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "postfix" > list/control/verp
        echo 2 > list/control/maxverprecips
@@ -3172,6 +3187,7 @@ normal_email_with_dot_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 
@@ -3233,6 +3249,7 @@ multi_line_headers_body()
        rmdir list/text
        ln -s ${top_srcdir}/listtexts/en list/text
        echo test@mlmmjtest > list/control/listaddress
+       touch list/control/nolistheaders
        start_fakesmtp list
        echo "heloname" > list/control/smtphelo
 

diff --git a/tests/mlmmj.c b/tests/mlmmj.c
index 5e4733cdb23423a33a7020e9cd6210fba546a7b7..f29a46397f4763edc8e8bb65d6207da031e7bdd3 100644 (file)
--- a/tests/mlmmj.c
+++ b/tests/mlmmj.c
@@ -191,6 +191,9 @@ ATF_TC_WITHOUT_HEAD(voodoo_prefix_dedup);
 ATF_TC_WITHOUT_HEAD(voodoo_header_manipulation);
 ATF_TC_WITHOUT_HEAD(voodoo_double_call);
 ATF_TC_WITHOUT_HEAD(voodoo_failure_cleans_queue);
+ATF_TC_WITHOUT_HEAD(voodoo_listheaders);
+ATF_TC_WITHOUT_HEAD(voodoo_listheaders_disabled);
+ATF_TC_WITHOUT_HEAD(voodoo_listheaders_owner);
 ATF_TC_WITHOUT_HEAD(checkwait_smtpreply_connect);
 ATF_TC_WITHOUT_HEAD(checkwait_smtpreply_ehlo_multiline);
 ATF_TC_WITHOUT_HEAD(checkwait_smtpreply_errors);
@@ -4057,7 +4060,7 @@ ATF_TC_BODY(voodoo_header_manipulation, tc)
        ATF_REQUIRE(outfd >= 0);
 
        ret = do_all_the_voodoo_here(infd, outfd, hdrfd, footfd,
-           &delhdrs, readhdrs, &allheaders, "[LIST]", 0);
+           &delhdrs, readhdrs, &allheaders, "[LIST]", 0, NULL);
        ATF_REQUIRE_EQ(ret, 0);
        close(infd);
        close(outfd);
@@ -4130,7 +4133,7 @@ ATF_TC_BODY(voodoo_replyto, tc)
 
        /* replyto=1 should add Reply-To from From header */
        ATF_REQUIRE_EQ(do_all_the_voodoo_here(infd, outfd, -1, -1,
-           &delhdrs, readhdrs, &allheaders, NULL, 1), 0);
+           &delhdrs, readhdrs, &allheaders, NULL, 1, NULL), 0);
        close(infd);
        close(outfd);
 
@@ -4174,7 +4177,7 @@ ATF_TC_BODY(voodoo_prefix_dedup, tc)
        ATF_REQUIRE(outfd >= 0);
 
        ATF_REQUIRE_EQ(do_all_the_voodoo_here(infd, outfd, -1, -1,
-           &delhdrs, readhdrs, &allheaders, "[LIST]", 0), 0);
+           &delhdrs, readhdrs, &allheaders, "[LIST]", 0, NULL), 0);
        close(infd);
        close(outfd);
 
@@ -4214,7 +4217,7 @@ ATF_TC_BODY(voodoo_prefix_dedup, tc)
        ATF_REQUIRE(outfd >= 0);
 
        ATF_REQUIRE_EQ(do_all_the_voodoo_here(infd, outfd, -1, -1,
-           &delhdrs, readhdrs2, &allheaders2, "[LIST]", 0), 0);
+           &delhdrs, readhdrs2, &allheaders2, "[LIST]", 0, NULL), 0);
        close(infd);
        close(outfd);
 
@@ -4268,7 +4271,7 @@ ATF_TC_BODY(voodoo_double_call, tc)
        outfd = open("out1.txt", O_RDWR|O_CREAT|O_TRUNC, 0600);
        ATF_REQUIRE(outfd >= 0);
        ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
-           &delhdrs, readhdrs, &allheaders, NULL, 0);
+           &delhdrs, readhdrs, &allheaders, NULL, 0, NULL);
        ATF_REQUIRE_EQ(ret, 0);
        close(infd);
        close(outfd);
@@ -4285,7 +4288,7 @@ ATF_TC_BODY(voodoo_double_call, tc)
        outfd = open("out2.txt", O_RDWR|O_CREAT|O_TRUNC, 0600);
        ATF_REQUIRE(outfd >= 0);
        ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
-           &delhdrs, NULL, &allheaders, NULL, 0);
+           &delhdrs, NULL, &allheaders, NULL, 0, NULL);
        ATF_REQUIRE_EQ(ret, 0);
        close(infd);
        close(outfd);
@@ -4338,7 +4341,7 @@ ATF_TC_BODY(voodoo_failure_cleans_queue, tc)
        ATF_REQUIRE(outfd >= 0);
 
        ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
-           NULL, readhdrs, &allheaders, NULL, 0);
+           NULL, readhdrs, &allheaders, NULL, 0, NULL);
        close(infd);
        close(outfd);
 
@@ -4357,6 +4360,188 @@ ATF_TC_BODY(voodoo_failure_cleans_queue, tc)
        vec_free_and_free(&allheaders, free);
 }
 
+ATF_TC_BODY(voodoo_listheaders, tc)
+{
+       struct ml ml;
+       strlist allheaders = vec_init();
+       struct mailhdr readhdrs[] = {
+               { "From:", 0, NULL },
+               { "To:", 0, NULL },
+               { "Subject:", 0, NULL },
+               { NULL, 0, NULL }
+       };
+       int infd, outfd, ret;
+
+       init_ml(true);
+       ml_init(&ml);
+       ml.dir = "list";
+       ml_open(&ml, false);
+
+       atf_utils_create_file("listhdr_in.txt",
+           "From: sender@example.org\n"
+           "To: test@test\n"
+           "Subject: Hello\n"
+           "\n"
+           "Body.\n");
+
+       infd = open("listhdr_in.txt", O_RDONLY);
+       ATF_REQUIRE(infd >= 0);
+       outfd = open("listhdr_out.txt", O_RDWR|O_CREAT|O_TRUNC, 0600);
+       ATF_REQUIRE(outfd >= 0);
+
+       ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
+           NULL, readhdrs, &allheaders, NULL, 0, &ml);
+       ATF_REQUIRE_EQ(ret, 0);
+       close(infd);
+       close(outfd);
+
+       const char *path = "listhdr_out.txt";
+       if (!atf_utils_grep_file("List-Id: <test\\.test>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Id header missing");
+       }
+       if (!atf_utils_grep_file("List-Post: <mailto:test@test>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Post header missing");
+       }
+       if (!atf_utils_grep_file("List-Help: <mailto:test\\+help@test>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Help header missing");
+       }
+       if (!atf_utils_grep_file("List-Subscribe: <mailto:test\\+subscribe@test>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Subscribe header missing");
+       }
+       if (!atf_utils_grep_file("List-Unsubscribe: <mailto:test\\+unsubscribe@test>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Unsubscribe header missing");
+       }
+       if (!atf_utils_grep_file("Precedence: list", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("Precedence header missing");
+       }
+       /* No owner file -> no List-Owner */
+       if (atf_utils_grep_file("List-Owner:", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Owner should not be present without control/owner");
+       }
+
+       vec_free_and_free(&allheaders, free);
+       ml_close(&ml);
+}
+
+ATF_TC_BODY(voodoo_listheaders_disabled, tc)
+{
+       struct ml ml;
+       strlist allheaders = vec_init();
+       struct mailhdr readhdrs[] = {
+               { "From:", 0, NULL },
+               { "To:", 0, NULL },
+               { "Subject:", 0, NULL },
+               { NULL, 0, NULL }
+       };
+       int infd, outfd, ret;
+
+       init_ml(true);
+       ml_init(&ml);
+       ml.dir = "list";
+       ml_open(&ml, false);
+
+       /* Create the nolistheaders control file */
+       atf_utils_create_file("list/control/nolistheaders", "");
+
+       /* Re-open to pick up new control file */
+       ml_open(&ml, false);
+
+       atf_utils_create_file("listhdr_dis_in.txt",
+           "From: sender@example.org\n"
+           "To: test@test\n"
+           "Subject: Hello\n"
+           "\n"
+           "Body.\n");
+
+       infd = open("listhdr_dis_in.txt", O_RDONLY);
+       ATF_REQUIRE(infd >= 0);
+       outfd = open("listhdr_dis_out.txt", O_RDWR|O_CREAT|O_TRUNC, 0600);
+       ATF_REQUIRE(outfd >= 0);
+
+       ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
+           NULL, readhdrs, &allheaders, NULL, 0, &ml);
+       ATF_REQUIRE_EQ(ret, 0);
+       close(infd);
+       close(outfd);
+
+       const char *path = "listhdr_dis_out.txt";
+       if (atf_utils_grep_file("List-Id:", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Id should not be present when nolistheaders is set");
+       }
+       if (atf_utils_grep_file("List-Post:", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Post should not be present when nolistheaders is set");
+       }
+       if (atf_utils_grep_file("Precedence:", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("Precedence should not be present when nolistheaders is set");
+       }
+
+       vec_free_and_free(&allheaders, free);
+       ml_close(&ml);
+}
+
+ATF_TC_BODY(voodoo_listheaders_owner, tc)
+{
+       struct ml ml;
+       strlist allheaders = vec_init();
+       struct mailhdr readhdrs[] = {
+               { "From:", 0, NULL },
+               { "To:", 0, NULL },
+               { "Subject:", 0, NULL },
+               { NULL, 0, NULL }
+       };
+       int infd, outfd, ret;
+
+       init_ml(true);
+       ml_init(&ml);
+       ml.dir = "list";
+       ml_open(&ml, false);
+
+       /* Create owner control file */
+       atf_utils_create_file("list/control/owner", "admin@example.com");
+       ml_open(&ml, false);
+
+       atf_utils_create_file("listhdr_own_in.txt",
+           "From: sender@example.org\n"
+           "To: test@test\n"
+           "Subject: Hello\n"
+           "\n"
+           "Body.\n");
+
+       infd = open("listhdr_own_in.txt", O_RDONLY);
+       ATF_REQUIRE(infd >= 0);
+       outfd = open("listhdr_own_out.txt", O_RDWR|O_CREAT|O_TRUNC, 0600);
+       ATF_REQUIRE(outfd >= 0);
+
+       ret = do_all_the_voodoo_here(infd, outfd, -1, -1,
+           NULL, readhdrs, &allheaders, NULL, 0, &ml);
+       ATF_REQUIRE_EQ(ret, 0);
+       close(infd);
+       close(outfd);
+
+       const char *path = "listhdr_own_out.txt";
+       if (!atf_utils_grep_file("List-Owner: <mailto:admin@example.com>", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Owner header missing");
+       }
+       if (!atf_utils_grep_file("List-Id:", path)) {
+               atf_utils_cat_file(path, ">");
+               atf_tc_fail("List-Id header missing");
+       }
+
+       vec_free_and_free(&allheaders, free);
+       ml_close(&ml);
+}
+
 ATF_TC_BODY(checkwait_smtpreply_connect, tc)
 {
        int sp[2];
@@ -4899,6 +5084,9 @@ ATF_TP_ADD_TCS(tp)
        ATF_TP_ADD_TC(tp, voodoo_header_manipulation);
        ATF_TP_ADD_TC(tp, voodoo_double_call);
        ATF_TP_ADD_TC(tp, voodoo_failure_cleans_queue);
+       ATF_TP_ADD_TC(tp, voodoo_listheaders);
+       ATF_TP_ADD_TC(tp, voodoo_listheaders_disabled);
+       ATF_TP_ADD_TC(tp, voodoo_listheaders_owner);
        ATF_TP_ADD_TC(tp, checkwait_smtpreply_connect);
        ATF_TP_ADD_TC(tp, checkwait_smtpreply_ehlo_multiline);
        ATF_TP_ADD_TC(tp, checkwait_smtpreply_errors);