From: Wietse Venema Date: Sun, 20 Feb 2022 05:00:00 +0000 (-0500) Subject: postfix-3.8-20220220 X-Git-Tag: v3.8.0-RC1~31 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=57dd3f38dd92c33abad4d0f0d05fe9e8964c462e;p=thirdparty%2Fpostfix.git postfix-3.8-20220220 --- diff --git a/postfix/HISTORY b/postfix/HISTORY index 0d900728b..dc80b6d45 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -26280,3 +26280,29 @@ Apologies for any names omitted. Bitrot: Berkeley DB 18 is like Berkeley DB 6. Yasuhiro Kimura. File: util/dict_db.c. + +20220204 + + Updated collate.pl script for better tracking when a + Milter rejects, discards, or quarantines a message. Viktor + Dukhovni. File: auxiliary/collate/collate.pl. + +20220212 + + Cleanup: removed WISHLIST items that were recently fixed. + +20220217 + + Typo: "pcre2 --libs" should be "pcre2 --libs8". Reported by + Carlos Velasco. File proto/PCRE_README.html. + + Future proofing: added comments on the purpose of address + sanitization. File: showq/showq.c. + +20220220 + + Added a hash_fnvz() function to eliminate unnecessary strlen() + calls, and added regression tests. File: util/hash_fnv.c, + util/htable.c. + + Cleanup: unused initialization. File: util/make_dirs.c diff --git a/postfix/README_FILES/PCRE_README b/postfix/README_FILES/PCRE_README index d8bf8d504..6dffb4b5c 100644 --- a/postfix/README_FILES/PCRE_README +++ b/postfix/README_FILES/PCRE_README @@ -41,7 +41,7 @@ To build Postfix explicitly with a pcre2 library (Postfix 3.7 and later): $ make -f Makefile.init makefiles \ "CCARGS=-DHAS_PCRE=2 `pcre2-config --cflags`" \ - "AUXLIBS_PCRE=`pcre2-config --libs`" + "AUXLIBS_PCRE=`pcre2-config --libs8`" $ make To build Postfix explicitly with a legacy pcre library (all Postfix versions): diff --git a/postfix/RELEASE_NOTES-3.7 b/postfix/RELEASE_NOTES-3.7 index a15bf4ed9..05ce65ac2 100644 --- a/postfix/RELEASE_NOTES-3.7 +++ b/postfix/RELEASE_NOTES-3.7 @@ -66,7 +66,7 @@ to do $name expansion as it evaluates the parameter value. Major changes - lmdb support ---------------------------- -[Feature 20210605] Overhauled the LMDB client implementation, and +[Feature 20210605] Overhauled the LMDB client's error handling, and added integration tests for future-proofing. There are no visible changes in documented behavior. diff --git a/postfix/WISHLIST b/postfix/WISHLIST index d0a0db45c..7e0bc51a8 100644 --- a/postfix/WISHLIST +++ b/postfix/WISHLIST @@ -6,9 +6,6 @@ Wish list: Disable -DSNAPSHOT and -DNONPROD in makedefs. - Alias htable(3) calls to equivalent binhash(3) calls, - and obsolete the htable(3) module. - FILTER_README needs some text on multi-instance implementations, and existing multi-instance references need to be updated. @@ -17,6 +14,10 @@ Wish list: A smart query service for live Postfix tables that outputs JSON? + JSON logging? + + default_transport_maps? This would simplify configuration. + Add a pointer to http://mmogilvi.users.sourceforge.net/software/oauthbearer.html in documentation or on-line howtos. @@ -264,10 +265,6 @@ Wish list: Maybe don't whitelist a client that has maxed out its per-MTA connection count limit. - Inline support for pcre:{/pattern/=action, ...} and ditto - support for regexp: and cidr: tables. Factor out and reuse - code that already exists in inline: and other tables. - Log command=good/bad statistics in postscreen? smtpd_checks tests either must use a DNS dummy resolver @@ -810,22 +807,6 @@ Wish list: Cleanup: make DNSBL query format configurable beyond the client's reversed IP address. - With 'final delivery' in the LMTP client, need an option - to also add delivered-to and other pipe(8) features. This - requires making mail_copy() functionality available in - non-mailbox context. - - Cleanup: modernize the "add missing From: header" code, to - ``phrase '' form. Most likely, quote the entire phrase - if it contains any text that is special, then rfc822_externalize - the whole thing. - - SMTP server: make the server_addr and server_port available - to policy server, Dovecot, and perhaps Milters. - - Med: local and remote source port and IP address for smtpd - policy hook. - Maybe change maps_rbl_reject_code default to 521, and update wording in STRESS_README. @@ -838,15 +819,6 @@ Wish list: Plan for time_t larger than long, or wait for LP64 to dominate the world? - Make "AUTH=<>" appendage to MAIL FROM configurable, enabled - by default. - - To support ternary operator without a huge parsing effort, - consider ${value?{xxx}:{yyy}} where ${name} is existing - syntax, and where ?{text} and :{text} are new syntax that - is unlikely to break existing configurations. Or perhaps - it's just too ugly. - Write delivery rate delay example (which _README?) and auth failure cache example (SASL_README). Then include them in SOHO_README. diff --git a/postfix/auxiliary/collate/collate.pl b/postfix/auxiliary/collate/collate.pl index 31b48d609..62d08b972 100755 --- a/postfix/auxiliary/collate/collate.pl +++ b/postfix/auxiliary/collate/collate.pl @@ -25,6 +25,7 @@ my %smtp; my %transaction; my $i = 0; my %seqno; +my %deleted; my %isagent = map { ($_, 1) } @agents; @@ -36,6 +37,7 @@ while (<>) { if (m{\Gconnect from }gc) { # Start new log $smtpd{$pid}->{"log"} = $_; next; + undef $smtpd{$pid}->{"qid"}; } $smtpd{$pid}->{"log"} .= $_; @@ -52,7 +54,14 @@ while (<>) { my $qid = $smtpd{$pid}->{"qid"}; $transaction{$qid} .= $_ if (defined($qid) && exists $transaction{$qid}); - delete $smtpd{$pid} if (m{\Gdisconnect from}gc); + if (m{\Gdisconnect from}gc) { + if (!defined($qid)) { + print $smtpd{$pid}->{"log"}, "\n"; + } elsif (delete $deleted{$qid}) { + print delete $transaction{$qid}, "\n"; + } + delete $smtpd{$pid}; + } next; } @@ -73,6 +82,9 @@ while (<>) { my $qid = "$inst/$1"; $transaction{$qid} .= $_; $seqno{$qid} = ++$i if (! exists $seqno{$qid}); + if (m{\G(?:milter(?:-(?:header|body))?-)?(?:reject|discard|hold): }) { + $deleted{$qid} = 1; + } next; } diff --git a/postfix/html/PCRE_README.html b/postfix/html/PCRE_README.html index 810e3aad0..0a89dfbda 100644 --- a/postfix/html/PCRE_README.html +++ b/postfix/html/PCRE_README.html @@ -66,7 +66,7 @@ and later): 

 $ make -f Makefile.init makefiles \
     "CCARGS=-DHAS_PCRE=2 `pcre2-config --cflags`" \
-    "AUXLIBS_PCRE=`pcre2-config --libs`"
+    "AUXLIBS_PCRE=`pcre2-config --libs8`"
 $ make
diff --git a/postfix/html/tlsproxy.8.html b/postfix/html/tlsproxy.8.html index d0b8b0ea1..c15322bf3 100644 --- a/postfix/html/tlsproxy.8.html +++ b/postfix/html/tlsproxy.8.html @@ -323,14 +323,6 @@ TLSPROXY(8) TLSPROXY(8) tlsproxy_client_scert_verifydepth ($smtp_tls_scert_verifydepth) The verification depth for remote TLS server certificates. - tlsproxy_client_level ($smtp_tls_security_level) - The default TLS security level for the Postfix tlsproxy(8) - client. - - tlsproxy_client_policy ($smtp_tls_policy_maps) - Optional lookup tables with the Postfix tlsproxy(8) client TLS - security policy by next-hop destination. - tlsproxy_client_use_tls ($smtp_use_tls) Opportunistic mode: use TLS when a remote server announces TLS support. @@ -343,6 +335,16 @@ TLSPROXY(8) TLSPROXY(8) usage policy by next-hop destination and by remote TLS server hostname. + Available in Postfix version 3.4-3.6: + + tlsproxy_client_level ($smtp_tls_security_level) + The default TLS security level for the Postfix tlsproxy(8) + client. + + tlsproxy_client_policy ($smtp_tls_policy_maps) + Optional lookup tables with the Postfix tlsproxy(8) client TLS + security policy by next-hop destination. + Available in Postfix version 3.7 and later: tlsproxy_client_security_level ($smtp_tls_security_level) diff --git a/postfix/man/man8/tlsproxy.8 b/postfix/man/man8/tlsproxy.8 index 10b447606..5b6167089 100644 --- a/postfix/man/man8/tlsproxy.8 +++ b/postfix/man/man8/tlsproxy.8 @@ -292,12 +292,6 @@ The name of the parameter that provides the tlsproxy_client_loglevel value. .IP "\fBtlsproxy_client_scert_verifydepth ($smtp_tls_scert_verifydepth)\fR" The verification depth for remote TLS server certificates. -.IP "\fBtlsproxy_client_level ($smtp_tls_security_level)\fR" -The default TLS security level for the Postfix \fBtlsproxy\fR(8) -client. -.IP "\fBtlsproxy_client_policy ($smtp_tls_policy_maps)\fR" -Optional lookup tables with the Postfix \fBtlsproxy\fR(8) client TLS -security policy by next\-hop destination. .IP "\fBtlsproxy_client_use_tls ($smtp_use_tls)\fR" Opportunistic mode: use TLS when a remote server announces TLS support. @@ -308,6 +302,14 @@ Optional lookup tables with the Postfix \fBtlsproxy\fR(8) client TLS usage policy by next\-hop destination and by remote TLS server hostname. .PP +Available in Postfix version 3.4\-3.6: +.IP "\fBtlsproxy_client_level ($smtp_tls_security_level)\fR" +The default TLS security level for the Postfix \fBtlsproxy\fR(8) +client. +.IP "\fBtlsproxy_client_policy ($smtp_tls_policy_maps)\fR" +Optional lookup tables with the Postfix \fBtlsproxy\fR(8) client TLS +security policy by next\-hop destination. +.PP Available in Postfix version 3.7 and later: .IP "\fBtlsproxy_client_security_level ($smtp_tls_security_level)\fR" The default TLS security level for the Postfix \fBtlsproxy\fR(8) diff --git a/postfix/proto/PCRE_README.html b/postfix/proto/PCRE_README.html index 896de9097..1741826a3 100644 --- a/postfix/proto/PCRE_README.html +++ b/postfix/proto/PCRE_README.html @@ -66,7 +66,7 @@ and later): 

 $ make -f Makefile.init makefiles \
     "CCARGS=-DHAS_PCRE=2 `pcre2-config --cflags`" \
-    "AUXLIBS_PCRE=`pcre2-config --libs`"
+    "AUXLIBS_PCRE=`pcre2-config --libs8`"
 $ make
diff --git a/postfix/proto/stop b/postfix/proto/stop index 8682f8e53..0189e961a 100644 --- a/postfix/proto/stop +++ b/postfix/proto/stop @@ -1562,3 +1562,4 @@ json postlogd proxied raf +fnvz diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index aa50f1c37..2542fa37e 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20220204" +#define MAIL_RELEASE_DATE "20220220" #define MAIL_VERSION_NUMBER "3.8" #ifdef SNAPSHOT diff --git a/postfix/src/showq/showq.c b/postfix/src/showq/showq.c index b8dd7e994..80e1e89e2 100644 --- a/postfix/src/showq/showq.c +++ b/postfix/src/showq/showq.c @@ -228,6 +228,7 @@ static void showq_report(VSTREAM *client, char *queue, char *id, if (*start == 0) start = var_empty_addr; quote_822_local(printable_quoted_addr, start); + /* For consistency with REC_TYPE_RCPT below. */ printable(STR(printable_quoted_addr), '?'); if (sender_seen++ > 0) { msg_warn("%s: duplicate sender address: %s " @@ -257,6 +258,7 @@ static void showq_report(VSTREAM *client, char *queue, char *id, if (*start == 0) /* can't happen? */ start = var_empty_addr; quote_822_local(printable_quoted_addr, start); + /* For consistency with recipients in bounce logfile. */ printable(STR(printable_quoted_addr), '?'); if (dup_filter == 0 || htable_locate(dup_filter, STR(printable_quoted_addr)) == 0) diff --git a/postfix/src/tlsproxy/tlsproxy.c b/postfix/src/tlsproxy/tlsproxy.c index 42b7a75e7..b8242df77 100644 --- a/postfix/src/tlsproxy/tlsproxy.c +++ b/postfix/src/tlsproxy/tlsproxy.c @@ -272,12 +272,6 @@ /* value. /* .IP "\fBtlsproxy_client_scert_verifydepth ($smtp_tls_scert_verifydepth)\fR" /* The verification depth for remote TLS server certificates. -/* .IP "\fBtlsproxy_client_level ($smtp_tls_security_level)\fR" -/* The default TLS security level for the Postfix \fBtlsproxy\fR(8) -/* client. -/* .IP "\fBtlsproxy_client_policy ($smtp_tls_policy_maps)\fR" -/* Optional lookup tables with the Postfix \fBtlsproxy\fR(8) client TLS -/* security policy by next-hop destination. /* .IP "\fBtlsproxy_client_use_tls ($smtp_use_tls)\fR" /* Opportunistic mode: use TLS when a remote server announces TLS /* support. @@ -288,6 +282,14 @@ /* usage policy by next-hop destination and by remote TLS server /* hostname. /* .PP +/* Available in Postfix version 3.4-3.6: +/* .IP "\fBtlsproxy_client_level ($smtp_tls_security_level)\fR" +/* The default TLS security level for the Postfix \fBtlsproxy\fR(8) +/* client. +/* .IP "\fBtlsproxy_client_policy ($smtp_tls_policy_maps)\fR" +/* Optional lookup tables with the Postfix \fBtlsproxy\fR(8) client TLS +/* security policy by next-hop destination. +/* .PP /* Available in Postfix version 3.7 and later: /* .IP "\fBtlsproxy_client_security_level ($smtp_tls_security_level)\fR" /* The default TLS security level for the Postfix \fBtlsproxy\fR(8) diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in index c59cdf986..bfe146f5c 100644 --- a/postfix/src/util/Makefile.in +++ b/postfix/src/util/Makefile.in @@ -140,7 +140,7 @@ TESTPROG= dict_open dup2_pass_on_exec events exec_command fifo_open \ valid_utf8_string ip_match base32_code msg_rate_delay netstring \ vstream timecmp dict_cache midna_domain casefold strcasecmp_utf8 \ vbuf_print split_qnameval vstream msg_logger byte_mask \ - known_tcp_ports dict_stream find_inet binhash + known_tcp_ports dict_stream find_inet binhash hash_fnv PLUGIN_MAP_SO = $(LIB_PREFIX)pcre$(LIB_SUFFIX) HTABLE_FIX = NORANDOMIZE=1 LIB_DIR = ../../lib @@ -432,6 +432,11 @@ binhash: $(LIB) $(CC) $(CFLAGS) -DTEST -o $@ $@.c $(LIB) $(SYSLIBS) mv junk $@.o +hash_fnv: $(LIB) + mv $@.o junk + $(CC) $(CFLAGS) -DTEST -o $@ $@.c $(LIB) $(SYSLIBS) + mv junk $@.o + unix_recv_fd: $(LIB) mv $@.o junk $(CC) $(CFLAGS) -DTEST -o $@ $@.c $(LIB) $(SYSLIBS) @@ -739,6 +744,9 @@ htable_test: htable /usr/share/dict/words binhash_test: binhash /usr/share/dict/words $(SHLIB_ENV) ${VALGRIND} ./binhash < /usr/share/dict/words +hash_fnv_test: hash_fnv + $(SHLIB_ENV) ${VALGRIND} ./hash_fnv + hex_code_test: hex_code $(SHLIB_ENV) ${VALGRIND} ./hex_code diff --git a/postfix/src/util/hash_fnv.c b/postfix/src/util/hash_fnv.c index 10e97f056..b56d176ac 100644 --- a/postfix/src/util/hash_fnv.c +++ b/postfix/src/util/hash_fnv.c @@ -9,9 +9,15 @@ /* HASH_FNV_T hash_fnv( /* const void *src, /* size_t len) +/* +/* HASH_FNV_T hash_fnvz( +/* const char *src) /* DESCRIPTION /* hash_fnv() implements a modified FNV type 1a hash function. /* +/* hash_fnvz() provides the same functionality for null-terminated +/* strings, avoiding an unnecessary strlen() call. +/* /* To thwart collision attacks, the hash function is seeded /* once from /dev/urandom, and if that is unavailable, from /* wallclock time, monotonic system clocks, and the process @@ -21,11 +27,10 @@ /* /* This function implements a workaround for a "sticky state" /* problem with FNV hash functions: when an input produces a -/* zero intermediate hash state, and the next input byte is -/* zero, then the operations "hash ^= 0" and "hash *= FNV_prime" -/* would not change the hash value. To avoid this, hash_fnv() -/* adds 1 to each input byte. Compile with -DSTRICT_FNV1A to -/* get the standard behavior. +/* zero hash state, and the next input byte is zero, then the +/* hash state would not change. To avoid this, hash_fnv() adds +/* 1 to each input value. Compile with -DSTRICT_FNV1A to get +/* the standard behavior. /* /* The default HASH_FNV_T result type is uint64_t. When compiled /* with -DUSE_FNV_32BIT, the result type is uint32_t. On ancient @@ -71,37 +76,164 @@ #define FNV_offset_basis 0xcbf29ce484222325ULL #endif + /* + * Workaround for the sticky all-zero hash state: when the next input byte + * is zero, then the operations "hash ^= 0" and "hash *= FNV_prime" would + * not change the hash state. To avoid that, add 1 to the every input value. + */ +#ifdef STRICT_FNV1A +#define HASH_FNV_NEW_BITS(new_bits) (new_bits) +#else +#define HASH_FNV_NEW_BITS(new_bits) (1 + (new_bits)) +#endif + +static HASH_FNV_T hash_fnv_basis = FNV_offset_basis; +static int hash_fnv_must_init = 1; + +/* hash_fnv_init - seed the hash */ + +static void hash_fnv_init(void) +{ + HASH_FNV_T seed; + + if (!getenv("NORANDOMIZE")) { + ldseed(&seed, sizeof(seed)); + hash_fnv_basis ^= seed; + } + hash_fnv_must_init = 0; +} + /* hash_fnv - modified FNV 1a hash */ HASH_FNV_T hash_fnv(const void *src, size_t len) { - static HASH_FNV_T basis = FNV_offset_basis; - static int randomize = 1; HASH_FNV_T hash; + HASH_FNV_T new_bits; - /* - * Initialize. - */ - if (randomize) { - if (!getenv("NORANDOMIZE")) { - HASH_FNV_T seed; + if (hash_fnv_must_init) + hash_fnv_init(); - ldseed(&seed, sizeof(seed)); - basis ^= seed; - } - randomize = 0; + hash = hash_fnv_basis; + while (len-- > 0) { + new_bits = *(unsigned char *) src++; + hash ^= HASH_FNV_NEW_BITS(new_bits); + hash *= FNV_prime; + } + return (hash); +} + +/* hash_fnvz - modified FNV 1a hash for null-terminated strings */ + +HASH_FNV_T hash_fnvz(const char *src) +{ + HASH_FNV_T hash; + HASH_FNV_T new_bits; + + if (hash_fnv_must_init) + hash_fnv_init(); + + hash = hash_fnv_basis; + while ((new_bits = *(unsigned char *) src++) != 0) { + hash ^= HASH_FNV_NEW_BITS(new_bits); + hash *= FNV_prime; } + return (hash); +} +#ifdef TEST +#include +#include +#include + +int main(void) +{ + int pass = 0; + int fail = 0; + + /* + * Sanity check. + */ #ifdef STRICT_FNV1A -#define FNV_NEXT_BYTE(s) ((HASH_FNV_T) * (const unsigned char *) s++) + msg_fatal("This test requires no STRICT_FNV1A"); +#endif + + /* + * Force unseeded hash, to make tests predictable. + */ + if (putenv("NORANDOMIZE=") != 0) + msg_fatal("putenv(\"NORANDOMIZE=\"): %m"); + + /* + * Test: hashing produces the expected results. + */ + { + struct testcase { + HASH_FNV_T hval; + const char *str; + }; + static struct testcase testcases[] = + { +#ifdef USE_FNV_32BIT + 0x1c00fc06UL, "overdeeply", + 0x1c00fc06UL, "undescript", + 0x1e1e52a4UL, "fanfold", + 0x1e1e52a4UL, "phrensied", #else -#define FNV_NEXT_BYTE(s) (1 + (HASH_FNV_T) * (const unsigned char *) s++) + 0xda19999ec0bda706ULL, "overdeeply", + 0xd7b9e43f26396a66ULL, "undescript", + 0xa50c585d385a2604ULL, "fanfold", + 0x1ec3ef9bb2b734a4ULL, "phrensied", #endif + 0, + }; + struct testcase *tp; + HASH_FNV_T hval; + int test_failed; - hash = basis; - while (len-- > 0) { - hash ^= FNV_NEXT_BYTE(src); - hash *= FNV_prime; + for (tp = testcases; tp->str; tp++) { + test_failed = 0; + if ((hval = hash_fnvz(tp->str)) != tp->hval) { + msg_warn("hash_fnv(\"%s\") want %lu, got: %lu", + tp->str, (unsigned long) tp->hval, + (unsigned long) hval); + test_failed = 1; + } + if (test_failed) { + fail += 1; + msg_info("FAIL: %s", tp->str); + } else { + pass += 1; + msg_info("PASS: %s", tp->str); + } + } } - return (hash); + + /* + * Test: hash_fnvz(s) is equivalent to hash_fnv(s, strlen(s)). No need to + * verify the actual result; we already did that above. + */ + { + const char *strval = "foobar"; + HASH_FNV_T h1 = hash_fnv(strval, strlen(strval)); + HASH_FNV_T h2 = hash_fnvz(strval); + + if (h1 == h2) { + pass += 1; + msg_info("PASS: hash_fnvz(\"%s\") == hash_fnv(\"%s\", %ld)", + strval, strval, (long) strlen(strval)); + } else { + fail += 1; + msg_info("FAIL: hash_fnvz(\"%s\") == hash_fnv(\"%s\", %ld)", + strval, strval, (long) strlen(strval)); + } + } + + + /* + * Wrap up. + */ + msg_info("PASS=%d FAIL=%d", pass, fail); + return (fail != 0); } + +#endif diff --git a/postfix/src/util/hash_fnv.h b/postfix/src/util/hash_fnv.h index dbbb383c5..0bf1771c8 100644 --- a/postfix/src/util/hash_fnv.h +++ b/postfix/src/util/hash_fnv.h @@ -24,6 +24,7 @@ #endif /* HASH_FNV_T */ extern HASH_FNV_T hash_fnv(const void *, size_t); +extern HASH_FNV_T hash_fnvz(const char *); /* LICENSE /* .ad diff --git a/postfix/src/util/htable.c b/postfix/src/util/htable.c index 1c08e9797..5d659aa18 100644 --- a/postfix/src/util/htable.c +++ b/postfix/src/util/htable.c @@ -137,7 +137,7 @@ #ifndef NO_HASH_FNV #include "hash_fnv.h" -#define htable_hash(s, size) (hash_fnv((s), strlen(s)) % (size)) +#define htable_hash(s, size) (hash_fnvz(s) % (size)) #else diff --git a/postfix/src/util/make_dirs.c b/postfix/src/util/make_dirs.c index 2e37f8fe6..75503504b 100644 --- a/postfix/src/util/make_dirs.c +++ b/postfix/src/util/make_dirs.c @@ -57,7 +57,7 @@ int make_dirs(const char *path, int perms) int saved_ch; struct stat st; int ret; - mode_t saved_mode = 0; + mode_t saved_mode; gid_t egid = -1; /*