From: Wietse Venema Date: Tue, 15 Jun 2021 05:00:00 +0000 (-0500) Subject: postfix-3.7-20210615 X-Git-Tag: v3.7.0-RC1~21 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1757990477eb40cdc5819a592422e2e07243e5c3;p=thirdparty%2Fpostfix.git postfix-3.7-20210615 --- diff --git a/postfix/HISTORY b/postfix/HISTORY index 29457aefc..862671a88 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -25624,3 +25624,16 @@ Apologies for any names omitted. Cleanup: simplify the LMDB error recovery code. File: util/slmdb.c. + +20210615 + + Bugfix (introduced: Postfix 3.4): the texthash: map + implementation did not support "postmap -F" behavior. + Reported by Christopher Gurnee, who also found the missing + code in the postmap source. File: util/dict_thash.c. + + Cleanup: documentation for the postmap -F option. File: + postmap/postmap.c. + + Cleanup: simplify the LMDB error recovery code. File: + util/slmdb.c. diff --git a/postfix/html/postmap.1.html b/postfix/html/postmap.1.html index 27380f2f3..3ce1ff838 100644 --- a/postfix/html/postmap.1.html +++ b/postfix/html/postmap.1.html @@ -101,7 +101,9 @@ POSTMAP(1) POSTMAP(1) When creating a map from source file, process each value as a list of filenames, concatenate the content of those files, and store the base64-encoded result instead of the value (see INPUT - FORMAT for details). + FILE FORMAT for details). + + This feature is available in Postfix version 3.4 and later. -h Enable message header query mode. When reading lookup keys from standard input with "-q -", process the input as if it is an diff --git a/postfix/man/man1/postmap.1 b/postfix/man/man1/postmap.1 index e4dd20e7b..d086d6ca2 100644 --- a/postfix/man/man1/postmap.1 +++ b/postfix/man/man1/postmap.1 @@ -115,7 +115,9 @@ When querying a map, or listing a map, base64\-decode each value. When creating a map from source file, process each value as a list of filenames, concatenate the content of those files, and store the base64\-encoded result instead -of the value (see INPUT FORMAT for details). +of the value (see INPUT FILE FORMAT for details). +.sp +This feature is available in Postfix version 3.4 and later. .IP \fB\-h\fR Enable message header query mode. When reading lookup keys from standard input with "\fB\-q \-\fR", process the input diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h index f57b8a1ef..989ba2a78 100644 --- a/postfix/src/global/mail_version.h +++ b/postfix/src/global/mail_version.h @@ -20,7 +20,7 @@ * Patches change both the patchlevel and the release date. Snapshots have no * patchlevel; they change the release date only. */ -#define MAIL_RELEASE_DATE "20210612" +#define MAIL_RELEASE_DATE "20210615" #define MAIL_VERSION_NUMBER "3.7" #ifdef SNAPSHOT diff --git a/postfix/src/postmap/Makefile.in b/postfix/src/postmap/Makefile.in index c28af2aea..dc5369d9b 100644 --- a/postfix/src/postmap/Makefile.in +++ b/postfix/src/postmap/Makefile.in @@ -85,7 +85,7 @@ lmdb_bulk_test: $(PROG) rm -f lmdb_retry.lmdb main.cf tr A-Z a-z < /usr/share/dict/words| \ sed -e 's/.*/& &/' -e 10000q| LANG=C sort -u >lmdb_retry - echo lmdb_map_size=10000 >main.cf + echo lmdb_map_size=10240 >main.cf ($(SHLIB_ENV) $(VALGRIND) ./postmap -c . lmdb:lmdb_retry; \ $(SHLIB_ENV) $(VALGRIND) ./postmap -s lmdb:lmdb_retry | \ LANG=C sort > lmdb_retry.tmp) @@ -95,8 +95,8 @@ lmdb_bulk_test: $(PROG) lmdb_incr_test: $(PROG) rm -f lmdb_retry.lmdb main.cf tr A-Z a-z < /usr/share/dict/words| \ - sed -e 's/.*/& &/' -e 10000q| LANG=C sort -u >lmdb_retry - echo lmdb_map_size=10000 >main.cf + sed -e 's/.*/& &/' -e 1000q| LANG=C sort -u >lmdb_retry + echo lmdb_map_size=10240 >main.cf ($(SHLIB_ENV) $(VALGRIND) ./postmap -ic . lmdb:lmdb_retry lmdb_retry.tmp) diff --git a/postfix/src/postmap/postmap.c b/postfix/src/postmap/postmap.c index 6d4f2413e..d97425c6c 100644 --- a/postfix/src/postmap/postmap.c +++ b/postfix/src/postmap/postmap.c @@ -105,7 +105,9 @@ /* value. When creating a map from source file, process each /* value as a list of filenames, concatenate the content of /* those files, and store the base64-encoded result instead -/* of the value (see INPUT FORMAT for details). +/* of the value (see INPUT FILE FORMAT for details). +/* .sp +/* This feature is available in Postfix version 3.4 and later. /* .IP \fB-h\fR /* Enable message header query mode. When reading lookup keys /* from standard input with "\fB-q -\fR", process the input diff --git a/postfix/src/util/Makefile.in b/postfix/src/util/Makefile.in index 3b2e55c04..a0bb00a91 100644 --- a/postfix/src/util/Makefile.in +++ b/postfix/src/util/Makefile.in @@ -1720,6 +1720,7 @@ dict_thash.o: htable.h dict_thash.o: iostuff.h dict_thash.o: msg.h dict_thash.o: myflock.h +dict_thash.o: mymalloc.h dict_thash.o: readlline.h dict_thash.o: stringops.h dict_thash.o: sys_defs.h diff --git a/postfix/src/util/dict_thash.c b/postfix/src/util/dict_thash.c index a121ead35..5012fb9e2 100644 --- a/postfix/src/util/dict_thash.c +++ b/postfix/src/util/dict_thash.c @@ -46,6 +46,7 @@ /* Utility library. */ #include +#include #include #include #include @@ -179,6 +180,24 @@ DICT *dict_thash_open(const char *path, int open_flags, int dict_flags) msg_warn("%s, line %d: record is in \"key: value\" format;" " is this an alias file?", path, lineno); + /* + * Optionally treat the value as a filename, and replace the value + * with the BASE64-encoded content of the named file. + */ + if (dict_flags & DICT_FLAG_SRC_RHS_IS_FILE) { + VSTRING *base64_buf; + char *err; + + if ((base64_buf = dict_file_to_b64(dict, value)) == 0) { + err = dict_file_get_error(dict); + msg_warn("%s, line %d: %s: skipping this entry", + VSTREAM_PATH(fp), lineno, err); + myfree(err); + continue; + } + value = vstring_str(base64_buf); + } + /* * Store the value under the key. Handle duplicates * appropriately. XXX Move this into dict_ht, but 1) that map diff --git a/postfix/src/util/slmdb.c b/postfix/src/util/slmdb.c index ac1594ff3..9f03b477d 100644 --- a/postfix/src/util/slmdb.c +++ b/postfix/src/util/slmdb.c @@ -275,6 +275,9 @@ * triggers a long jump back into the caller to some pre-arranged point (the * closest thing that C has to exception handling). The application is then * expected to repeat the bulk transaction from scratch. + * + * When any code aborts a bulk transaction, it must reset slmdb->txn to null + * to avoid a use-after-free problem in slmdb_close(). */ /* @@ -379,7 +382,9 @@ static int slmdb_prepare(SLMDB *slmdb) * This is called before accessing the database, or after recovery from * an LMDB error. Note: this code cannot recover from errors itself. * slmdb->txn is either the database open() transaction or a - * freshly-created bulk-mode transaction. + * freshly-created bulk-mode transaction. When slmdb_prepare() commits or + * aborts commits a transaction, it must set slmdb->txn to null to avoid + * a use-after-free error in slmdb_close(). * * - With O_TRUNC we make a "drop" request before updating the database. * @@ -410,6 +415,7 @@ static int slmdb_prepare(SLMDB *slmdb) static int slmdb_recover(SLMDB *slmdb, int status) { MDB_envinfo info; + int original_status = status; /* * This may be needed in non-MDB_NOLOCK mode. Recovery is rare enough @@ -425,19 +431,14 @@ static int slmdb_recover(SLMDB *slmdb, int status) return (status); /* - * Recover bulk transactions only if they can be restarted, but limit the - * number of recovery attempts. + * Limit the number of bulk transaction recovery attempts. */ if ((slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0 - && (slmdb->longjmp_fn == 0 - || (slmdb->bulk_retry_count += 1) > slmdb->bulk_retry_limit)) + && (slmdb->bulk_retry_count += 1) > slmdb->bulk_retry_limit) return (status); /* - * If we can recover from the error, we clear the error condition and the - * caller should retry the failed operation immediately. Otherwise, the - * caller should terminate with a fatal run-time error and the program - * should be re-run later. + * Try to clear the error condition. */ switch (status) { @@ -504,18 +505,33 @@ static int slmdb_recover(SLMDB *slmdb, int status) } /* - * If a bulk-transaction error is recoverable, build a new bulk - * transaction from scratch, by making a long jump back into the caller - * at some pre-arranged point. In MDB_NOLOCK mode, there is no need to - * upgrade the lock to "exclusive", because the failed write transaction - * has no side effects. + * If we cleared the error condition for a non-bulk transaction, return a + * success status. The caller should retry the failed operation + * immediately. */ - if (status == 0 && (slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0 - && (status = mdb_txn_begin(slmdb->env, (MDB_txn *) 0, - slmdb->lmdb_flags & MDB_RDONLY, - &slmdb->txn)) == 0 - && (status = slmdb_prepare(slmdb)) == 0) - slmdb->longjmp_fn(slmdb->cb_context, 1); + if (status == 0 && (slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0) { + + /* + * We cleared the error condition for a bulk transaction. If the + * transaction is not restartable, return the original error. The + * caller should terminate with a fatal run-time error, and the + * program should be re-run later. + */ + if (slmdb->longjmp_fn == 0) + return (original_status); + + /* + * Rebuild a bulk transaction from scratch, by making a long jump + * back into the caller at some pre-arranged point. In MDB_NOLOCK + * mode, there is no need to upgrade a lock to "exclusive", because a + * failed write transaction has no side effects. + */ + if ((status = mdb_txn_begin(slmdb->env, (MDB_txn *) 0, + slmdb->lmdb_flags & MDB_RDONLY, + &slmdb->txn)) == 0 + && (status = slmdb_prepare(slmdb)) == 0) + slmdb->longjmp_fn(slmdb->cb_context, 1); + } return (status); } @@ -667,8 +683,12 @@ int slmdb_cursor_get(SLMDB *slmdb, MDB_val *mdb_key, /* * TODO: figure how we would recover a failing bulk transaction. */ - if ((slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0) + if ((slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0) { + if (slmdb->assert_fn) + slmdb->assert_fn(slmdb->cb_context, + "slmdb_cursor_get: bulk transaction is not supported"); return (MDB_PANIC); + } /* * Open a read transaction and cursor if needed. @@ -790,7 +810,8 @@ int slmdb_close(SLMDB *slmdb) /* * Finish an open bulk transaction. If slmdb_recover() returns after a - * bulk-transaction error, then it was unable to recover. + * bulk-transaction error, then it was unable to clear the error + * condition, or unable to restart the bulk transaction. */ if ((slmdb->slmdb_flags & SLMDB_FLAG_BULK) != 0 && slmdb->txn != 0 && (status = mdb_txn_commit(slmdb->txn)) != 0)