From: Wietse Venema <wietse@porcupine.org>
Date: Fri, 6 Jun 2003 05:00:00 +0000 (-0500)
Subject: postfix-2.0.11-20030606
X-Git-Tag: v2.1-RC1-20040331~58
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0e858d959605355dec6e042c51bd4914fd991a21;p=thirdparty%2Fpostfix.git

postfix-2.0.11-20030606
---

diff --git a/postfix/HISTORY b/postfix/HISTORY
index d59c39046..dc0154521 100644
--- a/postfix/HISTORY
+++ b/postfix/HISTORY
@@ -8104,14 +8104,12 @@ Apologies for any names omitted.
 	descriptor is writable while write() transfers zero bytes.
 	File:  global/pipe_command.c.
 
-2003052[3-9]
-
-	Cleanup: qmgr_fudge_factor is gone.
+20030523-20030605
 
 	Cleanup: rewrote the queue file record processing loops in
-	cleanup and in [n]qmgr. This code had deteriorated a lot
-	as the result of small changes over the years. This change
-	brings the code closer to "obviously correct". Files:
+	pickup, cleanup and in [n]qmgr. This code had deteriorated
+	a lot as the result of small changes over the years. This
+	change brings the code closer to "obviously correct". Files:
 	cleanup/cleanup_envelope.c, cleanup/cleanup_extracted.c,
 	*qmgr/qmgr_message.c.
 
@@ -8130,9 +8128,10 @@ Apologies for any names omitted.
 
 20030528
 
-	Compatibility: "sendmail -q<time>" without -bd option causes
-	the command to exit instead of waiting for input on the
-	standard input stream. File: sendmail/sendmail.c.
+	Compatibility: "sendmail -q<time>" without -bd option now
+	exits immediately, instead of waiting for input on the
+	standard input stream and screwing up system boot sequences.
+	File: sendmail/sendmail.c.
 
 20030530
 
@@ -8149,9 +8148,9 @@ Apologies for any names omitted.
 
 	Cleanup: input checks moved from the pickup daemon to the
 	postdrop mail submission command; this is to prepare for
-	mail submission from postdrop->cleanup without going through
-	the maildrop directory. Files: pickup/pickup.c,
-	postdrop/postdrop.c.
+	direct mail submission from postdrop->cleanup without going
+	through the maildrop directory and the pickup service.
+	Files: pickup/pickup.c, postdrop/postdrop.c.
 
 	Bugfix: the "dead host" backoff timer in the MySQL client
 	didn't work.  Fix by Leandro Santi. File: util/dict_mysql.c.
diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES
index e5f168aa0..710866067 100644
--- a/postfix/RELEASE_NOTES
+++ b/postfix/RELEASE_NOTES
@@ -22,6 +22,15 @@ snapshot release).  Patches change the patchlevel and the release
 date. Snapshots change only the release date, unless they include
 the same bugfixes as a patch release.
 
+Major changes with Postfix snapshot 2.0.11-20030606
+===================================================
+
+Complete rewrite of the queue file record reading loops in the
+pickup, cleanup and in the queue manager daemons. This code had
+deteriorated over time.  The new code eliminates an old problem
+where the queue manager had to read most queue file records twice
+in the case of very large alias/include file expansions.
+
 Incompatible changes with Postfix snapshot 2.0.8-20030417
 =========================================================
 
diff --git a/postfix/examples/chroot-setup/Solaris8 b/postfix/examples/chroot-setup/Solaris8
index ae7e5b1b0..f9767fe61 100644
--- a/postfix/examples/chroot-setup/Solaris8
+++ b/postfix/examples/chroot-setup/Solaris8
@@ -4,52 +4,57 @@
 # too many files. There is no need to copy libc.so and other files
 # that are already linked in before a Postfix daemon chroots itself.
 
+COMMAND_DIRECTORY="/usr/sbin"
+DAEMON_DIRECTORY="/usr/libexec/postfix"
+QUEUE_DIRECTORY="/var/spool/postfix"
+
 ## Copy any shared libraries, device entries, or configuration files
 ## needed by Postfix into the jail.
 binlist="
-/usr/libexec/postfix/virtual
-/usr/libexec/postfix/trivial-rewrite
-/usr/libexec/postfix/spawn
-/usr/libexec/postfix/smtpd
-/usr/libexec/postfix/smtp
-/usr/libexec/postfix/showq
-/usr/libexec/postfix/qmqpd
-/usr/libexec/postfix/qmgr
-/usr/libexec/postfix/proxymap
-/usr/libexec/postfix/pipe
-/usr/libexec/postfix/pickup
-/usr/libexec/postfix/nqmgr
-/usr/libexec/postfix/master
-/usr/libexec/postfix/local
-/usr/libexec/postfix/lmtp
-/usr/libexec/postfix/flush
-/usr/libexec/postfix/error
-/usr/libexec/postfix/cleanup
-/usr/libexec/postfix/bounce
+$DAEMON_DIRECTORY/virtual
+$DAEMON_DIRECTORY/trivial-rewrite
+$DAEMON_DIRECTORY/spawn
+$DAEMON_DIRECTORY/smtpd
+$DAEMON_DIRECTORY/smtp
+$DAEMON_DIRECTORY/showq
+$DAEMON_DIRECTORY/qmqpd
+$DAEMON_DIRECTORY/qmgr
+$DAEMON_DIRECTORY/proxymap
+$DAEMON_DIRECTORY/pipe
+$DAEMON_DIRECTORY/pickup
+$DAEMON_DIRECTORY/nqmgr
+$DAEMON_DIRECTORY/master
+$DAEMON_DIRECTORY/local
+$DAEMON_DIRECTORY/lmtp
+$DAEMON_DIRECTORY/flush
+$DAEMON_DIRECTORY/error
+$DAEMON_DIRECTORY/cleanup
+$DAEMON_DIRECTORY/bounce
 /usr/lib/sendmail
-/usr/sbin/postsuper
-/usr/sbin/postqueue
-/usr/sbin/postmap
-/usr/sbin/postlog
-/usr/sbin/postlock
-/usr/sbin/postkick
-/usr/sbin/postfix
-/usr/sbin/postdrop
-/usr/sbin/postconf
-/usr/sbin/postcat
-/usr/sbin/postalias
+$COMMAND_DIRECTORY/postsuper
+$COMMAND_DIRECTORY/postqueue
+$COMMAND_DIRECTORY/postmap
+$COMMAND_DIRECTORY/postlog
+$COMMAND_DIRECTORY/postlock
+$COMMAND_DIRECTORY/postkick
+$COMMAND_DIRECTORY/postfix
+$COMMAND_DIRECTORY/postdrop
+$COMMAND_DIRECTORY/postconf
+$COMMAND_DIRECTORY/postcat
+$COMMAND_DIRECTORY/postalias
 "
-for i in `xargs ldd $binlist | grep -v '^[^:]*:' | sort | uniq | sed -e 's/^[^ ]* =>//' | awk '{print $1}'`; do
-    mkdir -p /var/spool/postfix`dirname $i`
+ldd $binlist | awk '/[=]>/ { print $3 }' | sort -u | while read i
+do
+    mkdir -p $QUEUE_DIRECTORY`dirname $i`
     ## Sun's version of tar sucks.  We'll have to remove the leading
     ## slashes from file names ourself, otherwise the copy doesn't
     ## work.
-    (cd / && tar cphf - `echo $i | sed -e 's/^\///'`) | (cd /var/spool/postfix && tar xpf -)
+    (cd / && tar cphf - `echo $i | sed -e 's/^\///'`) | (cd $QUEUE_DIRECTORY && tar xpf -)
 done
 
 ## More stuff for the jail, mostly discovered by inspection
 ## (e.g. strings, lsof).
-for i in "
+more="
 /dev/zero
 /dev/null
 /dev/udp6
@@ -89,9 +94,10 @@ for i in "
 /usr/lib/nss_files.so.1
 /usr/share/lib/zoneinfo
 /var/ld/ld.config
-"; do
-    mkdir -p /var/spool/postfix`dirname $i`
-    (cd / && tar cpf - `echo $i | sed -e 's/^\///'`) | (cd /var/spool/postfix && tar xpf -)
+"
+for i in $more; do
+    mkdir -p $QUEUE_DIRECTORY`dirname $i`
+    (cd / && tar cpf - `echo $i | sed -e 's/^\///'`) | (cd $QUEUE_DIRECTORY && tar xpf -)
 done
 
 exit 0
diff --git a/postfix/html/qmgr.8.html b/postfix/html/qmgr.8.html
index bf1197c21..48c8b188f 100644
--- a/postfix/html/qmgr.8.html
+++ b/postfix/html/qmgr.8.html
@@ -220,26 +220,42 @@ QMGR(8)                                                   QMGR(8)
        In the text below, <i>transport</i> is the first field in a  <b>mas-</b>
        <b>ter.cf</b> entry.
 
+       <b>qmgr</b><i>_</i><b>fudge</b><i>_</i><b>factor</b> (valid range: 10..100)
+              The  percentage  of  delivery resources that a busy
+              mail system will use up for  delivery  of  a  large
+              mailing  list  message.  With 100%, delivery of one
+              message does not begin before the previous  message
+              has  been  delivered.  This results in good perfor-
+              mance for large mailing lists, but results in  poor
+              response  time for one-to-one mail.  With less than
+              100%, response time for one-to-one  mail  improves,
+              but  large  mailing  list delivery performance suf-
+              fers. In the worst case, recipients near the begin-
+              ning  of  a  large list receive a burst of messages
+              immediately, while recipients near the end of  that
+              list  receive  that  same burst of messages a whole
+              day later.
+
        <b>initial</b><i>_</i><b>destination</b><i>_</i><b>concurrency</b>
-              Initial  per-destination concurrency level for par-
+              Initial per-destination concurrency level for  par-
               allel delivery to the same destination.
 
        <b>default</b><i>_</i><b>destination</b><i>_</i><b>concurrency</b><i>_</i><b>limit</b>
-              Default limit on the number of parallel  deliveries
+              Default  limit on the number of parallel deliveries
               to the same destination.
 
        <i>transport_</i><b>destination</b><i>_</i><b>concurrency</b><i>_</i><b>limit</b>
-              Limit  on  the number of parallel deliveries to the
-              same destination, for delivery via the  named  mes-
+              Limit on the number of parallel deliveries  to  the
+              same  destination,  for delivery via the named mes-
               sage <i>transport</i>.
 
 <b>Recipient</b> <b>controls</b>
        <b>default</b><i>_</i><b>destination</b><i>_</i><b>recipient</b><i>_</i><b>limit</b>
-              Default  limit on the number of recipients per mes-
+              Default limit on the number of recipients per  mes-
               sage transfer.
 
        <i>transport_</i><b>destination</b><i>_</i><b>recipient</b><i>_</i><b>limit</b>
-              Limit on  the  number  of  recipients  per  message
+              Limit  on  the  number  of  recipients  per message
               transfer, for the named message <i>transport</i>.
 
 <b>SEE</b> <b>ALSO</b>
@@ -248,7 +264,7 @@ QMGR(8)                                                   QMGR(8)
        <a href="trivial-rewrite.8.html">trivial-rewrite(8)</a>, address routing
 
 <b>LICENSE</b>
-       The  Secure  Mailer  license must be distributed with this
+       The Secure Mailer license must be  distributed  with  this
        software.
 
 <b>AUTHOR(S)</b>
diff --git a/postfix/man/man8/qmgr.8 b/postfix/man/man8/qmgr.8
index a42aba547..039112b61 100644
--- a/postfix/man/man8/qmgr.8
+++ b/postfix/man/man8/qmgr.8
@@ -211,6 +211,17 @@ delivery transport.
 .fi
 In the text below, \fItransport\fR is the first field in a
 \fBmaster.cf\fR entry.
+.IP "\fBqmgr_fudge_factor\fR (valid range: 10..100)"
+The percentage of delivery resources that a busy mail system will
+use up for delivery of a large mailing list message.
+With 100%, delivery of one message does not begin before the previous
+message has been delivered. This results in good performance for large
+mailing lists, but results in poor response time for one-to-one mail.
+With less than 100%, response time for one-to-one mail improves,
+but large mailing list delivery performance suffers. In the worst
+case, recipients near the beginning of a large list receive a burst
+of messages immediately, while recipients near the end of that list
+receive that same burst of messages a whole day later.
 .IP \fBinitial_destination_concurrency\fR
 Initial per-destination concurrency level for parallel delivery
 to the same destination.
diff --git a/postfix/src/cleanup/cleanup_output.txt b/postfix/src/cleanup/cleanup_output.txt
deleted file mode 100644
index f15d7b3ee..000000000
--- a/postfix/src/cleanup/cleanup_output.txt
+++ /dev/null
@@ -1,57 +0,0 @@
-
-    /*
-     * Postfix keeps all information related to an email message is in a
-     * write-once file, including the envelope sender and recipients, and the
-     * message content. This design maximizes robustness: one file is easier
-     * to keep track of than multiple files, and write-once means that no
-     * operation ever needs to be undone. This design also minimizes file
-     * system overhead, because creating and removing files is relatively
-     * expensive compared to writing files. Separate files are used for
-     * logging the causes of deferral or failed delivery.
-     * 
-     * A Postfix queue file consists of three segments.
-     * 
-     * 1) The initial envelope segment with the arrival time, sender address,
-     * recipients, and some other stuff that can be recorded before the
-     * message content is received, including non-recipient information that
-     * results from actions in Postfix SMTP server access tables. In this
-     * segment, recipient records may be preceded or followed by
-     * non-recipient records.
-     * 
-     * 2) The message content segment with the message headers and body. The
-     * message body includes all the MIME segments, if there are any.
-     * 
-     * 3) The extracted envelope segment with information that was extracted
-     * from message headers or from the message body, including recipient
-     * addresses that were extracted from message headers, and non-recipient
-     * information that results from actions in header/body_checks patterns.
-     * In this segment, all non-recipient records precede the recipient
-     * records. The last non-recipient records are return-receipt-to and
-     * errors-to.
-     * 
-     * There are two queue file layouts.
-     * 
-     * A) All recipient records are in the initial envelope segment, except for
-     * the optional always_bcc recipient which is always stored in the
-     * extracted envelope segment. The queue manager reads as many recipients
-     * as it can from the initial envelope segment, and then examines all
-     * remaining initial envelope records and all extracted envelope records,
-     * picking up non-recipient information. This organization favors
-     * messages with fewer than $qmgr_active_recipient_limit recipients.
-     * 
-     * B) All recipient records are stored in the extracted envelope segment,
-     * after all non-recipient records. The queue manager is guaranteed to
-     * have read all the non-recipient records before it sees the first
-     * recipient record. This organization can handle messages with very
-     * large numbers of recipients.
-     * 
-     * All this is the result of an evolutionary process, where compatibility
-     * between Postfix versions was a major goal as new features were added.
-     * Therefore the file organization is not optimal from a performance
-     * point of view. In hindsight, the non-recipient information that
-     * follows recipients in the initial envelope segment could be moved to
-     * the extracted envelope segment. This would improve file organization
-     * A)'s performance with very large numbers of recipients, by eliminating
-     * the need to examine all initial envelope records before starting
-     * deliveries.
-     */
diff --git a/postfix/src/global/mail_version.h b/postfix/src/global/mail_version.h
index 31de9eae7..a71ab8f1b 100644
--- a/postfix/src/global/mail_version.h
+++ b/postfix/src/global/mail_version.h
@@ -20,7 +20,7 @@
   * Patches change the patchlevel and the release date. Snapshots change the
   * release date only, unless they include the same bugfix as a patch release.
   */
-#define MAIL_RELEASE_DATE	"20030605"
+#define MAIL_RELEASE_DATE	"20030606"
 
 #define VAR_MAIL_VERSION	"mail_version"
 #define DEF_MAIL_VERSION	"2.0.11-" MAIL_RELEASE_DATE
diff --git a/postfix/src/global/rec_type.h b/postfix/src/global/rec_type.h
index 5bb910952..0e98998e7 100644
--- a/postfix/src/global/rec_type.h
+++ b/postfix/src/global/rec_type.h
@@ -87,19 +87,21 @@
   */
 #define REC_TYPE_POST_ENVELOPE	"MFSRVA"
 #define REC_TYPE_POST_CONTENT	"XLN"
-#define REC_TYPE_POST_EXTRACT	"ERA"
+#define REC_TYPE_POST_EXTRACT	"ER"
 
  /*
-  * The record at the beginning of the envelope segment specifies the message
-  * content size, data offset, recipient count, and processing flags. These
-  * are fixed-width fields so they can be updated in place. Flags are defined
-  * in cleanup_user.h
+  * The record at the start of the queue file specifies the message content
+  * size (number of bytes between the REC_TYPE_MESG and REC_TYPE_XTRA meta
+  * records), data offset (offset of the first REC_TYPE_NORM or REC_TYPE_CONT
+  * text record), recipient count, and queue manager hints. These are all
+  * fixed-width fields so they can be updated in place. Queue manager hints
+  * are defined in qmgr_user.h
   */
 #define REC_TYPE_SIZE_FORMAT	"%15ld %15ld %15ld %15ld"
 #define REC_TYPE_SIZE_CAST1	long
-#define REC_TYPE_SIZE_CAST2	long
-#define REC_TYPE_SIZE_CAST3	long
-#define REC_TYPE_SIZE_CAST4	long
+#define REC_TYPE_SIZE_CAST2	long	/* Postfix 1.0, a.k.a. 20010228 */
+#define REC_TYPE_SIZE_CAST3	long	/* Postfix 1.0, a.k.a. 20010228 */
+#define REC_TYPE_SIZE_CAST4	long	/* Postfix 2.1 */
 
  /*
   * The warn record specifies when the next warning that the message was
diff --git a/postfix/src/nqmgr/qmgr_message.c b/postfix/src/nqmgr/qmgr_message.c
index eb45a9c39..b1771b8f2 100644
--- a/postfix/src/nqmgr/qmgr_message.c
+++ b/postfix/src/nqmgr/qmgr_message.c
@@ -352,11 +352,11 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
      * 
      * On the first open, we must examine all non-recipient records.
      * 
-     * XXX We know how to skip over large numbers of recipient records in the
-     * initial envelope segment but we haven't yet implemented code to skip
-     * over large numbers of recipient records in the extracted envelope
-     * segment. This is not a problem as long as extracted segment recipients
-     * are not mixed with non-recipient information (sendmail -t, qmqpd).
+     * Optimization: when we know that recipient records are not mixed with
+     * non-recipient records, as is typical with mailing list mail, then we
+     * can avoid having to examine all the queue file records before we can
+     * start deliveries. This avoids some file system thrashing with huge
+     * mailing lists.
      */
     for (;;) {
 	if ((curr_offset = vstream_ftell(message->fp)) < 0)
@@ -367,16 +367,22 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    curr_offset += message->data_size;
 	}
 	rec_type = rec_get(message->fp, buf, 0);
-	if (rec_type <= 0)
-	    /* Report missing end record later. */
-	    break;
 	start = vstring_str(buf);
 	if (msg_verbose > 1)
 	    msg_info("record %c %s", rec_type, start);
+	if (rec_type <= 0) {
+	    msg_warn("%s: message rejected: missing end record",
+		     message->queue_id);
+	    break;
+	}
 	if (rec_type == REC_TYPE_END) {
 	    message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT;
 	    break;
 	}
+
+	/*
+	 * Process recipient records.
+	 */
 	if (rec_type == REC_TYPE_RCPT) {
 	    /* See also below for code setting orig_rcpt. */
 	    if (message->rcpt_offset == 0) {
@@ -391,14 +397,21 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		    if ((message->rcpt_offset = vstream_ftell(message->fp)) < 0)
 			msg_fatal("vstream_ftell %s: %m",
 				  VSTREAM_PATH(message->fp));
-		    /* We already examined all non-recipient records. */
 		    if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT)
+			/* We already examined all non-recipient records. */
 			break;
+		    if (message->rflags & QMGR_READ_FLAG_MIXED_RCPT_OTHER)
+			/* Examine all remaining non-recipient records. */
+			continue;
+		    /* Optimizations for "pure recipient" record sections. */
+		    if (curr_offset > message->data_offset) {
+			/* We already examined all non-recipient records. */
+			message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT;
+			break;
+		    }
 		    /* Examine non-recipient records in extracted segment. */
-		    if ((message->rflags & QMGR_READ_FLAG_MIXED_RCPT_OTHER) == 0
-			&& curr_offset < message->data_offset
-			&& vstream_fseek(message->fp, message->data_offset
-					 + message->data_size, SEEK_SET) < 0)
+		    if (vstream_fseek(message->fp, message->data_offset
+				      + message->data_size, SEEK_SET) < 0)
 			msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp));
 		    continue;
 		}
@@ -428,6 +441,10 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		orig_rcpt = mystrdup(start);
 	    continue;
 	}
+
+	/*
+	 * Process non-recipient records.
+	 */
 	if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT)
 	    /* We already examined all non-recipient records. */
 	    continue;
@@ -467,15 +484,6 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		message->arrival_time = atol(start);
 	    continue;
 	}
-	if (rec_type == REC_TYPE_FROM) {
-	    if (message->sender == 0) {
-		message->sender = mystrdup(start);
-		opened(message->queue_id, message->sender,
-		       message->data_size, message->rcpt_unread,
-		       "queue %s", message->queue_name);
-	    }
-	    continue;
-	}
 	if (rec_type == REC_TYPE_FILT) {
 	    if (message->filter_xport != 0)
 		myfree(message->filter_xport);
@@ -494,6 +502,15 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    message->redirect_addr = mystrdup(start);
 	    continue;
 	}
+	if (rec_type == REC_TYPE_FROM) {
+	    if (message->sender == 0) {
+		message->sender = mystrdup(start);
+		opened(message->queue_id, message->sender,
+		       message->data_size, message->rcpt_unread,
+		       "queue %s", message->queue_name);
+	    }
+	    continue;
+	}
 	if (rec_type == REC_TYPE_ATTR) {
 	    if ((error_text = split_nameval(start, &name, &value)) != 0) {
 		msg_warn("%s: bad attribute: %s: %.200s",
@@ -585,8 +602,7 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	message->rcpt_unread = 0;
     }
     if (rec_type <= 0) {
-	msg_warn("%s: message rejected: missing end record",
-		 message->queue_id);
+	/* Already logged warning. */
     } else if (message->arrival_time == 0) {
 	msg_warn("%s: message rejected: missing arrival time record",
 		 message->queue_id);
diff --git a/postfix/src/pickup/pickup.c b/postfix/src/pickup/pickup.c
index b4b116ef6..24f8dd1d3 100644
--- a/postfix/src/pickup/pickup.c
+++ b/postfix/src/pickup/pickup.c
@@ -181,6 +181,14 @@ static int copy_segment(VSTREAM *qfile, VSTREAM *cleanup, PICKUP_INFO *info,
 	    /* Use our own arrival time record instead. */
 	    continue;
 
+	/*
+	 * XXX Workaround: REC_TYPE_FILT (used in envelopes) == REC_TYPE_CONT
+	 * (used in message content).
+	 */
+	if (type == REC_TYPE_FILT && *expected != REC_TYPE_CONTENT[0])
+	    /* Use our own content filter settings instead. */
+	    continue;
+
 	/*
 	 * XXX Force an empty record when the queue file content begins with
 	 * whitespace, so that it won't be considered as being part of our
diff --git a/postfix/src/qmgr/qmgr.c b/postfix/src/qmgr/qmgr.c
index 771882eea..4ff798eb7 100644
--- a/postfix/src/qmgr/qmgr.c
+++ b/postfix/src/qmgr/qmgr.c
@@ -187,6 +187,17 @@
 /* .fi
 /*	In the text below, \fItransport\fR is the first field in a
 /*	\fBmaster.cf\fR entry.
+/* .IP "\fBqmgr_fudge_factor\fR (valid range: 10..100)"
+/*	The percentage of delivery resources that a busy mail system will
+/*	use up for delivery of a large mailing list message.
+/*	With 100%, delivery of one message does not begin before the previous
+/*	message has been delivered. This results in good performance for large
+/*	mailing lists, but results in poor response time for one-to-one mail.
+/*	With less than 100%, response time for one-to-one mail improves,
+/*	but large mailing list delivery performance suffers. In the worst
+/*	case, recipients near the beginning of a large list receive a burst
+/*	of messages immediately, while recipients near the end of that list
+/*	receive that same burst of messages a whole day later.
 /* .IP \fBinitial_destination_concurrency\fR
 /*	Initial per-destination concurrency level for parallel delivery
 /*	to the same destination.
@@ -266,6 +277,7 @@ int     var_dest_con_limit;
 int     var_dest_rcpt_limit;
 char   *var_defer_xports;
 bool    var_allow_min_user;
+int     var_qmgr_fudge;
 int     var_local_rcpt_lim;		/* XXX */
 int     var_local_con_lim;		/* XXX */
 int     var_proc_limit;
@@ -414,7 +426,7 @@ static int qmgr_loop(char *unused_name, char **unused_argv)
 static void pre_accept(char *unused_name, char **unused_argv)
 {
     const char *table;
- 
+
     if ((table = dict_changed_name()) != 0) {
 	msg_info("table %s has changed -- restarting", table);
 	exit(0);
@@ -479,6 +491,7 @@ int     main(int argc, char **argv)
 	VAR_INIT_DEST_CON, DEF_INIT_DEST_CON, &var_init_dest_concurrency, 1, 0,
 	VAR_DEST_CON_LIMIT, DEF_DEST_CON_LIMIT, &var_dest_con_limit, 0, 0,
 	VAR_DEST_RCPT_LIMIT, DEF_DEST_RCPT_LIMIT, &var_dest_rcpt_limit, 0, 0,
+	VAR_QMGR_FUDGE, DEF_QMGR_FUDGE, &var_qmgr_fudge, 10, 100,
 	VAR_LOCAL_RCPT_LIMIT, DEF_LOCAL_RCPT_LIMIT, &var_local_rcpt_lim, 0, 0,
 	VAR_LOCAL_CON_LIMIT, DEF_LOCAL_CON_LIMIT, &var_local_con_lim, 0, 0,
 	VAR_PROC_LIMIT, DEF_PROC_LIMIT, &var_proc_limit, 1, 0,
diff --git a/postfix/src/qmgr/qmgr_entry.c b/postfix/src/qmgr/qmgr_entry.c
index ef07b0b08..3d3f9f461 100644
--- a/postfix/src/qmgr/qmgr_entry.c
+++ b/postfix/src/qmgr/qmgr_entry.c
@@ -174,9 +174,10 @@ void    qmgr_entry_done(QMGR_ENTRY *entry, int which)
      * today, while people near the end of the list get that same burst of
      * postings a whole day later.
      */
+#define FUDGE(x)	((x) * (var_qmgr_fudge / 100.0))
     message->refcount--;
     if (message->rcpt_offset > 0
-	&& qmgr_recipient_count < var_qmgr_rcpt_limit)
+	&& qmgr_recipient_count < FUDGE(var_qmgr_rcpt_limit))
 	qmgr_message_realloc(message);
     if (message->refcount == 0)
 	qmgr_active_done(message);
diff --git a/postfix/src/qmgr/qmgr_message.c b/postfix/src/qmgr/qmgr_message.c
index ab0ad3b64..d72664012 100644
--- a/postfix/src/qmgr/qmgr_message.c
+++ b/postfix/src/qmgr/qmgr_message.c
@@ -321,11 +321,11 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
      * 
      * On the first open, we must examine all non-recipient records.
      * 
-     * XXX We know how to skip over large numbers of recipient records in the
-     * initial envelope segment but we haven't yet implemented code to skip
-     * over large numbers of recipient records in the extracted envelope
-     * segment. This is not a problem as long as extracted segment recipients
-     * are not mixed with non-recipient information (sendmail -t, qmqpd).
+     * Optimization: when we know that recipient records are not mixed with
+     * non-recipient records, as is typical with mailing list mail, then we
+     * can avoid having to examine all the queue file records before we can
+     * start deliveries. This avoids some file system thrashing with huge
+     * mailing lists.
      */
     for (;;) {
 	if ((curr_offset = vstream_ftell(message->fp)) < 0)
@@ -336,18 +336,25 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    curr_offset += message->data_size;
 	}
 	rec_type = rec_get(message->fp, buf, 0);
-	if (rec_type <= 0)
-	    /* Report missing end record later. */
-	    break;
 	start = vstring_str(buf);
 	if (msg_verbose > 1)
 	    msg_info("record %c %s", rec_type, start);
+	if (rec_type <= 0) {
+	    msg_warn("%s: message rejected: missing end record",
+		     message->queue_id);
+	    break;
+	}
 	if (rec_type == REC_TYPE_END) {
 	    message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT;
 	    break;
 	}
+
+	/*
+	 * Process recipient records.
+	 */
 	if (rec_type == REC_TYPE_RCPT) {
 	    /* See also below for code setting orig_rcpt. */
+#define FUDGE(x)	((x) * (var_qmgr_fudge / 100.0))
 	    if (message->rcpt_offset == 0) {
 		qmgr_rcpt_list_add(&message->rcpt_list, curr_offset,
 				   orig_rcpt ? orig_rcpt : "unknown", start);
@@ -355,18 +362,25 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		    myfree(orig_rcpt);
 		    orig_rcpt = 0;
 		}
-		if (message->rcpt_list.len >= var_qmgr_rcpt_limit) {
+		if (message->rcpt_list.len >= FUDGE(var_qmgr_rcpt_limit)) {
 		    if ((message->rcpt_offset = vstream_ftell(message->fp)) < 0)
 			msg_fatal("vstream_ftell %s: %m",
 				  VSTREAM_PATH(message->fp));
-		    /* We already examined all non-recipient records. */
 		    if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT)
+			/* We already examined all non-recipient records. */
 			break;
+		    if (message->rflags & QMGR_READ_FLAG_MIXED_RCPT_OTHER)
+			/* Examine all remaining non-recipient records. */
+			continue;
+		    /* Optimizations for "pure recipient" record sections. */
+		    if (curr_offset > message->data_offset) {
+			/* We already examined all non-recipient records. */
+			message->rflags |= QMGR_READ_FLAG_SEEN_ALL_NON_RCPT;
+			break;
+		    }
 		    /* Examine non-recipient records in extracted segment. */
-		    if ((message->rflags & QMGR_READ_FLAG_MIXED_RCPT_OTHER) == 0
-			&& curr_offset < message->data_offset
-			&& vstream_fseek(message->fp, message->data_offset
-					 + message->data_size, SEEK_SET) < 0)
+		    if (vstream_fseek(message->fp, message->data_offset
+				      + message->data_size, SEEK_SET) < 0)
 			msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp));
 		    continue;
 		}
@@ -393,6 +407,10 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		orig_rcpt = mystrdup(start);
 	    continue;
 	}
+
+	/*
+	 * Process non-recipient records.
+	 */
 	if (message->rflags & QMGR_READ_FLAG_SEEN_ALL_NON_RCPT)
 	    /* We already examined all non-recipient records. */
 	    continue;
@@ -432,15 +450,6 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 		message->arrival_time = atol(start);
 	    continue;
 	}
-	if (rec_type == REC_TYPE_FROM) {
-	    if (message->sender == 0) {
-		message->sender = mystrdup(start);
-		opened(message->queue_id, message->sender,
-		       message->data_size, nrcpt,
-		       "queue %s", message->queue_name);
-	    }
-	    continue;
-	}
 	if (rec_type == REC_TYPE_FILT) {
 	    if (message->filter_xport != 0)
 		myfree(message->filter_xport);
@@ -459,6 +468,15 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
 	    message->redirect_addr = mystrdup(start);
 	    continue;
 	}
+	if (rec_type == REC_TYPE_FROM) {
+	    if (message->sender == 0) {
+		message->sender = mystrdup(start);
+		opened(message->queue_id, message->sender,
+		       message->data_size, nrcpt,
+		       "queue %s", message->queue_name);
+	    }
+	    continue;
+	}
 	if (rec_type == REC_TYPE_ATTR) {
 	    if ((error_text = split_nameval(start, &name, &value)) != 0) {
 		msg_warn("%s: bad attribute: %s: %.200s",
@@ -544,8 +562,7 @@ static int qmgr_message_read(QMGR_MESSAGE *message)
      * including the queue file end marker.
      */
     if (rec_type <= 0) {
-	msg_warn("%s: message rejected: missing end record",
-		 message->queue_id);
+	/* Already logged warning. */
     } else if (message->arrival_time == 0) {
 	msg_warn("%s: message rejected: missing arrival time record",
 		 message->queue_id);
diff --git a/postfix/src/showq/showq.c b/postfix/src/showq/showq.c
index 536ddcd24..59d6ad3cf 100644
--- a/postfix/src/showq/showq.c
+++ b/postfix/src/showq/showq.c
@@ -57,7 +57,6 @@
 #include <time.h>
 #include <string.h>
 #include <ctype.h>
-#include <stdio.h>			/* sscanf() */
 
 /* Utility library. */
 
@@ -113,12 +112,11 @@ static void showq_report(VSTREAM *client, char *queue, char *id,
     time_t  arrival_time = 0;
     char   *start;
     long    msg_size = 0;
-    long    msg_offset = 0;
     BOUNCE_LOG *logfile;
     HTABLE *dup_filter = 0;
     char    status = (strcmp(queue, MAIL_QUEUE_ACTIVE) == 0 ? '*' :
 		      strcmp(queue, MAIL_QUEUE_HOLD) == 0 ? '!' : ' ');
-    long    offset;
+    int     msg_size_ok = 0;
 
     /*
      * XXX addresses in defer logfiles are in printable quoted form, while
@@ -126,18 +124,26 @@ static void showq_report(VSTREAM *client, char *queue, char *id,
      * may change once we replace the present ad-hoc bounce/defer logfile
      * format by one that is transparent for control etc. characters. See
      * also: bounce/bounce_append_service.c.
+     * 
+     * XXX With Postfix <= 2.0, "postsuper -r" results in obsolete size records
+     * from previous cleanup runs. Skip the obsolete size records.
      */
     while (!vstream_ferror(client) && (rec_type = rec_get(qfile, buf, 0)) > 0) {
 	start = vstring_str(buf);
+	if (msg_verbose)
+	    msg_info("record %c %s", rec_type, printable(start, '?'));
 	switch (rec_type) {
 	case REC_TYPE_TIME:
 	    arrival_time = atol(start);
 	    break;
 	case REC_TYPE_SIZE:
-	    if (sscanf(start, "%ld %ld", &msg_size, &msg_offset) == 2)
-		/* Postfix >= 1.0 (a.k.a. 20010228) style queue file. */
-		if (msg_size <= 0)
+	    if (msg_size == 0) {
+		if ((msg_size_ok = ((msg_size = atol(start)) > 0)) == 0) {
+		    msg_warn("%s: malformed size record: %.100s",
+			     id, printable(start, '?'));
 		    msg_size = size;
+		}
+	    }
 	    break;
 	case REC_TYPE_FROM:
 	    if (*start == 0)
@@ -160,11 +166,7 @@ static void showq_report(VSTREAM *client, char *queue, char *id,
 				"", "", "", STR(printable_quoted_addr));
 	    break;
 	case REC_TYPE_MESG:
-	    if (msg_size > 0 && msg_offset > 0
-		&& vstream_fseek(qfile, msg_size, SEEK_CUR) < 0)
-		msg_fatal("seek file %s: %m", VSTREAM_PATH(qfile));
-	    else if ((offset = atol(start)) > 0
-		     && vstream_fseek(qfile, offset, SEEK_SET) < 0)
+	    if (msg_size_ok && vstream_fseek(qfile, msg_size, SEEK_CUR) < 0)
 		msg_fatal("seek file %s: %m", VSTREAM_PATH(qfile));
 	    break;
 	case REC_TYPE_END: