From: Wietse Venema Date: Sun, 28 May 2000 00:00:00 +0000 (+0000) Subject: snapshot-20000528 X-Git-Tag: v20010228~52 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7da180a864ba8baca7cd1a300c424954c05810a7;p=thirdparty%2Fpostfix.git snapshot-20000528 --- diff --git a/postfix/HISTORY b/postfix/HISTORY index 22519392b..7db3e2817 100644 --- a/postfix/HISTORY +++ b/postfix/HISTORY @@ -3975,3 +3975,37 @@ Apologies for any names omitted. Finalized the delayed SMTP command/reply flushing code in the SMTP and LMTP clients after lots of testing and review. + +20000520 + + Robustness: upon receipt of mail, map the mailer-daemon + sender address back into the magic null string. File: + cleanup/cleanup_envelope.c. + +20000524 + + Bugfix: the code for masquerade_exceptions was case sensitive. + Reported by Eduard Vopicka. File: cleanup/cleanup_masquerade.c. + +20000526 + + Feature: experimental queue manager by Patrik Rak with a + fancy pre-emptive scheduling algorithm that improves delivery + performance of mail with few recipients. This queue manager + is made available as "nqmgr". + +20000528 + + Feature: the SMTP client SASL password file can contain + entries for destination domain names (the address remote + part) not just mail server hostnames. File: smtp_sasl_glue.c. + + Feature: smtpd_sasl_local_domain parameter (default: + $myhostname) to specify the local SASL authentication realm. + File: smtpd_sasl_glue.c. + + Feature: specify "body_checks=regexp:/file/name" for a very + crude one line at a time message body content filter. This + feature uses the same filtering syntax as the header_checks + feature. File: cleanup/cleanup_message.c. See also the + conf/sample-filter.cf file. diff --git a/postfix/Makefile.in b/postfix/Makefile.in index ce59fd1d1..37d18300b 100644 --- a/postfix/Makefile.in +++ b/postfix/Makefile.in @@ -4,7 +4,7 @@ OPTS = "CC=$(CC)" DIRS = util global dns master postfix smtpstone sendmail error \ pickup cleanup smtpd local lmtp trivial-rewrite qmgr smtp bounce pipe \ showq postalias postcat postconf postdrop postkick postlock postlog \ - postmap postsuper # spawn base64 proto man html + postmap postsuper nqmgr # spawn base64 proto man html default: update diff --git a/postfix/RELEASE_NOTES b/postfix/RELEASE_NOTES index 2c87b5e9d..0a00de788 100644 --- a/postfix/RELEASE_NOTES +++ b/postfix/RELEASE_NOTES @@ -1,3 +1,16 @@ +Major changes with snapshot-20000526 +==================================== + +Specify "body_checks = regexp:/etc/postfix/body_checks" for a quick +and dirty emergency content filter that looks at non-header lines +one line at a time (including MIME headers inside the message body). +Details in conf/sample-filter.cf. + +This version introduces a new queue manager with a clever scheduler +by Patrik Rak that allow mailing list deliveries be pre-empted by +non-list mail, while preserving correct average delivery delays. +The queue manager is build as nqmgr. It needs further testing. + Major changes with snapshot-20000514 ==================================== diff --git a/postfix/SASL_README b/postfix/SASL_README index 13ebf317d..7c9f18ef7 100644 --- a/postfix/SASL_README +++ b/postfix/SASL_README @@ -29,9 +29,10 @@ via message headers or via SMTP. It is no-one's business what username and authentication method the poster was using in order to access the mail server. -When sending mail, Postfix looks up the server hostname in a table, -and if a username/password is found, it will use that username and -password to authenticate to the server. +When sending mail, Postfix looks up the server hostname or destination +domain (the address remote part) in a table, and if a username/password +is found, it will use that username and password to authenticate +to the server. Building the SASL library ========================= @@ -134,11 +135,14 @@ Enabling SASL authentication in the Postfix SMTP client ======================================================= Turn on client-side SASL authentication, and specify a table with -per-host username and password information. +per-host or per-destination username and password information. +Postfix first looks up the server hostname; if no entry is found, +then Postfix looks up the destination domain name (the address +remote part). /etc/postfix/main.cf: smtp_sasl_auth_enable = yes - smtp_sasl_passwd_maps = hash:/etc/postfix/sasl_passwd + smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd /etc/postfix/sasl_passwd: foo.com username:password diff --git a/postfix/cleanup/cleanup.c b/postfix/cleanup/cleanup.c index 68c0fb137..732663126 100644 --- a/postfix/cleanup/cleanup.c +++ b/postfix/cleanup/cleanup.c @@ -60,6 +60,15 @@ /* this program. See the Postfix \fBmain.cf\fR file for syntax details /* and for default values. Use the \fBpostfix reload\fR command after /* a configuration change. +/* .SH Content filtering +/* .IP \fBbody_checks\fR +/* Lookup tables with content filters for message body lines. +/* These filters see physical lines one at a time, in chunks of +/* at most line_length_limit bytes. +/* .IP \fBheader_checks\fR +/* Lookup tables with content filters for message header lines. +/* These filters see logical headers one at a time, including headers +/* that span multiple lines. /* .SH Miscellaneous /* .ad /* .fi diff --git a/postfix/cleanup/cleanup.h b/postfix/cleanup/cleanup.h index 7c5eaed71..4bba6ada5 100644 --- a/postfix/cleanup/cleanup.h +++ b/postfix/cleanup/cleanup.h @@ -68,6 +68,7 @@ extern MAPS *cleanup_comm_canon_maps; extern MAPS *cleanup_send_canon_maps; extern MAPS *cleanup_rcpt_canon_maps; extern MAPS *cleanup_header_checks; +extern MAPS *cleanup_body_checks; extern MAPS *cleanup_virtual_maps; extern ARGV *cleanup_masq_domains; diff --git a/postfix/cleanup/cleanup_envelope.c b/postfix/cleanup/cleanup_envelope.c index 3541cb5d9..00aa74b6d 100644 --- a/postfix/cleanup/cleanup_envelope.c +++ b/postfix/cleanup/cleanup_envelope.c @@ -46,6 +46,10 @@ #include #include +#ifdef STRCASECMP_IN_STRINGS_H +#include +#endif + /* Utility library. */ #include @@ -61,6 +65,8 @@ #include #include #include +#include +#include /* Application-specific. */ @@ -130,6 +136,12 @@ static void cleanup_envelope_process(CLEANUP_STATE *state, int type, char *buf, VSTRING *clean_addr = vstring_alloc(100); cleanup_rewrite_internal(clean_addr, buf); + if (strncasecmp(STR(clean_addr), MAIL_ADDR_MAIL_DAEMON "@", + sizeof(MAIL_ADDR_MAIL_DAEMON)) == 0) { + canon_addr_internal(state->temp1, MAIL_ADDR_MAIL_DAEMON); + if (strcasecmp(STR(clean_addr), STR(state->temp1)) == 0) + vstring_strcpy(clean_addr, ""); + } if (cleanup_send_canon_maps) cleanup_map11_internal(state, clean_addr, cleanup_send_canon_maps, cleanup_ext_prop_mask & EXT_PROP_CANONICAL); diff --git a/postfix/cleanup/cleanup_init.c b/postfix/cleanup/cleanup_init.c index 325260bd1..31372dfdf 100644 --- a/postfix/cleanup/cleanup_init.c +++ b/postfix/cleanup/cleanup_init.c @@ -96,13 +96,14 @@ char *var_virtual_maps; /* virtual maps */ char *var_masq_domains; /* masquerade domains */ char *var_masq_exceptions; /* users not masqueraded */ char *var_header_checks; /* any header checks */ +char *var_body_checks; /* any body checks */ int var_dup_filter_limit; /* recipient dup filter */ char *var_empty_addr; /* destination of bounced bounces */ int var_delay_warn_time; /* delay that triggers warning */ char *var_prop_extension; /* propagate unmatched extension */ char *var_always_bcc; /* big brother */ int var_extra_rcpt_limit; /* recipient extract limit */ -char *var_rcpt_witheld; /* recipients not disclosed */ +char *var_rcpt_witheld; /* recipients not disclosed */ CONFIG_INT_TABLE cleanup_int_table[] = { VAR_HOPCOUNT_LIMIT, DEF_HOPCOUNT_LIMIT, &var_hopcount_limit, 1, 0, @@ -122,6 +123,7 @@ CONFIG_STR_TABLE cleanup_str_table[] = { VAR_EMPTY_ADDR, DEF_EMPTY_ADDR, &var_empty_addr, 1, 0, VAR_MASQ_EXCEPTIONS, DEF_MASQ_EXCEPTIONS, &var_masq_exceptions, 0, 0, VAR_HEADER_CHECKS, DEF_HEADER_CHECKS, &var_header_checks, 0, 0, + VAR_BODY_CHECKS, DEF_BODY_CHECKS, &var_body_checks, 0, 0, VAR_PROP_EXTENSION, DEF_PROP_EXTENSION, &var_prop_extension, 0, 0, VAR_ALWAYS_BCC, DEF_ALWAYS_BCC, &var_always_bcc, 0, 0, VAR_RCPT_WITHELD, DEF_RCPT_WITHELD, &var_rcpt_witheld, 1, 0, @@ -135,6 +137,7 @@ MAPS *cleanup_comm_canon_maps; MAPS *cleanup_send_canon_maps; MAPS *cleanup_rcpt_canon_maps; MAPS *cleanup_header_checks; +MAPS *cleanup_body_checks; MAPS *cleanup_virtual_maps; ARGV *cleanup_masq_domains; @@ -174,6 +177,9 @@ void cleanup_pre_jail(char *unused_name, char **unused_argv) if (*var_header_checks) cleanup_header_checks = maps_create(VAR_HEADER_CHECKS, var_header_checks, DICT_FLAG_LOCK); + if (*var_body_checks) + cleanup_body_checks = + maps_create(VAR_BODY_CHECKS, var_body_checks, DICT_FLAG_LOCK); } /* cleanup_post_jail - initialize after entering the chroot jail */ diff --git a/postfix/cleanup/cleanup_masquerade.c b/postfix/cleanup/cleanup_masquerade.c index d00ecb795..b742e25ca 100644 --- a/postfix/cleanup/cleanup_masquerade.c +++ b/postfix/cleanup/cleanup_masquerade.c @@ -99,7 +99,7 @@ void cleanup_masquerade_external(VSTRING *addr, ARGV *masq_domains) masq_except_table = htable_create(5); ptr = saved_names = mystrdup(var_masq_exceptions); while ((name = mystrtok(&ptr, ", \t\r\n")) != 0) - htable_enter(masq_except_table, name, (char *) 0); + htable_enter(masq_except_table, lowercase(name), (char *) 0); myfree(saved_names); } @@ -116,7 +116,7 @@ void cleanup_masquerade_external(VSTRING *addr, ARGV *masq_domains) */ if (masq_except_table) { name = mystrndup(STR(addr), domain - 1 - STR(addr)); - excluded = (htable_locate(masq_except_table, name) != 0); + excluded = (htable_locate(masq_except_table, lowercase(name)) != 0); myfree(name); if (excluded) return; diff --git a/postfix/cleanup/cleanup_message.c b/postfix/cleanup/cleanup_message.c index 1e5e5117d..3b6efc245 100644 --- a/postfix/cleanup/cleanup_message.c +++ b/postfix/cleanup/cleanup_message.c @@ -263,7 +263,7 @@ static void cleanup_header(CLEANUP_STATE *state) if ((value = maps_find(cleanup_header_checks, header, 0)) != 0) { if (strcasecmp(value, "REJECT") == 0) { - msg_info("%s: reject: header %.100s; from=<%s> to=<%s>", + msg_info("%s: reject: header %.200s; from=<%s> to=<%s>", state->queue_id, header, state->sender, state->recip ? state->recip : "unknown"); state->errs |= CLEANUP_STAT_CONT; @@ -490,6 +490,26 @@ static void cleanup_message_body(CLEANUP_STATE *state, int type, char *buf, int * Copy body record to the output. */ if (type == REC_TYPE_NORM || type == REC_TYPE_CONT) { + + /* + * Crude message body content filter for emergencies. This code has + * several problems: it sees one line at a time, and therefore does + * not recognize multi-line MIME headers in the body; it looks at + * long lines only in chunks of line_length_limit (2048) characters; + * it is easily bypassed with encodings and with multi-line tricks. + */ + if ((state->flags & CLEANUP_FLAG_FILTER) && cleanup_body_checks) { + const char *value; + + if ((value = maps_find(cleanup_body_checks, buf, 0)) != 0) { + if (strcasecmp(value, "REJECT") == 0) { + msg_info("%s: reject: body %.200s; from=<%s> to=<%s>", + state->queue_id, buf, state->sender, + state->recip ? state->recip : "unknown"); + state->errs |= CLEANUP_STAT_CONT; + } + } + } cleanup_out(state, type, buf, len); } diff --git a/postfix/conf/master.cf b/postfix/conf/master.cf index e3c05a1be..a2f3f9621 100644 --- a/postfix/conf/master.cf +++ b/postfix/conf/master.cf @@ -63,6 +63,7 @@ smtp inet n - n - - smtpd pickup fifo n n n 60 1 pickup cleanup unix - - n - 0 cleanup qmgr fifo n - n 300 1 qmgr +#qmgr fifo n - n 300 1 nqmgr rewrite unix - - n - - trivial-rewrite bounce unix - - n - 0 bounce defer unix - - n - 0 bounce diff --git a/postfix/conf/sample-auth.cf b/postfix/conf/sample-auth.cf index e4e30d4ef..be003437d 100644 --- a/postfix/conf/sample-auth.cf +++ b/postfix/conf/sample-auth.cf @@ -64,6 +64,15 @@ smtpd_sasl_auth_enable = no #smtpd_sasl_security_options = noanonymous, noplaintext smtpd_sasl_security_options = noanonymous +# The smtpd_sasl_local_domain parameter specifies the name of the +# local authentication realm. +# +# By default, the local authentication realm name is the name of the +# machine. +# +# smtpd_sasl_local_domain = $mydomain +smtpd_sasl_local_domain = $myhostname + # SMTP CLIENT CONTROLS # The smtp_sasl_auth_enable parameter controls whether authentication @@ -74,14 +83,15 @@ smtpd_sasl_security_options = noanonymous smtp_sasl_auth_enable = no # The smtp_sasl_password_maps parameter specifies the names of lookup -# tables with one username:password entry per remote hostname. If a -# remote host has no username:password entry, then the Postfix SMTP -# client will not attempt to authenticate to the remote host. +# tables with one username:password entry per remote hostname or +# domain. If a remote host or domain has no username:password entry, +# then the Postfix SMTP client will not attempt to authenticate to +# the remote host. # # The Postfix SMTP client opens the lookup table before going to # chroot jail, so you can leave the password file in /etc/postfix. # -smtp_auth_passwd_map = hash:/etc/postfix/saslpass +smtp_sasl_password_maps = hash:/etc/postfix/saslpass # The smtp_sasl_security_options parameter controls what authentication # mechanisms the local Postfix SMTP client is allowed to use. The @@ -96,5 +106,5 @@ smtp_auth_passwd_map = hash:/etc/postfix/saslpass # # By default, the Postfix SMTP client will not use plaintext passwords. # -#smtpd_sasl_security_options = -smtpd_sasl_security_options = noplaintext +#smtp_sasl_security_options = +smtp_sasl_security_options = noplaintext diff --git a/postfix/conf/sample-filter.cf b/postfix/conf/sample-filter.cf new file mode 100644 index 000000000..5c27373d0 --- /dev/null +++ b/postfix/conf/sample-filter.cf @@ -0,0 +1,24 @@ +# DO NOT EDIT THIS FILE. EDIT THE MAIN.CF FILE INSTEAD. THE STUFF +# HERE JUST SERVES AS AN EXAMPLE. +# +# This file contains example settings for miscellaneous Postfix +# content filtering parameters. + +# The header_checks parameter specifies an optional table with patterns +# that each logical message header is matched against, including +# headers that span multiple physical lines. Patterns are matched +# in the specified order, and the search stops upon the first match. +# When a pattern matches, and the associated action is REJECT, the +# entire message is rejected. +# +header_checks = regexp:/etc/postfix/header_checks + +# The body_checks parameter specifies an optional table with patterns +# that each physical non-header line is matched against (including +# MIME headers inside the message body). Lines are matched one at +# a time. Long lines are matched in chunks of at most $line_length_limit +# characters. Patterns are matched in the specified order, and the +# search stops upon the first match. When a pattern matches, and +# the associated action is REJECT, the entire message is rejected. +# +body_checks = regexp:/etc/postfix/body_checks diff --git a/postfix/global/mail_params.h b/postfix/global/mail_params.h index 16eb7b5ef..0238a04a5 100644 --- a/postfix/global/mail_params.h +++ b/postfix/global/mail_params.h @@ -425,6 +425,9 @@ extern int var_max_queue_time; #define DEF_DELAY_WARN_TIME 0 extern int var_delay_warn_time; + /* + * Queue manager: various in-core message and recipient limits. + */ #define VAR_QMGR_ACT_LIMIT "qmgr_message_active_limit" #define DEF_QMGR_ACT_LIMIT 1000 extern int var_qmgr_active_limit; @@ -433,6 +436,43 @@ extern int var_qmgr_active_limit; #define DEF_QMGR_RCPT_LIMIT 10000 extern int var_qmgr_rcpt_limit; +#define VAR_QMGR_MSG_RCPT_LIMIT "qmgr_message_recipient_minimum" +#define DEF_QMGR_MSG_RCPT_LIMIT 10 +extern int var_qmgr_msg_rcpt_limit; + +#define VAR_XPORT_RCPT_LIMIT "default_recipient_limit" +#define _XPORT_RCPT_LIMIT "_recipient_limit" +#define DEF_XPORT_RCPT_LIMIT 10000 +extern int var_xport_rcpt_limit; + +#define VAR_STACK_RCPT_LIMIT "default_extra_recipient_limit" +#define _STACK_RCPT_LIMIT "_extra_recipient_limit" +#define DEF_STACK_RCPT_LIMIT 1000 +extern int var_stack_rcpt_limit; + + /* + * Queue manager: default job scheduler parameters. + */ +#define VAR_DELIVERY_SLOT_COST "default_delivery_slot_cost" +#define _DELIVERY_SLOT_COST "_delivery_slot_cost" +#define DEF_DELIVERY_SLOT_COST 10 +extern int var_delivery_slot_cost; + +#define VAR_DELIVERY_SLOT_LOAN "default_delivery_slot_loan" +#define _DELIVERY_SLOT_LOAN "_delivery_slot_loan" +#define DEF_DELIVERY_SLOT_LOAN 5 +extern int var_delivery_slot_loan; + +#define VAR_DELIVERY_SLOT_DISCOUNT "default_delivery_slot_discount" +#define _DELIVERY_SLOT_DISCOUNT "_delivery_slot_discount" +#define DEF_DELIVERY_SLOT_DISCOUNT 50 +extern int var_delivery_slot_discount; + +#define VAR_MIN_DELIVERY_SLOTS "default_minimum_delivery_slots" +#define _MIN_DELIVERY_SLOTS "_minimum_delivery_slots" +#define DEF_MIN_DELIVERY_SLOTS 3 +extern int var_min_delivery_slots; + #define VAR_QMGR_FUDGE "qmgr_fudge_factor" #define DEF_QMGR_FUDGE 100 extern int var_qmgr_fudge; @@ -657,6 +697,10 @@ extern bool var_smtpd_sasl_enable; #define DEF_SMTPD_SASL_OPTS "noanonymous" extern char *var_smtpd_sasl_opts; +#define VAR_SMTPD_SASL_REALM "smtpd_sasl_local_domain" +#define DEF_SMTPD_SASL_REALM "$myhostname" +extern char *var_smtpd_sasl_realm; + /* * SASL authentication support, client side. */ @@ -759,6 +803,10 @@ extern int var_queue_minfree; #define DEF_HEADER_CHECKS "" extern char *var_header_checks; +#define VAR_BODY_CHECKS "body_checks" +#define DEF_BODY_CHECKS "" +extern char *var_body_checks; + /* * Bounce service: truncate bounce message that exceed $bounce_size_limit. */ diff --git a/postfix/global/mail_version.h b/postfix/global/mail_version.h index eaf587a2a..3883d430d 100644 --- a/postfix/global/mail_version.h +++ b/postfix/global/mail_version.h @@ -15,7 +15,7 @@ * Version of this program. */ #define VAR_MAIL_VERSION "mail_version" -#define DEF_MAIL_VERSION "Snapshot-20000514" +#define DEF_MAIL_VERSION "Snapshot-20000528" extern char *var_mail_version; /* LICENSE diff --git a/postfix/global/tok822_parse.c b/postfix/global/tok822_parse.c index 7ca5fc0ab..a07b75113 100644 --- a/postfix/global/tok822_parse.c +++ b/postfix/global/tok822_parse.c @@ -464,7 +464,7 @@ static void tok822_quote_atom(TOK822 *tp) /* tok822_comment - tokenize comment */ -const char *tok822_comment(TOK822 *tp, const char *str) +static const char *tok822_comment(TOK822 *tp, const char *str) { TOK822 *tc = 0; int ch; @@ -501,7 +501,7 @@ const char *tok822_comment(TOK822 *tp, const char *str) /* tok822_group - cluster a group of tokens */ -TOK822 *tok822_group(int group_type, TOK822 *left, TOK822 *right, int sync_type) +static TOK822 *tok822_group(int group_type, TOK822 *left, TOK822 *right, int sync_type) { TOK822 *group; TOK822 *sync; diff --git a/postfix/html/cleanup.8.html b/postfix/html/cleanup.8.html index 9ad4103f1..2c1722d5c 100644 --- a/postfix/html/cleanup.8.html +++ b/postfix/html/cleanup.8.html @@ -84,6 +84,18 @@ CLEANUP(8) CLEANUP(8) details and for default values. Use the postfix reload command after a configuration change. +Content filtering + body_checks + Lookup tables with content filters for message body + lines. These filters see physical lines one at a + time, in chunks of at most line_length_limit bytes. + + header_checks + Lookup tables with content filters for message + header lines. These filters see logical headers + one at a time, including headers that span multiple + lines. + Miscellaneous always_bcc Address to send a copy of each message that enters @@ -113,30 +125,30 @@ CLEANUP(8) CLEANUP(8) sender_canonical_maps Address mapping lookup table for envelope and - header sender addresses. - masquerade_domains - List of domains that hide their subdomain struc- - ture. - masquerade_exceptions - List of user names that are not subject to address - masquerading. - virtual_maps - Address mapping lookup table for envelope recipient + 2 - 2 +CLEANUP(8) CLEANUP(8) + header sender addresses. -CLEANUP(8) CLEANUP(8) + masquerade_domains + List of domains that hide their subdomain struc- + ture. + masquerade_exceptions + List of user names that are not subject to address + masquerading. + virtual_maps + Address mapping lookup table for envelope recipient addresses. Resource controls @@ -180,18 +192,6 @@ CLEANUP(8) CLEANUP(8) - - - - - - - - - - - - 3 diff --git a/postfix/html/resource.html b/postfix/html/resource.html index fb7a972a5..4c24021b3 100644 --- a/postfix/html/resource.html +++ b/postfix/html/resource.html @@ -98,8 +98,8 @@ The following parameters restrict the use of file system storage:
message_size_limit (default: 10240000 bytes) -
The maximal size of a Postfix queue file for inbound mail, -including envelope information (sender, recipient, etc.). +
The maximal size of a Postfix queue file, including envelope +information (sender, recipient, etc.).

diff --git a/postfix/html/smtpd.8.html b/postfix/html/smtpd.8.html index b7d0796db..c61c758e9 100644 --- a/postfix/html/smtpd.8.html +++ b/postfix/html/smtpd.8.html @@ -144,15 +144,15 @@ SMTPD(8) SMTPD(8) hopcount_limit Limit the number of Received: message headers. - notify_classes - List of error classes. Of special interest are: - local_recipient_maps List of maps with user names that are local to $myorigin or $inet_interfaces. If this parameter is defined, then the SMTP server rejects mail for unknown local users. + notify_classes + List of error classes. Of special interest are: + policy When a client violates any policy, mail a transcript of the entire SMTP session to the postmaster. diff --git a/postfix/man/man8/cleanup.8 b/postfix/man/man8/cleanup.8 index e96116fa8..2cbdb5a3a 100644 --- a/postfix/man/man8/cleanup.8 +++ b/postfix/man/man8/cleanup.8 @@ -74,6 +74,15 @@ The following \fBmain.cf\fR parameters are especially relevant to this program. See the Postfix \fBmain.cf\fR file for syntax details and for default values. Use the \fBpostfix reload\fR command after a configuration change. +.SH Content filtering +.IP \fBbody_checks\fR +Lookup tables with content filters for message body lines. +These filters see physical lines one at a time, in chunks of +at most line_length_limit bytes. +.IP \fBheader_checks\fR +Lookup tables with content filters for message header lines. +These filters see logical headers one at a time, including headers +that span multiple lines. .SH Miscellaneous .ad .fi diff --git a/postfix/man/man8/smtpd.8 b/postfix/man/man8/smtpd.8 index c0c7a5f25..82aa2336e 100644 --- a/postfix/man/man8/smtpd.8 +++ b/postfix/man/man8/smtpd.8 @@ -113,12 +113,12 @@ specified in the \fBdebug_peer_level\fR parameter. Recipient of protocol/policy/resource/software error notices. .IP \fBhopcount_limit\fR Limit the number of \fBReceived:\fR message headers. -.IP \fBnotify_classes\fR -List of error classes. Of special interest are: .IP \fBlocal_recipient_maps\fR List of maps with user names that are local to \fB$myorigin\fR or \fB$inet_interfaces\fR. If this parameter is defined, then the SMTP server rejects mail for unknown local users. +.IP \fBnotify_classes\fR +List of error classes. Of special interest are: .RS .IP \fBpolicy\fR When a client violates any policy, mail a transcript of the diff --git a/postfix/nqmgr/.indent.pro b/postfix/nqmgr/.indent.pro new file mode 100644 index 000000000..5fbb816df --- /dev/null +++ b/postfix/nqmgr/.indent.pro @@ -0,0 +1,120 @@ +-TALIAS_TOKEN +-TARGV +-TBH_TABLE +-TBINATTR +-TBINATTR_INFO +-TBINHASH +-TBINHASH_INFO +-TBOUNCE_STAT +-TCLEANUP_STATE +-TCLIENT_LIST +-TCLNT_STREAM +-TCONFIG_BOOL_FN_TABLE +-TCONFIG_BOOL_TABLE +-TCONFIG_INT_FN_TABLE +-TCONFIG_INT_TABLE +-TCONFIG_STR_FN_TABLE +-TCONFIG_STR_TABLE +-TDELIVER_ATTR +-TDELIVER_REQUEST +-TDICT +-TDICT_DB +-TDICT_DBM +-TDICT_ENV +-TDICT_HT +-TDICT_LDAP +-TDICT_MYSQL +-TDICT_NI +-TDICT_NIS +-TDICT_NISPLUS +-TDICT_NODE +-TDICT_OPEN_INFO +-TDICT_PCRE +-TDICT_REGEXP +-TDICT_REGEXP_RULE +-TDICT_UNIX +-TDNS_FIXED +-TDNS_REPLY +-TDNS_RR +-TDOMAIN_LIST +-TEXPAND_ATTR +-TFILE +-TFORWARD_INFO +-THEADER_OPTS +-THOST +-THTABLE +-THTABLE_INFO +-TINET_ADDR_LIST +-TINT_TABLE +-TJMP_BUF_WRAPPER +-TLMTP_ATTR +-TLMTP_RESP +-TLMTP_SESSION +-TLMTP_STATE +-TLOCAL_EXP +-TLOCAL_STATE +-TMAC_EXP +-TMAC_HEAD +-TMAC_PARSE +-TMAIL_PRINT +-TMAIL_SCAN +-TMAPS +-TMASTER_PROC +-TMASTER_SERV +-TMASTER_STATUS +-TMBLOCK +-TMKMAP +-TMKMAP_OPEN_INFO +-TMULTI_SERVER +-TMVECT +-TMYSQL_NAME +-TNAMADR_LIST +-TNAME_MASK +-TPEER_NAME +-TPICKUP_INFO +-TPIPE_ATTR +-TPIPE_PARAMS +-TPLMYSQL +-TQMGR_ENTRY +-TQMGR_MESSAGE +-TQMGR_QUEUE +-TQMGR_RCPT_LIST +-TQMGR_RECIPIENT +-TQMGR_SCAN +-TQMGR_TRANSPORT +-TRECIPIENT +-TRECIPIENT_LIST +-TREC_TYPE_NAME +-TRESOLVE_REPLY +-TRESPONSE +-TSCAN_DIR +-TSCAN_INFO +-TSCAN_OBJ +-TSESSION +-TSINGLE_SERVER +-TSINK_COMMAND +-TSINK_STATE +-TSMTPD_CMD +-TSMTPD_STATE +-TSMTPD_TOKEN +-TSMTP_ADDR +-TSMTP_CMD +-TSMTP_RESP +-TSMTP_SESSION +-TSMTP_STATE +-TSOCKADDR_SIZE +-TSPAWN_ATTR +-TSTRING_TABLE +-TSYS_EXITS_TABLE +-TTOK822 +-TTRIGGER_SERVER +-TUSER_ATTR +-TVBUF +-TVSTREAM +-TVSTREAM_POPEN_ARGS +-TVSTRING +-TWAIT_STATUS_T +-TWATCHDOG +-TWATCH_FD +-Tsasl_conn_t +-Tsasl_secret_t diff --git a/postfix/nqmgr/.printfck b/postfix/nqmgr/.printfck new file mode 100644 index 000000000..65eb6bfa6 --- /dev/null +++ b/postfix/nqmgr/.printfck @@ -0,0 +1,25 @@ +been_here_xt 2 0 +bounce_append 5 0 +cleanup_out_format 1 0 +defer_append 5 0 +mail_command 1 0 +mail_print 1 0 +msg_error 0 0 +msg_fatal 0 0 +msg_info 0 0 +msg_panic 0 0 +msg_warn 0 0 +opened 3 0 +post_mail_fprintf 1 0 +qmgr_message_bounce 2 0 +rec_fprintf 2 0 +sent 4 0 +smtp_cmd 1 0 +smtp_mesg_fail 2 0 +smtp_printf 1 0 +smtp_rcpt_fail 3 0 +smtp_site_fail 2 0 +udp_syslog 1 0 +vstream_fprintf 1 0 +vstream_printf 0 0 +vstring_sprintf 1 0 diff --git a/postfix/nqmgr/Makefile.in b/postfix/nqmgr/Makefile.in new file mode 100644 index 000000000..6423a22ab --- /dev/null +++ b/postfix/nqmgr/Makefile.in @@ -0,0 +1,262 @@ +SHELL = /bin/sh +SRCS = qmgr.c qmgr_active.c qmgr_transport.c qmgr_queue.c qmgr_entry.c \ + qmgr_message.c qmgr_deliver.c qmgr_move.c qmgr_rcpt_list.c \ + qmgr_job.c qmgr_peer.c \ + qmgr_defer.c qmgr_enable.c qmgr_scan.c qmgr_bounce.c +OBJS = qmgr.o qmgr_active.o qmgr_transport.o qmgr_queue.o qmgr_entry.o \ + qmgr_message.o qmgr_deliver.o qmgr_move.o qmgr_rcpt_list.o \ + qmgr_job.o qmgr_peer.o \ + qmgr_defer.o qmgr_enable.o qmgr_scan.o qmgr_bounce.o +HDRS = qmgr.h +TESTSRC = +WARN = -W -Wformat -Wimplicit -Wmissing-prototypes \ + -Wparentheses -Wstrict-prototypes -Wswitch -Wuninitialized \ + -Wunused +DEFS = -I. -I$(INC_DIR) -D$(SYSTYPE) +CFLAGS = $(DEBUG) $(OPT) $(DEFS) +TESTPROG= +PROG = qmgr +INC_DIR = ../include +LIBS = ../lib/libmaster.a ../lib/libglobal.a ../lib/libutil.a + +.c.o:; $(CC) $(CFLAGS) -c $*.c + +$(PROG): $(OBJS) $(LIBS) + $(CC) $(CFLAGS) -o $@ $(OBJS) $(LIBS) $(SYSLIBS) + +Makefile: Makefile.in + (set -e; echo "# DO NOT EDIT"; $(OPTS) $(SHELL) ../makedefs; cat $?) >$@ + +test: $(TESTPROG) + +update: ../libexec/n$(PROG) + +../libexec/n$(PROG): $(PROG) + cp $(PROG) ../libexec/n$(PROG) + +printfck: $(OBJS) $(PROG) + rm -rf printfck + mkdir printfck + cp *.h printfck + sed '1,/^# do not edit/!d' Makefile >printfck/Makefile + set -e; for i in *.c; do printfck -f .printfck $$i >printfck/$$i; done + cd printfck; make "INC_DIR=../../include" `cd ..; ls *.o` + +lint: + lint $(DEFS) $(SRCS) $(LINTFIX) + +clean: + rm -f *.o *core $(PROG) $(TESTPROG) junk + rm -rf printfck + +tidy: clean + +depend: $(MAKES) + (sed '1,/^# do not edit/!d' Makefile.in; \ + set -e; for i in [a-z][a-z0-9]*.c; do \ + $(CC) -E $(DEFS) $(INCL) $$i | sed -n -e '/^# *1 *"\([^"]*\)".*/{' \ + -e 's//'`echo $$i|sed 's/c$$/o/'`': \1/' -e 'p' -e '}'; \ + done) | grep -v '[.][o][:][ ][/]' >$$$$ && mv $$$$ Makefile.in + @$(EXPORT) make -f Makefile.in Makefile 1>&2 + +# do not edit below this line - it is generated by 'make depend' +qmgr.o: qmgr.c +qmgr.o: ../include/sys_defs.h +qmgr.o: ../include/msg.h +qmgr.o: ../include/events.h +qmgr.o: ../include/vstream.h +qmgr.o: ../include/vbuf.h +qmgr.o: ../include/dict.h +qmgr.o: ../include/argv.h +qmgr.o: ../include/mail_queue.h +qmgr.o: ../include/vstring.h +qmgr.o: ../include/recipient_list.h +qmgr.o: ../include/mail_conf.h +qmgr.o: ../include/mail_params.h +qmgr.o: ../include/mail_proto.h +qmgr.o: ../include/iostuff.h +qmgr.o: ../include/master_proto.h +qmgr.o: ../include/mail_server.h +qmgr.o: qmgr.h +qmgr.o: ../include/scan_dir.h +qmgr.o: ../include/maps.h +qmgr_active.o: qmgr_active.c +qmgr_active.o: ../include/sys_defs.h +qmgr_active.o: ../include/msg.h +qmgr_active.o: ../include/events.h +qmgr_active.o: ../include/mymalloc.h +qmgr_active.o: ../include/vstream.h +qmgr_active.o: ../include/vbuf.h +qmgr_active.o: ../include/mail_params.h +qmgr_active.o: ../include/mail_open_ok.h +qmgr_active.o: ../include/mail_queue.h +qmgr_active.o: ../include/vstring.h +qmgr_active.o: ../include/recipient_list.h +qmgr_active.o: ../include/bounce.h +qmgr_active.o: ../include/defer.h +qmgr_active.o: ../include/rec_type.h +qmgr_active.o: qmgr.h +qmgr_active.o: ../include/scan_dir.h +qmgr_active.o: ../include/maps.h +qmgr_active.o: ../include/dict.h +qmgr_active.o: ../include/argv.h +qmgr_bounce.o: qmgr_bounce.c +qmgr_bounce.o: ../include/sys_defs.h +qmgr_bounce.o: ../include/bounce.h +qmgr_bounce.o: ../include/deliver_completed.h +qmgr_bounce.o: ../include/vstream.h +qmgr_bounce.o: ../include/vbuf.h +qmgr_bounce.o: qmgr.h +qmgr_bounce.o: ../include/scan_dir.h +qmgr_bounce.o: ../include/maps.h +qmgr_bounce.o: ../include/dict.h +qmgr_bounce.o: ../include/argv.h +qmgr_defer.o: qmgr_defer.c +qmgr_defer.o: ../include/sys_defs.h +qmgr_defer.o: ../include/msg.h +qmgr_defer.o: ../include/vstream.h +qmgr_defer.o: ../include/vbuf.h +qmgr_defer.o: ../include/defer.h +qmgr_defer.o: ../include/bounce.h +qmgr_defer.o: qmgr.h +qmgr_defer.o: ../include/scan_dir.h +qmgr_defer.o: ../include/maps.h +qmgr_defer.o: ../include/dict.h +qmgr_defer.o: ../include/argv.h +qmgr_deliver.o: qmgr_deliver.c +qmgr_deliver.o: ../include/sys_defs.h +qmgr_deliver.o: ../include/msg.h +qmgr_deliver.o: ../include/vstring.h +qmgr_deliver.o: ../include/vbuf.h +qmgr_deliver.o: ../include/vstream.h +qmgr_deliver.o: ../include/vstring_vstream.h +qmgr_deliver.o: ../include/events.h +qmgr_deliver.o: ../include/iostuff.h +qmgr_deliver.o: ../include/mail_queue.h +qmgr_deliver.o: ../include/mail_proto.h +qmgr_deliver.o: ../include/recipient_list.h +qmgr_deliver.o: ../include/mail_params.h +qmgr_deliver.o: qmgr.h +qmgr_deliver.o: ../include/scan_dir.h +qmgr_deliver.o: ../include/maps.h +qmgr_deliver.o: ../include/dict.h +qmgr_deliver.o: ../include/argv.h +qmgr_enable.o: qmgr_enable.c +qmgr_enable.o: ../include/sys_defs.h +qmgr_enable.o: ../include/msg.h +qmgr_enable.o: ../include/vstream.h +qmgr_enable.o: ../include/vbuf.h +qmgr_enable.o: qmgr.h +qmgr_enable.o: ../include/scan_dir.h +qmgr_enable.o: ../include/maps.h +qmgr_enable.o: ../include/dict.h +qmgr_enable.o: ../include/argv.h +qmgr_entry.o: qmgr_entry.c +qmgr_entry.o: ../include/sys_defs.h +qmgr_entry.o: ../include/msg.h +qmgr_entry.o: ../include/mymalloc.h +qmgr_entry.o: ../include/events.h +qmgr_entry.o: ../include/vstream.h +qmgr_entry.o: ../include/vbuf.h +qmgr_entry.o: ../include/mail_params.h +qmgr_entry.o: qmgr.h +qmgr_entry.o: ../include/scan_dir.h +qmgr_entry.o: ../include/maps.h +qmgr_entry.o: ../include/dict.h +qmgr_entry.o: ../include/argv.h +qmgr_message.o: qmgr_message.c +qmgr_message.o: ../include/sys_defs.h +qmgr_message.o: ../include/msg.h +qmgr_message.o: ../include/mymalloc.h +qmgr_message.o: ../include/vstring.h +qmgr_message.o: ../include/vbuf.h +qmgr_message.o: ../include/vstream.h +qmgr_message.o: ../include/split_at.h +qmgr_message.o: ../include/valid_hostname.h +qmgr_message.o: ../include/argv.h +qmgr_message.o: ../include/stringops.h +qmgr_message.o: ../include/myflock.h +qmgr_message.o: ../include/dict.h +qmgr_message.o: ../include/mail_queue.h +qmgr_message.o: ../include/mail_params.h +qmgr_message.o: ../include/canon_addr.h +qmgr_message.o: ../include/record.h +qmgr_message.o: ../include/rec_type.h +qmgr_message.o: ../include/sent.h +qmgr_message.o: ../include/deliver_completed.h +qmgr_message.o: ../include/mail_addr_find.h +qmgr_message.o: ../include/maps.h +qmgr_message.o: ../include/opened.h +qmgr_message.o: ../include/resolve_local.h +qmgr_message.o: ../include/resolve_clnt.h +qmgr_message.o: qmgr.h +qmgr_message.o: ../include/scan_dir.h +qmgr_move.o: qmgr_move.c +qmgr_move.o: ../include/sys_defs.h +qmgr_move.o: ../include/msg.h +qmgr_move.o: ../include/scan_dir.h +qmgr_move.o: ../include/recipient_list.h +qmgr_move.o: ../include/mail_queue.h +qmgr_move.o: ../include/vstring.h +qmgr_move.o: ../include/vbuf.h +qmgr_move.o: ../include/vstream.h +qmgr_move.o: ../include/mail_scan_dir.h +qmgr_move.o: qmgr.h +qmgr_move.o: ../include/maps.h +qmgr_move.o: ../include/dict.h +qmgr_move.o: ../include/argv.h +qmgr_queue.o: qmgr_queue.c +qmgr_queue.o: ../include/sys_defs.h +qmgr_queue.o: ../include/msg.h +qmgr_queue.o: ../include/mymalloc.h +qmgr_queue.o: ../include/events.h +qmgr_queue.o: ../include/htable.h +qmgr_queue.o: ../include/mail_params.h +qmgr_queue.o: ../include/recipient_list.h +qmgr_queue.o: qmgr.h +qmgr_queue.o: ../include/vstream.h +qmgr_queue.o: ../include/vbuf.h +qmgr_queue.o: ../include/scan_dir.h +qmgr_queue.o: ../include/maps.h +qmgr_queue.o: ../include/dict.h +qmgr_queue.o: ../include/argv.h +qmgr_rcpt_list.o: qmgr_rcpt_list.c +qmgr_rcpt_list.o: ../include/sys_defs.h +qmgr_rcpt_list.o: ../include/mymalloc.h +qmgr_rcpt_list.o: qmgr.h +qmgr_rcpt_list.o: ../include/vstream.h +qmgr_rcpt_list.o: ../include/vbuf.h +qmgr_rcpt_list.o: ../include/scan_dir.h +qmgr_rcpt_list.o: ../include/maps.h +qmgr_rcpt_list.o: ../include/dict.h +qmgr_rcpt_list.o: ../include/argv.h +qmgr_scan.o: qmgr_scan.c +qmgr_scan.o: ../include/sys_defs.h +qmgr_scan.o: ../include/msg.h +qmgr_scan.o: ../include/mymalloc.h +qmgr_scan.o: ../include/scan_dir.h +qmgr_scan.o: ../include/mail_scan_dir.h +qmgr_scan.o: qmgr.h +qmgr_scan.o: ../include/vstream.h +qmgr_scan.o: ../include/vbuf.h +qmgr_scan.o: ../include/maps.h +qmgr_scan.o: ../include/dict.h +qmgr_scan.o: ../include/argv.h +qmgr_transport.o: qmgr_transport.c +qmgr_transport.o: ../include/sys_defs.h +qmgr_transport.o: ../include/msg.h +qmgr_transport.o: ../include/htable.h +qmgr_transport.o: ../include/events.h +qmgr_transport.o: ../include/mymalloc.h +qmgr_transport.o: ../include/vstream.h +qmgr_transport.o: ../include/vbuf.h +qmgr_transport.o: ../include/iostuff.h +qmgr_transport.o: ../include/mail_proto.h +qmgr_transport.o: ../include/recipient_list.h +qmgr_transport.o: ../include/mail_conf.h +qmgr_transport.o: ../include/mail_params.h +qmgr_transport.o: qmgr.h +qmgr_transport.o: ../include/scan_dir.h +qmgr_transport.o: ../include/maps.h +qmgr_transport.o: ../include/dict.h +qmgr_transport.o: ../include/argv.h diff --git a/postfix/nqmgr/qmgr.c b/postfix/nqmgr/qmgr.c new file mode 100644 index 000000000..aff5a8bd9 --- /dev/null +++ b/postfix/nqmgr/qmgr.c @@ -0,0 +1,554 @@ +/*++ +/* NAME +/* qmgr 8 +/* SUMMARY +/* Postfix queue manager +/* SYNOPSIS +/* \fBqmgr\fR [generic Postfix daemon options] +/* DESCRIPTION +/* The \fBqmgr\fR daemon awaits the arrival of incoming mail +/* and arranges for its delivery via Postfix delivery processes. +/* The actual mail routing strategy is delegated to the +/* \fBtrivial-rewrite\fR(8) daemon. +/* This program expects to be run from the \fBmaster\fR(8) process +/* manager. +/* +/* Mail addressed to the local \fBdouble-bounce\fR address is silently +/* discarded. This stops potential loops caused by undeliverable +/* bounce notifications. +/* +/* Mail addressed to a user listed in the optional \fBrelocated\fR +/* database is bounced with a "user has moved to \fInew_location\fR" +/* message. See \fBrelocated\fR(5) for a precise description. +/* MAIL QUEUES +/* .ad +/* .fi +/* The \fBqmgr\fR daemon maintains the following queues: +/* .IP \fBincoming\fR +/* Inbound mail from the network, or mail picked up by the +/* local \fBpickup\fR agent from the \fBmaildrop\fR directory. +/* .IP \fBactive\fR +/* Messages that the queue manager has opened for delivery. Only +/* a limited number of messages is allowed to enter the \fBactive\fR +/* queue (leaky bucket strategy, for a fixed delivery rate). +/* .IP \fBdeferred\fR +/* Mail that could not be delivered upon the first attempt. The queue +/* manager implements exponential backoff by doubling the time between +/* delivery attempts. +/* .IP \fBcorrupt\fR +/* Unreadable or damaged queue files are moved here for inspection. +/* DELIVERY STATUS REPORTS +/* .ad +/* .fi +/* The \fBqmgr\fR daemon keeps an eye on per-message delivery status +/* reports in the following directories. Each status report file has +/* the same name as the corresponding message file: +/* .IP \fBbounce\fR +/* Per-recipient status information about why mail is bounced. +/* These files are maintained by the \fBbounce\fR(8) daemon. +/* .IP \fBdefer\fR +/* Per-recipient status information about why mail is delayed. +/* These files are maintained by the \fBdefer\fR(8) daemon. +/* .PP +/* The \fBqmgr\fR daemon is responsible for asking the +/* \fBbounce\fR(8) or \fBdefer\fR(8) daemons to send non-delivery +/* reports. +/* STRATEGIES +/* .ad +/* .fi +/* The queue manager implements a variety of strategies for +/* either opening queue files (input) or for message delivery (output). +/* .IP "\fBleaky bucket\fR" +/* This strategy limits the number of messages in the \fBactive\fR queue +/* and prevents the queue manager from running out of memory under +/* heavy load. +/* .IP \fBfairness\fR +/* When the \fBactive\fR queue has room, the queue manager takes one +/* message from the \fBincoming\fR queue and one from the \fBdeferred\fR +/* queue. This prevents a large mail backlog from blocking the delivery +/* of new mail. +/* .IP "\fBslow start\fR" +/* This strategy eliminates "thundering herd" problems by slowly +/* adjusting the number of parallel deliveries to the same destination. +/* .IP "\fBround robin\fR +/* The queue manager sorts delivery requests by destination. +/* Round-robin selection prevents one destination from dominating +/* deliveries to other destinations. +/* .IP "\fBexponential backoff\fR" +/* Mail that cannot be delivered upon the first attempt is deferred. +/* The time interval between delivery attempts is doubled after each +/* attempt. +/* .IP "\fBdestination status cache\fR" +/* The queue manager avoids unnecessary delivery attempts by +/* maintaining a short-term, in-memory list of unreachable destinations. +/* .IP "\fBpreemptive message scheduling\fR" +/* The queue manager attempts to minimize the average per-recipient delay +/* while still assuring equality of average per-message delays, using +/* sophisticated preemptive message scheduling. +/* TRIGGERS +/* .ad +/* .fi +/* On an idle system, the queue manager waits for the arrival of +/* trigger events, or it waits for a timer to go off. A trigger +/* is a one-byte message. +/* Depending on the message received, the queue manager performs +/* one of the following actions (the message is followed by the +/* symbolic constant used internally by the software): +/* .IP "\fBD (QMGR_REQ_SCAN_DEFERRED)\fR" +/* Start a deferred queue scan. If a deferred queue scan is already +/* in progress, that scan will be restarted as soon as it finishes. +/* .IP "\fBI (QMGR_REQ_SCAN_INCOMING)\fR" +/* Start an incoming queue scan. If an incoming queue scan is already +/* in progress, that scan will be restarted as soon as it finishes. +/* .IP "\fBA (QMGR_REQ_SCAN_ALL)\fR" +/* Ignore deferred queue file time stamps. The request affects +/* the next deferred queue scan. +/* .IP "\fBF (QMGR_REQ_FLUSH_DEAD)\fR" +/* Purge all information about dead transports and destinations. +/* .IP "\fBW (TRIGGER_REQ_WAKEUP)\fR" +/* Wakeup call, This is used by the master server to instantiate +/* servers that should not go away forever. The action is to start +/* an incoming queue scan. +/* .PP +/* The \fBqmgr\fR daemon reads an entire buffer worth of triggers. +/* Multiple identical trigger requests are collapsed into one, and +/* trigger requests are sorted so that \fBA\fR and \fBF\fR precede +/* \fBD\fR and \fBI\fR. Thus, in order to force a deferred queue run, +/* one would request \fBA F D\fR; in order to notify the queue manager +/* of the arrival of new mail one would request \fBI\fR. +/* STANDARDS +/* .ad +/* .fi +/* None. The \fBqmgr\fR daemon does not interact with the outside world. +/* SECURITY +/* .ad +/* .fi +/* The \fBqmgr\fR daemon is not security sensitive. It reads +/* single-character messages from untrusted local users, and thus may +/* be susceptible to denial of service attacks. The \fBqmgr\fR daemon +/* does not talk to the outside world, and it can be run at fixed low +/* privilege in a chrooted environment. +/* DIAGNOSTICS +/* Problems and transactions are logged to the syslog daemon. +/* Corrupted message files are saved to the \fBcorrupt\fR queue +/* for further inspection. +/* +/* Depending on the setting of the \fBnotify_classes\fR parameter, +/* the postmaster is notified of bounces and of other trouble. +/* BUGS +/* A single queue manager process has to compete for disk access with +/* multiple front-end processes such as \fBsmtpd\fR. A sudden burst of +/* inbound mail can negatively impact outbound delivery rates. +/* CONFIGURATION PARAMETERS +/* .ad +/* .fi +/* The following \fBmain.cf\fR parameters are especially relevant to +/* this program. See the Postfix \fBmain.cf\fR file for syntax details +/* and for default values. Use the \fBpostfix reload\fR command after +/* a configuration change. +/* .SH Miscellaneous +/* .ad +/* .fi +/* .IP \fBallow_min_user\fR +/* Do not bounce recipient addresses that begin with '-'. +/* .IP \fBrelocated_maps\fR +/* Tables with contact information for users, hosts or domains +/* that no longer exist. See \fBrelocated\fR(5). +/* .IP \fBqueue_directory\fR +/* Top-level directory of the Postfix queue. +/* .SH "Active queue controls" +/* .ad +/* .fi +/* In the text below, \fItransport\fR is the first field in a +/* \fBmaster.cf\fR entry. +/* .IP \fBqmgr_message_active_limit\fR +/* Limit the number of messages in the active queue. +/* .IP \fBqmgr_message_recipient_limit\fR +/* Limit the number of in-memory recipients. +/* .sp +/* This parameter also limits the size of the short-term, in-memory +/* destination cache. +/* .IP \fBqmgr_message_recipient_minimum\fR +/* Per message minimum of in-memory recipients. +/* .IP \fBdefault_recipient_limit\fR +/* Default limit on the number of in-memory recipients per transport. +/* .IP \fItransport\fB_recipient_limit\fR +/* Limit on the number of in-memory recipients, for the named +/* message \fItransport\fR. +/* .IP \fBdefault_extra_recipient_limit\fR +/* Default limit on the total number of per transport in-memory +/* recipients that the preempting messages can have. +/* .IP \fItransport\fB_extra_recipient_limit\fR +/* Limit on the number of in-memory recipients which all preempting +/* messages delivered by transport \fItransport\fR can have. +/* .SH "Timing controls" +/* .ad +/* .fi +/* .IP \fBmin_backoff\fR +/* Minimal time in seconds between delivery attempts +/* of a deferred message. +/* .sp +/* This parameter also limits the time an unreachable destination +/* is kept in the short-term, in-memory destination status cache. +/* .IP \fBmax_backoff\fR +/* Maximal time in seconds between delivery attempts +/* of a deferred message. +/* .IP \fBmaximal_queue_lifetime\fR +/* Maximal time in days a message is queued +/* before it is sent back as undeliverable. +/* .IP \fBqueue_run_delay\fR +/* Time in seconds between deferred queue scans. Queue scans do +/* not overlap. +/* .IP \fBtransport_retry_time\fR +/* Time in seconds between attempts to contact a broken +/* delivery transport. +/* .SH "Concurrency controls" +/* .ad +/* .fi +/* .IP \fBinitial_destination_concurrency\fR +/* Initial per-destination concurrency level for parallel delivery +/* to the same destination. +/* .IP \fBdefault_destination_concurrency_limit\fR +/* Default limit on the number of parallel deliveries to the same +/* destination. +/* .IP \fItransport\fB_destination_concurrency_limit\fR +/* Limit on the number of parallel deliveries to the same destination, +/* for delivery via the named message \fItransport\fR. +/* .SH "Recipient controls" +/* .ad +/* .fi +/* .IP \fBdefault_destination_recipient_limit\fR +/* Default limit on the number of recipients per message transfer. +/* .IP \fItransport\fB_destination_recipient_limit\fR +/* Limit on the number of recipients per message transfer, for the +/* named message \fItransport\fR. +/* .SH "Message scheduling" +/* .ad +/* .fi +/* .IP "\fItransport\fB_delivery_slot_cost\fR (valid range: 0,2,3...) +/* This parameter basically controls how often a message +/* delivered by \fItransport\fB can be preempted by another +/* message. +/* An internal per-message/transport counter is incremented by one +/* for each \fItransport\fB_delivery_slot_cost\fR +/* deliveries handled by \fItransport\fR. This counter represents +/* the number of "available delivery slots" for use by other messages. +/* Current message can be preempted by another message when that +/* other message can be delivered using less \fItranpsort\fR agents +/* than the value of the "available delivery slots" counter. +/* .sp +/* Value equal to 0 disables the message preemption for \fItransport\fR. +/* .IP \fItransport\fB_minimum_delivery_slots\fR +/* Message preemption is not attempted at all whenever a message +/* that can't ever accumulate at least \fItransport\fB_minimum_delivery_slots\fR +/* available delivery slots is being delivered by \fItransport\fR. +/* .IP "\fItransport\fB_delivery_slot_discount\fR (valid range: 0..100)" +/* .IP \fItransport\fB_delivery_slot_loan\fR +/* These parameters speed up the moment when a message preemption can happen. +/* Instead of waiting till the full amount of delivery slots +/* required is available, the preemption can happen when +/* \fItransport\fB_delivery_slot_discount\fR percent of the required +/* amount plus \fItransport\fB_delivery_slot_loan\fR still remains to +/* be accumulated. Note that the full amount will still have to be +/* accumulated before another preemption can take place later. +/* .IP \fBdefault_delivery_slot_cost\fR +/* .IP \fBdefault_minimum_delivery_slots\fR +/* .IP \fBdefault_delivery_slot_discount\fR +/* .IP \fBdefault_delivery_slot_loan\fR +/* Default values for the transport specific parameters described above. +/* SEE ALSO +/* master(8), process manager +/* relocated(5), format of the "user has moved" table +/* syslogd(8) system logging +/* trivial-rewrite(8), address routing +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include + +/* Global library. */ + +#include +#include +#include +#include +#include /* QMGR_SCAN constants */ + +/* Master process interface */ + +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + + /* + * Tunables. + */ +int var_queue_run_delay; +int var_min_backoff_time; +int var_max_backoff_time; +int var_max_queue_time; +int var_qmgr_active_limit; +int var_qmgr_rcpt_limit; +int var_qmgr_msg_rcpt_limit; +int var_xport_rcpt_limit; +int var_stack_rcpt_limit; +int var_delivery_slot_cost; +int var_delivery_slot_loan; +int var_delivery_slot_discount; +int var_min_delivery_slots; +int var_init_dest_concurrency; +int var_transport_retry_time; +int var_dest_con_limit; +int var_dest_rcpt_limit; +char *var_relocated_maps; +char *var_virtual_maps; +char *var_defer_xports; +bool var_allow_min_user; +int var_qmgr_hog; /* XXX */ +int var_qmgr_fudge; /* XXX */ +int var_local_rcpt_lim; /* XXX */ + +static QMGR_SCAN *qmgr_incoming; +static QMGR_SCAN *qmgr_deferred; + +MAPS *qmgr_relocated; +MAPS *qmgr_virtual; + +/* qmgr_deferred_run_event - queue manager heartbeat */ + +static void qmgr_deferred_run_event(int unused_event, char *dummy) +{ + + /* + * This routine runs when it is time for another deferred queue scan. + * Make sure this routine gets called again in the future. + */ + qmgr_scan_request(qmgr_deferred, QMGR_SCAN_START); + event_request_timer(qmgr_deferred_run_event, dummy, var_queue_run_delay); +} + +/* qmgr_trigger_event - respond to external trigger(s) */ + +static void qmgr_trigger_event(char *buf, int len, + char *unused_service, char **argv) +{ + int incoming_flag = 0; + int deferred_flag = 0; + int i; + + /* + * Sanity check. This service takes no command-line arguments. + */ + if (argv[0]) + msg_fatal("unexpected command-line argument: %s", argv[0]); + + /* + * Collapse identical requests that have arrived since we looked last + * time. There is no client feedback so there is no need to process each + * request in order. And as long as we don't have conflicting requests we + * are free to sort them into the most suitable order. + */ + for (i = 0; i < len; i++) { + if (msg_verbose) + msg_info("request: %d (%c)", + buf[i], ISALNUM(buf[i]) ? buf[i] : '?'); + switch (buf[i]) { + case TRIGGER_REQ_WAKEUP: + case QMGR_REQ_SCAN_INCOMING: + incoming_flag |= QMGR_SCAN_START; + break; + case QMGR_REQ_SCAN_DEFERRED: + deferred_flag |= QMGR_SCAN_START; + break; + case QMGR_REQ_FLUSH_DEAD: + deferred_flag |= QMGR_FLUSH_DEAD; + incoming_flag |= QMGR_FLUSH_DEAD; + break; + case QMGR_REQ_SCAN_ALL: + deferred_flag |= QMGR_SCAN_ALL; + incoming_flag |= QMGR_SCAN_ALL; + break; + default: + if (msg_verbose) + msg_info("request ignored"); + break; + } + } + + /* + * Process each request type at most once. Modifiers take effect upon the + * next queue run. If no queue run is in progress, and a queue scan is + * requested, the request takes effect immediately. + */ + if (incoming_flag != 0) + qmgr_scan_request(qmgr_incoming, incoming_flag); + if (deferred_flag != 0) + qmgr_scan_request(qmgr_deferred, deferred_flag); +} + +/* qmgr_loop - queue manager main loop */ + +static int qmgr_loop(char *unused_name, char **unused_argv) +{ + char *in_path = 0; + char *df_path = 0; + + /* + * This routine runs as part of the event handling loop, after the event + * manager has delivered a timer or I/O event (including the completion + * of a connection to a delivery process), or after it has waited for a + * specified amount of time. The result value of qmgr_loop() specifies + * how long the event manager should wait for the next event. + */ +#define DONT_WAIT 0 +#define WAIT_FOR_EVENT (-1) + + /* + * Attempt to drain the active queue by allocating a suitable delivery + * process and by delivering mail via it. Delivery process allocation and + * mail delivery are asynchronous. + */ + qmgr_active_drain(); + + /* + * Let some new blood into the active queue when the queue size is + * smaller than some configurable limit, and when the number of in-core + * recipients does not exceed some configurable limit. When the system is + * under heavy load, favor new mail over old mail. + */ + if (qmgr_message_count < var_qmgr_active_limit) + if ((in_path = qmgr_scan_next(qmgr_incoming)) != 0) + qmgr_active_feed(qmgr_incoming, in_path); + if (qmgr_message_count < var_qmgr_active_limit) + if ((df_path = qmgr_scan_next(qmgr_deferred)) != 0) + qmgr_active_feed(qmgr_deferred, df_path); + if (in_path || df_path) + return (DONT_WAIT); + return (WAIT_FOR_EVENT); +} + +/* pre_accept - see if tables have changed */ + +static void pre_accept(char *unused_name, char **unused_argv) +{ + if (dict_changed()) { + msg_info("table has changed -- exiting"); + exit(0); + } +} + +/* qmgr_pre_init - pre-jail initialization */ + +static void qmgr_pre_init(char *unused_name, char **unused_argv) +{ + if (*var_relocated_maps) + qmgr_relocated = maps_create("relocated", var_relocated_maps, + DICT_FLAG_LOCK); + if (*var_virtual_maps) + qmgr_virtual = maps_create("virtual", var_virtual_maps, + DICT_FLAG_LOCK); +} + +/* qmgr_post_init - post-jail initialization */ + +static void qmgr_post_init(char *unused_name, char **unused_argv) +{ + + /* + * This routine runs after the skeleton code has entered the chroot jail. + * Prevent automatic process suicide after a limited number of client + * requests or after a limited amount of idle time. Move any left-over + * entries from the active queue to the incoming queue, and give them a + * time stamp into the future, in order to allow ongoing deliveries to + * finish first. Start scanning the incoming and deferred queues. + * Left-over active queue entries are moved to the incoming queue because + * the incoming queue has priority; moving left-overs to the deferred + * queue could cause anomalous delays when "postfix reload/start" are + * issued often. + */ + var_use_limit = 0; + var_idle_limit = 0; + qmgr_move(MAIL_QUEUE_ACTIVE, MAIL_QUEUE_INCOMING, + event_time() + var_min_backoff_time); + qmgr_incoming = qmgr_scan_create(MAIL_QUEUE_INCOMING); + qmgr_deferred = qmgr_scan_create(MAIL_QUEUE_DEFERRED); + qmgr_scan_request(qmgr_incoming, QMGR_SCAN_START); + qmgr_deferred_run_event(0, (char *) 0); +} + +/* main - the main program */ + +int main(int argc, char **argv) +{ + static CONFIG_STR_TABLE str_table[] = { + VAR_RELOCATED_MAPS, DEF_RELOCATED_MAPS, &var_relocated_maps, 0, 0, + VAR_VIRTUAL_MAPS, DEF_VIRTUAL_MAPS, &var_virtual_maps, 0, 0, + VAR_DEFER_XPORTS, DEF_DEFER_XPORTS, &var_defer_xports, 0, 0, + 0, + }; + static CONFIG_INT_TABLE int_table[] = { + VAR_QUEUE_RUN_DELAY, DEF_QUEUE_RUN_DELAY, &var_queue_run_delay, 1, 0, + VAR_MIN_BACKOFF_TIME, DEF_MIN_BACKOFF_TIME, &var_min_backoff_time, 1, 0, + VAR_MAX_BACKOFF_TIME, DEF_MAX_BACKOFF_TIME, &var_max_backoff_time, 1, 0, + VAR_MAX_QUEUE_TIME, DEF_MAX_QUEUE_TIME, &var_max_queue_time, 1, 0, + VAR_QMGR_ACT_LIMIT, DEF_QMGR_ACT_LIMIT, &var_qmgr_active_limit, 1, 0, + VAR_QMGR_RCPT_LIMIT, DEF_QMGR_RCPT_LIMIT, &var_qmgr_rcpt_limit, 0, 0, + VAR_QMGR_MSG_RCPT_LIMIT, DEF_QMGR_MSG_RCPT_LIMIT, &var_qmgr_msg_rcpt_limit, 1, 0, + VAR_XPORT_RCPT_LIMIT, DEF_XPORT_RCPT_LIMIT, &var_xport_rcpt_limit, 0, 0, + VAR_STACK_RCPT_LIMIT, DEF_STACK_RCPT_LIMIT, &var_stack_rcpt_limit, 0, 0, + VAR_DELIVERY_SLOT_COST, DEF_DELIVERY_SLOT_COST, &var_delivery_slot_cost, 0, 0, + VAR_DELIVERY_SLOT_LOAN, DEF_DELIVERY_SLOT_LOAN, &var_delivery_slot_loan, 0, 0, + VAR_DELIVERY_SLOT_DISCOUNT, DEF_DELIVERY_SLOT_DISCOUNT, &var_delivery_slot_discount, 0, 100, + VAR_MIN_DELIVERY_SLOTS, DEF_MIN_DELIVERY_SLOTS, &var_min_delivery_slots, 0, 0, + VAR_INIT_DEST_CON, DEF_INIT_DEST_CON, &var_init_dest_concurrency, 1, 0, + VAR_XPORT_RETRY_TIME, DEF_XPORT_RETRY_TIME, &var_transport_retry_time, 1, 0, + VAR_DEST_CON_LIMIT, DEF_DEST_CON_LIMIT, &var_dest_con_limit, 0, 0, + VAR_DEST_RCPT_LIMIT, DEF_DEST_RCPT_LIMIT, &var_dest_rcpt_limit, 0, 0, + VAR_QMGR_HOG, DEF_QMGR_HOG, &var_qmgr_hog, 10, 100, + VAR_QMGR_FUDGE, DEF_QMGR_FUDGE, &var_qmgr_fudge, 10, 100, + VAR_LOCAL_RCPT_LIMIT, DEF_LOCAL_RCPT_LIMIT, &var_local_rcpt_lim, 0, 0, + 0, + }; + static CONFIG_BOOL_TABLE bool_table[] = { + VAR_ALLOW_MIN_USER, DEF_ALLOW_MIN_USER, &var_allow_min_user, + 0, + }; + + /* + * Use the trigger service skeleton, because no-one else should be + * monitoring our service port while this process runs, and because we do + * not talk back to the client. + */ + trigger_server_main(argc, argv, qmgr_trigger_event, + MAIL_SERVER_INT_TABLE, int_table, + MAIL_SERVER_STR_TABLE, str_table, + MAIL_SERVER_BOOL_TABLE, bool_table, + MAIL_SERVER_PRE_INIT, qmgr_pre_init, + MAIL_SERVER_POST_INIT, qmgr_post_init, + MAIL_SERVER_LOOP, qmgr_loop, + MAIL_SERVER_PRE_ACCEPT, pre_accept, + 0); +} diff --git a/postfix/nqmgr/qmgr.h b/postfix/nqmgr/qmgr.h new file mode 100644 index 000000000..f806ba968 --- /dev/null +++ b/postfix/nqmgr/qmgr.h @@ -0,0 +1,393 @@ +/*++ +/* NAME +/* qmgr 3h +/* SUMMARY +/* queue manager data structures +/* SYNOPSIS +/* #include "qmgr.h" +/* DESCRIPTION +/* .nf + + /* + * Utility library. + */ +#include +#include + + /* + * Global library. + */ +#include + + /* + * The queue manager is built around lots of mutually-referring structures. + * These typedefs save some typing. + */ +typedef struct QMGR_TRANSPORT QMGR_TRANSPORT; +typedef struct QMGR_QUEUE QMGR_QUEUE; +typedef struct QMGR_ENTRY QMGR_ENTRY; +typedef struct QMGR_MESSAGE QMGR_MESSAGE; +typedef struct QMGR_JOB QMGR_JOB; +typedef struct QMGR_PEER QMGR_PEER ; +typedef struct QMGR_TRANSPORT_LIST QMGR_TRANSPORT_LIST; +typedef struct QMGR_QUEUE_LIST QMGR_QUEUE_LIST; +typedef struct QMGR_ENTRY_LIST QMGR_ENTRY_LIST; +typedef struct QMGR_JOB_LIST QMGR_JOB_LIST; +typedef struct QMGR_PEER_LIST QMGR_PEER_LIST; +typedef struct QMGR_RCPT QMGR_RCPT; +typedef struct QMGR_RCPT_LIST QMGR_RCPT_LIST; +typedef struct QMGR_SCAN QMGR_SCAN; + + /* + * Hairy macros to update doubly-linked lists. + */ +#define QMGR_LIST_ROTATE(head, object, peers) { \ + head.next->peers.prev = head.prev; \ + head.prev->peers.next = head.next; \ + head.next = object->peers.next; \ + head.next->peers.prev = 0; \ + head.prev = object; \ + object->peers.next = 0; \ +} + +#define QMGR_LIST_UNLINK(head, type, object, peers) { \ + type next = object->peers.next; \ + type prev = object->peers.prev; \ + if (prev) prev->peers.next = next; \ + else head.next = next; \ + if (next) next->peers.prev = prev; \ + else head.prev = prev; \ + object->peers.next = object->peers.prev = 0; \ +} + +#define QMGR_LIST_PREPEND(head, object, peers) { \ + object->peers.next = head.next; \ + object->peers.prev = 0; \ + if (head.next) { \ + head.next->peers.prev = object; \ + } else { \ + head.prev = object; \ + } \ + head.next = object; \ +} + +#define QMGR_LIST_APPEND(head, object, peers) { \ + object->peers.prev = head.prev; \ + object->peers.next = 0; \ + if (head.prev) { \ + head.prev->peers.next = object; \ + } else { \ + head.next = object; \ + } \ + head.prev = object; \ +} + +#define QMGR_LIST_INIT(head) { \ + head.prev = 0; \ + head.next = 0; \ +} + + /* + * Transports are looked up by name (when we have resolved a message), or + * round-robin wise (when we want to distribute resources fairly). + */ +struct QMGR_TRANSPORT_LIST { + QMGR_TRANSPORT *next; + QMGR_TRANSPORT *prev; +}; + +extern struct HTABLE *qmgr_transport_byname; /* transport by name */ +extern QMGR_TRANSPORT_LIST qmgr_transport_list; /* transports, round robin */ + + /* + * Each transport (local, smtp-out, bounce) can have one queue per next hop + * name. Queues are looked up by next hop name (when we have resolved a + * message destination), or round-robin wise (when we want to deliver + * messages fairly). + */ +struct QMGR_QUEUE_LIST { + QMGR_QUEUE *next; + QMGR_QUEUE *prev; +}; + +struct QMGR_JOB_LIST { + QMGR_JOB *next; + QMGR_JOB *prev; +}; + +struct QMGR_TRANSPORT { + int flags; /* blocked, etc. */ + char *name; /* transport name */ + int dest_concurrency_limit; /* concurrency per domain */ + int init_dest_concurrency; /* init. per-domain concurrency */ + int recipient_limit; /* recipients per transaction */ + int rcpt_per_stack; /* slots reserved for job on 1st stack level */ + int rcpt_unused; /* available in-core slots */ + int slot_cost; /* cost of new preemption slot (# selected entries) */ + int slot_loan; /* preemption boost offset and factor, see */ + int slot_loan_factor; /* qmgr_job_preempt() for more info */ + int min_slots; /* when preemption can take effect at all */ + struct HTABLE *queue_byname; /* queues indexed by domain */ + QMGR_QUEUE_LIST queue_list; /* queues, round robin order */ + struct HTABLE *job_byname; /* jobs indexed by queue id */ + QMGR_JOB_LIST job_list; /* list of message jobs (1 per message) */ + QMGR_JOB_LIST job_stack; /* job stack for tracking preemption */ + QMGR_JOB *job_next_unread; /* next job with unread recipients */ + QMGR_JOB *candidate_cache; /* cached result from qmgr_job_candidate() */ + time_t candidate_cache_time; /* when candidate_cache was last updated */ + QMGR_TRANSPORT_LIST peers; /* linkage */ + char *reason; /* why unavailable */ +}; + +#define QMGR_TRANSPORT_STAT_DEAD (1<<1) +#define QMGR_TRANSPORT_STAT_BUSY (1<<2) + +typedef void (*QMGR_TRANSPORT_ALLOC_NOTIFY) (QMGR_TRANSPORT *, VSTREAM *); +extern QMGR_TRANSPORT *qmgr_transport_select(void); +extern void qmgr_transport_alloc(QMGR_TRANSPORT *, QMGR_TRANSPORT_ALLOC_NOTIFY); +extern void qmgr_transport_throttle(QMGR_TRANSPORT *, const char *); +extern void qmgr_transport_unthrottle(QMGR_TRANSPORT *); +extern QMGR_TRANSPORT *qmgr_transport_create(const char *); +extern QMGR_TRANSPORT *qmgr_transport_find(const char *); + + /* + * Each next hop (e.g., a domain name) has its own queue of pending message + * transactions. The "todo" queue contains messages that are to be delivered + * to this next hop. When a message is elected for transmission, it is moved + * from the "todo" queue to the "busy" queue. Messages are taken from the + * "todo" queue in round-robin order. + */ +struct QMGR_ENTRY_LIST { + QMGR_ENTRY *next; + QMGR_ENTRY *prev; +}; + +struct QMGR_QUEUE { + char *name; /* domain name */ + int todo_refcount; /* queue entries (todo list) */ + int busy_refcount; /* queue entries (busy list) */ + int window; /* slow open algorithm */ + QMGR_TRANSPORT *transport; /* transport linkage */ + QMGR_ENTRY_LIST todo; /* todo queue entries */ + QMGR_ENTRY_LIST busy; /* messages on the wire */ + QMGR_QUEUE_LIST peers; /* neighbor queues */ + char *reason; /* why unavailable */ +}; + +#define QMGR_QUEUE_TODO 1 /* waiting for service */ +#define QMGR_QUEUE_BUSY 2 /* recipients on the wire */ + +extern int qmgr_queue_count; + +extern QMGR_QUEUE *qmgr_queue_create(QMGR_TRANSPORT *, const char *); +extern void qmgr_queue_done(QMGR_QUEUE *); +extern void qmgr_queue_throttle(QMGR_QUEUE *, const char *); +extern void qmgr_queue_unthrottle(QMGR_QUEUE *); +extern QMGR_QUEUE *qmgr_queue_find(QMGR_TRANSPORT *, const char *); + + /* + * Structure for a recipient list. Initially, it just contains recipient + * addresses and file offsets. After the address resolver has done its work, + * each recipient is accompanied by a reference to a specific queues (which + * implies a specific transport). This is an extended version of similar + * information maintained by the recipient_list(3) module. + */ +struct QMGR_RCPT { + long offset; /* REC_TYPE_RCPT byte */ + char *address; /* complete address */ + QMGR_QUEUE *queue; /* resolved queue */ +}; + +struct QMGR_RCPT_LIST { + QMGR_RCPT *info; + int len; + int avail; +}; + +extern void qmgr_rcpt_list_init(QMGR_RCPT_LIST *); +extern void qmgr_rcpt_list_add(QMGR_RCPT_LIST *, long, const char *); +extern void qmgr_rcpt_list_free(QMGR_RCPT_LIST *); + + /* + * Structure of one next-hop queue entry. In order to save some copying + * effort we allow multiple recipients per transaction. + */ +struct QMGR_ENTRY { + VSTREAM *stream; /* delivery process */ + QMGR_MESSAGE *message; /* message info */ + QMGR_RCPT_LIST rcpt_list; /* as many as it takes */ + QMGR_QUEUE *queue; /* parent linkage */ + QMGR_PEER *peer; /* parent linkage */ + QMGR_ENTRY_LIST queue_peers; /* per queue neighbor entries */ + QMGR_ENTRY_LIST peer_peers; /* per peer neighbor entries */ +}; + +extern QMGR_ENTRY *qmgr_entry_select(QMGR_PEER *); +extern void qmgr_entry_unselect(QMGR_ENTRY *); +extern void qmgr_entry_done(QMGR_ENTRY *, int); +extern QMGR_ENTRY *qmgr_entry_create(QMGR_PEER *, QMGR_MESSAGE *); + + /* + * All common in-core information about a message is kept here. When all + * recipients have been tried the message file is linked to the "deferred" + * queue (some hosts not reachable), to the "bounce" queue (some recipients + * were rejected), and is then removed from the "active" queue. + */ +struct QMGR_MESSAGE { + int flags; /* delivery problems */ + int qflags; /* queuing flags */ + VSTREAM *fp; /* open queue file or null */ + int refcount; /* queue entries */ + int single_rcpt; /* send one rcpt at a time */ + long arrival_time; /* time when queued */ + time_t queued_time; /* time when moved to the active queue */ + long warn_offset; /* warning bounce flag offset */ + time_t warn_time; /* time next warning to be sent */ + long data_offset; /* data seek offset */ + char *queue_name; /* queue name */ + char *queue_id; /* queue file */ + char *sender; /* complete address */ + char *errors_to; /* error report address */ + char *return_receipt; /* confirm receipt address */ + long data_size; /* message content size */ + long rcpt_offset; /* more recipients here */ + long unread_offset; /* more unread recipients here */ + QMGR_RCPT_LIST rcpt_list; /* complete addresses */ + int rcpt_count; /* used recipient slots */ + int rcpt_limit; /* maximum read in-core */ + int rcpt_unread; /* # of recipients left in queue file */ + QMGR_JOB_LIST job_list; /* jobs delivering this message (1 per transport) */ +}; + +#define QMGR_MESSAGE_LOCKED ((QMGR_MESSAGE *) 1) + +extern int qmgr_message_count; +extern int qmgr_recipient_count; +extern MAPS *qmgr_relocated; +extern MAPS *qmgr_virtual; + +extern void qmgr_message_free(QMGR_MESSAGE *); +extern void qmgr_message_update_warn(QMGR_MESSAGE *); +extern QMGR_MESSAGE *qmgr_message_alloc(const char *, const char *, int); +extern QMGR_MESSAGE *qmgr_message_realloc(QMGR_MESSAGE *); + + /* + * Sometimes it's required to access the transport queues and entries on per + * message basis. That's what the QMGR_JOB structure is for - it groups all per + * message information within each transport using a list of QMGR_PEER structures. + * These structures in turn correspond with per message QMGR_QUEUE structure + * and list all per message QMGR_ENTRY structures. + */ +struct QMGR_PEER_LIST { + QMGR_PEER *next; + QMGR_PEER *prev; +}; + +struct QMGR_JOB { + QMGR_MESSAGE *message; /* message delivered by this job */ + QMGR_TRANSPORT *transport; /* transport this job belongs to */ + QMGR_JOB_LIST message_peers; /* per message neighbor linkage */ + QMGR_JOB_LIST transport_peers; /* per transport neighbor linkage */ + QMGR_JOB_LIST stack_peers; /* transport stack linkage */ + int stack_level; /* job stack nesting level (-1 -> retired) */ + struct HTABLE *peer_byname; /* message job peers, indexed by domain */ + QMGR_PEER_LIST peer_list; /* list of message job peers */ + int slots_used; /* slots used during preemption */ + int slots_available; /* slots available for preemption (* slot_cost) */ + int selected_entries; /* # of entries selected for delivery so far */ + int read_entries; /* # of entries read in-core so far */ + int rcpt_count; /* used recipient slots */ + int rcpt_limit; /* available recipient slots */ +}; + +struct QMGR_PEER { + QMGR_JOB *job; /* job handling this peer */ + QMGR_QUEUE *queue; /* queue corresponding with this peer */ + int refcount; /* peer entries */ + QMGR_ENTRY_LIST entry_list; /* todo message entries queued for this peer */ + QMGR_PEER_LIST peers; /* neighbor linkage */ +}; + +extern QMGR_ENTRY *qmgr_job_entry_select(QMGR_TRANSPORT *); +extern QMGR_PEER *qmgr_peer_select(QMGR_JOB *); + +extern QMGR_JOB *qmgr_job_obtain(QMGR_MESSAGE *, QMGR_TRANSPORT *); +extern void qmgr_job_free(QMGR_JOB *); +extern void qmgr_job_move_limits(QMGR_JOB *); + +extern QMGR_PEER *qmgr_peer_create(QMGR_JOB *, QMGR_QUEUE *); +extern QMGR_PEER *qmgr_peer_find(QMGR_JOB *, QMGR_QUEUE *); +extern void qmgr_peer_free(QMGR_PEER *); + + /* + * qmgr_defer.c + */ +extern void qmgr_defer_transport(QMGR_TRANSPORT *, const char *); +extern void qmgr_defer_todo(QMGR_QUEUE *, const char *); +extern void qmgr_defer_recipient(QMGR_MESSAGE *, const char *, const char *); + + /* + * qmgr_bounce.c + */ +extern void qmgr_bounce_recipient(QMGR_MESSAGE *, QMGR_RCPT *, const char *,...); + + /* + * qmgr_deliver.c + */ +extern int qmgr_deliver_concurrency; +extern void qmgr_deliver(QMGR_TRANSPORT *, VSTREAM *); + + /* + * qmgr_active.c + */ +extern void qmgr_active_feed(QMGR_SCAN *, const char *); +extern void qmgr_active_drain(void); +extern void qmgr_active_done(QMGR_MESSAGE *); + + /* + * qmgr_move.c + */ +extern void qmgr_move(const char *, const char *, time_t); + + /* + * qmgr_enable.c + */ +extern void qmgr_enable_all(void); +extern void qmgr_enable_transport(QMGR_TRANSPORT *); +extern void qmgr_enable_queue(QMGR_QUEUE *); + + /* + * Queue scan context. + */ +struct QMGR_SCAN { + char *queue; /* queue name */ + int flags; /* private, this run */ + int nflags; /* private, next run */ + struct SCAN_DIR *handle; /* scan */ +}; + + /* + * Flags that control queue scans or destination selection. These are + * similar to the QMGR_REQ_XXX request codes. + */ +#define QMGR_SCAN_START (1<<0) /* start now/restart when done */ +#define QMGR_SCAN_ALL (1<<1) /* all queue file time stamps */ +#define QMGR_FLUSH_DEAD (1<<2) /* all sites, all transports */ + + /* + * qmgr_scan.c + */ +extern QMGR_SCAN *qmgr_scan_create(const char *); +extern void qmgr_scan_request(QMGR_SCAN *, int); +extern char *qmgr_scan_next(QMGR_SCAN *); + +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ diff --git a/postfix/nqmgr/qmgr_active.c b/postfix/nqmgr/qmgr_active.c new file mode 100644 index 000000000..9ba5a7dab --- /dev/null +++ b/postfix/nqmgr/qmgr_active.c @@ -0,0 +1,392 @@ +/*++ +/* NAME +/* qmgr_active 3 +/* SUMMARY +/* active queue management +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* void qmgr_active_feed(scan_info, queue_id) +/* QMGR_SCAN *scan_info; +/* const char *queue_id; +/* +/* void qmgr_active_drain() +/* +/* int qmgr_active_done(message) +/* QMGR_MESSAGE *message; +/* DESCRIPTION +/* These functions maintain the active message queue: the set +/* of messages that the queue manager is actually working on. +/* The active queue is limited in size. Messages are drained +/* from the active queue by allocating a delivery process and +/* by delivering mail via that process. Messages leak into the +/* active queue only when the active queue is small enough. +/* Damaged message files are saved to the "corrupt" directory. +/* +/* qmgr_active_feed() inserts the named message file into +/* the active queue. Message files with the wrong name or +/* with other wrong properties are skipped but not removed. +/* The following queue flags are recognized, other flags being +/* ignored: +/* .IP QMGR_SCAN_ALL +/* Examine all queue files. Normally, deferred queue files with +/* future time stamps are ignored, and incoming queue files with +/* future time stamps are frowned upon. +/* .PP +/* qmgr_active_drain() allocates one delivery process. +/* Process allocation is asynchronous. Once the delivery +/* process is available, an attempt is made to deliver +/* a message via it. Message delivery is asynchronous, too. +/* +/* qmgr_active_done() deals with a message after delivery +/* has been tried for all in-core recipients. If the message +/* was bounced, a bounce message is sent to the sender, or +/* to the Errors-To: address if one was specified. +/* If there are more on-file recipients, a new batch of +/* in-core recipients is read from the queue file. Otherwise, +/* if a delivery agent marked the queue file as corrupt, +/* the queue file is moved to the "corrupt" queue (surprise); +/* if at least one delivery failed, the message is moved +/* to the deferred queue. The time stamps of a deferred queue +/* file are set to the nearest wakeup time of its recipient +/* sites (if delivery failed due to a problem with a next-hop +/* host), are set into the future by the amount of time the +/* message was queued (per-message exponential backoff), or are set +/* into the future by a minimal backoff time, whichever is more. +/* The minimal_backoff_time parameter specifies the minimal +/* amount of time between delivery attempts; maximal_backoff_time +/* specifies an upper limit. +/* DIAGNOSTICS +/* Fatal: queue file access failures, out of memory. +/* Panic: interface violations, internal consistency errors. +/* Warnings: corrupt message file. A corrupt message is saved +/* to the "corrupt" queue for further inspection. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef S_IRWXU /* What? no POSIX system? */ +#define S_IRWXU 0700 +#endif + +/* Utility library. */ + +#include +#include +#include +#include + +/* Global library. */ + +#include +#include +#include +#include +#include +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_active_corrupt - move corrupted file out of the way */ + +static void qmgr_active_corrupt(const char *queue_id) +{ + char *myname = "qmgr_active_corrupt"; + + if (mail_queue_rename(queue_id, MAIL_QUEUE_ACTIVE, MAIL_QUEUE_CORRUPT)) { + if (errno != ENOENT) + msg_fatal("%s: save corrupt file queue %s id %s: %m", + myname, MAIL_QUEUE_ACTIVE, queue_id); + msg_warn("%s: save corrupt file queue %s id %s: %m", + myname, MAIL_QUEUE_ACTIVE, queue_id); + } else { + msg_warn("corrupt file queue %s id %s", MAIL_QUEUE_ACTIVE, queue_id); + } +} + +/* qmgr_active_defer - defer queue file */ + +static void qmgr_active_defer(const char *queue_name, const char *queue_id, + int delay) +{ + char *myname = "qmgr_active_defer"; + const char *path; + struct utimbuf tbuf; + + if (msg_verbose) + msg_info("wakeup %s after %ld secs", queue_id, (long) delay); + + tbuf.actime = tbuf.modtime = event_time() + delay; + path = mail_queue_path((VSTRING *) 0, queue_name, queue_id); + if (utime(path, &tbuf) < 0) + msg_fatal("%s: update %s time stamps: %m", myname, path); + if (mail_queue_rename(queue_id, queue_name, MAIL_QUEUE_DEFERRED)) { + if (errno != ENOENT) + msg_fatal("%s: rename %s from %s to %s: %m", myname, + queue_id, queue_name, MAIL_QUEUE_DEFERRED); + msg_warn("%s: rename %s from %s to %s: %m", myname, + queue_id, queue_name, MAIL_QUEUE_DEFERRED); + } else if (msg_verbose) { + msg_info("%s: defer %s", myname, queue_id); + } +} + +/* qmgr_active_feed - feed one message into active queue */ + +void qmgr_active_feed(QMGR_SCAN *scan_info, const char *queue_id) +{ + char *myname = "qmgr_active_feed"; + QMGR_MESSAGE *message; + struct stat st; + const char *path; + + if (strcmp(scan_info->queue, MAIL_QUEUE_ACTIVE) == 0) + msg_panic("%s: bad queue %s", myname, scan_info->queue); + if (msg_verbose) + msg_info("%s: queue %s", myname, scan_info->queue); + + /* + * Make sure this is something we are willing to open. + */ + if (mail_open_ok(scan_info->queue, queue_id, &st, &path) == MAIL_OPEN_NO) + return; + + if (msg_verbose) + msg_info("%s: %s", myname, path); + + /* + * Skip files that have time stamps into the future. They need to cool + * down. Incoming and deferred files can have future time stamps. + */ + if ((scan_info->flags & QMGR_SCAN_ALL) == 0 + && st.st_mtime > time((time_t *) 0) + 1) { + if (msg_verbose) + msg_info("%s: skip %s (%ld seconds)", myname, queue_id, + (long) (st.st_mtime - event_time())); + return; + } + + /* + * Move the message to the active queue. File access errors are fatal. + */ + if (mail_queue_rename(queue_id, scan_info->queue, MAIL_QUEUE_ACTIVE)) { + if (errno != ENOENT) + msg_fatal("%s: %s: rename from %s to %s: %m", myname, + queue_id, scan_info->queue, MAIL_QUEUE_ACTIVE); + msg_warn("%s: %s: rename from %s to %s: %m", myname, + queue_id, scan_info->queue, MAIL_QUEUE_ACTIVE); + return; + } + + /* + * Extract envelope information: sender and recipients. At this point, + * mail addresses have been processed by the cleanup service so they + * should be in canonical form. Generate requests to deliver this + * message. + * + * Throwing away queue files seems bad, especially when they made it this + * far into the mail system. Therefore we save bad files to a separate + * directory for further inspection. + * + * After queue manager restart it is possible that a queue file is still + * being delivered. In that case (the file is locked), defer delivery by + * a minimal amount of time. + */ + if ((message = qmgr_message_alloc(MAIL_QUEUE_ACTIVE, queue_id, + scan_info->flags)) == 0) { + qmgr_active_corrupt(queue_id); + } else if (message == QMGR_MESSAGE_LOCKED) { + qmgr_active_defer(MAIL_QUEUE_ACTIVE, queue_id, var_min_backoff_time); + } else { + + /* + * Special case if all recipients were already delivered. Send any + * bounces and clean up. + */ + if (message->refcount == 0) + qmgr_active_done(message); + } +} + +/* qmgr_active_done - dispose of message after recipients have been tried */ + +void qmgr_active_done(QMGR_MESSAGE *message) +{ + char *myname = "qmgr_active_done"; + struct stat st; + const char *path; + int delay; + + if (msg_verbose) + msg_info("%s: %s", myname, message->queue_id); + + /* + * During a previous iteration, an attempt to bounce this message may + * have failed, so there may still be a bounce log lying around. XXX By + * groping around in the bounce queue, we're trespassing on the bounce + * service's territory. But doing so is more robust than depending on the + * bounce daemon to do the lookup for us, and for us to do the deleting + * after we have received a successful status from the bounce service. + * The bounce queue directory blocks are most likely in memory anyway. If + * these lookups become a performance problem we will have to build an + * in-core cache into the bounce daemon. + * + * Don't bounce when the bounce log is empty. The bounce process obviously + * failed, and the delivery agent will have requested that the message be + * deferred. + */ + if (stat(mail_queue_path((VSTRING *) 0, MAIL_QUEUE_BOUNCE, message->queue_id), &st) == 0) { + if (st.st_size == 0) { + if (mail_queue_remove(MAIL_QUEUE_BOUNCE, message->queue_id)) + msg_fatal("remove %s %s: %m", + MAIL_QUEUE_BOUNCE, message->queue_id); + } else { + if (msg_verbose) + msg_info("%s: bounce %s", myname, message->queue_id); + message->flags |= bounce_flush(BOUNCE_FLAG_KEEP, + message->queue_name, + message->queue_id, + message->errors_to); + } + } + + /* + * A delivery agent marks a queue file as corrupt by changing its + * attributes, and by pretending that delivery was deferred. + */ + if (message->flags + && !mail_open_ok(MAIL_QUEUE_ACTIVE, message->queue_id, &st, &path)) { + qmgr_active_corrupt(message->queue_id); + qmgr_message_free(message); + return; + } + + /* + * If we did not read all recipients from this file, go read some more, + * but remember whether some recipients have to be tried again. + * + * Throwing away queue files seems bad, especially when they made it this + * far into the mail system. Therefore we save bad files to a separate + * directory for further inspection by a human being. + */ + if (message->rcpt_offset > 0) { + if (qmgr_message_realloc(message) == 0) { + qmgr_active_corrupt(message->queue_id); + qmgr_message_free(message); + } else { + if (message->refcount == 0) + qmgr_active_done(message); /* recurse for consistency */ + } + return; + } + + /* + * If we get to this point we have tried all recipients for this message. + * If the message is too old, try to bounce it. + */ +#define HOUR 3600 +#define DAY 86400 + + if (message->flags) { + if (event_time() > message->arrival_time + var_max_queue_time * DAY) { + if (msg_verbose) + msg_info("%s: too old, bouncing %s", myname, message->queue_id); + message->flags = defer_flush(BOUNCE_FLAG_KEEP, + message->queue_name, + message->queue_id, + message->errors_to); + } else if (message->warn_time > 0 + && event_time() > message->warn_time) { + if (msg_verbose) + msg_info("%s: sending defer warning for %s", myname, message->queue_id); + if (defer_warn(BOUNCE_FLAG_KEEP, + message->queue_name, + message->queue_id, + message->errors_to) == 0) { + qmgr_message_update_warn(message); + } + } + } + + /* + * Some recipients need to be tried again. Move the queue file time + * stamps into the future by the amount of time that the message is + * delayed, and move the message to the deferred queue. Impose minimal + * and maximal backoff times. + * + * Since we look at actual time in queue, not time since last delivery + * attempt, backoff times will be distributed. However, we can still see + * spikes in delivery activity because the interval between deferred + * queue scans is finite. + */ + if (message->flags) { + if (message->arrival_time > 0) { + delay = event_time() - message->arrival_time; + if (delay > var_max_backoff_time) + delay = var_max_backoff_time; + if (delay < var_min_backoff_time) + delay = var_min_backoff_time; + } else { + delay = var_min_backoff_time; + } + qmgr_active_defer(message->queue_name, message->queue_id, delay); + } + + /* + * All recipients done. Remove the queue file. + */ + else { + if (mail_queue_remove(message->queue_name, message->queue_id)) { + if (errno != ENOENT) + msg_fatal("%s: remove %s from %s: %m", myname, + message->queue_id, message->queue_name); + msg_warn("%s: remove %s from %s: %m", myname, + message->queue_id, message->queue_name); + } else if (msg_verbose) { + msg_info("%s: remove %s", myname, message->queue_id); + } + } + + /* + * Finally, delete the in-core message structure. + */ + qmgr_message_free(message); +} + +/* qmgr_active_drain - drain active queue by allocating a delivery process */ + +void qmgr_active_drain(void) +{ + QMGR_TRANSPORT *transport; + + /* + * Use round-robin search to find a transport with pending mail. Allocate + * a delivery process. The process allocation completes asynchronously. + */ + if ((transport = qmgr_transport_select()) != 0) { + if (msg_verbose) + msg_info("qmgr_active_drain: allocate %s", transport->name); + qmgr_transport_alloc(transport, qmgr_deliver); + } +} diff --git a/postfix/nqmgr/qmgr_bounce.c b/postfix/nqmgr/qmgr_bounce.c new file mode 100644 index 000000000..e96126266 --- /dev/null +++ b/postfix/nqmgr/qmgr_bounce.c @@ -0,0 +1,74 @@ +/*++ +/* NAME +/* qmgr_bounce +/* SUMMARY +/* deal with mail that will not be delivered +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_QUEUE *qmgr_bounce_recipient(message, recipient, format, ...) +/* QMGR_MESSAGE *message; +/* QMGR_RCPT *recipient; +/* const char *format; +/* DESCRIPTION +/* qmgr_bounce_recipient() produces a bounce log record. +/* Once the bounce record is written successfully, the recipient +/* is marked as done. When the bounce record cannot be written, +/* the message structure is updated to reflect that the mail is +/* deferred. +/* +/* Arguments: +/* .IP message +/* Open queue file with the message being bounced. +/* .IP recipient +/* The recipient that will not be delivered. +/* .IP format +/* Free-format text that describes why delivery will not happen. +/* DIAGNOSTICS +/* Panic: consistency check failure. Fatal: out of memory. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include + +/* Utility library. */ + +/* Global library. */ + +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_bounce_recipient - bounce one message recipient */ + +void qmgr_bounce_recipient(QMGR_MESSAGE *message, QMGR_RCPT * recipient, + const char *format,...) +{ + va_list ap; + int status; + + va_start(ap, format); + status = vbounce_append(BOUNCE_FLAG_KEEP, message->queue_id, + recipient->address, "none", + message->arrival_time, format, ap); + va_end(ap); + + if (status == 0) + deliver_completed(message->fp, recipient->offset); + else + message->flags |= status; +} diff --git a/postfix/nqmgr/qmgr_defer.c b/postfix/nqmgr/qmgr_defer.c new file mode 100644 index 000000000..26560347c --- /dev/null +++ b/postfix/nqmgr/qmgr_defer.c @@ -0,0 +1,159 @@ +/*++ +/* NAME +/* qmgr_defer +/* SUMMARY +/* deal with mail that must be delivered later +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* void qmgr_defer_recipient(message, address, reason) +/* QMGR_MESSAGE *message; +/* const char *address; +/* const char *reason; +/* +/* void qmgr_defer_todo(queue, reason) +/* QMGR_QUEUE *queue; +/* const char *reason; +/* +/* QMGR_QUEUE *qmgr_defer_transport(transport, reason) +/* QMGR_TRANSPORT *transport; +/* const char *reason; +/* DESCRIPTION +/* qmgr_defer_recipient() defers delivery of the named message to +/* the named recipient. It updates the message structure and writes +/* a log entry. +/* +/* qmgr_defer_todo() iterates over all "todo" deliveries queued for +/* the named site, and calls qmgr_defer_recipient() for each recipient +/* found. Side effects caused by qmgr_entry_done(), qmgr_queue_done(), +/* and by qmgr_active_done(): in-core queue entries will disappear, +/* in-core queues may disappear, in-core and on-disk messages may +/* disappear, bounces may be sent, new in-core queues, queue entries +/* and recipients may appear. +/* +/* qmgr_defer_transport() calls qmgr_defer_todo() for each queue +/* that depends on the named transport. See there for side effects. +/* +/* Arguments: +/* .IP recipient +/* A recipient address; used for logging purposes, and for updating +/* the message-specific \fIdefer\fR log. +/* .IP queue +/* Specifies a queue with delivery requests for a specific next-hop +/* host (or local user). +/* .IP transport +/* Specifies a message delivery transport. +/* .IP reason +/* Free-format text that describes why delivery is deferred; this +/* used for logging purposes, and for updating the message-specific +/* \fIdefer\fR log. +/* BUGS +/* The side effects of calling this routine are quite dramatic. +/* DIAGNOSTICS +/* Panic: consistency check failure. Fatal: out of memory. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include +#include + +/* Global library. */ + +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_defer_transport - defer todo entries for named transport */ + +void qmgr_defer_transport(QMGR_TRANSPORT *transport, const char *reason) +{ + char *myname = "qmgr_defer_transport"; + QMGR_QUEUE *queue; + QMGR_QUEUE *next; + + /* + * Sanity checks. + */ + if (reason == 0) + msg_panic("%s: null reason", myname); + if (msg_verbose) + msg_info("defer transport %s: %s", transport->name, reason); + + /* + * Proceed carefully. Queues may disappear as a side effect. + */ + for (queue = transport->queue_list.next; queue; queue = next) { + next = queue->peers.next; + qmgr_defer_todo(queue, reason); + } +} + +/* qmgr_defer_todo - defer all todo queue entries for specific site */ + +void qmgr_defer_todo(QMGR_QUEUE *queue, const char *reason) +{ + char *myname = "qmgr_defer_todo"; + QMGR_ENTRY *entry; + QMGR_ENTRY *next; + QMGR_MESSAGE *message; + QMGR_RCPT *recipient; + int nrcpt; + + /* + * Sanity checks. + */ + if (reason == 0) + msg_panic("%s: null reason", myname); + if (msg_verbose) + msg_info("defer site %s: %s", queue->name, reason); + + /* + * Proceed carefully. Queue entries will disappear as a side effect. + */ + for (entry = queue->todo.next; entry != 0; entry = next) { + next = entry->queue_peers.next; + message = entry->message; + for (nrcpt = 0; nrcpt < entry->rcpt_list.len; nrcpt++) { + recipient = entry->rcpt_list.info + nrcpt; + qmgr_defer_recipient(message, recipient->address, reason); + } + qmgr_entry_done(entry, QMGR_QUEUE_TODO); + } +} + +/* qmgr_defer_recipient - defer delivery of specific recipient */ + +void qmgr_defer_recipient(QMGR_MESSAGE *message, const char *address, + const char *reason) +{ + char *myname = "qmgr_defer_recipient"; + + /* + * Sanity checks. + */ + if (reason == 0) + msg_panic("%s: reason 0", myname); + + /* + * Update the message structure and log the message disposition. + */ + message->flags |= defer_append(BOUNCE_FLAG_KEEP, message->queue_id, + address, "none", message->arrival_time, + "%s", reason); +} diff --git a/postfix/nqmgr/qmgr_deliver.c b/postfix/nqmgr/qmgr_deliver.c new file mode 100644 index 000000000..60ad0f6a5 --- /dev/null +++ b/postfix/nqmgr/qmgr_deliver.c @@ -0,0 +1,292 @@ +/*++ +/* NAME +/* qmgr_deliver 3 +/* SUMMARY +/* deliver one pe-site queue entry to that site +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* int qmgr_deliver_concurrency; +/* +/* int qmgr_deliver(transport, fp) +/* QMGR_TRANSPORT *transport; +/* VSTREAM *fp; +/* DESCRIPTION +/* This module implements the client side of the `queue manager +/* to delivery agent' protocol. The queue manager uses +/* asynchronous I/O so that it can drive multiple delivery +/* agents in parallel. Depending on the outcome of a delivery +/* attempt, the status of messages, queues and transports is +/* updated. +/* +/* qmgr_deliver_concurrency is a global counter that says how +/* many delivery processes are in use. This can be used, for +/* example, to control the size of the `active' message queue. +/* +/* qmgr_deliver() executes when a delivery process announces its +/* availability for the named transport. It arranges for delivery +/* of a suitable queue entry. The \fIfp\fR argument specifies a +/* stream that is connected to the delivery process. Upon completion +/* of delivery (successful or not), the stream is closed, so that the +/* delivery process is released. +/* DIAGNOSTICS +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include +#include +#include + +/* Global library. */ + +#include +#include +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +int qmgr_deliver_concurrency; + + /* + * Message delivery status codes. + */ +#define DELIVER_STAT_OK 0 /* all recipients delivered */ +#define DELIVER_STAT_DEFER 1 /* try some recipients later */ +#define DELIVER_STAT_CRASH 2 /* mailer internal problem */ + +/* qmgr_deliver_initial_reply - retrieve initial delivery process response */ + +static int qmgr_deliver_initial_reply(VSTREAM *stream) +{ + int stat; + + if (peekfd(vstream_fileno(stream)) < 0) { + msg_warn("%s: premature disconnect", VSTREAM_PATH(stream)); + return (DELIVER_STAT_CRASH); + } else if (mail_scan(stream, "%d", &stat) != 1) { + msg_warn("%s: malformed response", VSTREAM_PATH(stream)); + return (DELIVER_STAT_CRASH); + } else { + return (stat ? DELIVER_STAT_DEFER : 0); + } +} + +/* qmgr_deliver_final_reply - retrieve final delivery process response */ + +static int qmgr_deliver_final_reply(VSTREAM *stream, VSTRING *reason) +{ + int stat; + + if (peekfd(vstream_fileno(stream)) < 0) { + msg_warn("%s: premature disconnect", VSTREAM_PATH(stream)); + return (DELIVER_STAT_CRASH); + } else if (mail_scan(stream, "%s %d", reason, &stat) != 2) { + msg_warn("%s: malformed response", VSTREAM_PATH(stream)); + return (DELIVER_STAT_CRASH); + } else { + return (stat ? DELIVER_STAT_DEFER : 0); + } +} + +/* qmgr_deliver_send_request - send delivery request to delivery process */ + +static int qmgr_deliver_send_request(QMGR_ENTRY *entry, VSTREAM *stream) +{ + QMGR_RCPT_LIST list = entry->rcpt_list; + QMGR_RCPT *recipient; + QMGR_MESSAGE *message = entry->message; + + mail_print(stream, "%s %s %ld %ld %s %s %s %s %ld", + message->queue_name, message->queue_id, + message->data_offset, message->data_size, + entry->queue->name, message->sender, + message->errors_to, message->return_receipt, + message->arrival_time); + for (recipient = list.info; recipient < list.info + list.len; recipient++) + mail_print(stream, "%ld %s", recipient->offset, recipient->address); + mail_print(stream, "%s", "0"); + if (vstream_fflush(stream) != 0) { + msg_warn("write to process (%s): %m", entry->queue->transport->name); + return (-1); + } else { + if (msg_verbose) + msg_info("qmgr_deliver: site `%s'", entry->queue->name); + return (0); + } +} + +/* qmgr_deliver_abort - transport response watchdog */ + +static void qmgr_deliver_abort(int unused_event, char *context) +{ + QMGR_ENTRY *entry = (QMGR_ENTRY *) context; + QMGR_QUEUE *queue = entry->queue; + QMGR_TRANSPORT *transport = queue->transport; + QMGR_MESSAGE *message = entry->message; + + msg_fatal("%s: timeout receiving delivery status from transport: %s", + message->queue_id, transport->name); +} + +/* qmgr_deliver_update - process delivery status report */ + +static void qmgr_deliver_update(int unused_event, char *context) +{ + QMGR_ENTRY *entry = (QMGR_ENTRY *) context; + QMGR_QUEUE *queue = entry->queue; + QMGR_TRANSPORT *transport = queue->transport; + QMGR_MESSAGE *message = entry->message; + VSTRING *reason = vstring_alloc(1); + int status; + + /* + * The message transport has responded. Stop the watchdog timer. + */ + event_cancel_timer(qmgr_deliver_abort, context); + + /* + * Retrieve the delivery agent status report. The numerical status code + * indicates if delivery should be tried again. The reason text is sent + * only when a site should be avoided for a while, so that the queue + * manager can log why it does not even try to schedule delivery to the + * affected recipients. + */ + status = qmgr_deliver_final_reply(entry->stream, reason); + + /* + * The mail delivery process failed for some reason (although delivery + * may have been successful). Back off with this transport type for a + * while. Dispose of queue entries for this transport that await + * selection (the todo lists). Stay away from queue entries that have + * been selected (the busy lists), or we would have dangling pointers. + * The queue itself won't go away before we dispose of the current queue + * entry. + */ + if (status == DELIVER_STAT_CRASH) { + message->flags |= DELIVER_STAT_DEFER; + qmgr_transport_throttle(transport, "unknown mail transport error"); + qmgr_defer_transport(transport, transport->reason); + } + + /* + * This message must be tried again. + * + * If we have a problem talking to this site, back off with this site for a + * while; dispose of queue entries for this site that await selection + * (the todo list); stay away from queue entries that have been selected + * (the busy list), or we would have dangling pointers. The queue itself + * won't go away before we dispose of the current queue entry. + */ + if (status == DELIVER_STAT_DEFER) { + message->flags |= DELIVER_STAT_DEFER; + if (VSTRING_LEN(reason)) { + qmgr_queue_throttle(queue, vstring_str(reason)); + if (queue->window == 0) + qmgr_defer_todo(queue, queue->reason); + } + } + + /* + * No problems detected. Mark the transport and queue as alive. The queue + * itself won't go away before we dispose of the current queue entry. + */ + if (status == 0) { + qmgr_transport_unthrottle(transport); + qmgr_queue_unthrottle(queue); + } + + /* + * Release the delivery process, and give some other queue entry a chance + * to be delivered. When all recipients for a message have been tried, + * decide what to do next with this message: defer, bounce, delete. + */ + event_disable_readwrite(vstream_fileno(entry->stream)); + if (vstream_fclose(entry->stream) != 0) + msg_warn("qmgr_deliver_update: close delivery stream: %m"); + entry->stream = 0; + qmgr_deliver_concurrency--; + qmgr_entry_done(entry, QMGR_QUEUE_BUSY); + vstring_free(reason); +} + +/* qmgr_deliver - deliver one per-site queue entry */ + +void qmgr_deliver(QMGR_TRANSPORT *transport, VSTREAM *stream) +{ + QMGR_ENTRY *entry; + + /* + * Find out if this delivery process is really available. Once elected, + * the delivery process is supposed to express its happiness. If there is + * a problem, wipe the pending deliveries for this transport. This + * routine runs in response to an external event, so it does not run + * while some other queue manipulation is happening. + */ + if (qmgr_deliver_initial_reply(stream) != 0) { + qmgr_transport_throttle(transport, "mail transport unavailable"); + qmgr_defer_transport(transport, transport->reason); + (void) vstream_fclose(stream); + return; + } + + /* + * Find a suitable queue entry. Things may have changed since this + * transport was allocated. If no suitable entry is found, + * unceremoniously disconnect from the delivery process. The delivery + * agent request reading routine is prepared for the queue manager to + * change its mind for no apparent reason. + */ + if ((entry = qmgr_job_entry_select(transport)) == 0) { + (void) vstream_fclose(stream); + return; + } + + /* + * Send the queue file info and recipient info to the delivery process. + * If there is a problem, wipe the pending deliveries for this transport. + * This routine runs in response to an external event, so it does not run + * while some other queue manipulation is happening. + */ + if (qmgr_deliver_send_request(entry, stream) < 0) { + qmgr_entry_unselect(entry); + qmgr_transport_throttle(transport, "mail transport unavailable"); + qmgr_defer_transport(transport, transport->reason); + /* warning: entry may be a dangling pointer here */ + (void) vstream_fclose(stream); + return; + } + + /* + * If we get this far, go wait for the delivery status report. + */ + qmgr_deliver_concurrency++; + entry->stream = stream; + event_enable_read(vstream_fileno(stream), + qmgr_deliver_update, (char *) entry); + + /* + * Guard against broken systems. + */ + event_request_timer(qmgr_deliver_abort, (char *) entry, var_daemon_timeout); +} diff --git a/postfix/nqmgr/qmgr_enable.c b/postfix/nqmgr/qmgr_enable.c new file mode 100644 index 000000000..5f0c3841b --- /dev/null +++ b/postfix/nqmgr/qmgr_enable.c @@ -0,0 +1,107 @@ +/*++ +/* NAME +/* qmgr_enable +/* SUMMARY +/* enable dead transports or sites +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* void qmgr_enable_queue(queue) +/* QMGR_QUEUE *queue; +/* +/* QMGR_QUEUE *qmgr_enable_transport(transport) +/* QMGR_TRANSPORT *transport; +/* +/* void qmgr_enable_all(void) +/* DESCRIPTION +/* This module purges dead in-core state information, effectively +/* re-enabling delivery. +/* +/* qmgr_enable_queue() enables deliveries to the named dead site. +/* Empty queues are destroyed. The existed solely to indicate that +/* a site is dead. +/* +/* qmgr_enable_transport() enables deliveries via the specified +/* transport, and calls qmgr_enable_queue() for each destination +/* on that transport. Empty queues are destroyed. +/* +/* qmgr_enable_all() enables all transports and queues. +/* See above for the side effects caused by doing this. +/* BUGS +/* The side effects of calling this module can be quite dramatic. +/* DIAGNOSTICS +/* Panic: consistency check failure. Fatal: out of memory. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_enable_all - enable transports and queues */ + +void qmgr_enable_all(void) +{ + QMGR_TRANSPORT *xport; + + if (msg_verbose) + msg_info("qmgr_enable_all"); + + /* + * The number of transports does not change as a side effect, so this can + * be a straightforward loop. + */ + for (xport = qmgr_transport_list.next; xport; xport = xport->peers.next) + qmgr_enable_transport(xport); +} + +/* qmgr_enable_transport - defer todo entries for named transport */ + +void qmgr_enable_transport(QMGR_TRANSPORT *transport) +{ + QMGR_QUEUE *queue; + QMGR_QUEUE *next; + + /* + * Proceed carefully. Queues may disappear as a side effect. + */ + if (transport->flags & QMGR_TRANSPORT_STAT_DEAD) { + if (msg_verbose) + msg_info("enable transport %s", transport->name); + qmgr_transport_unthrottle(transport); + } + for (queue = transport->queue_list.next; queue; queue = next) { + next = queue->peers.next; + qmgr_enable_queue(queue); + } +} + +/* qmgr_enable_queue - enable and possibly delete queue */ + +void qmgr_enable_queue(QMGR_QUEUE *queue) +{ + if (queue->window == 0) { + if (msg_verbose) + msg_info("enable site %s/%s", queue->transport->name, queue->name); + qmgr_queue_unthrottle(queue); + } + if (queue->todo.next == 0 && queue->busy.next == 0) + qmgr_queue_done(queue); +} diff --git a/postfix/nqmgr/qmgr_entry.c b/postfix/nqmgr/qmgr_entry.c new file mode 100644 index 000000000..57b4631ec --- /dev/null +++ b/postfix/nqmgr/qmgr_entry.c @@ -0,0 +1,234 @@ +/*++ +/* NAME +/* qmgr_entry 3 +/* SUMMARY +/* per-site queue entries +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_ENTRY *qmgr_entry_create(peer, message) +/* QMGR_PEER *peer; +/* QMGR_MESSAGE *message; +/* +/* void qmgr_entry_done(entry, which) +/* QMGR_ENTRY *entry; +/* int which; +/* +/* QMGR_ENTRY *qmgr_entry_select(queue) +/* QMGR_QUEUE *queue; +/* +/* void qmgr_entry_unselect(queue, entry) +/* QMGR_QUEUE *queue; +/* QMGR_ENTRY *entry; +/* DESCRIPTION +/* These routines add/delete/manipulate per-site message +/* delivery requests. +/* +/* qmgr_entry_create() creates an entry for the named peer and message, +/* and appends the entry to the peer's list and its queue's todo list. +/* Filling in and cleaning up the recipients is the responsibility +/* of the caller. +/* +/* qmgr_entry_done() discards a per-site queue entry. The +/* \fIwhich\fR argument is either QMGR_QUEUE_BUSY for an entry +/* of the site's `busy' list (i.e. queue entries that have been +/* selected for actual delivery), or QMGR_QUEUE_TODO for an entry +/* of the site's `todo' list (i.e. queue entries awaiting selection +/* for actual delivery). +/* +/* qmgr_entry_done() discards its peer structure when the peer +/* is not referenced anymore. +/* +/* qmgr_entry_done() triggers cleanup of the per-site queue when +/* the site has no pending deliveries, and the site is either +/* alive, or the site is dead and the number of in-core queues +/* exceeds a configurable limit (see qmgr_queue_done()). +/* +/* qmgr_entry_done() triggers special action when the last in-core +/* queue entry for a message is done with: either read more +/* recipients from the queue file, delete the queue file, or move +/* the queue file to the deferred queue; send bounce reports to the +/* message originator (see qmgr_active_done()). +/* +/* qmgr_entry_select() selects first entry from the named +/* per-site queue's `todo' list for actual delivery. The entry is +/* moved to the queue's `busy' list: the list of messages being +/* delivered. The entry is also removed from its peer list. +/* +/* qmgr_entry_unselect() takes the named entry off the named +/* per-site queue's `busy' list and moves it to the queue's +/* `todo' list. The entry is also appended to its peer list again. +/* DIAGNOSTICS +/* Panic: interface violations, internal inconsistencies. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include + +/* Global library. */ + +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_entry_select - select queue entry for delivery */ + +QMGR_ENTRY *qmgr_entry_select(QMGR_PEER *peer) +{ + QMGR_ENTRY *entry; + QMGR_QUEUE *queue; + + if ((entry = peer->entry_list.next) != 0) { + queue = entry->queue; + QMGR_LIST_UNLINK(queue->todo, QMGR_ENTRY *, entry, queue_peers); + queue->todo_refcount--; + QMGR_LIST_APPEND(queue->busy, entry, queue_peers); + queue->busy_refcount++; + QMGR_LIST_UNLINK(peer->entry_list, QMGR_ENTRY *, entry, peer_peers); + peer->job->selected_entries++; + } + return (entry); +} + +/* qmgr_entry_unselect - unselect queue entry for delivery */ + +void qmgr_entry_unselect(QMGR_ENTRY *entry) +{ + QMGR_PEER *peer = entry->peer; + QMGR_QUEUE *queue = entry->queue; + + QMGR_LIST_UNLINK(queue->busy, QMGR_ENTRY *, entry, queue_peers); + queue->busy_refcount--; + QMGR_LIST_APPEND(queue->todo, entry, queue_peers); + queue->todo_refcount++; + QMGR_LIST_APPEND(peer->entry_list, entry, peer_peers); + peer->job->selected_entries--; +} + +/* qmgr_entry_done - dispose of queue entry */ + +void qmgr_entry_done(QMGR_ENTRY *entry, int which) +{ + QMGR_QUEUE *queue = entry->queue; + QMGR_MESSAGE *message = entry->message; + QMGR_PEER *peer = entry->peer; + QMGR_JOB *sponsor, *job = peer->job; + + /* + * Take this entry off the in-core queue. + */ + if (entry->stream != 0) + msg_panic("qmgr_entry_done: file is open"); + if (which == QMGR_QUEUE_BUSY) { + QMGR_LIST_UNLINK(queue->busy, QMGR_ENTRY *, entry, queue_peers); + queue->busy_refcount--; + } else if (which == QMGR_QUEUE_TODO) { + QMGR_LIST_UNLINK(peer->entry_list, QMGR_ENTRY *, entry, peer_peers); + QMGR_LIST_UNLINK(queue->todo, QMGR_ENTRY *, entry, queue_peers); + queue->todo_refcount--; + } else { + msg_panic("qmgr_entry_done: bad queue spec: %d", which); + } + + /* + * Decrease the in-core recipient counts and free the recipient list and + * the structure itself. + */ + job->rcpt_count -= entry->rcpt_list.len; + message->rcpt_count -= entry->rcpt_list.len; + qmgr_recipient_count -= entry->rcpt_list.len; + qmgr_rcpt_list_free(&entry->rcpt_list); + myfree((char *) entry); + + /* + * Make sure that transport of any retired or finishing jobs that + * donated recipient slots to this job's message gets them back first. + * Then, if possible, pass the remaining unused recipient slots to the + * next job in the queue. + */ + for (sponsor = message->job_list.next; sponsor; sponsor = sponsor->message_peers.next) { + if (sponsor->rcpt_count >= sponsor->rcpt_limit || sponsor == job) + continue; + if (sponsor->stack_level < 0 || message->rcpt_offset == 0) + qmgr_job_move_limits(sponsor); + } + if (message->rcpt_offset == 0) { + qmgr_job_move_limits(job); + } + + /* + * When there are no more entries for this peer, discard the peer structure. + */ + peer->refcount--; + if (peer->refcount == 0) + qmgr_peer_free(peer); + + /* + * When the in-core queue for this site is empty and when this site is + * not dead, discard the in-core queue. When this site is dead, but the + * number of in-core queues exceeds some threshold, get rid of this + * in-core queue anyway, in order to avoid running out of memory. + */ + if (queue->todo.next == 0 && queue->busy.next == 0) { + if (queue->window == 0 && qmgr_queue_count > 2 * var_qmgr_rcpt_limit) + qmgr_queue_unthrottle(queue); + if (queue->window > 0) + qmgr_queue_done(queue); + } + + /* + * Update the in-core message reference count. When the in-core message + * structure has no more references, dispose of the message. + */ + message->refcount--; + if (message->refcount == 0) + qmgr_active_done(message); +} + +/* qmgr_entry_create - create queue todo entry */ + +QMGR_ENTRY *qmgr_entry_create(QMGR_PEER *peer, QMGR_MESSAGE *message) +{ + QMGR_ENTRY *entry; + QMGR_QUEUE *queue = peer->queue; + + /* + * Sanity check. + */ + if (queue->window == 0) + msg_panic("qmgr_entry_create: dead queue: %s", queue->name); + + entry = (QMGR_ENTRY *) mymalloc(sizeof(QMGR_ENTRY)); + entry->stream = 0; + entry->message = message; + qmgr_rcpt_list_init(&entry->rcpt_list); + message->refcount++; + entry->peer = peer; + QMGR_LIST_APPEND(peer->entry_list, entry, peer_peers); + peer->refcount++ ; + entry->queue = queue; + QMGR_LIST_APPEND(queue->todo, entry, queue_peers); + queue->todo_refcount++; + return (entry); +} diff --git a/postfix/nqmgr/qmgr_job.c b/postfix/nqmgr/qmgr_job.c new file mode 100644 index 000000000..46bb8ee5d --- /dev/null +++ b/postfix/nqmgr/qmgr_job.c @@ -0,0 +1,729 @@ +/*++ +/* NAME +/* qmgr_job 3 +/* SUMMARY +/* per-transport jobs +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_JOB *qmgr_job_obtain(message, transport) +/* QMGR_MESSAGE *message; +/* QMGR_TRANSPORT *transport; +/* +/* void qmgr_job_free(job) +/* QMGR_JOB *job; +/* +/* void qmgr_job_move_limits(job) +/* QMGR_JOB *job; +/* +/* QMGR_ENTRY *qmgr_job_entry_select(transport) +/* QMGR_TRANSPORT *transport; +/* DESCRIPTION +/* These routines add/delete/manipulate per-transport jobs. +/* Each job corresponds to a specific transport and message. +/* Each job has a peer list containing all pending delivery +/* requests for that message. +/* +/* qmgr_job_obtain() finds an existing job for named message and +/* transport combination. New empty job is created if no existing can +/* be found.In either case, the job is prepared for assignement of +/* (more) message recipients +/* +/* qmgr_job_free() disposes of a per-transport job after all +/* its entries have been taken care of. It is an error to dispose +/* of a job that is still in use. +/* +/* qmgr_job_entry_select() attempts to find the next entry suitable +/* for delivery. The job preempting algorithm is also exercised. +/* If necessary, an attempt to read more recipients into core is made. +/* This can result in creation of more job, queue and entry structures. +/* +/* qmgr_job_move_limits() takes care of proper distribution of the +/* per-transport recipients limit among the per-transport jobs. +/* Should be called whenever a job's recipient slot becomes available. +/* DIAGNOSTICS +/* Panic: consistency check failure. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Patrik Rak +/* patrik@raxoft.cz +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include +#include +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* Forward declarations */ + +static void qmgr_job_pop(QMGR_JOB *); + +/* Helper macros */ + +#define HAS_ENTRIES(job) ((job)->selected_entries < (job)->read_entries) + +/* + * The MIN_ENTRIES macro may underestimate a lot but we can't use message->rcpt_unread + * because we don't know if all those unread recipients go to our transport yet. + */ + +#define MIN_ENTRIES(job) ((job)->read_entries) +#define MAX_ENTRIES(job) ((job)->read_entries + (job)->message->rcpt_unread) + +#define RESET_CANDIDATE_CACHE(transport) do { \ + (transport)->candidate_cache_time = (time_t) 0; \ + (transport)->candidate_cache = 0; \ + } while(0) + +/* qmgr_job_create - create and initialize message job structure */ + +static QMGR_JOB *qmgr_job_create(QMGR_MESSAGE *message, QMGR_TRANSPORT *transport) +{ + QMGR_JOB *job; + + job = (QMGR_JOB *) mymalloc(sizeof(QMGR_JOB)); + job->message = message; + QMGR_LIST_APPEND(message->job_list, job, message_peers); + htable_enter(transport->job_byname, message->queue_id, (char *) job); + job->transport = transport; + QMGR_LIST_INIT(job->transport_peers); + job->peer_byname = htable_create(0); + QMGR_LIST_INIT(job->peer_list); + job->stack_level = 0; + job->slots_used = 0; + job->slots_available = 0; + job->selected_entries = 0; + job->read_entries = 0; + job->rcpt_count = 0; + job->rcpt_limit = 0; + return (job); +} + +/* qmgr_job_link - append the job to the job list, according to the time it was queued */ + +static void qmgr_job_link(QMGR_JOB *job) +{ + QMGR_TRANSPORT *transport = job->transport; + QMGR_MESSAGE *message = job->message; + QMGR_JOB *prev,*next,*unread; + int delay; + + unread = transport->job_next_unread; + + /* + * This may look inefficient but under normal operation it is expected + * that the loop will stop right away, resulting in normal list append below. + * However, this code is necessary for reviving retired jobs and for jobs + * which are created long after the first chunk of recipients was read in-core + * (either of these can happen only for multi-transport messages). + * + * In case this is found unsatisfactory one day, it's possible to deploy some + * smarter technique (using some form of lookup trees perhaps). + */ + for (next = 0, prev = transport->job_list.prev; prev; + next = prev, prev = prev->transport_peers.prev) + { + delay = message->queued_time - prev->message->queued_time; + if (delay >= 0) + break; + if (unread == prev) + unread = 0; + } + + /* + * Don't link the new job in front of the first job on the job list + * if that job was already used for the regular delivery. + * This seems like a subtle difference but it helps many invariants + * used at various other places to remain true. + */ + if (prev == 0 && next != 0 && next->slots_used != 0) { + prev = next; + next = next->transport_peers.next; + /* + * The following is not currently necessary but is done anyway + * for the sake of consistency. + */ + if (prev == transport->job_next_unread) + unread = prev; + } + + /* + * Link the job into the proper place on the job list. + */ + job->transport_peers.prev = prev; + job->transport_peers.next = next; + if (prev != 0) + prev->transport_peers.next = job; + else + transport->job_list.next = job; + if (next != 0) + next->transport_peers.prev = job; + else + transport->job_list.prev = job; + + /* + * Update the pointer to the first unread job on the job list + * and steal the unused recipient slots from the old one. + */ + if (unread == 0) { + unread = transport->job_next_unread; + transport->job_next_unread = job; + if (unread != 0) + qmgr_job_move_limits(unread); + } + + /* + * Get as much recipient slots as possible. The excess will be returned + * to the transport pool as soon as the exact amount required is known + * (which is usually after all recipients have been read in core). + */ + if (transport->rcpt_unused > 0) { + job->rcpt_limit += transport->rcpt_unused; + message->rcpt_limit += transport->rcpt_unused; + transport->rcpt_unused = 0; + } +} + +/* qmgr_job_find - lookup job associated with named message and transport */ + +static QMGR_JOB *qmgr_job_find(QMGR_MESSAGE *message, QMGR_TRANSPORT *transport) +{ + /* + * Instead of traversing the message job list, we use single per transport + * hash table. This is better (at least with respect to memory usage) + * than having single hash table (usually almost empty) for each message. + */ + return ((QMGR_JOB *) htable_find(transport->job_byname, message->queue_id)); +} + +/* qmgr_job_obtain - find/create the appropriate job and make it ready for new recipients */ + +QMGR_JOB * qmgr_job_obtain(QMGR_MESSAGE *message, QMGR_TRANSPORT *transport) +{ + QMGR_JOB *job; + + /* + * Try finding the existing job and revive it if it was already retired. + * Create the new job for this transport/message combination otherwise. + */ + if ((job = qmgr_job_find(message, transport)) != 0) { + if (job->stack_level < 0) { + job->stack_level = 0; + qmgr_job_link(job); + } + } + else { + job = qmgr_job_create(message, transport); + qmgr_job_link(job); + } + + /* + * Reset the candidate cache because of the new expected recipients. + */ + RESET_CANDIDATE_CACHE(transport); + + return(job); +} + +/* qmgr_job_move_limits - move unused recipient slots to the next job */ + +void qmgr_job_move_limits(QMGR_JOB *job) +{ + QMGR_TRANSPORT *transport = job->transport; + QMGR_MESSAGE *message = job->message; + QMGR_JOB *next = transport->job_next_unread; + int rcpt_unused, msg_rcpt_unused; + + /* + * Find next unread job on the job list if necessary. Cache it for later. + * This makes the amortized efficiency of this routine O(1) per job. + */ + if (job == next) { + for (next = next->transport_peers.next; next; next = next->transport_peers.next) + if (next->message->rcpt_offset != 0) + break; + transport->job_next_unread = next; + } + + /* + * Calculate the number of available unused slots. + */ + rcpt_unused = job->rcpt_limit - job->rcpt_count; + msg_rcpt_unused = message->rcpt_limit - message->rcpt_count; + if( msg_rcpt_unused < rcpt_unused ) + rcpt_unused = msg_rcpt_unused; + + /* + * Transfer the unused recipient slots back to the transport pool and + * to the next not-fully-read job. Job's message limits are adjusted accordingly. + */ + if (rcpt_unused > 0) { + job->rcpt_limit -= rcpt_unused; + message->rcpt_limit -= rcpt_unused; + transport->rcpt_unused += rcpt_unused; + if (next != 0 && (rcpt_unused = transport->rcpt_unused) > 0) { + next->rcpt_limit += rcpt_unused; + next->message->rcpt_limit += rcpt_unused; + transport->rcpt_unused = 0; + } + } +} + +/* qmgr_job_retire - remove the job from the job list while waiting for recipients to deliver */ + +static void qmgr_job_retire(QMGR_JOB *job) +{ + char *myname = "qmgr_job_retire"; + QMGR_TRANSPORT *transport = job->transport; + + if (msg_verbose) + msg_info("%s: %s", myname, job->message->queue_id); + + /* + * Sanity checks. + */ + if (job->stack_level != 0) + msg_panic("%s: non-zero stack level (%d)", myname, job->stack_level); + + /* + * Make sure this job is not cached as the next unread job for this transport. + * The qmgr_entry_done() will make sure that the slots donated by this job + * are moved back to the transport pool as soon as possible. + */ + qmgr_job_move_limits(job); + + /* + * Invalidate the candidate selection cache if necessary. + */ + if (job == transport->candidate_cache + || (transport->job_stack.next == 0 && job == transport->job_list.next)) + RESET_CANDIDATE_CACHE(transport); + + /* + * Remove the job from the job list and mark it as retired. + */ + QMGR_LIST_UNLINK(transport->job_list, QMGR_JOB *, job, transport_peers); + job->stack_level = -1; +} + +/* qmgr_job_free - release the job structure */ + +void qmgr_job_free(QMGR_JOB *job) +{ + char *myname = "qmgr_job_free"; + QMGR_MESSAGE *message = job->message; + QMGR_TRANSPORT *transport = job->transport; + + if (msg_verbose) + msg_info("%s: %s %s", myname, message->queue_id, transport->name); + + /* + * Sanity checks. + */ + if (job->rcpt_count) + msg_panic("%s: non-zero recipient count (%d)", myname, job->rcpt_count); + + /* + * Remove the job from the job stack if necessary. + */ + if (job->stack_level > 0) + qmgr_job_pop(job); + + /* + * Return any remaining recipient slots back to the recipient slots pool. + */ + qmgr_job_move_limits(job); + if (job->rcpt_limit) + msg_panic("%s: recipient slots leak (%d)", myname, job->rcpt_limit); + + /* + * Invalidate the candidate selection cache if necessary. + */ + if (job == transport->candidate_cache + || (transport->job_stack.next == 0 && job == transport->job_list.next)) + RESET_CANDIDATE_CACHE(transport); + + /* + * Unlink and discard the structure. Check if the job is still on the transport + * job list or if it was already retired before unlinking it. + */ + if (job->stack_level >= 0) + QMGR_LIST_UNLINK(transport->job_list, QMGR_JOB *, job, transport_peers); + QMGR_LIST_UNLINK(message->job_list, QMGR_JOB *, job, message_peers); + htable_delete(transport->job_byname, message->queue_id, (void (*) (char *)) 0) ; + htable_free(job->peer_byname, (void (*) (char *)) 0); + myfree((char *) job); +} + +/* qmgr_job_count_slots - maintain the delivery slots' counters */ + +static void qmgr_job_count_slots(QMGR_JOB *current, QMGR_JOB *job) +{ + /* + * Count the number of delivery slots used during the delivery + * of the selected job and also the number of delivery slots + * available for preemption. + * + * However, suppress any slot counting if we didn't start regular delivery + * of the selected job yet. + */ + if (job == current || job->slots_used > 0) { + job->slots_used++; + job->slots_available++; + } + + /* + * If the selected job is not the current job, its chance to be chosen by + * qmgr_job_candidate() has slightly changed. If we would like to make + * the candidate cache completely transparent, we should invalidate it now. + * + * However, this case should usually happen only at "end of current job" + * phase, when it's unlikely that the current job can be preempted + * anyway. And because it's likely to happen quite often then, we + * intentionally don't reset the cache, to safe some cycles. + * Furthermore, the cache times out every second anyway. + */ +#if 0 + if (job != current) + RESET_CANDIDATE_CACHE(job->transport); +#endif +} + +/* qmgr_job_candidate - find best job candidate for preempting given job */ + +static QMGR_JOB *qmgr_job_candidate(QMGR_JOB *current) +{ + QMGR_TRANSPORT *transport = current->transport; + QMGR_JOB *job, *best_job = 0; + float score, best_score = 0.0; + int max_slots, max_needed_entries, max_total_entries; + int delay; + time_t now = event_time(); + + /* + * Fetch the result directly from the cache if the cache is still valid. + * + * Note that we cache negative results too, so the cache must be + * invalidated by resetting the cache time, not the candidate pointer itself. + */ + if (transport->candidate_cache_time == now) + return (transport->candidate_cache); + + /* + * Estimate the minimum amount of delivery slots that can ever be accumulated + * for the given job. All jobs that won't fit into these slots are excluded + * from the candidate selection. + */ + max_slots = (MIN_ENTRIES(current) - current->selected_entries + + current->slots_available) / transport->slot_cost; + + /* + * Select the candidate with best time_since_queued/total_recipients score. + * In addition to jobs which don't meet the max_slots limit, skip also jobs + * which don't have any selectable entries at the moment. + * + * By the way, the selection is reasonably resistant to OS time warping, too. + * + * However, don't bother searching if we can't find anything suitable anyway. + */ + if (max_slots > 0) { + for (job = transport->job_list.next; job; job = job->transport_peers.next) { + if (job->stack_level != 0 || job == current) + continue; + max_total_entries = MAX_ENTRIES(job); + max_needed_entries = max_total_entries - job->selected_entries; + delay = now - job->message->queued_time + 1; + if (max_needed_entries > 0 && max_needed_entries <= max_slots) { + score = (float) delay / max_total_entries; + if (score > best_score) { + best_score = score; + best_job = job; + } + } + /* + * Stop early if the best score is as good as it can get. + */ + if (delay <= best_score) + break; + } + } + + /* + * Cache the result for later use. + */ + transport->candidate_cache = best_job; + transport->candidate_cache_time = now; + + return (best_job); +} + +/* qmgr_job_preempt - preempt large message with smaller one */ + +static QMGR_JOB * qmgr_job_preempt(QMGR_JOB *current) +{ + char *myname = "qmgr_job_preempt"; + QMGR_TRANSPORT *transport = current->transport; + QMGR_JOB *job; + int rcpt_slots; + + /* + * Suppress preempting completely if the current job is not big enough + * to accumulate even the mimimal number of slots required. + * + * Also, don't look for better job candidate if there are no available + * slots yet (the count can get negative due to the slot loans below). + */ + if (current->slots_available <= 0 + || MAX_ENTRIES(current) < transport->min_slots * transport->slot_cost) + return (current); + + /* + * Find best candidate for preempting the current job. + * + * Note that the function also takes care that the candidate fits within + * the number of delivery slots the current job can ever accumulate yet. + */ + if ((job = qmgr_job_candidate(current)) == 0) + return (current); + + /* + * Sanity checks. + */ + if (job == current) + msg_panic("%s: attempt to preempt itself", myname); + if (job->stack_level != 0) + msg_panic("%s: already on the job stack (%d)", myname, job->stack_level); + + /* + * Check if there is enough available delivery slots accumulated + * to preempt the current job. + * + * The slot loaning scheme improves the average message response time. + * Note that the loan only allows the preemption happen earlier, though. + * It doesn't affect how many slots have to be "paid" - the full number + * of slots required will have to accumulate later before next + * preemption on the same stack level can happen anyway. + */ + if (current->slots_available / transport->slot_cost + + transport->slot_loan + < (MAX_ENTRIES(job) - job->selected_entries) + * transport->slot_loan_factor / 100.0) + return (current); + + /* + * Preempt the current job. + */ + QMGR_LIST_PREPEND(transport->job_stack, job, stack_peers); + job->stack_level = current->stack_level + 1; + + /* + * Add part of extra recipient slots reserved for preempting jobs + * to the new current job if necessary. + * + * Note that transport->rcpt_unused is within <-rcpt_per_stack,0> in such case. + */ + if (job->message->rcpt_offset != 0) { + rcpt_slots = (transport->rcpt_per_stack + transport->rcpt_unused + 1) / 2; + job->rcpt_limit += rcpt_slots; + job->message->rcpt_limit += rcpt_slots; + transport->rcpt_unused -= rcpt_slots; + } + + /* + * Candidate cache must be reset because the current job has changed completely. + */ + RESET_CANDIDATE_CACHE(transport); + + if (msg_verbose) + msg_info("%s: %s by %s", myname, current->message->queue_id, + job->message->queue_id); + + return (job); +} + +/* qmgr_job_pop - remove the job from the job preemption stack */ + +static void qmgr_job_pop(QMGR_JOB *job) +{ + QMGR_TRANSPORT *transport = job->transport; + QMGR_JOB *parent; + + if (msg_verbose) + msg_info("qmgr_job_pop: %s", job->message->queue_id); + + /* + * Adjust the number of delivery slots available to preempt + * job's parent. + * + * Note that we intentionally do not adjust slots_used of the parent. + * Doing so would decrease the maximum per message inflation factor + * if the preemption appeared near the end of parent delivery. + * + * For the same reason we do not adjust parent's slots_available + * if the parent is not the original parent preempted by the + * selected job (i.e., the original parent job has already completed). + * + * The special case when the head of the job list was preempted and + * then delivered before the preempting job itself is taken care of too. + * Otherwise we would decrease available slot counter of some job that + * was not in fact preempted yet. + */ + if (((parent = job->stack_peers.next) != 0 + || ((parent = transport->job_list.next) != 0 && parent->slots_used > 0)) + && job->stack_level == parent->stack_level + 1) + parent->slots_available -= job->slots_used * transport->slot_cost; + + /* + * Invalidate the candidate selection cache if necessary. + */ + if (job == transport->job_stack.next) + RESET_CANDIDATE_CACHE(transport); + + /* + * Remove the job from the job stack and reinitialize the slot counters. + */ + QMGR_LIST_UNLINK(transport->job_stack, QMGR_JOB *, job, stack_peers); + job->stack_level = 0; + job->slots_used = 0; + job->slots_available = 0; +} + +/* qmgr_job_peer_select - select next peer suitable for delivery */ + +static QMGR_PEER *qmgr_job_peer_select(QMGR_JOB *job) +{ + QMGR_PEER *peer; + QMGR_MESSAGE *message = job->message; + + if (HAS_ENTRIES(job) && (peer = qmgr_peer_select(job)) != 0) + return (peer); + + /* + * Try reading in more recipients. Note that we do not try to read them + * as soon as possible as that would decrease the chance of per-site + * recipient grouping. We waited until reading more is really necessary. + */ + if (message->rcpt_offset != 0 && message->rcpt_limit > message->rcpt_count) { + qmgr_message_realloc(message); + if (HAS_ENTRIES(job)) + return (qmgr_peer_select(job)); + } + return (0); +} + +/* qmgr_job_entry_select - select next entry suitable for delivery */ + +QMGR_ENTRY *qmgr_job_entry_select(QMGR_TRANSPORT *transport) +{ + QMGR_JOB *job, *current, *next; + QMGR_PEER *peer; + QMGR_ENTRY *entry; + + /* + * Select the "current" job. + */ + if ((current = transport->job_stack.next) == 0 + && (current = transport->job_list.next) == 0) + return (0); + + /* + * Exercise the preempting algorithm if enabled. + * + * The slot_cost equal to 1 causes the algorithm to degenerate and is + * therefore disabled too. + */ + if (transport->slot_cost >= 2) + current = qmgr_job_preempt(current); + + /* + * Select next entry suitable for delivery. First check the stack of + * preempting jobs, then the list of all remaining jobs in FIFO order. + * + * Note that although the loops may look inefficient, they only serve as + * a recovery mechanism when an entry of the current job itself can't be + * selected due peer concurrency restrictions. In most cases some entry + * of the current job itself is selected. + * + * Note that both loops also take care of getting the "stall" current job + * (job with no entries currently available) out of the way if necessary. + * Stall jobs can appear in case of multi-transport messages whose recipients + * don't fit in-core at once. Some jobs created by such message may have + * only few recipients and would block the job queue until all other + * jobs of the message are delivered. Trying to read in more recipients + * of such jobs each selection would also break the per peer recipient + * grouping of the other jobs. That's why we retire such jobs below. + */ + for (job = transport->job_stack.next; job; job = next) { + next = job->stack_peers.next; + if ((peer = qmgr_job_peer_select(job)) != 0) { + entry = qmgr_entry_select(peer); + qmgr_job_count_slots(current, job); + + /* + * In case we selected the very last job entry, remove the job + * from the job stack and the job list right now. + * + * This action uses the assumption that once the job entry + * has been selected, it can be unselected only before the + * message ifself is deferred. Thus the job with all entries + * selected can't re-appear with more entries available for + * selection again (without reading in more entries from + * the queue file, which in turn invokes qmgr_job_obtain() + * which re-links the job back on the list if necessary). + * + * Note that qmgr_job_move_limits() transfers the recipients slots + * correctly even if the job is unlinked from the job list + * thanks to the job_next_unread caching. + */ + if (!HAS_ENTRIES(job) && job->message->rcpt_offset == 0) { + qmgr_job_pop(job); + qmgr_job_retire(job); + } + return (entry); + } + else if (job == current && !HAS_ENTRIES(job)) { + qmgr_job_pop(job); + qmgr_job_retire(job); + current = next ? next : transport->job_list.next; + } + } + + /* + * Try the regular job list if there is nothing (suitable) on the job stack. + */ + for (job = transport->job_list.next; job; job = next) { + next = job->transport_peers.next; + if (job->stack_level != 0) + continue; + if ((peer = qmgr_job_peer_select(job)) != 0) { + entry = qmgr_entry_select(peer); + qmgr_job_count_slots(current, job); + + /* + * In case we selected the very last job entry, remove the job + * from the job list right away. + */ + if (!HAS_ENTRIES(job) && job->message->rcpt_offset == 0) + qmgr_job_retire(job); + return (entry); + } + else if (job == current && !HAS_ENTRIES(job)) { + qmgr_job_retire(job); + current = next; + } + } + return (0); +} diff --git a/postfix/nqmgr/qmgr_message.c b/postfix/nqmgr/qmgr_message.c new file mode 100644 index 000000000..07a530e86 --- /dev/null +++ b/postfix/nqmgr/qmgr_message.c @@ -0,0 +1,916 @@ +/*++ +/* NAME +/* qmgr_message 3 +/* SUMMARY +/* in-core message structures +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* int qmgr_message_count; +/* int qmgr_recipient_count; +/* +/* QMGR_MESSAGE *qmgr_message_alloc(class, name, qflags) +/* const char *class; +/* const char *name; +/* int qflags; +/* +/* QMGR_MESSAGE *qmgr_message_realloc(message) +/* QMGR_MESSAGE *message; +/* +/* void qmgr_message_free(message) +/* QMGR_MESSAGE *message; +/* +/* void qmgr_message_update_warn(message) +/* QMGR_MESSAGE *message; +/* DESCRIPTION +/* This module performs en-gross operations on queue messages. +/* +/* qmgr_message_count is a global counter for the total number +/* of in-core message structures (i.e. the total size of the +/* `active' message queue). +/* +/* qmgr_recipient_count is a global counter for the total number +/* of in-core recipient structures (i.e. the sum of all recipients +/* in all in-core message structures). +/* +/* qmgr_message_alloc() creates an in-core message structure +/* with sender and recipient information taken from the named queue +/* file. A null result means the queue file could not be read or +/* that the queue file contained incorrect information. A result +/* QMGR_MESSAGE_LOCKED means delivery must be deferred. The number +/* of recipients read from a queue file is limited by the global +/* var_qmgr_rcpt_limit configuration parameter. When the limit +/* is reached, the \fIrcpt_offset\fR structure member is set to +/* the position where the read was terminated. Recipients are +/* run through the resolver, and are assigned to destination +/* queues. Recipients that cannot be assigned are deferred or +/* bounced. Mail that has bounced twice is silently absorbed. +/* +/* qmgr_message_realloc() resumes reading recipients from the queue +/* file, and updates the recipient list and \fIrcpt_offset\fR message +/* structure members. A null result means that the file could not be +/* read or that the file contained incorrect information. Recipient +/* limit imposed this time is based on the position of the message +/* job(s) on corresponding job list(s). +/* +/* qmgr_message_free() destroys an in-core message structure and makes +/* the resources available for reuse. It is an error to destroy +/* a message structure that is still referenced by queue entry structures. +/* +/* qmgr_message_update_warn() takes a closed message, opens it, updates +/* the warning field, and closes it again. +/* DIAGNOSTICS +/* Warnings: malformed message file. Fatal errors: out of memory. +/* SEE ALSO +/* envelope(3) message envelope parser +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include +#include +#include +#include +#include +#include /* sscanf() */ + +#ifdef STRCASECMP_IN_STRINGS_H +#include +#endif + +/* Utility library. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Global library. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Client stubs. */ + +#include + +/* Application-specific. */ + +#include "qmgr.h" + +int qmgr_message_count; +int qmgr_recipient_count; + +/* qmgr_message_create - create in-core message structure */ + +static QMGR_MESSAGE *qmgr_message_create(const char *queue_name, + const char *queue_id, int qflags) +{ + QMGR_MESSAGE *message; + + message = (QMGR_MESSAGE *) mymalloc(sizeof(QMGR_MESSAGE)); + qmgr_message_count++; + message->flags = 0; + message->qflags = qflags; + message->fp = 0; + message->refcount = 0; + message->single_rcpt = 0; + message->arrival_time = 0; + message->queued_time = event_time(); + message->data_offset = 0; + message->queue_id = mystrdup(queue_id); + message->queue_name = mystrdup(queue_name); + message->sender = 0; + message->errors_to = 0; + message->return_receipt = 0; + message->data_size = 0; + message->warn_offset = 0; + message->warn_time = 0; + message->rcpt_offset = 0; + message->unread_offset = 0; + qmgr_rcpt_list_init(&message->rcpt_list); + message->rcpt_count = 0; + message->rcpt_limit = var_qmgr_msg_rcpt_limit; + message->rcpt_unread = 0; + QMGR_LIST_INIT(message->job_list); + return (message); +} + +/* qmgr_message_close - close queue file */ + +static void qmgr_message_close(QMGR_MESSAGE *message) +{ + vstream_fclose(message->fp); + message->fp = 0; +} + +/* qmgr_message_open - open queue file */ + +static int qmgr_message_open(QMGR_MESSAGE *message) +{ + + /* + * Sanity check. + */ + if (message->fp) + msg_panic("%s: queue file is open", message->queue_id); + + /* + * Open this queue file. Skip files that we cannot open. Back off when + * the system appears to be running out of resources. + */ + if ((message->fp = mail_queue_open(message->queue_name, + message->queue_id, + O_RDWR, 0)) == 0) { + if (errno != ENOENT) + msg_fatal("open %s %s: %m", message->queue_name, message->queue_id); + msg_warn("open %s %s: %m", message->queue_name, message->queue_id); + return (-1); + } + return (0); +} + +/* qmgr_message_oldstyle_scan - extract required information from old style queue file */ + +static void qmgr_message_oldstyle_scan(QMGR_MESSAGE *message) +{ + VSTRING *buf; + long orig_offset, curr_offset, extra_offset; + int rec_type; + char *start; + + /* + * Initialize. No early returns or we have a memory leak. + */ + buf = vstring_alloc(100); + if ((orig_offset = vstream_ftell(message->fp)) < 0) + msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); + + /* + * Rewind to the very begining to make sure we see all records. + */ + if (vstream_fseek(message->fp, 0, SEEK_SET) < 0) + msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); + + /* + * Scan through the old style queue file. Count the total number + * of recipients and find the data/extra sections offsets. + * Note that the new queue files require that data_size equals + * extra_offset - data_offset, so we set data_size to this as well + * and ignore the size record itself completely. + */ + message->rcpt_unread = 0; + do { + if ((curr_offset = vstream_ftell(message->fp)) < 0) + msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); + rec_type = rec_get(message->fp, buf, 0); + start = vstring_str(buf); + if (rec_type == REC_TYPE_DONE || rec_type == REC_TYPE_RCPT) { + message->rcpt_unread++; + } else if (rec_type == REC_TYPE_MESG) { + if ((message->data_offset = vstream_ftell(message->fp)) < 0) + msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); + if ((extra_offset = atol(start)) <= curr_offset) + msg_fatal("bad extra offset %s file %s", + start, VSTREAM_PATH(message->fp)); + if (vstream_fseek(message->fp, extra_offset, SEEK_SET) < 0) + msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); + message->data_size = extra_offset - message->data_offset; + } + } while (rec_type > 0 && rec_type != REC_TYPE_END); + + /* + * Clean up. + */ + if (vstream_fseek(message->fp, orig_offset, SEEK_SET) < 0) + msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); + vstring_free(buf); + + /* + * Sanity checks. Verify that all required information was found, + * including the queue file end marker. + */ + if (message->data_offset == 0 || rec_type != REC_TYPE_END) + msg_fatal("%s: envelope records out of order", message->queue_id); +} + +/* qmgr_message_read - read envelope records */ + +static int qmgr_message_read(QMGR_MESSAGE *message) +{ + VSTRING *buf; + long extra_offset; + int rec_type; + long curr_offset; + long save_offset = message->rcpt_offset; /* save a flag */ + char *start; + int recipient_limit; + + /* + * Initialize. No early returns or we have a memory leak. + */ + buf = vstring_alloc(100); + + /* + * If we re-open this file, skip over on-file recipient records that we + * already looked at, and reset the in-core recipient address list. + * + * For the first time, the message recipient limit is calculated from + * the global recipient limit. This is to avoid reading little recipients + * when the active queue is near empty. When the queue becomes full, only + * the necessary amount is read in core. Such priming is necessary + * because there are no message jobs yet. + * + * For the next time, the recipient limit is based solely on the message + * jobs' positions in the job queues and/or job stacks. + */ + if (message->rcpt_offset) { + if (message->rcpt_list.len) + msg_panic("%s: recipient list not empty on recipient reload", message->queue_id); + if (vstream_fseek(message->fp, message->rcpt_offset, SEEK_SET) < 0) + msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); + message->rcpt_offset = 0; + recipient_limit = message->rcpt_limit - message->rcpt_count; + } + else { + recipient_limit = var_qmgr_rcpt_limit - qmgr_recipient_count; + if (recipient_limit < message->rcpt_limit) + recipient_limit = message->rcpt_limit; + } + + /* + * Read envelope records. XXX Rely on the front-end programs to enforce + * record size limits. Read up to recipient_limit recipients from the + * queue file, to protect against memory exhaustion. Recipient records + * may appear before or after the message content, so we keep reading + * from the queue file until we have enough recipients (rcpt_offset != 0) + * and until we know where the message content starts (data_offset != 0). + * + * Note that the total recipient count record is accurate only for fresh + * queue files. After some of the recipients are marked as done and + * the queue file is deferred, it can be used as upper bound estimate + * only. Fortunately, this poses no major problem on the scheduling + * algorithm, as the only impact is that the already deferred messages + * are not chosen by qmgr_job_candidate() as often as they could. + */ + do { + if ((curr_offset = vstream_ftell(message->fp)) < 0) + msg_fatal("vstream_ftell %s: %m", VSTREAM_PATH(message->fp)); + if (curr_offset == message->data_offset && curr_offset > 0) { + extra_offset = curr_offset + message->data_size; + if (extra_offset <= curr_offset) + msg_fatal("bad extra offset %ld file %s", + extra_offset, VSTREAM_PATH(message->fp)); + if (vstream_fseek(message->fp, extra_offset, SEEK_SET) < 0) + msg_fatal("seek file %s: %m", VSTREAM_PATH(message->fp)); + curr_offset = extra_offset; + } + rec_type = rec_get(message->fp, buf, 0); + start = vstring_str(buf); + if (rec_type == REC_TYPE_SIZE) { + if (message->data_size == 0) { + switch (sscanf(start, "%ld %ld %d", &message->data_size, + &message->data_offset, &message->rcpt_unread)) + { + case 1: + /* + * Gather data_size, data_offset and rcpt_unread values + * from the old style queue file. + */ + qmgr_message_oldstyle_scan(message); + break; + case 3: + /* + * No extra work for new style queue files. + */ + break; + default: + msg_fatal("%s: weird size record", message->queue_id); + break; + } + } + } else if (rec_type == REC_TYPE_TIME) { + if (message->arrival_time == 0) + message->arrival_time = atol(start); + } else if (rec_type == REC_TYPE_FROM) { + if (message->sender == 0) { + message->sender = mystrdup(start); + opened(message->queue_id, message->sender, + message->data_size, "queue %s", message->queue_name); + } + } else if (rec_type == REC_TYPE_DONE) { + if (curr_offset > message->unread_offset) { + message->unread_offset = curr_offset; + message->rcpt_unread--; + } + } else if (rec_type == REC_TYPE_RCPT) { + if (message->rcpt_list.len < recipient_limit) { + message->rcpt_unread--; + qmgr_rcpt_list_add(&message->rcpt_list, curr_offset, start); + if (message->rcpt_list.len >= recipient_limit) { + if ((message->rcpt_offset = vstream_ftell(message->fp)) < 0) + msg_fatal("vstream_ftell %s: %m", + VSTREAM_PATH(message->fp)); + if (message->data_offset != 0 + && message->errors_to != 0 + && message->return_receipt != 0) + break; + } + } + } else if (rec_type == REC_TYPE_ERTO) { + if (message->errors_to == 0) { + message->errors_to = mystrdup(start); + if (message->data_offset != 0 + && message->rcpt_offset != 0 + && message->return_receipt != 0) + break; + } + } else if (rec_type == REC_TYPE_RRTO) { + if (message->return_receipt == 0) { + message->return_receipt = mystrdup(start); + if (message->data_offset != 0 + && message->rcpt_offset != 0 + && message->errors_to != 0) + break; + } + } else if (rec_type == REC_TYPE_WARN) { + if (message->warn_offset == 0) { + message->warn_offset = curr_offset; + message->warn_time = atol(start); + } + } + } while (rec_type > 0 && rec_type != REC_TYPE_END); + + /* + * Avoid clumsiness elsewhere in the program. When sending data across an + * IPC channel, sending an empty string is more convenient than sending a + * null pointer. + */ + if (message->errors_to == 0) + message->errors_to = mystrdup(""); + if (message->return_receipt == 0) + message->return_receipt = mystrdup(""); + + /* + * Clean up. + */ + vstring_free(buf); + + /* + * Sanity checks. Verify that all required information was found, + * including the queue file end marker. + */ + if (message->rcpt_unread < 0 + || (message->rcpt_offset == 0 && message->rcpt_unread != 0)) { + msg_warn("%s: rcpt count mismatch (%d)", + message->queue_id, message->rcpt_unread); + message->rcpt_unread = 0; + } + if (message->arrival_time == 0 + || message->sender == 0 + || message->data_offset == 0 + || (message->rcpt_offset == 0 && rec_type != REC_TYPE_END)) { + msg_warn("%s: envelope records out of order", message->queue_id); + message->rcpt_offset = save_offset; /* restore flag */ + message->rcpt_unread += message->rcpt_list.len; + qmgr_rcpt_list_free(&message->rcpt_list); + qmgr_rcpt_list_init(&message->rcpt_list); + return (-1); + } else { + return (0); + } +} + +/* qmgr_message_update_warn - update the time of next delay warning */ + +void qmgr_message_update_warn(QMGR_MESSAGE *message) +{ + + /* + * XXX eventually this should let us schedule multiple warnings, right + * now it just allows for one. + */ + if (qmgr_message_open(message) + || vstream_fseek(message->fp, message->warn_offset, SEEK_SET) < 0 + || rec_fprintf(message->fp, REC_TYPE_WARN, REC_TYPE_WARN_FORMAT, 0L) < 0 + || vstream_fflush(message->fp)) + msg_fatal("update queue file %s: %m", VSTREAM_PATH(message->fp)); + qmgr_message_close(message); +} + +/* qmgr_message_sort_compare - compare recipient information */ + +static int qmgr_message_sort_compare(const void *p1, const void *p2) +{ + QMGR_RCPT *rcpt1 = (QMGR_RCPT *) p1; + QMGR_RCPT *rcpt2 = (QMGR_RCPT *) p2; + QMGR_QUEUE *queue1; + QMGR_QUEUE *queue2; + char *at1; + char *at2; + int result; + + /* + * Compare most significant to least significant recipient attributes. + */ + if ((queue1 = rcpt1->queue) != 0 && (queue2 = rcpt2->queue) != 0) { + + /* + * Compare message transport. + */ + if ((result = strcasecmp(queue1->transport->name, + queue2->transport->name)) != 0) + return (result); + + /* + * Compare next-hop hostname. + */ + if ((result = strcasecmp(queue1->name, queue2->name)) != 0) + return (result); + } + + /* + * Compare recipient domain. + */ + if ((at1 = strrchr(rcpt1->address, '@')) != 0 + && (at2 = strrchr(rcpt2->address, '@')) != 0 + && (result = strcasecmp(at1, at2)) != 0) + return (result); + + /* + * Compare recipient address. + */ + return (strcasecmp(rcpt1->address, rcpt2->address)); +} + +/* qmgr_message_sort - sort message recipient addresses by domain */ + +static void qmgr_message_sort(QMGR_MESSAGE *message) +{ + qsort((char *) message->rcpt_list.info, message->rcpt_list.len, + sizeof(message->rcpt_list.info[0]), qmgr_message_sort_compare); + if (msg_verbose) { + QMGR_RCPT_LIST list = message->rcpt_list; + QMGR_RCPT *rcpt; + + msg_info("start sorted recipient list"); + for (rcpt = list.info; rcpt < list.info + list.len; rcpt++) + msg_info("qmgr_message_sort: %s", rcpt->address); + msg_info("end sorted recipient list"); + } +} + +/* qmgr_message_resolve - resolve recipients */ + +static void qmgr_message_resolve(QMGR_MESSAGE *message) +{ + static ARGV *defer_xport_argv; + QMGR_RCPT_LIST list = message->rcpt_list; + QMGR_RCPT *recipient; + QMGR_TRANSPORT *transport = 0; + QMGR_QUEUE *queue = 0; + RESOLVE_REPLY reply; + const char *newloc; + char *at; + char **cpp; + char *domain; + const char *junk; + +#define STREQ(x,y) (strcasecmp(x,y) == 0) +#define STR vstring_str +#define UPDATE(ptr,new) { myfree(ptr); ptr = mystrdup(new); } + + resolve_clnt_init(&reply); + for (recipient = list.info; recipient < list.info + list.len; recipient++) { + + /* + * This may be a bit late in the game, but it is the most convenient + * place to scrutinize the destination address syntax. We have a + * complete queue file, so bouncing is easy. That luxury is not + * available to the cleanup service. The main issue is that we want + * to have this test in one place, instead of having to do this in + * every front-ent program. + */ + if ((at = strrchr(recipient->address, '@')) != 0 + && (at + 1)[strspn(at + 1, "[]0123456789.")] != 0 + && valid_hostname(at + 1) == 0) { + qmgr_bounce_recipient(message, recipient, + "bad host/domain syntax: \"%s\"", at + 1); + continue; + } + + /* + * Resolve the destination to (transport, nexthop, address). The + * result address may differ from the one specified by the sender. + */ + resolve_clnt_query(recipient->address, &reply); + if (!STREQ(recipient->address, STR(reply.recipient))) + UPDATE(recipient->address, STR(reply.recipient)); + + + /* + * Bounce recipients that have moved. We do it here instead of in the + * local delivery agent. The benefit is that we can bounce mail for + * virtual addresses, not just local addresses only, and that there + * is no need to run a local delivery agent just for the sake of + * relocation notices. The downside is that this table has no effect + * on local alias expansion results, so that mail will have to make + * almost an entire iteration through the mail system. + */ +#define IGNORE_ADDR_EXTENSION ((char **) 0) + + if (qmgr_relocated != 0) { + if ((newloc = mail_addr_find(qmgr_relocated, recipient->address, + IGNORE_ADDR_EXTENSION)) != 0) { + qmgr_bounce_recipient(message, recipient, + "user has moved to %s", newloc); + continue; + } else if (dict_errno != 0) { + qmgr_defer_recipient(message, recipient->address, + "relocated map lookup failure"); + continue; + } + } + + /* + * Bounce mail to non-existent users in virtual domains. + */ + if (qmgr_virtual != 0 + && (at = strrchr(recipient->address, '@')) != 0 + && !resolve_local(at + 1)) { + domain = lowercase(mystrdup(at + 1)); + junk = maps_find(qmgr_virtual, domain, 0); + myfree(domain); + if (junk) { + qmgr_bounce_recipient(message, recipient, + "unknown user: \"%s\"", recipient->address); + continue; + } + } + + /* + * Bounce recipient addresses that start with `-'. External commands + * may misinterpret such addresses as command-line options. + * + * In theory I could say people should always carefully set up their + * master.cf pipe mailer entries with `--' before the first + * non-option argument, but mistakes will happen regardless. + * + * Therefore the protection is put in place here, in the queue manager, + * where it cannot be bypassed. + */ + if (var_allow_min_user == 0 && recipient->address[0] == '-') { + qmgr_bounce_recipient(message, recipient, + "invalid recipient syntax: \"%s\"", + recipient->address); + continue; + } + + /* + * Queues are identified by the transport name and by the next-hop + * hostname. When the destination is local (no next hop), derive the + * queue name from the recipient name. XXX Should split the address + * on the recipient delimiter if one is defined, but doing a proper + * job requires knowledge of local aliases. Yuck! I don't want to + * duplicate delivery-agent specific knowledge in the queue manager. + */ + if ((at = strrchr(STR(reply.recipient), '@')) == 0 + || resolve_local(at + 1)) { +#if 0 + vstring_strcpy(reply.nexthop, STR(reply.recipient)); + (void) split_at_right(STR(reply.nexthop), '@'); +#endif +#if 0 + if (*var_rcpt_delim) + (void) split_addr(STR(reply.nexthop), *var_rcpt_delim); +#endif + + /* + * Discard mail to the local double bounce address here, so this + * system can run without a local delivery agent. They'd still + * have to configure something for mail directed to the local + * postmaster, though, but that is an RFC requirement anyway. + */ + if (strncasecmp(STR(reply.recipient), var_double_bounce_sender, + at - STR(reply.recipient)) == 0 + && !var_double_bounce_sender[at - STR(reply.recipient)]) { + sent(message->queue_id, recipient->address, + "none", message->arrival_time, "discarded"); + deliver_completed(message->fp, recipient->offset); + msg_warn("%s: undeliverable postmaster notification discarded", + message->queue_id); + continue; + } + } + + /* + * Optionally defer deliveries over specific transports, unless the + * restriction is lifted temporarily. + */ + if (*var_defer_xports && (message->qflags & QMGR_SCAN_ALL) == 0) { + if (defer_xport_argv == 0) + defer_xport_argv = argv_split(var_defer_xports, " \t\r\n,"); + for (cpp = defer_xport_argv->argv; *cpp; cpp++) + if (strcasecmp(*cpp, STR(reply.transport)) == 0) + break; + if (*cpp) { + qmgr_defer_recipient(message, recipient->address, + "deferred transport"); + continue; + } + } + + /* + * XXX Gross hack alert. We want to group recipients by transport and + * by next-hop hostname, in order to minimize the number of network + * transactions. However, it would be wasteful to have an in-memory + * resolver reply structure for each in-core recipient. Instead, we + * bind each recipient to an in-core queue instance which is needed + * anyway. That gives all information needed for recipient grouping. + */ + + /* + * Look up or instantiate the proper transport. + */ + if (transport == 0 || !STREQ(transport->name, STR(reply.transport))) { + if ((transport = qmgr_transport_find(STR(reply.transport))) == 0) + transport = qmgr_transport_create(STR(reply.transport)); + queue = 0; + } + + /* + * This transport is dead. Defer delivery to this recipient. + */ + if ((transport->flags & QMGR_TRANSPORT_STAT_DEAD) != 0) { + qmgr_defer_recipient(message, recipient->address, transport->reason); + continue; + } + + /* + * This transport is alive. Find or instantiate a queue for this + * recipient. + */ + if (queue == 0 || !STREQ(queue->name, STR(reply.nexthop))) { + if ((queue = qmgr_queue_find(transport, STR(reply.nexthop))) == 0) + queue = qmgr_queue_create(transport, STR(reply.nexthop)); + } + + /* + * This queue is dead. Defer delivery to this recipient. + */ + if (queue->window == 0) { + qmgr_defer_recipient(message, recipient->address, queue->reason); + continue; + } + + /* + * This queue is alive. Bind this recipient to this queue instance. + */ + recipient->queue = queue; + } + resolve_clnt_free(&reply); +} + +/* qmgr_message_assign - assign recipients to specific delivery requests */ + +static void qmgr_message_assign(QMGR_MESSAGE *message) +{ + QMGR_RCPT_LIST list = message->rcpt_list; + QMGR_RCPT *recipient; + QMGR_ENTRY *entry = 0; + QMGR_QUEUE *queue; + QMGR_JOB *job = 0; + QMGR_PEER *peer = 0; + + /* + * Try to bundle as many recipients in a delivery request as we can. When + * the recipient resolves to the same site and transport as the previous + * recipient, do not create a new queue entry, just move that recipient + * to the recipient list of the existing queue entry. All this provided + * that we do not exceed the transport-specific limit on the number of + * recipients per transaction. Skip recipients with a dead transport or + * destination. + */ +#define LIMIT_OK(limit, count) ((limit) == 0 || ((count) < (limit))) + + for (recipient = list.info; recipient < list.info + list.len; recipient++) { + if ((queue = recipient->queue) != 0) { + if (message->single_rcpt || entry == 0 || entry->queue != queue + || !LIMIT_OK(queue->transport->recipient_limit, + entry->rcpt_list.len)) { + if (job == 0 || queue->transport != job->transport) { + /* FIXME: randomly rotate the peer list of previous job */ + job = qmgr_job_obtain(message, queue->transport); + peer = 0; + } + if (peer == 0 || queue != peer->queue) { + if ((peer = qmgr_peer_find(job, queue)) == 0) + peer = qmgr_peer_create(job, queue); + } + entry = qmgr_entry_create(peer, message); + job->read_entries++; + } + qmgr_rcpt_list_add(&entry->rcpt_list, recipient->offset, recipient->address); + job->rcpt_count++; + message->rcpt_count++; + qmgr_recipient_count++; + } + } + qmgr_rcpt_list_free(&message->rcpt_list); + qmgr_rcpt_list_init(&message->rcpt_list); +} + +/* qmgr_message_move_limits - recycle unused recipient slots */ + +static void qmgr_message_move_limits(QMGR_MESSAGE *message) +{ + QMGR_JOB *job; + + for (job = message->job_list.next; job; job = job->message_peers.next) + qmgr_job_move_limits(job); +} + +/* qmgr_message_free - release memory for in-core message structure */ + +void qmgr_message_free(QMGR_MESSAGE *message) +{ + QMGR_JOB *job; + if (message->refcount != 0) + msg_panic("qmgr_message_free: reference len: %d", message->refcount); + if (message->fp) + msg_panic("qmgr_message_free: queue file is open"); + while ((job = message->job_list.next) != 0) + qmgr_job_free(job); + myfree(message->queue_id); + myfree(message->queue_name); + if (message->sender) + myfree(message->sender); + if (message->errors_to) + myfree(message->errors_to); + if (message->return_receipt) + myfree(message->return_receipt); + qmgr_rcpt_list_free(&message->rcpt_list); + qmgr_message_count--; + myfree((char *) message); +} + +/* qmgr_message_alloc - create in-core message structure */ + +QMGR_MESSAGE *qmgr_message_alloc(const char *queue_name, const char *queue_id, + int qflags) +{ + char *myname = "qmgr_message_alloc"; + QMGR_MESSAGE *message; + + if (msg_verbose) + msg_info("%s: %s %s", myname, queue_name, queue_id); + + /* + * Create an in-core message structure. + */ + message = qmgr_message_create(queue_name, queue_id, qflags); + + /* + * Extract message envelope information: time of arrival, sender address, + * recipient addresses. Skip files with malformed envelope information. + */ +#define QMGR_LOCK_MODE (MYFLOCK_EXCLUSIVE | MYFLOCK_NOWAIT) + + if (qmgr_message_open(message) < 0) { + qmgr_message_free(message); + return (0); + } + if (myflock(vstream_fileno(message->fp), QMGR_LOCK_MODE) < 0) { + msg_info("%s: skipped, still being delivered", queue_id); + qmgr_message_close(message); + qmgr_message_free(message); + return (QMGR_MESSAGE_LOCKED); + } + if (qmgr_message_read(message) < 0) { + qmgr_message_close(message); + qmgr_message_free(message); + return (0); + } else { + + /* + * Reset the defer log. This code should not be here, but we must + * reset the defer log *after* acquiring the exclusive lock on the + * queue file and *before* resolving new recipients. Since all those + * operations are encapsulated so nicely by this routine, the defer + * log reset has to be done here as well. + */ + if (mail_queue_remove(MAIL_QUEUE_DEFER, queue_id) && errno != ENOENT) + msg_fatal("%s: %s: remove %s %s: %m", myname, + queue_id, MAIL_QUEUE_DEFER, queue_id); + qmgr_message_sort(message); + qmgr_message_resolve(message); + qmgr_message_sort(message); + qmgr_message_assign(message); + qmgr_message_close(message); + if(message->rcpt_offset == 0) + qmgr_message_move_limits(message); + return (message); + } +} + +/* qmgr_message_realloc - refresh in-core message structure */ + +QMGR_MESSAGE *qmgr_message_realloc(QMGR_MESSAGE *message) +{ + char *myname = "qmgr_message_realloc"; + + /* + * Sanity checks. + */ + if (message->rcpt_offset <= 0) + msg_panic("%s: invalid offset: %ld", myname, message->rcpt_offset); + if (msg_verbose) + msg_info("%s: %s %s offset %ld", myname, message->queue_name, + message->queue_id, message->rcpt_offset); + + /* + * Extract recipient addresses. Skip files with malformed envelope + * information. + */ + if (qmgr_message_open(message) < 0) + return (0); + if (qmgr_message_read(message) < 0) { + qmgr_message_close(message); + return (0); + } else { + qmgr_message_sort(message); + qmgr_message_resolve(message); + qmgr_message_sort(message); + qmgr_message_assign(message); + qmgr_message_close(message); + if(message->rcpt_offset == 0) + qmgr_message_move_limits(message); + return (message); + } +} diff --git a/postfix/nqmgr/qmgr_move.c b/postfix/nqmgr/qmgr_move.c new file mode 100644 index 000000000..7f86197ec --- /dev/null +++ b/postfix/nqmgr/qmgr_move.c @@ -0,0 +1,94 @@ +/*++ +/* NAME +/* qmgr_move 3 +/* SUMMARY +/* move queue entries to another queue +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* void qmgr_move(from, to, time_stamp) +/* const char *from; +/* const char *to; +/* time_t time_stamp; +/* DESCRIPTION +/* The \fBqmgr_move\fR routine scans the \fIfrom\fR queue for entries +/* with valid queue names and moves them to the \fIto\fR queue. +/* If \fItime_stamp\fR is non-zero, the queue file time stamps are +/* set to the specified value. +/* Entries with invalid names are left alone. No attempt is made to +/* look for other badness such as multiple links or weird file types. +/* These issues are dealt with when a queue file is actually opened. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include +#include +#include + +/* Utility library. */ + +#include +#include +#include + +/* Global library. */ + +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_move - move queue entries to another queue, leave bad files alone */ + +void qmgr_move(const char *src_queue, const char *dst_queue, + time_t time_stamp) +{ + char *myname = "qmgr_move"; + SCAN_DIR *queue_dir; + char *queue_id; + struct utimbuf tbuf; + const char *path; + + if (strcmp(src_queue, dst_queue) == 0) + msg_panic("%s: source queue %s is destination", myname, src_queue); + if (msg_verbose) + msg_info("start move queue %s -> %s", src_queue, dst_queue); + + queue_dir = scan_dir_open(src_queue); + while ((queue_id = mail_scan_dir_next(queue_dir)) != 0) { + if (mail_queue_id_ok(queue_id)) { + if (time_stamp > 0) { + tbuf.actime = tbuf.modtime = time_stamp; + path = mail_queue_path((VSTRING *) 0, src_queue, queue_id); + if (utime(path, &tbuf) < 0) + msg_fatal("%s: update %s time stamps: %m", myname, path); + } + if (mail_queue_rename(queue_id, src_queue, dst_queue)) + msg_fatal("%s: rename %s from %s to %s: %m", + myname, queue_id, src_queue, dst_queue); + if (msg_verbose) + msg_info("%s: moved %s from %s to %s", + myname, queue_id, src_queue, dst_queue); + } else { + msg_warn("%s: ignored: queue %s id %s", + myname, src_queue, queue_id); + } + } + scan_dir_close(queue_dir); + + if (msg_verbose) + msg_info("end move queue %s -> %s", src_queue, dst_queue); +} diff --git a/postfix/nqmgr/qmgr_peer.c b/postfix/nqmgr/qmgr_peer.c new file mode 100644 index 000000000..72953d3a4 --- /dev/null +++ b/postfix/nqmgr/qmgr_peer.c @@ -0,0 +1,124 @@ +/*++ +/* NAME +/* qmgr_peer 3 +/* SUMMARY +/* per-job peers +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_PEER *qmgr_peer_create(job, queue) +/* QMGR_JOB *job; +/* QMGR_QUEUE *queue; +/* +/* QMGR_PEER *qmgr_peer_find(job, queue) +/* QMGR_JOB *job; +/* QMGR_QUEUE *queue; +/* +/* void qmgr_peer_free(peer) +/* QMGR_PEER *peer; +/* +/* QMGR_PEER *qmgr_peer_select(job) +/* QMGR_JOB *job; +/* +/* DESCRIPTION +/* These routines add/delete/manipulate per-job peers. +/* Each queue corresponds to a specific job and destination. +/* It is similar to per-transport queue structure, but groups +/* only the entries of the given job. +/* +/* qmgr_peer_create() creates an empty peer structure for the named +/* job and destination. It is an error to call this function +/* if an peer for given combination already exists. +/* +/* qmgr_peer_find() looks up the peer for the named destination +/* for the named job. A null result means that the queue +/* was not found. +/* +/* qmgr_peer_free() disposes of a per-job queue after all +/* its entries have been taken care of. It is an error to dispose +/* of a peer still in use. +/* +/* qmgr_peer_select() attempts to find a peer of named job that +/* has messages pending delivery. This routine implements +/* round-robin search among job's peers. +/* DIAGNOSTICS +/* Panic: consistency check failure. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Patrik Rak +/* patrik@raxoft.cz +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_peer_create - create and initialize message peer structure */ + +QMGR_PEER *qmgr_peer_create(QMGR_JOB *job, QMGR_QUEUE *queue) +{ + QMGR_PEER *peer; + + peer = (QMGR_PEER *) mymalloc(sizeof(QMGR_PEER)); + peer->queue = queue; + peer->job = job; + QMGR_LIST_APPEND(job->peer_list, peer, peers); + htable_enter(job->peer_byname, queue->name, (char *) peer); + peer->refcount = 0; + QMGR_LIST_INIT(peer->entry_list); + return (peer); +} + +/* qmgr_peer_free - release peer structure */ + +void qmgr_peer_free(QMGR_PEER *peer) +{ + QMGR_JOB *job = peer->job; + + QMGR_LIST_UNLINK(job->peer_list, QMGR_PEER *, peer, peers); + htable_delete(job->peer_byname, peer->queue->name, (void (*) (char *)) 0) ; + myfree((char *) peer); +} + +/* qmgr_peer_find - lookup peer associated with given job and queue */ + +QMGR_PEER *qmgr_peer_find(QMGR_JOB *job, QMGR_QUEUE *queue) +{ + return ((QMGR_PEER *) htable_find(job->peer_byname, queue->name)); +} + +/* qmgr_peer_select - select next peer suitable for delivery within given job */ + +QMGR_PEER *qmgr_peer_select(QMGR_JOB *job) +{ + QMGR_PEER *peer; + QMGR_QUEUE *queue; + + /* + * If we find a suitable site, rotate the list to enforce round-robin + * selection. See similar selection code in qmgr_transport_select(). + */ + for (peer = job->peer_list.next; peer; peer = peer->peers.next) { + queue = peer->queue; + if (queue->window > queue->busy_refcount && peer->entry_list.next != 0) { + QMGR_LIST_ROTATE(job->peer_list, peer, peers); + if (msg_verbose) + msg_info("qmgr_peer_select: %s %s", job->message->queue_id, queue->name); + return (peer); + } + } + return (0); +} diff --git a/postfix/nqmgr/qmgr_queue.c b/postfix/nqmgr/qmgr_queue.c new file mode 100644 index 000000000..9eeb49248 --- /dev/null +++ b/postfix/nqmgr/qmgr_queue.c @@ -0,0 +1,245 @@ +/*++ +/* NAME +/* qmgr_queue 3 +/* SUMMARY +/* per-destination queues +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* int qmgr_queue_count; +/* +/* QMGR_QUEUE *qmgr_queue_create(transport, site) +/* QMGR_TRANSPORT *transport; +/* const char *site; +/* +/* void qmgr_queue_done(queue) +/* QMGR_QUEUE *queue; +/* +/* QMGR_QUEUE *qmgr_queue_find(transport, site) +/* QMGR_TRANSPORT *transport; +/* const char *site; +/* +/* void qmgr_queue_throttle(queue, reason) +/* QMGR_QUEUE *queue; +/* const char *reason; +/* +/* void qmgr_queue_unthrottle(queue) +/* QMGR_QUEUE *queue; +/* DESCRIPTION +/* These routines add/delete/manipulate per-destination queues. +/* Each queue corresponds to a specific transport and destination. +/* Each queue has a `todo' list of delivery requests for that +/* destination, and a `busy' list of delivery requests in progress. +/* +/* qmgr_queue_count is a global counter for the total number +/* of in-core queue structures. +/* +/* qmgr_queue_create() creates an empty queue for the named +/* transport and destination. The queue is given an initial +/* concurrency limit as specified with the +/* \fIinitial_destination_concurrency\fR configuration parameter, +/* provided that it does not exceed the transport-specific +/* concurrency limit. +/* +/* qmgr_queue_done() disposes of a per-destination queue after all +/* its entries have been taken care of. It is an error to dispose +/* of a dead queue. +/* +/* qmgr_queue_find() looks up the queue for the named destination +/* for the named transport. A null result means that the queue +/* was not found. +/* +/* qmgr_queue_throttle() handles a delivery error, and decrements the +/* concurrency limit for the destination. When the concurrency limit +/* for a destination becomes zero, qmgr_queue_throttle() starts a timer +/* to re-enable delivery to the destination after a configurable delay. +/* +/* qmgr_queue_unthrottle() undoes qmgr_queue_throttle()'s effects. +/* The concurrency limit for the destination is incremented, +/* provided that it does not exceed the destination concurrency +/* limit specified for the transport. This routine implements +/* "slow open" mode, and eliminates the "thundering herd" problem. +/* DIAGNOSTICS +/* None +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include + +/* Global library. */ + +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +int qmgr_queue_count; + +/* qmgr_queue_unthrottle_wrapper - in case (char *) != (struct *) */ + +static void qmgr_queue_unthrottle_wrapper(int unused_event, char *context) +{ + QMGR_QUEUE *queue = (QMGR_QUEUE *) context; + + /* + * This routine runs when a wakeup timer goes off; it does not run in the + * context of some queue manipulation. Therefore, it is safe to discard + * this in-core queue when it is empty and when this site is not dead. + */ + qmgr_queue_unthrottle(queue); + if (queue->window > 0 && queue->todo.next == 0 && queue->busy.next == 0) + qmgr_queue_done(queue); +} + +/* qmgr_queue_unthrottle - give this destination another chance */ + +void qmgr_queue_unthrottle(QMGR_QUEUE *queue) +{ + char *myname = "qmgr_queue_unthrottle"; + QMGR_TRANSPORT *transport = queue->transport; + + if (msg_verbose) + msg_info("%s: queue %s", myname, queue->name); + + /* + * Special case when this site was dead. + */ + if (queue->window == 0) { + event_cancel_timer(qmgr_queue_unthrottle_wrapper, (char *) queue); + if (queue->reason == 0) + msg_panic("%s: queue %s: window 0 reason 0", myname, queue->name); + myfree(queue->reason); + queue->reason = 0; + queue->window = transport->init_dest_concurrency; + return; + } + + /* + * Increase the destination's concurrency limit until we reach the + * transport's concurrency limit. Set the destination's concurrency limit + * to the actual concurrency + 1, so that qmgr_queue_throttle() takes + * effect quickly. + */ + if (transport->dest_concurrency_limit == 0 + || transport->dest_concurrency_limit > queue->busy_refcount) + queue->window = queue->busy_refcount + 1; +} + +/* qmgr_queue_throttle - handle destination delivery failure */ + +void qmgr_queue_throttle(QMGR_QUEUE *queue, const char *reason) +{ + char *myname = "qmgr_queue_throttle"; + + /* + * Sanity checks. + */ + if (queue->reason) + msg_panic("%s: queue %s: spurious reason %s", + myname, queue->name, queue->reason); + if (msg_verbose) + msg_info("%s: queue %s: %s", myname, queue->name, reason); + + /* + * Decrease the destination's concurrency limit until we reach zero, at + * which point the destination is declared dead. Decrease the concurrency + * limit by one, instead of using actual concurrency - 1, to avoid + * declaring a host dead after just one single delivery failure. + */ + if (queue->window > 0) + queue->window--; + + /* + * Special case for a site that just was declared dead. + */ + if (queue->window == 0) { + queue->reason = mystrdup(reason); + event_request_timer(qmgr_queue_unthrottle_wrapper, + (char *) queue, var_min_backoff_time); + } +} + +/* qmgr_queue_done - delete in-core queue for site */ + +void qmgr_queue_done(QMGR_QUEUE *queue) +{ + char *myname = "qmgr_queue_done"; + QMGR_TRANSPORT *transport = queue->transport; + + /* + * Sanity checks. It is an error to delete an in-core queue with pending + * messages or timers. + */ + if (queue->busy_refcount != 0 || queue->todo_refcount != 0) + msg_panic("%s: refcount: %d", myname, + queue->busy_refcount + queue->todo_refcount); + if (queue->todo.next || queue->busy.next) + msg_panic("%s: queue not empty: %s", myname, queue->name); + if (queue->window <= 0) + msg_panic("%s: window %d", myname, queue->window); + if (queue->reason) + msg_panic("%s: queue %s: spurious reason %s", + myname, queue->name, queue->reason); + + /* + * Clean up this in-core queue. + */ + QMGR_LIST_UNLINK(transport->queue_list, QMGR_QUEUE *, queue, peers); + htable_delete(transport->queue_byname, queue->name, (void (*) (char *)) 0); + myfree(queue->name); + qmgr_queue_count--; + myfree((char *) queue); +} + +/* qmgr_queue_create - create in-core queue for site */ + +QMGR_QUEUE *qmgr_queue_create(QMGR_TRANSPORT *transport, const char *site) +{ + QMGR_QUEUE *queue; + + /* + * If possible, choose an initial concurrency of > 1 so that one bad + * message or one bad network won't slow us down unnecessarily. + */ + + queue = (QMGR_QUEUE *) mymalloc(sizeof(QMGR_QUEUE)); + qmgr_queue_count++; + queue->name = mystrdup(site); + queue->todo_refcount = 0; + queue->busy_refcount = 0; + queue->transport = transport; + queue->window = transport->init_dest_concurrency; + QMGR_LIST_INIT(queue->todo); + QMGR_LIST_INIT(queue->busy); + queue->reason = 0; + QMGR_LIST_APPEND(transport->queue_list, queue, peers); + htable_enter(transport->queue_byname, site, (char *) queue); + return (queue); +} + +/* qmgr_queue_find - find in-core queue for site */ + +QMGR_QUEUE *qmgr_queue_find(QMGR_TRANSPORT *transport, const char *site) +{ + return ((QMGR_QUEUE *) htable_find(transport->queue_byname, site)); +} diff --git a/postfix/nqmgr/qmgr_rcpt_list.c b/postfix/nqmgr/qmgr_rcpt_list.c new file mode 100644 index 000000000..2bb292e01 --- /dev/null +++ b/postfix/nqmgr/qmgr_rcpt_list.c @@ -0,0 +1,96 @@ +/*++ +/* NAME +/* qmgr_rcpt_list 3 +/* SUMMARY +/* in-core recipient structures +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* void qmgr_rcpt_list_init(list) +/* QMGR_RCPT_LIST *list; +/* +/* void qmgr_rcpt_list_add(list, offset, recipient) +/* QMGR_RCPT_LIST *list; +/* long offset; +/* const char *recipient; +/* +/* void qmgr_rcpt_list_free(list) +/* QMGR_RCPT_LIST *list; +/* DESCRIPTION +/* This module maintains lists of queue manager recipient structures. +/* These structures are extended versions of the structures maintained +/* by the recipient_list(3) module. The extension is that the queue +/* manager version of a recipient can have a reference to a queue +/* structure. +/* +/* qmgr_rcpt_list_init() creates an empty recipient structure list. +/* The list argument is initialized such that it can be given to +/* qmgr_rcpt_list_add() and qmgr_rcpt_list_free(). +/* +/* qmgr_rcpt_list_add() adds a recipient to the specified list. +/* The recipient name is copied. +/* +/* qmgr_rcpt_list_free() releases memory for the specified list +/* of recipient structures. +/* SEE ALSO +/* qmgr_rcpt_list(3h) data structure +/* recipient_list(3) same code, different data structure. +/* DIAGNOSTICS +/* Fatal errors: memory allocation. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_rcpt_list_init - initialize */ + +void qmgr_rcpt_list_init(QMGR_RCPT_LIST *list) +{ + list->avail = 1; + list->len = 0; + list->info = (QMGR_RCPT *) mymalloc(sizeof(QMGR_RCPT)); +} + +/* qmgr_rcpt_list_add - add rcpt to list */ + +void qmgr_rcpt_list_add(QMGR_RCPT_LIST *list, long offset, const char *rcpt) +{ + if (list->len >= list->avail) { + list->avail *= 2; + list->info = (QMGR_RCPT *) + myrealloc((char *) list->info, list->avail * sizeof(QMGR_RCPT)); + } + list->info[list->len].address = mystrdup(rcpt); + list->info[list->len].offset = offset; + list->info[list->len].queue = 0; + list->len++; +} + +/* qmgr_rcpt_list_free - release memory for in-core recipient structure */ + +void qmgr_rcpt_list_free(QMGR_RCPT_LIST *list) +{ + QMGR_RCPT *rcpt; + + for (rcpt = list->info; rcpt < list->info + list->len; rcpt++) + myfree(rcpt->address); + myfree((char *) list->info); +} diff --git a/postfix/nqmgr/qmgr_scan.c b/postfix/nqmgr/qmgr_scan.c new file mode 100644 index 000000000..e0d9fc079 --- /dev/null +++ b/postfix/nqmgr/qmgr_scan.c @@ -0,0 +1,159 @@ +/*++ +/* NAME +/* qmgr_scan 3 +/* SUMMARY +/* queue scanning +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_SCAN *qmgr_scan_create(queue_name) +/* const char *queue_name; +/* +/* char *qmgr_scan_next(scan_info) +/* QMGR_SCAN *scan_info; +/* +/* void qmgr_scan_request(scan_info, flags) +/* QMGR_SCAN *scan_info; +/* int flags; +/* DESCRIPTION +/* This module implements queue scans. A queue scan always runs +/* to completion, so that all files get a fair chance. The caller +/* can request that a queue scan be restarted once it completes. +/* +/* qmgr_scan_create() creates a context for scanning the named queue, +/* but does not start a queue scan. +/* +/* qmgr_scan_next() returns the base name of the next queue file. +/* A null pointer means that no file was found. qmgr_scan_next() +/* automagically restarts a queue scan when a scan request had +/* arrived while the scan was in progress. +/* +/* qmgr_scan_request() records a request for the next queue scan. The +/* flags argument is the bit-wise OR of zero or more of the following, +/* unrecognized flags being ignored: +/* .IP QMGR_FLUSH_DEAD +/* Forget state information about dead hosts or transports. This +/* request takes effect upon the next queue scan. +/* .IP QMGR_SCAN_ALL +/* Ignore queue file time stamps. +/* This flag is passed on to the qmgr_active_feed() routine. +/* .IP QMGR_SCAN_START +/* Start a queue scan when none is in progress, or restart the +/* current scan upon completion. +/* DIAGNOSTICS +/* Fatal: out of memory. +/* Panic: interface violations, internal consistency errors. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include + +/* Utility library. */ + +#include +#include +#include + +/* Global library. */ + +#include + +/* Application-specific. */ + +#include "qmgr.h" + +/* qmgr_scan_start - start queue scan */ + +static void qmgr_scan_start(QMGR_SCAN *scan_info) +{ + char *myname = "qmgr_scan_start"; + + /* + * Sanity check. + */ + if (scan_info->handle) + msg_panic("%s: %s queue scan in progress", + myname, scan_info->queue); + + /* + * Give the poor tester a clue. + */ + if (msg_verbose) + msg_info("%s: %sstart %s queue scan", + myname, + scan_info->nflags & QMGR_SCAN_START ? "re" : "", + scan_info->queue); + + /* + * Optionally forget all dead host information. + */ + if (scan_info->nflags & QMGR_FLUSH_DEAD) + qmgr_enable_all(); + + /* + * Start or restart the scan. + */ + scan_info->flags = scan_info->nflags; + scan_info->nflags = 0; + scan_info->handle = scan_dir_open(scan_info->queue); +} + +/* qmgr_scan_request - request for future scan */ + +void qmgr_scan_request(QMGR_SCAN *scan_info, int flags) +{ + + /* + * If a scan is in progress, just record the request. + */ + scan_info->nflags |= flags; + if (scan_info->handle == 0 && (flags & QMGR_SCAN_START) != 0) { + scan_info->nflags &= ~QMGR_SCAN_START; + qmgr_scan_start(scan_info); + } +} + +/* qmgr_scan_next - look for next queue file */ + +char *qmgr_scan_next(QMGR_SCAN *scan_info) +{ + char *path = 0; + + /* + * Restart the scan if we reach the end and a queue scan request has + * arrived in the mean time. + */ + if (scan_info->handle && (path = mail_scan_dir_next(scan_info->handle)) == 0) { + scan_info->handle = scan_dir_close(scan_info->handle); + if (msg_verbose && (scan_info->nflags & QMGR_SCAN_START) == 0) + msg_info("done %s queue scan", scan_info->queue); + } + if (!scan_info->handle && (scan_info->nflags & QMGR_SCAN_START)) { + qmgr_scan_start(scan_info); + path = mail_scan_dir_next(scan_info->handle); + } + return (path); +} + +/* qmgr_scan_create - create queue scan context */ + +QMGR_SCAN *qmgr_scan_create(const char *queue) +{ + QMGR_SCAN *scan_info; + + scan_info = (QMGR_SCAN *) mymalloc(sizeof(*scan_info)); + scan_info->queue = mystrdup(queue); + scan_info->flags = scan_info->nflags = 0; + scan_info->handle = 0; + return (scan_info); +} diff --git a/postfix/nqmgr/qmgr_transport.c b/postfix/nqmgr/qmgr_transport.c new file mode 100644 index 000000000..9fc1aabf5 --- /dev/null +++ b/postfix/nqmgr/qmgr_transport.c @@ -0,0 +1,373 @@ +/*++ +/* NAME +/* qmgr_transport 3 +/* SUMMARY +/* per-transport data structures +/* SYNOPSIS +/* #include "qmgr.h" +/* +/* QMGR_TRANSPORT *qmgr_transport_create(name) +/* const char *name; +/* +/* QMGR_TRANSPORT *qmgr_transport_find(name) +/* const char *name; +/* +/* QMGR_TRANSPORT *qmgr_transport_select() +/* +/* void qmgr_transport_alloc(transport, notify) +/* QMGR_TRANSPORT *transport; +/* void (*notify)(QMGR_TRANSPORT *transport, VSTREAM *fp); +/* +/* void qmgr_transport_throttle(transport, reason) +/* QMGR_TRANSPORT *transport; +/* const char *reason; +/* +/* void qmgr_transport_unthrottle(transport) +/* QMGR_TRANSPORT *transport; +/* DESCRIPTION +/* This module organizes the world by message transport type. +/* Each transport can have zero or more destination queues +/* associated with it. +/* +/* qmgr_transport_create() instantiates a data structure for the +/* named transport type. +/* +/* qmgr_transport_find() looks up an existing message transport +/* data structure. +/* +/* qmgr_transport_select() attempts to find a transport that +/* has messages pending delivery. This routine implements +/* round-robin search among transports. +/* +/* qmgr_transport_alloc() allocates a delivery process for the +/* specified transport type. Allocation is performed asynchronously. +/* When a process becomes available, the application callback routine +/* is invoked with as arguments the transport and a stream that +/* is connected to a delivery process. It is an error to call +/* qmgr_transport_alloc() while delivery process allocation for +/* the same transport is in progress. +/* +/* qmgr_transport_throttle blocks further allocation of delivery +/* processes for the named transport. Attempts to throttle a +/* throttled transport are ignored. +/* +/* qmgr_transport_unthrottle() undoes qmgr_transport_throttle(). +/* Attempts to unthrottle a non-throttled transport are ignored. +/* DIAGNOSTICS +/* Panic: consistency check failure. Fatal: out of memory. +/* LICENSE +/* .ad +/* .fi +/* The Secure Mailer license must be distributed with this software. +/* AUTHOR(S) +/* Wietse Venema +/* IBM T.J. Watson Research +/* P.O. Box 704 +/* Yorktown Heights, NY 10598, USA +/*--*/ + +/* System library. */ + +#include +#include + +/* Utility library. */ + +#include +#include +#include +#include +#include +#include + +/* Global library. */ + +#include +#include +#include +#include + +/* Application-specific. */ + +#include "qmgr.h" + +HTABLE *qmgr_transport_byname; /* transport by name */ +QMGR_TRANSPORT_LIST qmgr_transport_list;/* transports, round robin */ + + /* + * A local structure to remember a delivery process allocation request. + */ +typedef struct QMGR_TRANSPORT_ALLOC QMGR_TRANSPORT_ALLOC; + +struct QMGR_TRANSPORT_ALLOC { + QMGR_TRANSPORT *transport; /* transport context */ + VSTREAM *stream; /* delivery service stream */ + QMGR_TRANSPORT_ALLOC_NOTIFY notify; /* application call-back routine */ +}; + +/* qmgr_transport_unthrottle_wrapper - in case (char *) != (struct *) */ + +static void qmgr_transport_unthrottle_wrapper(int unused_event, char *context) +{ + qmgr_transport_unthrottle((QMGR_TRANSPORT *) context); +} + +/* qmgr_transport_unthrottle - open the throttle */ + +void qmgr_transport_unthrottle(QMGR_TRANSPORT *transport) +{ + char *myname = "qmgr_transport_unthrottle"; + + /* + * This routine runs after expiration of the timer set by + * qmgr_transport_throttle(), or whenever a delivery transport has been + * used without malfunction. In either case, we enable delivery again if + * the transport was blocked, otherwise the request is ignored. + */ + if ((transport->flags & QMGR_TRANSPORT_STAT_DEAD) != 0) { + if (msg_verbose) + msg_info("%s: transport %s", myname, transport->name); + transport->flags &= ~QMGR_TRANSPORT_STAT_DEAD; + if (transport->reason == 0) + msg_panic("%s: transport %s: null reason", myname, transport->name); + myfree(transport->reason); + transport->reason = 0; + event_cancel_timer(qmgr_transport_unthrottle_wrapper, + (char *) transport); + } +} + +/* qmgr_transport_throttle - disable delivery process allocation */ + +void qmgr_transport_throttle(QMGR_TRANSPORT *transport, const char *reason) +{ + char *myname = "qmgr_transport_throttle"; + + /* + * We are unable to connect to a deliver process for this type of message + * transport. Instead of hosing the system by retrying in a tight loop, + * back off and disable this transport type for a while. + */ + if ((transport->flags & QMGR_TRANSPORT_STAT_DEAD) == 0) { + if (msg_verbose) + msg_info("%s: transport %s: reason: %s", + myname, transport->name, reason); + transport->flags |= QMGR_TRANSPORT_STAT_DEAD; + if (transport->reason) + msg_panic("%s: transport %s: spurious reason: %s", + myname, transport->name, transport->reason); + transport->reason = mystrdup(reason); + event_request_timer(qmgr_transport_unthrottle_wrapper, + (char *) transport, var_transport_retry_time); + } +} + +/* qmgr_transport_abort - transport connect watchdog */ + +static void qmgr_transport_abort(int unused_event, char *context) +{ + QMGR_TRANSPORT_ALLOC *alloc = (QMGR_TRANSPORT_ALLOC *) context; + + msg_fatal("timeout connecting to transport: %s", alloc->transport->name); +} + +/* qmgr_transport_event - delivery process availability notice */ + +static void qmgr_transport_event(int unused_event, char *context) +{ + QMGR_TRANSPORT_ALLOC *alloc = (QMGR_TRANSPORT_ALLOC *) context; + + /* + * This routine notifies the application when the request given to + * qmgr_transport_alloc() completes. + */ + if (msg_verbose) + msg_info("transport_event: %s", alloc->transport->name); + + /* + * Connection request completed. Stop the watchdog timer. + */ + event_cancel_timer(qmgr_transport_abort, context); + + /* + * Disable further read events that end up calling this function. + */ + event_disable_readwrite(vstream_fileno(alloc->stream)); + alloc->transport->flags &= ~QMGR_TRANSPORT_STAT_BUSY; + + /* + * Notify the requestor. + */ + alloc->notify(alloc->transport, alloc->stream); + myfree((char *) alloc); +} + +#ifdef UNIX_DOMAIN_CONNECT_BLOCKS_FOR_ACCEPT + +/* qmgr_transport_connect - handle connection request completion */ + +static void qmgr_transport_connect(int unused_event, char *context) +{ + QMGR_TRANSPORT_ALLOC *alloc = (QMGR_TRANSPORT_ALLOC *) context; + + /* + * This code is necessary for some versions of LINUX, where connect(2) + * blocks until the application performs an accept(2). Reportedly, the + * same can happen on Solaris 2.5.1. + */ + event_disable_readwrite(vstream_fileno(alloc->stream)); + non_blocking(vstream_fileno(alloc->stream), BLOCKING); + event_enable_read(vstream_fileno(alloc->stream), + qmgr_transport_event, (char *) alloc); +} + +#endif + +/* qmgr_transport_select - select transport for allocation */ + +QMGR_TRANSPORT *qmgr_transport_select(void) +{ + QMGR_TRANSPORT *xport; + QMGR_QUEUE *queue; + + /* + * If we find a suitable transport, rotate the list of transports to + * effectuate round-robin selection. See similar selection code in + * qmgr_queue_select(). + */ +#define STAY_AWAY (QMGR_TRANSPORT_STAT_BUSY | QMGR_TRANSPORT_STAT_DEAD) + + for (xport = qmgr_transport_list.next; xport; xport = xport->peers.next) { + if (xport->flags & STAY_AWAY) + continue; + for (queue = xport->queue_list.next; queue; queue = queue->peers.next) { + if (queue->window > queue->busy_refcount && queue->todo.next != 0) { + QMGR_LIST_ROTATE(qmgr_transport_list, xport, peers); + if (msg_verbose) + msg_info("qmgr_transport_select: %s", xport->name); + return (xport); + } + } + } + return (0); +} + +/* qmgr_transport_alloc - allocate delivery process */ + +void qmgr_transport_alloc(QMGR_TRANSPORT *transport, QMGR_TRANSPORT_ALLOC_NOTIFY notify) +{ + QMGR_TRANSPORT_ALLOC *alloc; + VSTREAM *stream; + + /* + * Sanity checks. + */ + if (transport->flags & QMGR_TRANSPORT_STAT_DEAD) + msg_panic("qmgr_transport: dead transport: %s", transport->name); + if (transport->flags & QMGR_TRANSPORT_STAT_BUSY) + msg_panic("qmgr_transport: nested allocation: %s", transport->name); + + /* + * Connect to the well-known port for this delivery service, and wake up + * when a process announces its availability. In the mean time, block out + * other delivery process allocation attempts for this transport. In case + * of problems, back off. Do not hose the system when it is in trouble + * already. + */ +#ifdef UNIX_DOMAIN_CONNECT_BLOCKS_FOR_ACCEPT +#define BLOCK_MODE NON_BLOCKING +#define ENABLE_EVENTS event_enable_write +#define EVENT_HANDLER qmgr_transport_connect +#else +#define BLOCK_MODE BLOCKING +#define ENABLE_EVENTS event_enable_read +#define EVENT_HANDLER qmgr_transport_event +#endif + + if ((stream = mail_connect(MAIL_CLASS_PRIVATE, transport->name, BLOCK_MODE)) == 0) { + msg_warn("connect to transport %s: %m", transport->name); + qmgr_transport_throttle(transport, "transport is unavailable"); + return; + } + alloc = (QMGR_TRANSPORT_ALLOC *) mymalloc(sizeof(*alloc)); + alloc->stream = stream; + alloc->transport = transport; + alloc->notify = notify; + transport->flags |= QMGR_TRANSPORT_STAT_BUSY; + ENABLE_EVENTS(vstream_fileno(alloc->stream), EVENT_HANDLER, (char *) alloc); + + /* + * Guard against broken systems. + */ + event_request_timer(qmgr_transport_abort, (char *) alloc, + var_daemon_timeout); +} + +/* qmgr_transport_create - create transport instance */ + +QMGR_TRANSPORT *qmgr_transport_create(const char *name) +{ + QMGR_TRANSPORT *transport; + + if (htable_find(qmgr_transport_byname, name) != 0) + msg_panic("qmgr_transport_create: transport exists: %s", name); + transport = (QMGR_TRANSPORT *) mymalloc(sizeof(QMGR_TRANSPORT)); + transport->flags = 0; + transport->name = mystrdup(name); + + /* + * Use global configuration settings or transport-specific settings. + */ + transport->dest_concurrency_limit = + get_mail_conf_int2(name, _DEST_CON_LIMIT, + var_dest_con_limit, 0, 0); + transport->recipient_limit = + get_mail_conf_int2(name, _DEST_RCPT_LIMIT, + var_dest_rcpt_limit, 0, 0); + + if (transport->dest_concurrency_limit == 0 + || transport->dest_concurrency_limit >= var_init_dest_concurrency) + transport->init_dest_concurrency = var_init_dest_concurrency; + else + transport->init_dest_concurrency = transport->dest_concurrency_limit; + + transport->slot_cost = get_mail_conf_int2(name, _DELIVERY_SLOT_COST, + var_delivery_slot_cost, 0, 0); + transport->slot_loan = get_mail_conf_int2(name, _DELIVERY_SLOT_LOAN, + var_delivery_slot_loan, 0, 0); + transport->slot_loan_factor = + 100 - get_mail_conf_int2(name, _DELIVERY_SLOT_DISCOUNT, + var_delivery_slot_discount, 0, 100); + transport->min_slots = get_mail_conf_int2(name, _MIN_DELIVERY_SLOTS, + var_min_delivery_slots, 0, 0); + transport->rcpt_unused = get_mail_conf_int2(name, _XPORT_RCPT_LIMIT, + var_xport_rcpt_limit, 0, 0); + transport->rcpt_per_stack = get_mail_conf_int2(name, _STACK_RCPT_LIMIT, + var_stack_rcpt_limit, 0, 0); + + transport->queue_byname = htable_create(0); + QMGR_LIST_INIT(transport->queue_list); + transport->job_byname = htable_create(0); + QMGR_LIST_INIT(transport->job_list); + QMGR_LIST_INIT(transport->job_stack); + transport->job_next_unread = 0; + transport->candidate_cache = 0; + transport->candidate_cache_time = (time_t) 0; + transport->reason = 0; + if (qmgr_transport_byname == 0) + qmgr_transport_byname = htable_create(10); + htable_enter(qmgr_transport_byname, name, (char *) transport); + QMGR_LIST_PREPEND(qmgr_transport_list, transport, peers); + if (msg_verbose) + msg_info("qmgr_transport_create: %s concurrency %d recipients %d", + transport->name, transport->dest_concurrency_limit, + transport->recipient_limit); + return (transport); +} + +/* qmgr_transport_find - find transport instance */ + +QMGR_TRANSPORT *qmgr_transport_find(const char *name) +{ + return ((QMGR_TRANSPORT *) htable_find(qmgr_transport_byname, name)); +} diff --git a/postfix/smtp/smtp_sasl_glue.c b/postfix/smtp/smtp_sasl_glue.c index de1bfaff3..a5e40392f 100644 --- a/postfix/smtp/smtp_sasl_glue.c +++ b/postfix/smtp/smtp_sasl_glue.c @@ -221,9 +221,11 @@ int smtp_sasl_passwd_lookup(SMTP_STATE *state) msg_panic("%s: passwd map not initialized", myname); /* - * Look up the per-server password information. + * Look up the per-server password information. Try the hostname first, + * then try the destination. */ - if ((value = maps_find(smtp_sasl_passwd_map, state->session->host, 0)) != 0) { + if ((value = maps_find(smtp_sasl_passwd_map, state->session->host, 0)) != 0 + || (value = maps_find(smtp_sasl_passwd_map, state->request->nexthop, 0)) != 0) { state->sasl_username = mystrdup(value); passwd = split_at(state->sasl_username, ':'); state->sasl_passwd = mystrdup(passwd ? passwd : ""); diff --git a/postfix/smtpd/smtpd.c b/postfix/smtpd/smtpd.c index 9d58c62ca..4ce6788ed 100644 --- a/postfix/smtpd/smtpd.c +++ b/postfix/smtpd/smtpd.c @@ -97,12 +97,12 @@ /* Recipient of protocol/policy/resource/software error notices. /* .IP \fBhopcount_limit\fR /* Limit the number of \fBReceived:\fR message headers. -/* .IP \fBnotify_classes\fR -/* List of error classes. Of special interest are: /* .IP \fBlocal_recipient_maps\fR /* List of maps with user names that are local to \fB$myorigin\fR /* or \fB$inet_interfaces\fR. If this parameter is defined, /* then the SMTP server rejects mail for unknown local users. +/* .IP \fBnotify_classes\fR +/* List of error classes. Of special interest are: /* .RS /* .IP \fBpolicy\fR /* When a client violates any policy, mail a transcript of the @@ -334,6 +334,7 @@ bool var_allow_untrust_route; int var_smtpd_junk_cmd_limit; bool var_smtpd_sasl_enable; char *var_smtpd_sasl_opts; +char *var_smtpd_sasl_realm; /* * Global state, for stand-alone mode queue file cleanup. When this is @@ -1423,6 +1424,7 @@ int main(int argc, char **argv) VAR_ALIAS_MAPS, DEF_ALIAS_MAPS, &var_alias_maps, 0, 0, VAR_LOCAL_RCPT_MAPS, DEF_LOCAL_RCPT_MAPS, &var_local_rcpt_maps, 0, 0, VAR_SMTPD_SASL_OPTS, DEF_SMTPD_SASL_OPTS, &var_smtpd_sasl_opts, 0, 0, + VAR_SMTPD_SASL_REALM, DEF_SMTPD_SASL_REALM, &var_smtpd_sasl_realm, 1, 0, 0, }; diff --git a/postfix/smtpd/smtpd_check.c b/postfix/smtpd/smtpd_check.c index a1d6eaa5b..03e2ee4fe 100644 --- a/postfix/smtpd/smtpd_check.c +++ b/postfix/smtpd/smtpd_check.c @@ -768,13 +768,14 @@ static int reject_unknown_mailhost(SMTPD_STATE *state, char *name, return (SMTPD_CHECK_DUNNO); } +static int permit_auth_destination(char *recipient); + /* check_relay_domains - OK/FAIL for message relaying */ static int check_relay_domains(SMTPD_STATE *state, char *recipient, char *reply_name, char *reply_class) { char *myname = "check_relay_domains"; - static int permit_auth_destination(char *recipient); if (msg_verbose) msg_info("%s: %s", myname, recipient); diff --git a/postfix/smtpd/smtpd_sasl_glue.c b/postfix/smtpd/smtpd_sasl_glue.c index 6423a5108..c703e5370 100644 --- a/postfix/smtpd/smtpd_sasl_glue.c +++ b/postfix/smtpd/smtpd_sasl_glue.c @@ -197,7 +197,7 @@ void smtpd_sasl_connect(SMTPD_STATE *state) #define NO_SECURITY_LAYERS (0) #define NO_SESSION_CALLBACKS ((sasl_callback_t *) 0) - if (sasl_server_new("smtp", var_myhostname, DEFAULT_USER_REALM, + if (sasl_server_new("smtp", var_smtpd_sasl_realm, DEFAULT_USER_REALM, NO_SESSION_CALLBACKS, NO_SECURITY_LAYERS, &state->sasl_conn) != SASL_OK) msg_fatal("SASL per-connection server initialization");