Logging: the queue manager now logs a "status=expired"
record when it returns a message that is too old. Files:
*qmgr/qmgr_active.c.
+
+20010719
+
+ Feature: stiffer coupling between mail arrival rates and
+ mail delivery rates, using a trivial token-based scheme
+ where qmgr provides a token when it retrieves mail from
+ the incoming queue, and where the cleanup daemon wants a
+ token after enqueueing mail. If no token is available the
+ cleanup server pauses briefly and proceeds anyway. This
+ allows input rates to gradually exceed output rates. After
+ sustained exposure to high input loads Postfix reverts to
+ the old situation where it sacrifices output rates in favor
+ of receiving mail.
# address.
#
# You can also specify the absolute pathname of a pattern file instead
-# of listing the patterns here.
+# of listing the patterns here. Specify type:name for table-based lookups
+# (the value on the table right-hand side is not used).
#
#mynetworks = 168.100.189.0/28, 127.0.0.0/8
#mynetworks = $config_directory/mynetworks
# address.
#
# You can also specify the absolute pathname of a pattern file instead
-# of listing the patterns here.
+# of listing the patterns here. Specify type:name for table-based lookups
+# (the value on the table right-hand side is not used).
#
#mynetworks = 168.100.189.0/28, 127.0.0.0/8
#mynetworks = $config_directory/mynetworks
cleanup.o: ../../include/mail_params.h
cleanup.o: ../../include/record.h
cleanup.o: ../../include/rec_type.h
+cleanup.o: ../../include/mail_flow.h
cleanup.o: ../../include/mail_server.h
cleanup.o: cleanup.h
cleanup.o: ../../include/maps.h
MAIL_SERVER_PRE_INIT, cleanup_pre_jail,
MAIL_SERVER_POST_INIT, cleanup_post_jail,
MAIL_SERVER_PRE_ACCEPT, pre_accept,
+ MAIL_SERVER_FLOW_CTL,
0);
}
/* char *var_export_environ;
/* char *var_debug_peer_list;
/* int var_debug_peer_level;
+/* int var_glob_flow_ctl;
/*
/* void mail_params_init()
/* DESCRIPTION
char *var_mynetworks_style;
char *var_verp_delims;
char *var_verp_filter;
+int var_glob_flow_ctl;
char *var_import_environ;
char *var_export_environ;
VAR_FORK_TRIES, DEF_FORK_TRIES, &var_fork_tries, 1, 0,
VAR_FLOCK_TRIES, DEF_FLOCK_TRIES, &var_flock_tries, 1, 0,
VAR_DEBUG_PEER_LEVEL, DEF_DEBUG_PEER_LEVEL, &var_debug_peer_level, 1, 0,
+ VAR_GLOB_FLOW_CTL, DEF_GLOB_FLOW_CTL, &var_glob_flow_ctl, 0, 0,
0,
};
static CONFIG_TIME_TABLE time_defaults[] = {
#define DEF_VERP_FILTER "-=+"
extern char *var_verp_filter;
+ /*
+ * Global flow control. This allows for a stiffer coupling between receiving
+ * programs and the queue manager, so that receiving processes cannot easily
+ * overwhelm the file system. The coupling is not so tight that Postfix
+ * stops receiving mail althogether. It just slows down a bit so that
+ * sending processes get a chance to read from the disk.
+ */
+#define VAR_GLOB_FLOW_CTL "global_mail_flow_control"
+#define DEF_GLOB_FLOW_CTL 0
+extern int var_glob_flow_ctl;
+
/* LICENSE
/* .ad
/* .fi
* Version of this program.
*/
#define VAR_MAIL_VERSION "mail_version"
-#define DEF_MAIL_VERSION "Snapshot-20010714"
+#define DEF_MAIL_VERSION "Snapshot-20010720"
extern char *var_mail_version;
/* LICENSE
SRCS = master.c master_conf.c master_ent.c master_sig.c master_avail.c \
master_spawn.c master_service.c master_status.o master_listen.c \
master_proto.c single_server.c multi_server.c master_vars.c \
- master_wakeup.c
+ master_wakeup.c master_flow.c mail_flow.c
OBJS = master.o master_conf.o master_ent.o master_sig.o master_avail.o \
master_spawn.o master_service.o master_status.o master_listen.o \
- master_vars.o master_wakeup.o
-LIB_OBJ = single_server.o multi_server.o trigger_server.o master_proto.o
-HDRS = mail_server.h master_proto.h
+ master_vars.o master_wakeup.o master_flow.o
+LIB_OBJ = single_server.o multi_server.o trigger_server.o master_proto.o \
+ mail_flow.o
+HDRS = mail_server.h master_proto.h mail_flow.h
INT_HDR = master.h
WARN = -W -Wformat -Wimplicit -Wmissing-prototypes \
-Wparentheses -Wstrict-prototypes -Wswitch -Wuninitialized \
@$(EXPORT) make -f Makefile.in Makefile 1>&2
# do not edit below this line - it is generated by 'make depend'
+mail_flow.o: mail_flow.c
+mail_flow.o: ../../include/sys_defs.h
+mail_flow.o: ../../include/msg.h
+mail_flow.o: mail_flow.h
+mail_flow.o: master_proto.h
master.o: master.c
master.o: ../../include/sys_defs.h
master.o: ../../include/events.h
master_ent.o: ../../include/own_inet_addr.h
master_ent.o: master_proto.h
master_ent.o: master.h
+master_flow.o: master_flow.c
+master_flow.o: ../../include/sys_defs.h
+master_flow.o: ../../include/msg.h
+master_flow.o: ../../include/iostuff.h
+master_flow.o: master.h
+master_flow.o: master_proto.h
master_listen.o: master_listen.c
master_listen.o: ../../include/sys_defs.h
master_listen.o: ../../include/msg.h
--- /dev/null
+/*++
+/* NAME
+/* mail_flow 3
+/* SUMMARY
+/* global mail flow control
+/* SYNOPSIS
+/* #include <mail_flow.h>
+/*
+/* int mail_flow_get(count)
+/* int count;
+/*
+/* void mail_flow_put(count)
+/* int count;
+/* DESCRIPTION
+/* This module implements a simple flow control mechanism that
+/* is based on tokens that are consumed by mail receiving processes
+/* and that are produced by mail sending processes.
+/*
+/* mail_flow_get() attempts to read specified number of tokens. The
+/* result is > 0 for success, < 0 for failure. In the latter case,
+/* the process is expected to slow down a little.
+/*
+/* mail_flow_put() produces the specified number of tokens. The
+/* token producing process is expected to produce new tokens
+/* whenever it falls idle and no more tokens are available.
+/* BUGS
+/* The producer needs to wake up periodically to ensure that
+/* tokens are not lost due to leakage.
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Wietse Venema
+/* IBM T.J. Watson Research
+/* P.O. Box 704
+/* Yorktown Heights, NY 10598, USA
+/*--*/
+
+/* System library. */
+
+#include <sys_defs.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+/* Utility library. */
+
+#include <msg.h>
+
+/* Global library. */
+
+#include <mail_flow.h>
+
+/* Master library. */
+
+#include <master_proto.h>
+
+#define BUFFER_SIZE 1024
+
+/* mail_flow_get - read N tokens */
+
+int mail_flow_get(int len)
+{
+ char *myname = "mail_flow_get";
+ char buf[BUFFER_SIZE];
+ int count;
+ int n = 0;
+
+ /*
+ * Sanity check.
+ */
+ if (len <= 0)
+ msg_panic("%s: bad length %d", myname, len);
+
+ /*
+ * Read and discard N bytes.
+ */
+ for (count = len; count > 0; count -= n)
+ if ((n = read(MASTER_FLOW_READ, buf, count > BUFFER_SIZE ?
+ BUFFER_SIZE : count)) < 0)
+ return (-1);
+ if (msg_verbose)
+ msg_info("%s: %d %d", myname, len, len - count);
+ return (len - count);
+}
+
+/* mail_flow_put - put N tokens */
+
+int mail_flow_put(int len)
+{
+ char *myname = "mail_flow_put";
+ char buf[BUFFER_SIZE];
+ int count;
+ int n = 0;
+
+ /*
+ * Sanity check.
+ */
+ if (len <= 0)
+ msg_panic("%s: bad length %d", myname, len);
+
+ /*
+ * Read and discard N bytes.
+ */
+ for (count = len; count > 0; count -= n)
+ if ((n = read(MASTER_FLOW_READ, buf, count > BUFFER_SIZE ?
+ BUFFER_SIZE : count)) < 0)
+ return (-1);
+ if (msg_verbose)
+ msg_info("%s: %d %d", myname, len, len - count);
+ return (len - count);
+}
--- /dev/null
+#ifndef _MAIL_FLOW_H_INCLUDED_
+#define _MAIL_FLOW_H_INCLUDED_
+
+/*++
+/* NAME
+/* mail_flow 3h
+/* SUMMARY
+/* global mail flow control
+/* SYNOPSIS
+/* #include <mail_flow.h>
+/* DESCRIPTION
+/* .nf
+
+ /*
+ * Functional interface.
+ */
+extern int mail_flow_get(int);
+extern int mail_flow_put(int);
+
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Wietse Venema
+/* IBM T.J. Watson Research
+/* P.O. Box 704
+/* Yorktown Heights, NY 10598, USA
+/*--*/
+
+#endif
#define MAIL_SERVER_EXIT 13
#define MAIL_SERVER_PRE_ACCEPT 14
+#define MAIL_SERVER_FLOW_CTL 20
+
typedef void (*MAIL_SERVER_INIT_FN) (char *, char **);
typedef int (*MAIL_SERVER_LOOP_FN) (char *, char **);
typedef void (*MAIL_SERVER_EXIT_FN) (char *, char **);
* when a service listens on many ports. In order to do this right we
* must change the master-child interface so that descriptors do not need
* to have fixed numbers.
+ *
+ * In a child we need two descriptors for the flow control pipe, one for
+ * child->master status updates and at least one for listening.
*/
- for (n = 0; n < 3; n++) {
+ for (n = 0; n < 5; n++) {
if (close_on_exec(dup(0), CLOSE_ON_EXEC) < 0)
msg_fatal("dup(0): %m");
}
*/
master_config();
master_sigsetup();
+ master_flow_init();
msg_info("daemon started");
/*
extern void master_reap_child(void);
extern void master_delete_children(MASTER_SERV *);
+ /*
+ * master_flow.c
+ */
+void master_flow_init(void);
+int master_flow_pipe[2];
+
/* DIAGNOSTICS
/* BUGS
/* SEE ALSO
--- /dev/null
+/* System library. */
+
+#include <sys_defs.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+/* Utility library. */
+
+#include <msg.h>
+#include <iostuff.h>
+
+/* Application-specific. */
+
+#include <master.h>
+#include <master_proto.h>
+
+int master_flow_pipe[2];
+
+/* master_flow_init - initialize the flow control channel */
+
+void master_flow_init(void)
+{
+ char *myname = "master_flow_init";
+
+ if (pipe(master_flow_pipe) < 0)
+ msg_fatal("%s: pipe: %m", myname);
+
+ non_blocking(master_flow_pipe[0], NON_BLOCKING);
+ non_blocking(master_flow_pipe[1], NON_BLOCKING);
+}
extern int master_notify(int, int); /* encapsulate status msg */
+ /*
+ * File descriptors inherited from the master process. The flow control pipe
+ * is read by receive processes and is written to by send processes. If
+ * receive processes get too far ahead they will pause for a brief moment.
+ */
+#define MASTER_FLOW_READ 3
+#define MASTER_FLOW_WRITE 4
+
/*
* File descriptors inherited from the master process. All processes that
* provide a given service share the same status file descriptor, and listen
* actually the lowest-numbered descriptor of a sequence of descriptors to
* listen on.
*/
-#define MASTER_STATUS_FD 3 /* shared channel to parent */
-#define MASTER_LISTEN_FD 4 /* accept connections here */
+#define MASTER_STATUS_FD 5 /* shared channel to parent */
+#define MASTER_LISTEN_FD 6 /* accept connections here */
/* LICENSE
/* .ad
case 0:
msg_cleanup((void (*) (void)) 0); /* disable exit handler */
closelog(); /* avoid filedes leak */
+
+ if (master_flow_pipe[0] <= MASTER_FLOW_READ)
+ msg_fatal("%s: flow pipe read descriptor <= %d",
+ myname, MASTER_FLOW_READ);
+ if (DUP2(master_flow_pipe[0], MASTER_FLOW_READ) < 0)
+ msg_fatal("%s: dup2: %m", myname);
+ if (close(master_flow_pipe[0]) < 0)
+ msg_fatal("close %d: %m", master_flow_pipe[0]);
+
+ if (master_flow_pipe[1] <= MASTER_FLOW_WRITE)
+ msg_fatal("%s: flow pipe read descriptor <= %d",
+ myname, MASTER_FLOW_WRITE);
+ if (DUP2(master_flow_pipe[1], MASTER_FLOW_WRITE) < 0)
+ msg_fatal("%s: dup2: %m", myname);
+ if (close(master_flow_pipe[1]) < 0)
+ msg_fatal("close %d: %m", master_flow_pipe[1]);
+
close(serv->status_fd[0]); /* status channel */
if (serv->status_fd[1] <= MASTER_STATUS_FD)
msg_fatal("%s: status file descriptor collision", myname);
}
event_enable_read(MASTER_STATUS_FD, multi_server_abort, (char *) 0);
close_on_exec(MASTER_STATUS_FD, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_READ, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_WRITE, CLOSE_ON_EXEC);
watchdog = watchdog_create(var_daemon_timeout, (WATCHDOG_FN) 0, (char *) 0);
/*
#include <mail_conf.h>
#include <timed_ipc.h>
#include <resolve_local.h>
+#include <mail_flow.h>
/* Process manager. */
static void (*single_server_onexit) (char *, char **);
static void (*single_server_pre_accept) (char *, char **);
static VSTREAM *single_server_lock;
+static int single_server_flow_ctl;
/* single_server_exit - normal termination */
if (msg_verbose)
msg_info("connection closed");
use_count++;
+ if (single_server_flow_ctl)
+ if (mail_flow_get(1) < 0)
+ rand_sleep(single_server_flow_ctl, 0);
if (var_idle_limit > 0)
event_request_timer(single_server_timeout, (char *) 0, var_idle_limit);
}
case MAIL_SERVER_PRE_ACCEPT:
single_server_pre_accept = va_arg(ap, MAIL_SERVER_ACCEPT_FN);
break;
+ case MAIL_SERVER_FLOW_CTL:
+ single_server_flow_ctl = var_glob_flow_ctl;
+ break;
default:
msg_panic("%s: unknown argument type: %d", myname, key);
}
}
event_enable_read(MASTER_STATUS_FD, single_server_abort, (char *) 0);
close_on_exec(MASTER_STATUS_FD, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_READ, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_WRITE, CLOSE_ON_EXEC);
watchdog = watchdog_create(var_daemon_timeout, (WATCHDOG_FN) 0, (char *) 0);
/*
}
event_enable_read(MASTER_STATUS_FD, trigger_server_abort, (char *) 0);
close_on_exec(MASTER_STATUS_FD, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_READ, CLOSE_ON_EXEC);
+ close_on_exec(MASTER_FLOW_WRITE, CLOSE_ON_EXEC);
watchdog = watchdog_create(1000, (WATCHDOG_FN) 0, (char *) 0);
/*
#include <mail_conf.h>
#include <mail_params.h>
#include <mail_proto.h> /* QMGR_SCAN constants */
+#include <mail_flow.h>
/* Master process interface */
if (qmgr_message_count < var_qmgr_active_limit)
if ((df_path = qmgr_scan_next(qmgr_deferred)) != 0)
qmgr_active_feed(qmgr_deferred, df_path);
+
+ /*
+ * Global flow control. If enabled, slow down receiving processes that
+ * get ahead of the queue manager, but don't block them completely.
+ */
+ if (var_glob_flow_ctl) {
+ if (in_path)
+ mail_flow_put(1);
+ else
+ mail_flow_get(1000);
+ }
if (in_path || df_path)
return (DONT_WAIT);
return (WAIT_FOR_EVENT);
/* \fBpostalias\fR terminates with zero exit status in case of success
/* (including successful \fBpostmap -q\fR lookup) and terminates
/* with non-zero exit status in case of failure.
-/* BUGS
-/* The "delete key" support is limited to one delete operation
-/* per command invocation.
/* ENVIRONMENT
/* .ad
/* .fi
/* \fBpostmap\fR terminates with zero exit status in case of success
/* (including successful \fBpostmap -q\fR lookup) and terminates
/* with non-zero exit status in case of failure.
-/* BUGS
-/* The "delete key" support is limited to one delete operation
-/* per command invocation.
/* ENVIRONMENT
/* .ad
/* .fi
#include <mail_conf.h>
#include <mail_params.h>
#include <mail_proto.h> /* QMGR_SCAN constants */
+#include <mail_flow.h>
/* Master process interface */
&& qmgr_recipient_count < var_qmgr_rcpt_limit)
if ((df_path = qmgr_scan_next(qmgr_deferred)) != 0)
qmgr_active_feed(qmgr_deferred, df_path);
+
+ /*
+ * Global flow control. If enabled, slow down receiving processes that
+ * get ahead of the queue manager, but don't block them completely.
+ */
+ if (var_glob_flow_ctl) {
+ if (in_path)
+ mail_flow_put(1);
+ else
+ mail_flow_get(1000);
+ }
if (in_path || df_path)
return (DONT_WAIT);
return (WAIT_FOR_EVENT);
projects / thirdparty / postfix.git / commitdiff
