src/postlog/postlog.c, src/postmap/postmap.c,
src/postmulti/postmulti.c, src/postqueue/postqueue.c,
src/postsuper/postsuper.c, src/sendmail/sendmail.c.
+
+20111211
+
+ Feature: first/next (sequence) support in the proxymap
+ protocol. This is needed for cache cleanup of a proxied
+ postscreen or verify persistent cache. Files:
+ global/dict_proxy.[hc], proxymap/proxymap.c.
+
+ Feature: memcache client support without libmemcache
+ dependencies. Files: global/memcache_proto.[hc],
+ global/dict_memcache.c.
+
+ Feature: support for persistent backup database in the
+ memcache client. The database can be shared with the proxymap
+ service, but it needs to be listed as "proxy:maptype:mapname"
+ in the proxy_read_maps or proxy_write_maps parameter value
+ (depending on whether the access is read-only or read-write).
+ Support for proxymap-over-tcp (proxy:maptype:mapname@host:port)
+ is under development. File: global/dict_memcache.c.
I\bIn\bnt\btr\bro\bod\bdu\buc\bct\bti\bio\bon\bn
-The Postfix memcache client type allows you to hook up Postfix to a memcache
-server. This implementation supports multiple memcache servers for redundancy,
-and multiple memcache clients that you can use for different table lookups. The
-Postfix memcache client supports both lookup and update operations.
-
-Typically, a memcache map is used to reduce query load on a database server, or
-to share a low-latency database among different Postfix instances.
+The Postfix memcache client allows you to hook up Postfix to a memcache server.
+The current implementation supports one memcache server per Postfix table, with
+one optional Postfix database that provides persistent backup. The Postfix
+memcache client supports the lookup, update, delete and sequence operations.
+The sequence (i.e. first/next) operation requires a backup database that
+supports this operation.
+
+Typically, the Postfix memcache client is used to reduce query load on a
+persistent database, but it may also be used to query a memory-only database
+for low-value, easy-to-create, information such as a reputation cache for
+postscreen(8), verify(8) or greylisting.
L\bLi\bim\bmi\bit\bta\bat\bti\bio\bon\bns\bs
- * The Postfix memcache client is based on libmemcache, which will terminate
- its process after a memcache server goes down. To avoid this, set up
- redundant memcache servers that have no common source of failure.
-
* The Postfix memcache client cannot be used for security-sensitive tables
such as alias_maps (these may contain "|command" and "/file/name"
- destinations), or virtual_uid_maps and virtual_gid_maps (these specify UNIX
- process privileges). Typically, a memcache database is shared via a TCP
- socket, and is writable not only by Postfix, but by any process that can
- talk to the memcache server.
-
- * The Postfix memcache client requires additional configuration when used
- with the postscreen(8) and verify(8) daemons. For details see the ttl
- parameter discussion in the memcache_table(5) manual page.
+ destinations), or virtual_uid_maps, virtual_gid_maps and
+ virtual_mailbox_maps (these specify UNIX process privileges or "/file/name"
+ destinations). Typically, a memcache database is writable by any process
+ that can talk to the memcache server; in contrast, security-sensitive
+ tables must not be writable by the unprivileged Postfix user.
- * The Postfix memcache client is supported only with libmemcache version
- 1.4.0. Some libmemcache features are documented by reading libmemcache
- source code, instead of a proper API.
+ * The Postfix memcache client requires additional configuration when used as
+ postscreen(8) or verify(8) cache. For details see the backup and ttl
+ parameter discussions in the memcache_table(5) manual page.
B\bBu\bui\bil\bld\bdi\bin\bng\bg P\bPo\bos\bst\btf\bfi\bix\bx w\bwi\bit\bth\bh m\bme\bem\bmc\bca\bac\bch\bhe\be s\bsu\bup\bpp\bpo\bor\brt\bt
-To build Postfix with memcache client support, specify -DHAS_MEMCACHE, the
-location of the libmemcache include files, and the location of the libmemcache
-object library.
-
-For example:
-
- % make -f Makefile.init makefiles \
- 'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \
- 'AUXLIBS=-L/usr/local/lib -lmemcache'
-
-Then run 'make'.
-
-If the build fails with "undefined reference to `mcm_buf_len'" (and with a
-similar error message for mcm_buf_remain_off), then you need to edit
-libmemcache source code.
-
-The following instructions apply to libmemcache 1.4.0.rc2.
-
- * Open the libmemcache source file include/memcache/buffer.h.
-
- * Delete the "inline" words before the functions that were reported in the
- "undefined reference" error messages.
-
- * Recompile and reinstall libmemcache.
-
-Then, continue building Postfix by running 'make'.
+The Postfix memcache client has no external dependencies, and is therefore
+built into Postfix by default.
C\bCo\bon\bnf\bfi\big\bgu\bur\bri\bin\bng\bg m\bme\bem\bmc\bca\bac\bch\bhe\be l\blo\boo\bok\bku\bup\bp t\bta\bab\bbl\ble\bes\bs
C\bCr\bre\bed\bdi\bit\bts\bs
-The first memcache client for Postfix was written by Omar Kilani.
+The first memcache client for Postfix was written by Omar Kilani, and was based
+on the libmemcache library.
-Wietse wrote a new memcache client from the ground up. Besides also using
-libmemcache, the current implementation bears no resemblance to Omar's work.
+Wietse wrote the current memcache client from the ground up. This
+implementation does not use libmemcache, and bears no resemblance to earlier
+work.
If you upgrade from Postfix 2.7 or earlier, read RELEASE_NOTES-2.8
before proceeding.
+Major changes with snapshot 20111213
+====================================
+
+Support for a persistent backup database in the memcache client.
+The memcache client updates the memcache whenever it looks up or
+modifies information in the persistent database.
+
+The persistent database can be shared with the proxymap service,
+but it needs to be listed as "proxy:maptype:mapname" in the
+proxy_read_maps or proxy_write_maps parameter value (depending on
+whether the access is read-only or read-write).
+
+Support for proxymap-over-tcp (proxy:maptype:mapname@host:port) is
+under development.
+
+Elimination of dependencies on the libmemcache library. Postfix
+memcache support is now compiled in by default.
+
+Major changes with snapshot 20111209
+====================================
+
+memcache lookup and update support. This provides a way to share
+postscreen(8) or verify(8) caches between Postfix instances. The
+Postfix memcache client can't be used for security-sensitive
+information, and it supports only libmemcache version 1.4.0. See
+MEMCACHE_README and memcache_table(5) for details and limitations.
+
Incompatible changes with snapshot 20111205
===========================================
or require that they reset dict_errno on entry, either exit
with a fatal error or set dict_errno on error.
+ dict_memcache: treat "bad" key as cache miss, i.e. read/write
+ the database as if the cache did not exist.
+
Is it possible to replace msg_fatal calls in match_ops.c
by msg_warn and longjmp? The callers will have to specify
if they want the code to return instead of terminate.
<h2>Introduction</h2>
-<p>The Postfix memcache client type allows you to hook up Postfix to
-a memcache server. This implementation supports multiple memcache
-servers for redundancy, and multiple memcache clients that you can
-use for different table lookups. The Postfix memcache client
-supports both lookup and update operations. </p>
-
-<p> Typically, a memcache map is used to reduce query load on a
-database server, or to share a low-latency database among different
-Postfix instances. </p>
+<p>The Postfix memcache client allows you to hook up Postfix to a
+memcache server. The current implementation supports one memcache
+server per Postfix table, with one optional Postfix database that
+provides persistent backup. The Postfix memcache client supports
+the lookup, update, delete and sequence operations. The sequence
+(i.e. first/next) operation requires a backup database that supports
+this operation. </p>
+
+<p> Typically, the Postfix memcache client is used to reduce query
+load on a persistent database, but it may also be used to query a
+memory-only database for low-value, easy-to-create, information
+such as a reputation cache for <a href="postscreen.8.html">postscreen(8)</a>, <a href="verify.8.html">verify(8)</a> or greylisting.
+</p>
<h2>Limitations</h2>
<ul>
-<li> <p> The Postfix memcache client is based on libmemcache, which
-will terminate its process after a memcache server goes down. To
-avoid this, set up redundant memcache servers that have no common
-source of failure. </p>
-
<li> <p> The Postfix memcache client cannot be used for security-sensitive
tables such as <tt><a href="postconf.5.html#alias_maps">alias_maps</a></tt> (these may contain "<tt>|command</tt>"
-and "<tt>/file/name</tt>" destinations), or <tt><a href="postconf.5.html#virtual_uid_maps">virtual_uid_maps</a></tt>
-and <tt><a href="postconf.5.html#virtual_gid_maps">virtual_gid_maps</a></tt> (these specify UNIX process privileges).
-Typically, a memcache database is shared via a TCP socket, and is
-writable not only by Postfix, but by any process that can talk to
-the memcache server. </p>
+and "<tt>/file/name</tt>" destinations), or <tt><a href="postconf.5.html#virtual_uid_maps">virtual_uid_maps</a></tt>,
+<tt><a href="postconf.5.html#virtual_gid_maps">virtual_gid_maps</a></tt> and <tt><a href="postconf.5.html#virtual_mailbox_maps">virtual_mailbox_maps</a></tt> (these
+specify UNIX process privileges or "<tt>/file/name</tt>" destinations).
+Typically, a memcache database is writable by any process that can
+talk to the memcache server; in contrast, security-sensitive tables
+must not be writable by the unprivileged Postfix user. </p>
<li> <p> The Postfix memcache client requires additional configuration
-when used with the <a href="postscreen.8.html">postscreen(8)</a> and <a href="verify.8.html">verify(8)</a> daemons. For details
-see the <tt>ttl</tt> parameter discussion in the <a href="memcache_table.5.html">memcache_table(5)</a>
-manual page. </p>
-
-<li> <p> The Postfix memcache client is supported only with libmemcache
-version 1.4.0. Some libmemcache features are documented by reading
-libmemcache source code, instead of a proper API. </p>
+when used as <a href="postscreen.8.html">postscreen(8)</a> or <a href="verify.8.html">verify(8)</a> cache. For details see the
+<tt>backup</tt> and <tt>ttl</tt> parameter discussions in the
+<a href="memcache_table.5.html">memcache_table(5)</a> manual page. </p>
</ul>
<h2>Building Postfix with memcache support</h2>
-<p>To build Postfix with memcache client support, specify
-<tt>-DHAS_MEMCACHE</tt>, the location of the libmemcache include
-files, and the location of the libmemcache object library. </p>
-
-<p> For example: </p>
-
-<blockquote>
-<pre>
-% make -f Makefile.init makefiles \
- 'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \
- 'AUXLIBS=-L/usr/local/lib -lmemcache'
-</pre>
-</blockquote>
-
-<p> Then run 'make'. </p>
-
-<p> If the build fails with "<tt>undefined reference to `mcm_buf_len'</tt>"
-(and with a similar error message for <tt>mcm_buf_remain_off</tt>),
-then you need to edit libmemcache source code. </p>
-
-<p> The following instructions apply to libmemcache 1.4.0.rc2. </p>
-
-<ul>
-
-<li> <p> Open the libmemcache source file
-<tt>include/memcache/buffer.h</tt>. </p>
-
-<li> <p> Delete the "<tt>inline</tt>" words before the functions
-that were reported in the "<tt>undefined reference</tt>" error
-messages. </p>
-
-<li> <p> Recompile and reinstall libmemcache. </p>
-
-</ul>
-
-<p> Then, continue building Postfix by running 'make'. </p>
+<p>The Postfix memcache client has no external dependencies,
+and is therefore built into Postfix by default. </p>
<h2>Configuring memcache lookup tables</h2>
<h2>Credits</h2>
-<p> The first memcache client for Postfix was written by Omar Kilani. </p>
+<p> The first memcache client for Postfix was written by Omar Kilani,
+and was based on the libmemcache library. </p>
-<p> Wietse wrote a new memcache client from the ground up. Besides
-also using libmemcache, the current implementation bears no resemblance
-to Omar's work. </p>
+<p> Wietse wrote the current memcache client from the ground up.
+This implementation does not use libmemcache, and bears no resemblance
+to earlier work. </p>
</body>
or <b>db</b> format.
Alternatively, lookup tables can be specified as memcache
- instances. In order to use memcache lookups, define a
-
memcache source as a lookup table in <a href="postconf.5.html">main.cf</a>, for example:
+ instances. To use memcache lookups, define a memcache
+ source as a lookup table in <a href="postconf.5.html">main.cf</a>, for example:
<a href="postconf.5.html#virtual_alias_maps">virtual_alias_maps</a> = <a href="memcache_table.5.html">memcache</a>:/etc/postfix/memcache-aliases.cf
- The file /etc/postfix/memcache-aliases.cf has the same
- format as the Postfix <a href="postconf.5.html">main.cf</a> file, and specifies the
+ The file /etc/postfix/memcache-aliases.cf has the same
+ format as the Postfix <a href="postconf.5.html">main.cf</a> file, and specifies the
parameters described below.
- The Postfix memcache client supports the lookup and update
- operations.
+ The Postfix memcache client supports the lookup, update,
+ delete and sequence (first/next) operations. The sequence
+ operation requires a backup database that supports the
+ operation.
<b>MEMCACHE PARAMETERS</b>
- <b>hosts (default: localhost:11211)</b>
- The memcache servers that Postfix will try to con-
- nect to. Specify a hostname or address, optionally
- followed by ":" and a port name or number. The
- default port is 11211. Examples:
-
- hosts = memcache01.example.com
- memcache02.example.com
+ <b>backup</b> An optional Postfix database that provides persis-
+ tent backup for the memcache database. The Postfix
+ memcache client will update the memcache database
+ whenever it looks up or changes information in the
+ persistent database. Specify a Postfix "<a href="DATABASE_README.html">type:table</a>"
+ database. Example:
+
+ backup = btree:/var/lib/postfix/<a href="postconf.5.html#postscreen_cache_map">postscreen_cache_map</a>
+
+ Access to remote proxymap servers is under develop-
+ ment.
+
+ NOTE 1: When using memcache with persistent backup
+ as <a href="postscreen.8.html"><b>postscreen</b>(8)</a> or <a href="verify.8.html"><b>verify</b>(8)</a> cache, disable auto-
+ matic cache cleanup (*_cache_cleanup_interval = 0)
+ in all Postfix instances except for one instance
+ that will be responsible for cache cleanup.
+
+ NOTE 2: In the case of a proxied database, the full
+ database name (including the "<a href="proxymap.8.html">proxy</a>:" prefix) must
+ be specified in the proxymap server's
+ <a href="postconf.5.html#proxy_read_maps">proxy_read_maps</a> or <a href="postconf.5.html#proxy_write_maps">proxy_write_maps</a> setting
+ (depending on whether the access is read-only or
+ read-write).
+
+ <b>memcache (default: inet:localhost:11211)</b>
+ The memcache server (note: singular) that Postfix
+ will try to connect to. For a TCP server specify
+ "inet:" followed by a hostname or address, ":", and
+ a port name or number. For a UNIX-domain server
+ specify "unix:" followed by the socket pathname.
+ Examples:
+
+ memcache = inet:memcache.example.com
+ memcache = unix:/path/to/socket
+
+ NOTE: In the case of a UNIX-domain socket, it must
+ be accessible by the unprivileged postfix user and
+ by the memcached process.
<b>key_format (default: %s)</b>
Format of the lookup and update keys in memcache
Optional flags that should be stored along with a
memcache update.
- <b>ttl (default: 604800)</b>
+ <b>ttl (default: 3600)</b>
The expiration time in seconds of memcache updates.
- The default is one week.
- When using memcache tables with <a href="postscreen.8.html"><b>postscreen</b>(8)</a> or
- <a href="verify.8.html"><b>verify</b>(8)</a>, specify a zero *_cache_cleanup_interval
- value, and specify the largest <a href="postscreen.8.html"><b>postscreen</b>(8)</a> *_ttl
+ NOTE 1: When using a memcache table as
+ <a href="postscreen.8.html"><b>postscreen</b>(8)</a> or <a href="verify.8.html"><b>verify</b>(8)</a> cache without persistent
+ backup, specify a zero *_cache_cleanup_interval
+ value with all Postfix instances that use the mem-
+ cache, and specify the largest <a href="postscreen.8.html"><b>postscreen</b>(8)</a> *_ttl
value or <a href="verify.8.html"><b>verify</b>(8)</a> *_expire_time value as the mem-
- cache map's <b>ttl</b> value.
+ cache table's <b>ttl</b> value.
- Note: according to memcache protocol documentation,
- a value greater than 30 days (2592000 seconds)
- specifies absolute UNIX time. Smaller values are
- relative to the time of the update.
+ NOTE 2: According to memcache protocol documenta-
+ tion, a value greater than 30 days (2592000 sec-
+ onds) specifies absolute UNIX time. Smaller values
+ are relative to the time of the update.
<b>BUGS</b>
- The Postfix memcache client is based on libmemcache, which
- will terminate its process after a memcache server goes
- down. To avoid this, set up redundant memcache servers
- that have no common source of failure.
-
- The Postfix memcache client cannot be used for security-
- sensitive tables such as <b><a href="postconf.5.html#alias_maps">alias_maps</a></b> (these may contain
- "<i>|command</i> and "<i>/file/name</i>" destinations), or <b><a href="postconf.5.html#virtual_uid_maps">vir</a>-</b>
- <b><a href="postconf.5.html#virtual_uid_maps">tual_uid_maps</a></b> and <b><a href="postconf.5.html#virtual_gid_maps">virtual_gid_maps</a></b> (these specify UNIX
- process privileges). In a typical deployment a memcache
- database is shared via a TCP socket, and is therefore
- writable not only by Postfix, but by any process that can
- talk to the memcache server.
+ The Postfix memcache client cannot be used for security-
+ sensitive tables such as <b><a href="postconf.5.html#alias_maps">alias_maps</a></b> (these may contain
+ "<i>|command</i> and "<i>/file/name</i>" destinations), or <b><a href="postconf.5.html#virtual_uid_maps">vir</a>-</b>
+ <b><a href="postconf.5.html#virtual_uid_maps">tual_uid_maps</a></b>, <b><a href="postconf.5.html#virtual_gid_maps">virtual_gid_maps</a></b> and <b><a href="postconf.5.html#virtual_mailbox_maps">virtual_mailbox_maps</a></b>
+ (these specify UNIX process privileges or "<i>/file/name</i>"
+ destinations). In a typical deployment a memcache data-
+ base is writable by any process that can talk to the mem-
+ cache server; in contrast, security-sensitive tables must
+ not be writable by the unprivileged Postfix user.
The Postfix memcache client requires additional configura-
- tion when used with the <a href="postscreen.8.html"><b>postscreen</b>(8)</a> and <a href="verify.8.html"><b>verify</b>(8)</a> dae-
- mons. For details see the <b>ttl</b> parameter discussion at the
- end of the MEMCACHE PARAMETERS section in this document.
-
- The Postfix memcache client is supported only with libmem-
- cache version 1.4.0. Some libmemcache features are docu-
- mented by reading libmemcache source code, instead a
- proper API.
+ tion when used as <a href="postscreen.8.html"><b>postscreen</b>(8)</a> or <a href="verify.8.html"><b>verify</b>(8)</a> cache. For
+ details see the <b>backup</b> and <b>ttl</b> parameter discussions in
+ the MEMCACHE PARAMETERS section above.
<b>SEE ALSO</b>
<a href="postmap.1.html">postmap(1)</a>, Postfix lookup table manager
<a href="MEMCACHE_README.html">MEMCACHE_README</a>, Postfix memcache client guide
<b>LICENSE</b>
- The Secure Mailer license must be distributed with this
+ The Secure Mailer license must be distributed with this
software.
<b>HISTORY</b>
- The first memcache client for Postfix was written by Omar
- Kilani. Besides being implemented on libmemcache, this
- implementation bears no resemblance to his work.
+ The first memcache client for Postfix was written by Omar
+ Kilani, and was based on libmemcache. The Postfix imple-
+ mentation does not use libmemcache, and bears no resem-
+ blance to earlier work.
<b>AUTHOR(S)</b>
Wietse Venema
This request is supported in Postfix 2.5 and later.
+ <b>sequence</b> <i>maptype:mapname flags function</i>
+ Iterate over the specified database. The <i>function</i>
+ is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT.
+ The reply is the request completion status code and
+ a lookup key and result value, if found.
+
+ This request is supported in Postfix 2.9 and later.
+
The request completion status is one of OK, RETRY, NOKEY
(lookup failed because the key was not found), BAD (mal-
formed request) or DENY (the table is not approved for
\fBdbm\fR or \fBdb\fR format.
Alternatively, lookup tables can be specified as memcache
-instances. In order to use memcache lookups, define a
-memcache source as a lookup table in main.cf, for example:
+instances. To use memcache lookups, define a memcache
+source as a lookup table in main.cf, for example:
.nf
virtual_alias_maps = memcache:/etc/postfix/memcache-aliases.cf
format as the Postfix main.cf file, and specifies the
parameters described below.
-The Postfix memcache client supports the lookup and update
-operations.
+The Postfix memcache client supports the lookup, update,
+delete and sequence (first/next) operations. The sequence
+operation requires a backup database that supports the
+operation.
.SH "MEMCACHE PARAMETERS"
.na
.nf
.ad
.fi
-.IP "\fBhosts (default: localhost:11211)\fR"
-The memcache servers that Postfix will try to connect to.
-Specify a hostname or address, optionally followed by ":"
-and a port name or number. The default port is 11211.
-Examples:
+.IP \fBbackup\fR
+An optional Postfix database that provides persistent backup
+for the memcache database. The Postfix memcache client will
+update the memcache database whenever it looks up or changes
+information in the persistent database. Specify a Postfix
+"type:table" database. Example:
.nf
- hosts = memcache01.example.com
- memcache02.example.com
+ backup = btree:/var/lib/postfix/postscreen_cache_map
.fi
+
+Access to remote proxymap servers is under development.
+
+NOTE 1: When using memcache with persistent backup as
+\fBpostscreen\fR(8) or \fBverify\fR(8) cache, disable
+automatic cache cleanup (*_cache_cleanup_interval = 0) in
+all Postfix instances except for one instance that will be
+responsible for cache cleanup.
+
+NOTE 2: In the case of a proxied database, the full database
+name (including the "proxy:" prefix) must be specified in
+the proxymap server's proxy_read_maps or proxy_write_maps
+setting (depending on whether the access is read-only or
+read-write).
+.IP "\fBmemcache (default: inet:localhost:11211)\fR"
+The memcache server (note: singular) that Postfix will try
+to connect to. For a TCP server specify "inet:" followed by
+a hostname or address, ":", and a port name or number.
+For a UNIX-domain server specify "unix:" followed by the
+socket pathname. Examples:
+
+.nf
+ memcache = inet:memcache.example.com
+ memcache = unix:/path/to/socket
+.fi
+
+NOTE: In the case of a UNIX-domain socket, it must be accessible
+by the unprivileged postfix user and by the memcached process.
.IP "\fBkey_format (default: %s)\fB"
Format of the lookup and update keys in memcache queries.
By default, these are the same as the lookup and update
.IP "\fBflags (default: 0)\fR"
Optional flags that should be stored along with a memcache
update.
-.IP "\fBttl (default: 604800)\fR"
+.IP "\fBttl (default: 3600)\fR"
The expiration time in seconds of memcache updates.
-The default is one week.
-When using memcache tables with \fBpostscreen\fR(8) or
-\fBverify\fR(8), specify a zero *_cache_cleanup_interval
-value, and specify the largest \fBpostscreen\fR(8) *_ttl
-value or \fBverify\fR(8) *_expire_time value as the memcache
-map's \fBttl\fR value.
+NOTE 1: When using a memcache table as \fBpostscreen\fR(8)
+or \fBverify\fR(8) cache without persistent backup, specify
+a zero *_cache_cleanup_interval value with all Postfix
+instances that use the memcache, and specify the largest
+\fBpostscreen\fR(8) *_ttl value or \fBverify\fR(8) *_expire_time
+value as the memcache table's \fBttl\fR value.
-Note: according to memcache protocol documentation, a value
-greater than 30 days (2592000 seconds) specifies absolute UNIX
+NOTE 2: According to memcache protocol documentation, a
+value greater than 30 days (2592000 seconds) specifies
+absolute UNIX
time. Smaller values are relative to the time of the update.
.SH BUGS
.ad
.fi
-The Postfix memcache client is based on libmemcache, which
-will terminate its process after a memcache server goes
-down. To avoid this, set up redundant memcache servers that
-have no common source of failure.
-
The Postfix memcache client cannot be used for security-sensitive
tables such as \fBalias_maps\fR (these may contain
"\fI|command\fR and "\fI/file/name\fR" destinations), or
-\fBvirtual_uid_maps\fR and \fBvirtual_gid_maps\fR (these
-specify UNIX process privileges). In a typical deployment
-a memcache database is shared via a TCP socket, and is
-therefore writable not only by Postfix, but by any process
-that can talk to the memcache server.
+\fBvirtual_uid_maps\fR, \fBvirtual_gid_maps\fR and
+\fBvirtual_mailbox_maps\fR (these specify UNIX process
+privileges or "\fI/file/name\fR" destinations). In a typical
+deployment a memcache database is writable by any process
+that can talk to the memcache server; in contrast,
+security-sensitive tables must not be writable by the
+unprivileged Postfix user.
The Postfix memcache client requires additional configuration
-when used with the \fBpostscreen\fR(8) and \fBverify\fR(8)
-daemons. For details see the \fBttl\fR parameter discussion
-at the end of the MEMCACHE PARAMETERS section in this
-document.
-
-The Postfix memcache client is supported only with libmemcache
-version 1.4.0. Some libmemcache features are documented
-by reading libmemcache source code, instead a proper API.
+when used as \fBpostscreen\fR(8) or \fBverify\fR(8) cache.
+For details see the \fBbackup\fR and \fBttl\fR parameter
+discussions in the MEMCACHE PARAMETERS section above.
.SH "SEE ALSO"
.na
.nf
.ad
.fi
The first memcache client for Postfix was written by Omar
-Kilani. Besides being implemented on libmemcache, this
-implementation bears no resemblance to his work.
+Kilani, and was based on libmemcache.
+The Postfix implementation does not use libmemcache, and
+bears no resemblance to earlier work.
.SH "AUTHOR(S)"
.na
.nf
as with the \fBopen\fR request.
.sp
This request is supported in Postfix 2.5 and later.
+.IP "\fBsequence\fR \fImaptype:mapname flags function\fR"
+Iterate over the specified database. The \fIfunction\fR
+is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT.
+The reply is the request completion status code and
+a lookup key and result value, if found.
+.sp
+This request is supported in Postfix 2.9 and later.
.PP
The request completion status is one of OK, RETRY, NOKEY
(lookup failed because the key was not found), BAD (malformed
<h2>Introduction</h2>
-<p>The Postfix memcache client type allows you to hook up Postfix to
-a memcache server. This implementation supports multiple memcache
-servers for redundancy, and multiple memcache clients that you can
-use for different table lookups. The Postfix memcache client
-supports both lookup and update operations. </p>
-
-<p> Typically, a memcache map is used to reduce query load on a
-database server, or to share a low-latency database among different
-Postfix instances. </p>
+<p>The Postfix memcache client allows you to hook up Postfix to a
+memcache server. The current implementation supports one memcache
+server per Postfix table, with one optional Postfix database that
+provides persistent backup. The Postfix memcache client supports
+the lookup, update, delete and sequence operations. The sequence
+(i.e. first/next) operation requires a backup database that supports
+this operation. </p>
+
+<p> Typically, the Postfix memcache client is used to reduce query
+load on a persistent database, but it may also be used to query a
+memory-only database for low-value, easy-to-create, information
+such as a reputation cache for postscreen(8), verify(8) or greylisting.
+</p>
<h2>Limitations</h2>
<ul>
-<li> <p> The Postfix memcache client is based on libmemcache, which
-will terminate its process after a memcache server goes down. To
-avoid this, set up redundant memcache servers that have no common
-source of failure. </p>
-
<li> <p> The Postfix memcache client cannot be used for security-sensitive
tables such as <tt>alias_maps</tt> (these may contain "<tt>|command</tt>"
-and "<tt>/file/name</tt>" destinations), or <tt>virtual_uid_maps</tt>
-and <tt>virtual_gid_maps</tt> (these specify UNIX process privileges).
-Typically, a memcache database is shared via a TCP socket, and is
-writable not only by Postfix, but by any process that can talk to
-the memcache server. </p>
+and "<tt>/file/name</tt>" destinations), or <tt>virtual_uid_maps</tt>,
+<tt>virtual_gid_maps</tt> and <tt>virtual_mailbox_maps</tt> (these
+specify UNIX process privileges or "<tt>/file/name</tt>" destinations).
+Typically, a memcache database is writable by any process that can
+talk to the memcache server; in contrast, security-sensitive tables
+must not be writable by the unprivileged Postfix user. </p>
<li> <p> The Postfix memcache client requires additional configuration
-when used with the postscreen(8) and verify(8) daemons. For details
-see the <tt>ttl</tt> parameter discussion in the memcache_table(5)
-manual page. </p>
-
-<li> <p> The Postfix memcache client is supported only with libmemcache
-version 1.4.0. Some libmemcache features are documented by reading
-libmemcache source code, instead of a proper API. </p>
+when used as postscreen(8) or verify(8) cache. For details see the
+<tt>backup</tt> and <tt>ttl</tt> parameter discussions in the
+memcache_table(5) manual page. </p>
</ul>
<h2>Building Postfix with memcache support</h2>
-<p>To build Postfix with memcache client support, specify
-<tt>-DHAS_MEMCACHE</tt>, the location of the libmemcache include
-files, and the location of the libmemcache object library. </p>
-
-<p> For example: </p>
-
-<blockquote>
-<pre>
-% make -f Makefile.init makefiles \
- 'CCARGS=-DHAS_MEMCACHE -I/usr/local/include' \
- 'AUXLIBS=-L/usr/local/lib -lmemcache'
-</pre>
-</blockquote>
-
-<p> Then run 'make'. </p>
-
-<p> If the build fails with "<tt>undefined reference to `mcm_buf_len'</tt>"
-(and with a similar error message for <tt>mcm_buf_remain_off</tt>),
-then you need to edit libmemcache source code. </p>
-
-<p> The following instructions apply to libmemcache 1.4.0.rc2. </p>
-
-<ul>
-
-<li> <p> Open the libmemcache source file
-<tt>include/memcache/buffer.h</tt>. </p>
-
-<li> <p> Delete the "<tt>inline</tt>" words before the functions
-that were reported in the "<tt>undefined reference</tt>" error
-messages. </p>
-
-<li> <p> Recompile and reinstall libmemcache. </p>
-
-</ul>
-
-<p> Then, continue building Postfix by running 'make'. </p>
+<p>The Postfix memcache client has no external dependencies,
+and is therefore built into Postfix by default. </p>
<h2>Configuring memcache lookup tables</h2>
<h2>Credits</h2>
-<p> The first memcache client for Postfix was written by Omar Kilani. </p>
+<p> The first memcache client for Postfix was written by Omar Kilani,
+and was based on the libmemcache library. </p>
-<p> Wietse wrote a new memcache client from the ground up. Besides
-also using libmemcache, the current implementation bears no resemblance
-to Omar's work. </p>
+<p> Wietse wrote the current memcache client from the ground up.
+This implementation does not use libmemcache, and bears no resemblance
+to earlier work. </p>
</body>
# \fBdbm\fR or \fBdb\fR format.
#
# Alternatively, lookup tables can be specified as memcache
-# instances. In order to use memcache lookups, define a
-# memcache source as a lookup table in main.cf, for example:
+# instances. To use memcache lookups, define a memcache
+# source as a lookup table in main.cf, for example:
#
# .nf
# virtual_alias_maps = memcache:/etc/postfix/memcache-aliases.cf
# format as the Postfix main.cf file, and specifies the
# parameters described below.
#
-# The Postfix memcache client supports the lookup and update
-# operations.
+# The Postfix memcache client supports the lookup, update,
+# delete and sequence (first/next) operations. The sequence
+# operation requires a backup database that supports the
+# operation.
# MEMCACHE PARAMETERS
# .ad
# .fi
-# .IP "\fBhosts (default: localhost:11211)\fR"
-# The memcache servers that Postfix will try to connect to.
-# Specify a hostname or address, optionally followed by ":"
-# and a port name or number. The default port is 11211.
-# Examples:
+# .IP \fBbackup\fR
+# An optional Postfix database that provides persistent backup
+# for the memcache database. The Postfix memcache client will
+# update the memcache database whenever it looks up or changes
+# information in the persistent database. Specify a Postfix
+# "type:table" database. Example:
#
# .nf
-# hosts = memcache01.example.com
-# memcache02.example.com
+# backup = btree:/var/lib/postfix/postscreen_cache_map
# .fi
+#
+# Access to remote proxymap servers is under development.
+#
+# NOTE 1: When using memcache with persistent backup as
+# \fBpostscreen\fR(8) or \fBverify\fR(8) cache, disable
+# automatic cache cleanup (*_cache_cleanup_interval = 0) in
+# all Postfix instances except for one instance that will be
+# responsible for cache cleanup.
+#
+# NOTE 2: In the case of a proxied database, the full database
+# name (including the "proxy:" prefix) must be specified in
+# the proxymap server's proxy_read_maps or proxy_write_maps
+# setting (depending on whether the access is read-only or
+# read-write).
+# .IP "\fBmemcache (default: inet:localhost:11211)\fR"
+# The memcache server (note: singular) that Postfix will try
+# to connect to. For a TCP server specify "inet:" followed by
+# a hostname or address, ":", and a port name or number.
+# For a UNIX-domain server specify "unix:" followed by the
+# socket pathname. Examples:
+#
+# .nf
+# memcache = inet:memcache.example.com
+# memcache = unix:/path/to/socket
+# .fi
+#
+# NOTE: In the case of a UNIX-domain socket, it must be accessible
+# by the unprivileged postfix user and by the memcached process.
# .IP "\fBkey_format (default: %s)\fB"
# Format of the lookup and update keys in memcache queries.
# By default, these are the same as the lookup and update
# .IP "\fBflags (default: 0)\fR"
# Optional flags that should be stored along with a memcache
# update.
-# .IP "\fBttl (default: 604800)\fR"
+# .IP "\fBttl (default: 3600)\fR"
# The expiration time in seconds of memcache updates.
-# The default is one week.
#
-# When using memcache tables with \fBpostscreen\fR(8) or
-# \fBverify\fR(8), specify a zero *_cache_cleanup_interval
-# value, and specify the largest \fBpostscreen\fR(8) *_ttl
-# value or \fBverify\fR(8) *_expire_time value as the memcache
-# map's \fBttl\fR value.
+# NOTE 1: When using a memcache table as \fBpostscreen\fR(8)
+# or \fBverify\fR(8) cache without persistent backup, specify
+# a zero *_cache_cleanup_interval value with all Postfix
+# instances that use the memcache, and specify the largest
+# \fBpostscreen\fR(8) *_ttl value or \fBverify\fR(8) *_expire_time
+# value as the memcache table's \fBttl\fR value.
#
-# Note: according to memcache protocol documentation, a value
-# greater than 30 days (2592000 seconds) specifies absolute UNIX
+# NOTE 2: According to memcache protocol documentation, a
+# value greater than 30 days (2592000 seconds) specifies
+# absolute UNIX
# time. Smaller values are relative to the time of the update.
# BUGS
-# The Postfix memcache client is based on libmemcache, which
-# will terminate its process after a memcache server goes
-# down. To avoid this, set up redundant memcache servers that
-# have no common source of failure.
-#
# The Postfix memcache client cannot be used for security-sensitive
# tables such as \fBalias_maps\fR (these may contain
# "\fI|command\fR and "\fI/file/name\fR" destinations), or
-# \fBvirtual_uid_maps\fR and \fBvirtual_gid_maps\fR (these
-# specify UNIX process privileges). In a typical deployment
-# a memcache database is shared via a TCP socket, and is
-# therefore writable not only by Postfix, but by any process
-# that can talk to the memcache server.
+# \fBvirtual_uid_maps\fR, \fBvirtual_gid_maps\fR and
+# \fBvirtual_mailbox_maps\fR (these specify UNIX process
+# privileges or "\fI/file/name\fR" destinations). In a typical
+# deployment a memcache database is writable by any process
+# that can talk to the memcache server; in contrast,
+# security-sensitive tables must not be writable by the
+# unprivileged Postfix user.
#
# The Postfix memcache client requires additional configuration
-# when used with the \fBpostscreen\fR(8) and \fBverify\fR(8)
-# daemons. For details see the \fBttl\fR parameter discussion
-# at the end of the MEMCACHE PARAMETERS section in this
-# document.
-#
-# The Postfix memcache client is supported only with libmemcache
-# version 1.4.0. Some libmemcache features are documented
-# by reading libmemcache source code, instead a proper API.
+# when used as \fBpostscreen\fR(8) or \fBverify\fR(8) cache.
+# For details see the \fBbackup\fR and \fBttl\fR parameter
+# discussions in the MEMCACHE PARAMETERS section above.
# SEE ALSO
# postmap(1), Postfix lookup table manager
# postconf(5), configuration parameters
# .ad
# .fi
# The first memcache client for Postfix was written by Omar
-# Kilani. Besides being implemented on libmemcache, this
-# implementation bears no resemblance to his work.
+# Kilani, and was based on libmemcache.
+# The Postfix implementation does not use libmemcache, and
+# bears no resemblance to earlier work.
# AUTHOR(S)
# Wietse Venema
# IBM T.J. Watson Research
fold_addr.c header_body_checks.c mkmap_proxy.c data_redirect.c \
match_service.c mail_conf_nint.c addr_match_list.c mail_conf_nbool.c \
smtp_reply_footer.c safe_ultostr.c verify_sender_addr.c \
- dict_memcache.c mail_version.c
+ dict_memcache.c mail_version.c memcache_proto.c
OBJS = abounce.o anvil_clnt.o been_here.o bounce.o bounce_log.o \
canon_addr.o cfg_parser.o cleanup_strerror.o cleanup_strflags.o \
clnt_stream.o conv_time.o db_common.o debug_peer.o debug_process.o \
fold_addr.o header_body_checks.o mkmap_proxy.o data_redirect.o \
match_service.o mail_conf_nint.o addr_match_list.o mail_conf_nbool.o \
smtp_reply_footer.o safe_ultostr.o verify_sender_addr.o \
- dict_memcache.o mail_version.o
+ dict_memcache.o mail_version.o memcache_proto.o
HDRS = abounce.h anvil_clnt.h been_here.h bounce.h bounce_log.h \
canon_addr.h cfg_parser.h cleanup_user.h clnt_stream.h config.h \
conv_time.h db_common.h debug_peer.h debug_process.h defer.h \
verp_sender.h wildcard_inet_addr.h xtext.h delivered_hdr.h \
fold_addr.h header_body_checks.h data_redirect.h match_service.h \
addr_match_list.h smtp_reply_footer.h safe_ultostr.h \
- verify_sender_addr.h dict_memcache.h
+ verify_sender_addr.h dict_memcache.h memcache_proto.h
TESTSRC = rec2stream.c stream2rec.c recdump.c
DEFS = -I. -I$(INC_DIR) -D$(SYSTYPE)
CFLAGS = $(DEBUG) $(OPT) $(DEFS)
dict_ldap.o: mail_conf.h
dict_ldap.o: string_list.h
dict_memcache.o: ../../include/argv.h
-dict_memcache.o: ../../include/binhash.h
+dict_memcache.o: ../../include/auto_clnt.h
dict_memcache.o: ../../include/dict.h
dict_memcache.o: ../../include/match_list.h
dict_memcache.o: ../../include/match_ops.h
dict_memcache.o: db_common.h
dict_memcache.o: dict_memcache.c
dict_memcache.o: dict_memcache.h
+dict_memcache.o: memcache_proto.h
dict_memcache.o: string_list.h
dict_mysql.o: ../../include/argv.h
dict_mysql.o: ../../include/dict.h
mail_trigger.o: mail_params.h
mail_trigger.o: mail_proto.h
mail_trigger.o: mail_trigger.c
+mail_version.o: ../../include/msg.h
mail_version.o: ../../include/mymalloc.h
mail_version.o: ../../include/split_at.h
mail_version.o: ../../include/stringops.h
mbox_open.o: mbox_conf.h
mbox_open.o: mbox_open.c
mbox_open.o: mbox_open.h
+memcache_proto.o: ../../include/msg.h
+memcache_proto.o: ../../include/sys_defs.h
+memcache_proto.o: ../../include/vbuf.h
+memcache_proto.o: ../../include/vstream.h
+memcache_proto.o: ../../include/vstring.h
+memcache_proto.o: ../../include/vstring_vstream.h
+memcache_proto.o: memcache_proto.c
+memcache_proto.o: memcache_proto.h
mime_state.o: ../../include/msg.h
mime_state.o: ../../include/mymalloc.h
mime_state.o: ../../include/sys_defs.h
mime_state.o: mime_state.c
mime_state.o: mime_state.h
mime_state.o: rec_type.h
-mkmap_cdb.o: ../../include/argv.h
-mkmap_cdb.o: ../../include/dict.h
-mkmap_cdb.o: ../../include/dict_cdb.h
-mkmap_cdb.o: ../../include/mymalloc.h
mkmap_cdb.o: ../../include/sys_defs.h
-mkmap_cdb.o: ../../include/vbuf.h
-mkmap_cdb.o: ../../include/vstream.h
-mkmap_cdb.o: ../../include/vstring.h
-mkmap_cdb.o: mkmap.h
mkmap_cdb.o: mkmap_cdb.c
mkmap_db.o: ../../include/argv.h
mkmap_db.o: ../../include/dict.h
/* NAME
/* dict_memcache 3
/* SUMMARY
-/* dictionary interface to memcache databases
+/* dictionary interface to memcaches
/* SYNOPSIS
/* #include <dict_memcache.h>
/*
/* int open_flags;
/* int dict_flags;
/* DESCRIPTION
-/* dict_memcache_open() opens a memcache database, providing
+/* dict_memcache_open() opens a memcache, providing
/* a dictionary interface for Postfix key->value mappings.
/* The result is a pointer to the installed dictionary.
/*
/*
/* Arguments:
/* .IP name
-/* Either the path to the Postfix memcache configuration file
-/* (if it starts with '/' or '.'), or the parameter name prefix
-/* which will be used to obtain main.cf configuration parameters.
+/* The path to the Postfix memcache configuration file.
/* .IP open_flags
/* O_RDONLY or O_RDWR. This function ignores flags that don't
/* specify a read, write or append mode.
/* See dict_open(3).
/* SEE ALSO
/* dict(3) generic dictionary manager
-/* BUGS
-/* This code requires libmemcache 1.4.0, because some parts
-/* of their API are documented by looking at the implementation.
/* HISTORY
-/* The first memcache client for Postfix was written by:
-/* Omar Kilani
-/* omar@tinysofa.com
-/* This implementation bears no resemblance to his work.
+/* .ad
+/* .fi
+/* The first memcache client for Postfix was written by Omar
+/* Kilani, and was based on libmemcache. The current
+/* implementation implements the memcache protocol directly,
+/* and bears no resemblance to earlier work.
/* AUTHOR(S)
/* Wietse Venema
/* IBM T.J. Watson Research
/* System library. */
-#include "sys_defs.h"
-
-#ifdef HAS_MEMCACHE
+#include <sys_defs.h>
#include <string.h>
-#include <memcache.h>
-
-#if !defined(MEMCACHE_VERNUM) || MEMCACHE_VERNUM != 10400
-#error "Postfix memcache supports only libmemcache version 1.4.0"
-#endif
+#include <ctype.h>
+#include <stdio.h> /* XXX sscanf() */
/* Utility library. */
#include <dict.h>
#include <vstring.h>
#include <stringops.h>
-#include <binhash.h>
+#include <auto_clnt.h>
+#include <vstream.h>
/* Global library. */
#include <cfg_parser.h>
#include <db_common.h>
+#include <memcache_proto.h>
/* Application-specific. */
#include <dict_memcache.h>
- /*
- * Robustness tests (with a single memcache server) proved disappointing.
- *
- * After failure to connect to the memcache server, libmemcache reports the
- * error once. From then on it silently discards all updates and always
- * reports "not found" for all lookups, without ever reporting an error. To
- * avoid this, we destroy the memcache client and create a new one after
- * libmemcache reports an error.
- *
- * Even more problematic is that libmemcache will terminate the process when
- * the memcache server connection is lost (the libmemcache error message is:
- * "read(2) failed: Socket is already connected"). Unfortunately, telling
- * libmemcache not to terminate the process will result in an assertion
- * failure followed by core dump.
- *
- * Conclusion: if we want robust code, then we should use our own memcache
- * protocol implementation instead of libmemcache.
- */
-
/*
* Structure of one memcache dictionary handle.
*/
typedef struct {
DICT dict; /* parent class */
- struct memcache_ctxt *mc_ctxt; /* libmemcache context */
- struct memcache *mc; /* libmemcache object */
CFG_PARSER *parser; /* common parameter parser */
void *dbc_ctxt; /* db_common context */
char *key_format; /* query key translation */
+ int timeout; /* client timeout */
int mc_ttl; /* memcache expiration */
int mc_flags; /* memcache flags */
+ int mc_pause; /* sleep between errors */
+ int mc_maxtry; /* number of tries */
+ char *memcache; /* memcache server spec */
+ AUTO_CLNT *clnt; /* memcache client stream */
+ VSTRING *clnt_buf; /* memcache client buffer */
VSTRING *key_buf; /* lookup key */
VSTRING *res_buf; /* lookup result */
+ int mc_errno; /* memcache dict_errno */
+ DICT *backup; /* persistent backup */
} DICT_MC;
/*
*/
#define DICT_MC_DEF_HOST "localhost"
#define DICT_MC_DEF_PORT "11211"
-#define DICT_MC_DEF_HOST_PORT DICT_MC_DEF_HOST ":" DICT_MC_DEF_PORT
+#define DICT_MC_DEF_MEMCACHE "inet:" DICT_MC_DEF_HOST ":" DICT_MC_DEF_PORT
#define DICT_MC_DEF_KEY_FMT "%s"
-#define DICT_MC_DEF_TTL (7 * 86400)
-#define DICT_MC_DEF_FLAGS 0
-
- /*
- * libmemcache can report errors through an application call-back function,
- * but there is no support for passing application context to the call-back.
- * The call-back API has two documented arguments: pointer to memcache_ctxt,
- * and pointer to memcache_ectxt. The memcache_ctxt data structure has no
- * space for application context, and the mcm_err() function zero-fills the
- * memcache_ectxt data structure, making it useless for application context.
- *
- * We use our own hash table to find our dictionary handle, so that we can
- * report errors in the proper context.
- */
-static BINHASH *dict_mc_hash;
+#define DICT_MC_DEF_MC_TTL 3600
+#define DICT_MC_DEF_MC_TIMEOUT 2
+#define DICT_MC_DEF_MC_FLAGS 0
+#define DICT_MC_DEF_MC_MAXTRY 2
+#define DICT_MC_DEF_MC_PAUSE 1
/*
* SLMs.
#define STR(x) vstring_str(x)
#define LEN(x) VSTRING_LEN(x)
-/*#define msg_verbose 1*/
+#define msg_verbose 1
-/* dict_memcache_error_cb - error call-back */
+/* dict_memcache_set - set memcache key/value */
-static int dict_memcache_error_cb(MCM_ERR_FUNC_ARGS)
+static void dict_memcache_set(DICT_MC *dict_mc, const char *value, int ttl)
{
- const char *myname = "dict_memcache_error_cb";
- const struct memcache_ctxt *ctxt;
- struct memcache_err_ctxt *ectxt;
- DICT_MC *dict_mc;
- void (*log_fn) (const char *,...);
-
- /*
- * Play by the rules of the libmemcache API.
- */
- MCM_ERR_INIT_CTXT(ctxt, ectxt);
-
- /*
- * Locate our own dictionary handle for error reporting context.
- * Unfortunately, the ctxt structure does not store application context,
- * and mcm_err() zero-fills the ectxt structure, making it useless for
- * storing application context. We use our own hash table instead.
- */
- if ((dict_mc = (DICT_MC *) binhash_find(dict_mc_hash, (char *) &ctxt,
- sizeof(ctxt))) == 0)
- msg_panic("%s: can't locate DICT_MC database handle", myname);
-
- /*
- * Report the error in our context, and set dict_errno for possible
- * errors. We override dict_errno when an error was recoverable.
- */
- switch (ectxt->severity) {
- default:
-#ifdef DICT_MC_RECOVER_FROM_DISCONNECT
- /* Code below causes an assert failure and core dump. */
- if (ectxt->errcode == MCM_ERR_SYS_READ)
- /* Also: MCM_ERR_SYS_WRITEV, MCM_ERR_SYS_SETSOCKOPT */
- ectxt->cont = 'y';
-#endif
- /* FALLTHROUGH */
- case MCM_ERR_LVL_NOTICE:
- log_fn = msg_warn;
- dict_errno = 1;
- break;
- case MCM_ERR_LVL_INFO:
- log_fn = msg_info;
- break;
+ VSTREAM *fp;
+ int count;
+
+#define MC_LINE_LIMIT 1024
+
+ dict_mc->mc_errno = DICT_ERR_RETRY;
+ for (count = 0; count < dict_mc->mc_maxtry; count++) {
+ if (count > 0)
+ sleep(1);
+ if ((fp = auto_clnt_access(dict_mc->clnt)) != 0) {
+ if (memcache_printf(fp, "set %s %d %d %ld",
+ STR(dict_mc->key_buf), dict_mc->mc_flags,
+ ttl, strlen(value)) < 0
+ || memcache_fwrite(fp, value, strlen(value)) < 0
+ || memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0) {
+ if (count > 0)
+ msg_warn("database %s:%s: I/O error: %m",
+ DICT_TYPE_MEMCACHE, dict_mc->dict.name);
+ auto_clnt_recover(dict_mc->clnt);
+ } else if (strcmp(STR(dict_mc->clnt_buf), "STORED") != 0) {
+ if (count > 0)
+ msg_warn("database %s:%s: update failed: %.30s",
+ DICT_TYPE_MEMCACHE, dict_mc->dict.name,
+ STR(dict_mc->clnt_buf));
+ auto_clnt_recover(dict_mc->clnt);
+ } else {
+ /* Victory! */
+ dict_mc->mc_errno = 0;
+ break;
+ }
+ }
}
- log_fn((ectxt)->errnum ? "database %s:%s: libmemcache error: %s: %m" :
- "database %s:%s: libmemcache error: %s",
- DICT_TYPE_MEMCACHE, dict_mc->dict.name, (ectxt)->errstr);
- return (0);
}
-static void dict_memcache_mc_free(DICT_MC *);
-static void dict_memcache_mc_init(DICT_MC *);
+/* dict_memcache_get - get memcache key/value */
-/* dict_memcache_recover - recover after libmemcache error */
-
-static void dict_memcache_recover(DICT_MC *dict_mc)
+static const char *dict_memcache_get(DICT_MC *dict_mc)
{
- int saved_dict_errno;
-
- /*
- * XXX If we don't try to recover from the first error, libmemcache will
- * silently skip all subsequent database operations.
- */
- saved_dict_errno = dict_errno;
- dict_memcache_mc_free(dict_mc);
- dict_memcache_mc_init(dict_mc);
- dict_errno = saved_dict_errno;
+ VSTREAM *fp;
+ long todo;
+ const char *retval;
+ int count;
+
+ dict_mc->mc_errno = DICT_ERR_RETRY;
+ retval = 0;
+ for (count = 0; count < dict_mc->mc_maxtry; count++) {
+ if (count > 0)
+ sleep(1);
+ if ((fp = auto_clnt_access(dict_mc->clnt)) != 0) {
+ if (memcache_printf(fp, "get %s", STR(dict_mc->key_buf)) < 0
+ || memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0) {
+ if (count > 0)
+ msg_warn("database %s:%s: I/O error: %m",
+ DICT_TYPE_MEMCACHE, dict_mc->dict.name);
+ auto_clnt_recover(dict_mc->clnt);
+ } else if (strcmp(STR(dict_mc->clnt_buf), "END") == 0) {
+ /* Not found. */
+ dict_mc->mc_errno = 0;
+ break;
+ } else if (sscanf(STR(dict_mc->clnt_buf),
+ "VALUE %*s %*s %ld", &todo) != 1 || todo < 0) {
+ if (count > 0)
+ msg_warn("%s: unexpected memcache server reply: %.30s",
+ dict_mc->dict.name, STR(dict_mc->clnt_buf));
+ auto_clnt_recover(dict_mc->clnt);
+ } else if (memcache_fread(fp, dict_mc->res_buf, todo) < 0) {
+ if (count > 0)
+ msg_warn("%s: EOF receiving memcache server reply",
+ dict_mc->dict.name);
+ auto_clnt_recover(dict_mc->clnt);
+ } else {
+ /* Victory! */
+ retval = STR(dict_mc->res_buf);
+ dict_mc->mc_errno = 0;
+ if (memcache_get(fp, dict_mc->clnt_buf, MC_LINE_LIMIT) < 0
+ || strcmp(STR(dict_mc->clnt_buf), "END") != 0)
+ auto_clnt_recover(dict_mc->clnt);
+ break;
+ }
+ }
+ }
+ return (retval);
}
/* dict_memcache_prepare_key - prepare lookup key */
return (LEN(dict_mc->key_buf));
}
-/* dict_memcache_update - update memcache database */
+/* dict_memcache_valid_key - validate key */
+
+static int dict_memcache_valid_key(DICT_MC *dict_mc,
+ const char *name,
+ const char *operation,
+ void (*log_func) (const char *,...))
+{
+ unsigned char *cp;
+
+#define DICT_MC_SKIP(why) do { \
+ if (msg_verbose || log_func != msg_info) \
+ log_func("%s: skipping %s for name \"%s\": %s", \
+ dict_mc->dict.name, operation, name, (why)); \
+ return(0); \
+ } while (0)
+
+ if (*name == 0)
+ DICT_MC_SKIP("empty lookup key");
+ if (db_common_check_domain(dict_mc->dbc_ctxt, name) == 0)
+ DICT_MC_SKIP("domain mismatch");
+ if (dict_memcache_prepare_key(dict_mc, name) == 0)
+ DICT_MC_SKIP("empty lookup key expansion");
+ for (cp = (unsigned char *) STR(dict_mc->key_buf); *cp; cp++)
+ if (isascii(*cp) && isspace(*cp))
+ DICT_MC_SKIP("name contains space");
+
+ return (1);
+}
+
+/* dict_memcache_update - update memcache */
static void dict_memcache_update(DICT *dict, const char *name,
const char *value)
{
const char *myname = "dict_memcache_update";
DICT_MC *dict_mc = (DICT_MC *) dict;
+ int backup_errno = 0;
/*
- * Skip updates with a null key, noisily. This would result in loss of
- * information.
+ * Skip updates with an inapplicable key, noisily. This results in loss
+ * of information.
*/
- if (dict_memcache_prepare_key(dict_mc, name) == 0) {
- dict_errno = 1;
- msg_warn("database %s:%s: name \"%s\" expands to empty lookup key "
- "-- skipping update", DICT_TYPE_MEMCACHE,
- dict_mc->dict.name, name);
+ dict_errno = DICT_ERR_RETRY;
+ if (dict_memcache_valid_key(dict_mc, name, "update", msg_warn) == 0)
return;
+
+ /*
+ * Update the backup database first.
+ */
+ if (dict_mc->backup) {
+ dict_errno = 0;
+ dict_mc->backup->update(dict_mc->backup, name, value);
+ backup_errno = dict_errno;
}
/*
- * Our error call-back routine will report errors and set dict_errno.
+ * Update the memcache last.
*/
- dict_errno = (mcm_set(dict_mc->mc_ctxt, dict_mc->mc, STR(dict_mc->key_buf),
- LEN(dict_mc->key_buf), value, strlen(value),
- dict_mc->mc_ttl, dict_mc->mc_flags) != 0);
+ dict_memcache_set(dict_mc, value, dict_mc->mc_ttl);
+
if (msg_verbose)
msg_info("%s: %s: update key \"%s\" => \"%s\" %s",
myname, dict_mc->dict.name, STR(dict_mc->key_buf), value,
- dict_errno ? "(error)" : "(no error)");
+ dict_mc->mc_errno ? "(memcache error)" :
+ backup_errno ? "(backup error)" : "(no error)");
- /*
- * Recover after server failure.
- */
- if (dict_errno)
- dict_memcache_recover(dict_mc);
+ dict_errno = (backup_errno ? backup_errno : dict_mc->mc_errno);
}
-/* dict_memcache_lookup - lookup memcache database */
+/* dict_memcache_lookup - lookup memcache */
static const char *dict_memcache_lookup(DICT *dict, const char *name)
{
const char *myname = "dict_memcache_lookup";
DICT_MC *dict_mc = (DICT_MC *) dict;
- struct memcache_req *req;
- struct memcache_res *res;
const char *retval;
+ int backup_errno = 0;
/*
- * Skip lookups with a null key, silently. This is just asking for
- * information that cannot exist.
+ * Skip lookups with an inapplicable key, silently. This is just asking
+ * for information that cannot exist.
*/
-#define DICT_MC_SKIP(why, map_name, key) do { \
- if (msg_verbose) \
- msg_info("%s: %s: skipping lookup of key \"%s\": %s", \
- myname, (map_name), (key), (why)); \
- return (0); \
- } while (0)
-
- if (*name == 0)
- DICT_MC_SKIP("empty lookup key", dict_mc->dict.name, name);
- if (db_common_check_domain(dict_mc->dbc_ctxt, name) == 0)
- DICT_MC_SKIP("domain mismatch", dict_mc->dict.name, name);
- if (dict_memcache_prepare_key(dict_mc, name) == 0)
- DICT_MC_SKIP("empty lookup key expansion", dict_mc->dict.name, name);
+ dict_errno = 0;
+ if (dict_memcache_valid_key(dict_mc, name, "lookup", msg_info) == 0)
+ return (0);
/*
- * Our error call-back routine will report errors and set dict_errno. We
- * reset dict_errno after an error turns out to be recoverable.
+ * Search the memcache first.
*/
- if ((req = mcm_req_new(dict_mc->mc_ctxt)) == 0)
- msg_fatal("%s: can't create new request: %m", myname); /* XXX */
- /* Not: mcm_req_add(), because that makes unnecessary copy of the key. */
- if ((res = mcm_req_add_ref(dict_mc->mc_ctxt, req, STR(dict_mc->key_buf),
- LEN(dict_mc->key_buf))) == 0)
- msg_fatal("%s: can't create new result: %m", myname); /* XXX */
+ retval = dict_memcache_get(dict_mc);
- dict_errno = 0;
- mcm_get(dict_mc->mc_ctxt, dict_mc->mc, req);
- if (mcm_res_found(dict_mc->mc_ctxt, res) && res->bytes) {
- vstring_strncpy(dict_mc->res_buf, res->val, res->bytes);
- retval = STR(dict_mc->res_buf);
- dict_errno = 0;
- } else {
- retval = 0;
+ /*
+ * Search the backup database last. Update the memcache if the data is
+ * found.
+ */
+ if (retval == 0 && dict_mc->backup) {
+ retval = dict_mc->backup->lookup(dict_mc->backup, name);
+ backup_errno = dict_errno;
+ /* Update the cache. */
+ if (retval != 0)
+ dict_memcache_set(dict_mc, retval, dict_mc->mc_ttl);
}
- mcm_res_free(dict_mc->mc_ctxt, req, res);
- mcm_req_free(dict_mc->mc_ctxt, req);
-
if (msg_verbose)
msg_info("%s: %s: key %s => %s",
myname, dict_mc->dict.name, STR(dict_mc->key_buf),
- retval ? STR(dict_mc->res_buf) :
- dict_errno ? "(error)" : "(not found)");
-
- /*
- * Recover after server failure.
- */
- if (dict_errno)
- dict_memcache_recover(dict_mc);
+ retval ? retval :
+ dict_mc->mc_errno ? "(memcache error)" :
+ backup_errno ? "(backup error)" : "(not found)");
return (retval);
}
-/* dict_memcache_mc_free - destroy libmemcache objects */
-
-static void dict_memcache_mc_free(DICT_MC *dict_mc)
-{
- binhash_delete(dict_mc_hash, (char *) &dict_mc->mc_ctxt,
- sizeof(dict_mc->mc_ctxt), (void (*) (char *)) 0);
- mcm_free(dict_mc->mc_ctxt, dict_mc->mc);
- mcMemFreeCtxt(dict_mc->mc_ctxt);
-}
-
-/* dict_memcache_mc_init - create libmemcache objects */
+/* dict_memcache_delete - delete memcache entry */
-static void dict_memcache_mc_init(DICT_MC *dict_mc)
+static int dict_memcache_delete(DICT *dict, const char *name)
{
- const char *myname = "dict_memcache_mc_init";
- char *servers;
- char *server;
- char *cp;
+ const char *myname = "dict_memcache_delete";
+ DICT_MC *dict_mc = (DICT_MC *) dict;
+ const char *retval;
+ int backup_errno = 0;
+ int del_res = 0;
/*
- * Create the libmemcache objects.
+ * Skip lookups with an inapplicable key, silently. This is just deleting
+ * information that cannot exist.
*/
- dict_mc->mc_ctxt =
- mcMemNewCtxt((mcFreeFunc) myfree, (mcMallocFunc) mymalloc,
- (mcMallocFunc) mymalloc, (mcReallocFunc) myrealloc);
- if (dict_mc->mc_ctxt == 0)
- msg_fatal("error creating memcache context: %m"); /* XXX */
- dict_mc->mc = mcm_new(dict_mc->mc_ctxt);
- if (dict_mc->mc == 0)
- msg_fatal("error creating memcache object: %m"); /* XXX */
+ dict_errno = 0;
+ if (dict_memcache_valid_key(dict_mc, name, "delete", msg_info) == 0)
+ return (1);
/*
- * Set up call-back info for error reporting.
+ * Update the persistent database first.
*/
- if (dict_mc_hash == 0)
- dict_mc_hash = binhash_create(1);
- binhash_enter(dict_mc_hash, (char *) &dict_mc->mc_ctxt,
- sizeof(dict_mc->mc_ctxt), (char *) dict_mc);
- mcErrSetupCtxt(dict_mc->mc_ctxt, dict_memcache_error_cb);
+ if (dict_mc->backup) {
+ dict_errno = 0;
+ del_res = dict_mc->backup->delete(dict_mc->backup, name);
+ backup_errno = dict_errno;
+ }
/*
- * Add the server list.
+ * Update the memcache last. There is no memcache delete operation.
+ * Instead, we set a short expiration time if the data exists.
*/
- cp = servers = cfg_get_str(dict_mc->parser, "hosts",
- DICT_MC_DEF_HOST_PORT, 0, 0);
- while ((server = mystrtok(&cp, " ,\t\r\n")) != 0) {
- if (msg_verbose)
- msg_info("%s: database %s:%s: adding server %s",
- myname, DICT_TYPE_MEMCACHE, dict_mc->dict.name, server);
- if (mcm_server_add4(dict_mc->mc_ctxt, dict_mc->mc, server) < 0)
- msg_warn("database %s:%s: error adding server %s",
- DICT_TYPE_MEMCACHE, dict_mc->dict.name, server);
- }
- myfree(servers);
+ if ((retval = dict_memcache_get(dict_mc)) != 0)
+ dict_memcache_set(dict_mc, retval, 1);
+
+ if (msg_verbose)
+ msg_info("%s: %s: delete key %s => %s",
+ myname, dict_mc->dict.name, STR(dict_mc->key_buf),
+ dict_mc->mc_errno ? "(memcache error)" :
+ backup_errno ? "(backup error)" : "(no error)");
+
+ dict_errno = (backup_errno ? backup_errno : dict_mc->mc_errno);
+
+ return (del_res);
+}
+
+/* dict_memcache_sequence - first/next lookup */
+
+static int dict_memcache_sequence(DICT *dict, int function, const char **key,
+ const char **value)
+{
+ DICT_MC *dict_mc = (DICT_MC *) dict;
+
+ if (dict_mc->backup == 0)
+ msg_fatal("database %s:%s: first/next support requires backup database",
+ DICT_TYPE_MEMCACHE, dict_mc->dict.name);
+ return (dict_mc->backup->sequence(dict_mc->backup, function, key, value));
}
-/* dict_memcache_close - close memcache database */
+/* dict_memcache_close - close memcache */
static void dict_memcache_close(DICT *dict)
{
DICT_MC *dict_mc = (DICT_MC *) dict;
- dict_memcache_mc_free(dict_mc);
cfg_parser_free(dict_mc->parser);
db_common_free_ctx(dict_mc->dbc_ctxt);
- vstring_free(dict_mc->key_buf);
- vstring_free(dict_mc->res_buf);
if (dict_mc->key_format)
myfree(dict_mc->key_format);
+ myfree(dict_mc->memcache);
+ auto_clnt_free(dict_mc->clnt);
+ vstring_free(dict_mc->clnt_buf);
+ vstring_free(dict_mc->key_buf);
+ vstring_free(dict_mc->res_buf);
if (dict->fold_buf)
vstring_free(dict->fold_buf);
+ if (dict_mc->backup)
+ dict_close(dict_mc->backup);
dict_free(dict);
}
-/* dict_memcache_open - open memcache database */
+/* dict_memcache_open - open memcache */
DICT *dict_memcache_open(const char *name, int open_flags, int dict_flags)
{
DICT_MC *dict_mc;
+ char *backup;
/*
* Sanity checks.
dict_mc = (DICT_MC *) dict_alloc(DICT_TYPE_MEMCACHE, name,
sizeof(*dict_mc));
dict_mc->dict.lookup = dict_memcache_lookup;
- if (open_flags == O_RDWR)
+ if (open_flags == O_RDWR) {
dict_mc->dict.update = dict_memcache_update;
+ dict_mc->dict.delete = dict_memcache_delete;
+ }
+ dict_mc->dict.sequence = dict_memcache_sequence;
dict_mc->dict.close = dict_memcache_close;
dict_mc->dict.flags = dict_flags;
dict_mc->key_buf = vstring_alloc(10);
dict_mc->parser = cfg_parser_alloc(name);
dict_mc->key_format = cfg_get_str(dict_mc->parser, "key_format",
DICT_MC_DEF_KEY_FMT, 0, 0);
+ dict_mc->timeout = cfg_get_int(dict_mc->parser, "timeout",
+ DICT_MC_DEF_MC_TIMEOUT, 0, 0);
dict_mc->mc_ttl = cfg_get_int(dict_mc->parser, "ttl",
- DICT_MC_DEF_TTL, 0, 0);
+ DICT_MC_DEF_MC_TTL, 0, 0);
dict_mc->mc_flags = cfg_get_int(dict_mc->parser, "flags",
- DICT_MC_DEF_FLAGS, 0, 0);
+ DICT_MC_DEF_MC_FLAGS, 0, 0);
+ dict_mc->mc_pause = cfg_get_int(dict_mc->parser, "error_pause",
+ DICT_MC_DEF_MC_PAUSE, 1, 0);
+ dict_mc->mc_maxtry = cfg_get_int(dict_mc->parser, "maxtry",
+ DICT_MC_DEF_MC_MAXTRY, 1, 0);
+ dict_mc->memcache = cfg_get_str(dict_mc->parser, "memcache",
+ DICT_MC_DEF_MEMCACHE, 0, 0);
/*
- * Initialize the memcache objects.
+ * Initialize the memcache client.
*/
- dict_memcache_mc_init(dict_mc);
+ dict_mc->clnt = auto_clnt_create(dict_mc->memcache, dict_mc->timeout, 0, 0);
+ dict_mc->clnt_buf = vstring_alloc(100);
+
+ /*
+ * Open the optional backup database.
+ */
+ backup = cfg_get_str(dict_mc->parser, "backup", (char *) 0, 0, 0);
+ if (backup) {
+ dict_mc->backup = dict_open(backup, open_flags, dict_flags);
+ myfree(backup);
+ } else
+ dict_mc->backup = 0;
/*
* Parse templates and common database parameters. Maps that use
return (&dict_mc->dict);
}
-
-#endif
/* int open_flags;
/* int dict_flags;
/* DESCRIPTION
-/* dict_proxy_open() relays read-only operations through
-/* the Postfix proxymap server.
+/* dict_proxy_open() relays read-only or read-write operations
+/* through the Postfix proxymap server.
/*
/* The \fIopen_flags\fR argument must specify O_RDONLY
-/* or O_RDWR|O_CREAT. Depending on this, the client
+/* or O_RDWR. Depending on this, the client
/* connects to the proxymap multiserver or to the
/* proxywrite single updater.
/*
CLNT_STREAM *clnt; /* client handle (shared) */
const char *service; /* service name */
int in_flags; /* caller-specified flags */
+ VSTRING *reskey; /* result key storage */
VSTRING *result; /* storage */
} DICT_PROXY;
static CLNT_STREAM *proxymap_stream; /* read-only maps */
static CLNT_STREAM *proxywrite_stream; /* read-write maps */
+/* dict_proxy_sequence - find first/next entry */
+
+static int dict_proxy_sequence(DICT *dict, int function,
+ const char **key, const char **value)
+{
+ const char *myname = "dict_proxy_sequence";
+ DICT_PROXY *dict_proxy = (DICT_PROXY *) dict;
+ VSTREAM *stream;
+ int status;
+ int count = 0;
+ int request_flags;
+
+ /*
+ * The client and server live in separate processes that may start and
+ * terminate independently. We cannot rely on a persistent connection,
+ * let alone on persistent state (such as a specific open table) that is
+ * associated with a specific connection. Each lookup needs to specify
+ * the table and the flags that were specified to dict_proxy_open().
+ */
+ VSTRING_RESET(dict_proxy->reskey);
+ VSTRING_TERMINATE(dict_proxy->reskey);
+ VSTRING_RESET(dict_proxy->result);
+ VSTRING_TERMINATE(dict_proxy->result);
+ request_flags = (dict_proxy->in_flags & DICT_FLAG_RQST_MASK)
+ | (dict->flags & DICT_FLAG_RQST_MASK);
+ for (;;) {
+ stream = clnt_stream_access(dict_proxy->clnt);
+ errno = 0;
+ count += 1;
+ if (attr_print(stream, ATTR_FLAG_NONE,
+ ATTR_TYPE_STR, MAIL_ATTR_REQ, PROXY_REQ_SEQUENCE,
+ ATTR_TYPE_STR, MAIL_ATTR_TABLE, dict->name,
+ ATTR_TYPE_INT, MAIL_ATTR_FLAGS, request_flags,
+ ATTR_TYPE_INT, MAIL_ATTR_FUNC, function,
+ ATTR_TYPE_END) != 0
+ || vstream_fflush(stream)
+ || attr_scan(stream, ATTR_FLAG_STRICT,
+ ATTR_TYPE_INT, MAIL_ATTR_STATUS, &status,
+ ATTR_TYPE_STR, MAIL_ATTR_KEY, dict_proxy->reskey,
+ ATTR_TYPE_STR, MAIL_ATTR_VALUE, dict_proxy->result,
+ ATTR_TYPE_END) != 3) {
+ if (msg_verbose || count > 1 || (errno && errno != EPIPE && errno != ENOENT))
+ msg_warn("%s: service %s: %m", myname, VSTREAM_PATH(stream));
+ } else {
+ if (msg_verbose)
+ msg_info("%s: table=%s flags=%s func=%d -> status=%d key=%s val=%s",
+ myname, dict->name, dict_flags_str(request_flags),
+ function, status, STR(dict_proxy->reskey),
+ STR(dict_proxy->result));
+ switch (status) {
+ case PROXY_STAT_BAD:
+ msg_fatal("%s sequence failed for table \"%s\" function %d: "
+ "invalid request",
+ dict_proxy->service, dict->name, function);
+ case PROXY_STAT_DENY:
+ msg_fatal("%s service is not configured for table \"%s\"",
+ dict_proxy->service, dict->name);
+ case PROXY_STAT_OK:
+ *key = STR(dict_proxy->reskey);
+ *value = STR(dict_proxy->result);
+ return (0);
+ case PROXY_STAT_NOKEY:
+ dict_errno = 0;
+ *key = *value = 0;
+ return (1);
+ case PROXY_STAT_RETRY:
+ dict_errno = DICT_ERR_RETRY;
+ *key = *value = 0;
+ return (1);
+ default:
+ msg_warn("%s sequence failed for table \"%s\" function %d: "
+ "unexpected reply status %d",
+ dict_proxy->service, dict->name, function, status);
+ }
+ }
+ clnt_stream_recover(dict_proxy->clnt);
+ sleep(1); /* XXX make configurable */
+ }
+}
+
/* dict_proxy_lookup - find table entry */
static const char *dict_proxy_lookup(DICT *dict, const char *key)
dict_proxy->service, dict->name);
case PROXY_STAT_OK:
return;
+ case PROXY_STAT_RETRY:
+ dict_errno = DICT_ERR_RETRY;
+ return;
default:
msg_warn("%s update failed for table \"%s\" key \"%s\": "
"unexpected reply status %d",
dict_proxy->service, dict->name);
case PROXY_STAT_OK:
return 0;
+ case PROXY_STAT_RETRY:
+ dict_errno = DICT_ERR_RETRY;
+ return (-1);
case PROXY_STAT_NOKEY:
- return 1;
+ dict_errno = 0;
+ return (1);
default:
msg_warn("%s delete failed for table \"%s\" key \"%s\": "
"unexpected reply status %d",
{
DICT_PROXY *dict_proxy = (DICT_PROXY *) dict;
+ vstring_free(dict_proxy->reskey);
vstring_free(dict_proxy->result);
dict_free(dict);
}
if (open_flags == O_RDONLY) {
pstream = &proxymap_stream;
service = var_proxymap_service;
- } else if (open_flags == (O_RDWR | O_CREAT)) {
+ } else if ((open_flags & O_RDWR) == O_RDWR) {
pstream = &proxywrite_stream;
service = var_proxywrite_service;
} else
- msg_fatal("%s: %s map open requires O_RDONLY or O_RDWR|O_CREAT mode",
+ msg_fatal("%s: %s map open requires O_RDONLY or O_RDWR mode",
map, DICT_TYPE_PROXY);
if (*pstream == 0) {
dict_proxy->dict.lookup = dict_proxy_lookup;
dict_proxy->dict.update = dict_proxy_update;
dict_proxy->dict.delete = dict_proxy_delete;
+ dict_proxy->dict.sequence = dict_proxy_sequence;
dict_proxy->dict.close = dict_proxy_close;
dict_proxy->in_flags = dict_flags;
+ dict_proxy->reskey = vstring_alloc(10);
dict_proxy->result = vstring_alloc(10);
dict_proxy->clnt = *pstream;
dict_proxy->service = service;
#define PROXY_REQ_LOOKUP "lookup"
#define PROXY_REQ_UPDATE "update"
#define PROXY_REQ_DELETE "delete"
+#define PROXY_REQ_SEQUENCE "sequence"
#define PROXY_STAT_OK 0 /* operation succeeded */
#define PROXY_STAT_NOKEY 1 /* requested key not found */
#ifdef HAS_SQLITE
DICT_TYPE_SQLITE, dict_sqlite_open,
#endif
-#ifdef HAS_MEMCACHE
DICT_TYPE_MEMCACHE, dict_memcache_open,
-#endif
0,
};
}
#ifdef TEST
-
/*
* Proof-of-concept test program.
*/
+
+#include <mail_proto.h>
+#include <mail_params.h>
+
int main(int argc, char **argv)
{
+ var_queue_dir = DEF_QUEUE_DIR;
+ var_proxymap_service = DEF_PROXYMAP_SERVICE;
+ var_proxywrite_service = DEF_PROXYWRITE_SERVICE;
+ var_ipc_timeout = 3600;
mail_dict_init();
dict_test(argc, argv);
return (0);
#define MAIL_ATTR_TTL "ttl"
#define MAIL_ATTR_LABEL "label"
#define MAIL_ATTR_PROP "property"
+#define MAIL_ATTR_FUNC "function"
#define MAIL_ATTR_CCERT_SUBJECT "ccert_subject"
#define MAIL_ATTR_CCERT_ISSUER "ccert_issuer"
#define MAIL_ATTR_CCERT_FINGERPRINT "ccert_fingerprint"
* Patches change both the patchlevel and the release date. Snapshots have no
* patchlevel; they change the release date only.
*/
-#define MAIL_RELEASE_DATE "20111209"
+#define MAIL_RELEASE_DATE "20111213"
#define MAIL_VERSION_NUMBER "2.9"
#ifdef SNAPSHOT
--- /dev/null
+/*++
+/* NAME
+/* memcache_proto 3
+/* SUMMARY
+/* memcache low-level protocol
+/* SYNOPSIS
+/* #include <memcache_proto.h>
+/*
+/* int memcache_get(fp, buf, len)
+/* VSTREAM *fp;
+/* VSTRING *buf;
+/* ssize_t len;
+/*
+/* int memcache_printf(fp, format, ...)
+/* VSTREAM *fp;
+/* const char *format;
+/*
+/* int memcache_vprintf(fp, format, ap)
+/* VSTREAM *fp;
+/* const char *format;
+/* va_list ap;
+/*
+/* int memcache_fread(fp, buf, len)
+/* VSTREAM *fp;
+/* VSTRING *buf;
+/* ssize_t len;
+/*
+/* int memcache_fwrite(fp, buf, len)
+/* VSTREAM *fp;
+/* const char *buf;
+/* ssize_t len;
+/* DESCRIPTION
+/* This module implements the low-level memcache protocol.
+/* All functions return -1 on error and 0 on succcess.
+/* SEE ALSO
+/* smtp_proto(3) SMTP low-level protocol.
+/* AUTHOR(S)
+/* Wietse Venema
+/* IBM T.J. Watson Research
+/* P.O. Box 704
+/* Yorktown Heights, NY 10598, USA
+/*--*/
+
+#include <sys_defs.h>
+
+/* Utility library. */
+
+#include <msg.h>
+#include <vstream.h>
+#include <vstring.h>
+#include <vstring_vstream.h>
+
+/* Application-specific. */
+
+#include <memcache_proto.h>
+
+#define STR(x) vstring_str(x)
+#define LEN(x) VSTRING_LEN(x)
+
+/* memcache_get - read one line from peer */
+
+int memcache_get(VSTREAM *stream, VSTRING *vp, ssize_t bound)
+{
+ int last_char;
+ int next_char;
+
+ last_char = (bound == 0 ? vstring_get(vp, stream) :
+ vstring_get_bound(vp, stream, bound));
+
+ switch (last_char) {
+
+ /*
+ * Do some repair in the rare case that we stopped reading in the
+ * middle of the CRLF record terminator.
+ */
+ case '\r':
+ if ((next_char = VSTREAM_GETC(stream)) == '\n') {
+ VSTRING_ADDCH(vp, '\n');
+ /* FALLTRHOUGH */
+ } else {
+ if (next_char != VSTREAM_EOF)
+ vstream_ungetc(stream, next_char);
+
+ /*
+ * Input too long, or EOF
+ */
+ default:
+ if (msg_verbose)
+ msg_info("%s got %s", VSTREAM_PATH(stream),
+ LEN(vp) < bound ? "EOF" : "input too long");
+ return (-1);
+ }
+
+ /*
+ * Strip off the record terminator: either CRLF or just bare LF.
+ */
+ case '\n':
+ vstring_truncate(vp, VSTRING_LEN(vp) - 1);
+ if (VSTRING_LEN(vp) > 0 && vstring_end(vp)[-1] == '\r')
+ vstring_truncate(vp, VSTRING_LEN(vp) - 1);
+ VSTRING_TERMINATE(vp);
+ if (msg_verbose)
+ msg_info("%s got: %s", VSTREAM_PATH(stream), STR(vp));
+ return (0);
+ }
+}
+
+/* memcache_fwrite - write one blob to peer */
+
+int memcache_fwrite(VSTREAM *stream, const char *cp, ssize_t todo)
+{
+
+ /*
+ * Sanity check.
+ */
+ if (todo < 0)
+ msg_panic("memcache_fwrite: negative todo %ld", (long) todo);
+
+ /*
+ * Do the I/O.
+ */
+ if (msg_verbose)
+ msg_info("%s write: %.*s", VSTREAM_PATH(stream), (int) todo, cp);
+ if (vstream_fwrite(stream, cp, todo) != todo
+ || vstream_fputs("\r\n", stream) == VSTREAM_EOF)
+ return (-1);
+ else
+ return (0);
+}
+
+/* memcache_fread - read one blob from peer */
+
+int memcache_fread(VSTREAM *stream, VSTRING *buf, ssize_t todo)
+{
+
+ /*
+ * Sanity check.
+ */
+ if (todo < 0)
+ msg_panic("memcache_fread: negative todo %ld", (long) todo);
+
+ /*
+ * Do the I/O.
+ */
+ VSTRING_SPACE(buf, todo);
+ VSTRING_AT_OFFSET(buf, todo);
+ if (vstream_fread(stream, STR(buf), todo) != todo
+ || VSTREAM_GETC(stream) != '\r'
+ || VSTREAM_GETC(stream) != '\n') {
+ if (msg_verbose)
+ msg_info("%s read: error", VSTREAM_PATH(stream));
+ return (-1);
+ } else {
+ vstring_truncate(buf, todo);
+ VSTRING_TERMINATE(buf);
+ if (msg_verbose)
+ msg_info("%s read: %s", VSTREAM_PATH(stream), STR(buf));
+ return (0);
+ }
+}
+
+/* memcache_vprintf - write one line to peer */
+
+int memcache_vprintf(VSTREAM *stream, const char *fmt, va_list ap)
+{
+
+ /*
+ * Do the I/O.
+ */
+ vstream_vfprintf(stream, fmt, ap);
+ vstream_fputs("\r\n", stream);
+ if (vstream_ferror(stream))
+ return (-1);
+ else
+ return (0);
+}
+
+/* memcache_printf - write one line to peer */
+
+int memcache_printf(VSTREAM *stream, const char *fmt,...)
+{
+ va_list ap;
+ int ret;
+
+ if (msg_verbose) {
+ VSTRING *buf = vstring_alloc(100);
+
+ va_start(ap, fmt);
+ vstring_vsprintf(buf, fmt, ap);
+ va_end(ap);
+ msg_info("%s write: %s", VSTREAM_PATH(stream), STR(buf));
+ vstring_free(buf);
+ }
+
+ /*
+ * Do the I/O.
+ */
+ va_start(ap, fmt);
+ ret = memcache_vprintf(stream, fmt, ap);
+ va_end(ap);
+ return (ret);
+}
--- /dev/null
+#ifndef _MEMCACHE_PROTO_H_INCLUDED_
+#define _MEMCACHE_PROTO_H_INCLUDED_
+
+/*++
+/* NAME
+/* memcache_proto 3h
+/* SUMMARY
+/* memcache low-level protocol
+/* SYNOPSIS
+/* #include <memcache_proto.h>
+/* DESCRIPTION
+/* .nf
+
+ /*
+ * External interface.
+ */
+extern int memcache_get(VSTREAM *, VSTRING *, ssize_t);
+extern int memcache_vprintf(VSTREAM *, const char *, va_list);
+extern int memcache_printf(VSTREAM *, const char *fmt,...);
+extern int memcache_fread(VSTREAM *, VSTRING *, ssize_t);
+extern int memcache_fwrite(VSTREAM *, const char *, ssize_t);
+
+/* LICENSE
+/* .ad
+/* .fi
+/* The Secure Mailer license must be distributed with this software.
+/* AUTHOR(S)
+/* Wietse Venema
+/* IBM T.J. Watson Research
+/* P.O. Box 704
+/* Yorktown Heights, NY 10598, USA
+/*--*/
+
+#endif
/* as with the \fBopen\fR request.
/* .sp
/* This request is supported in Postfix 2.5 and later.
+/* .IP "\fBsequence\fR \fImaptype:mapname flags function\fR"
+/* Iterate over the specified database. The \fIfunction\fR
+/* is one of DICT_SEQ_FUN_FIRST or DICT_SEQ_FUN_NEXT.
+/* The reply is the request completion status code and
+/* a lookup key and result value, if found.
+/* .sp
+/* This request is supported in Postfix 2.9 and later.
/* .PP
/* The request completion status is one of OK, RETRY, NOKEY
/* (lookup failed because the key was not found), BAD (malformed
* aren't too broken. The fix is to gather all parameter default settings in
* one place.
*/
+char *var_alias_maps;
char *var_local_rcpt_maps;
char *var_virt_alias_maps;
char *var_virt_alias_doms;
return (dict);
}
+/* proxymap_sequence_service - remote sequence service */
+
+static void proxymap_sequence_service(VSTREAM *client_stream)
+{
+ int request_flags;
+ DICT *dict;
+ int request_func;
+ const char *reply_key;
+ const char *reply_value;
+ int reply_status;
+
+ /*
+ * Process the request.
+ */
+ if (attr_scan(client_stream, ATTR_FLAG_STRICT,
+ ATTR_TYPE_STR, MAIL_ATTR_TABLE, request_map,
+ ATTR_TYPE_INT, MAIL_ATTR_FLAGS, &request_flags,
+ ATTR_TYPE_INT, MAIL_ATTR_FUNC, &request_func,
+ ATTR_TYPE_END) != 3
+ || (request_func != DICT_SEQ_FUN_FIRST
+ && request_func != DICT_SEQ_FUN_NEXT)) {
+ reply_status = PROXY_STAT_BAD;
+ reply_key = reply_value = "";
+ } else if ((dict = proxy_map_find(STR(request_map), request_flags,
+ &reply_status)) == 0) {
+ reply_key = reply_value = "";
+ } else if (dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK)
+ | (request_flags & DICT_FLAG_RQST_MASK)),
+ dict_seq(dict, request_func, &reply_key, &reply_value) == 0) {
+ reply_status = PROXY_STAT_OK;
+ } else if (dict_errno == 0) {
+ reply_status = PROXY_STAT_NOKEY;
+ reply_key = reply_value = "";
+ } else {
+ reply_status = PROXY_STAT_RETRY;
+ reply_key = reply_value = "";
+ }
+
+ /*
+ * Respond to the client.
+ */
+ attr_print(client_stream, ATTR_FLAG_NONE,
+ ATTR_TYPE_INT, MAIL_ATTR_STATUS, reply_status,
+ ATTR_TYPE_STR, MAIL_ATTR_KEY, reply_key,
+ ATTR_TYPE_STR, MAIL_ATTR_VALUE, reply_value,
+ ATTR_TYPE_END);
+}
+
/* proxymap_lookup_service - remote lookup service */
static void proxymap_lookup_service(VSTREAM *client_stream)
dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK)
| (request_flags & DICT_FLAG_RQST_MASK)
| DICT_FLAG_SYNC_UPDATE | DICT_FLAG_DUP_REPLACE);
+ dict_errno = 0;
dict_put(dict, STR(request_key), STR(request_value));
- reply_status = PROXY_STAT_OK;
+ reply_status = (dict_errno ? PROXY_STAT_RETRY : PROXY_STAT_OK);
}
/*
{
int request_flags;
DICT *dict;
+ int dict_status;
int reply_status;
/*
dict->flags = ((dict->flags & ~DICT_FLAG_RQST_MASK)
| (request_flags & DICT_FLAG_RQST_MASK)
| DICT_FLAG_SYNC_UPDATE);
- reply_status =
- dict_del(dict, STR(request_key)) ? PROXY_STAT_OK : PROXY_STAT_NOKEY;
+ dict_errno = 0;
+ dict_status = dict_del(dict, STR(request_key));
+ reply_status = (dict_status == 0 ? PROXY_STAT_OK :
+ dict_status > 0 ? PROXY_STAT_NOKEY :
+ PROXY_STAT_RETRY);
}
/*
proxymap_update_service(client_stream);
} else if (VSTREQ(request, PROXY_REQ_DELETE)) {
proxymap_delete_service(client_stream);
+ } else if (VSTREQ(request, PROXY_REQ_SEQUENCE)) {
+ proxymap_sequence_service(client_stream);
} else if (VSTREQ(request, PROXY_REQ_OPEN)) {
proxymap_open_service(client_stream);
} else {
int main(int argc, char **argv)
{
static const CONFIG_STR_TABLE str_table[] = {
+ VAR_ALIAS_MAPS, DEF_ALIAS_MAPS, &var_alias_maps, 0, 0,
VAR_LOCAL_RCPT_MAPS, DEF_LOCAL_RCPT_MAPS, &var_local_rcpt_maps, 0, 0,
VAR_VIRT_ALIAS_MAPS, DEF_VIRT_ALIAS_MAPS, &var_virt_alias_maps, 0, 0,
VAR_VIRT_ALIAS_DOMS, DEF_VIRT_ALIAS_DOMS, &var_virt_alias_doms, 0, 0,
/* modified, or if the result is to survive multiple dict_lookup() calls.
/*
/* dict_delete() removes the named member from the named dictionary.
-/* The result value is zero when the member was found.
+/* The result value is zero when the member was found, > 0 if
+/* it was not found, and < 0 in case of error (a database may
+/* not return after error).
/*
/* dict_sequence() steps through the named dictionary and returns
/* keys and values in some implementation-defined order. The func
if (*bufp == '#')
continue;
if ((cmd = mystrtok(&bufp, " ")) == 0) {
- vstream_printf("usage: del key|get key|put key=value|first|next\n");
+ vstream_printf("usage: verbose|del key|get key|put key=value|first|next\n");
vstream_fflush(VSTREAM_OUT);
continue;
}
msg_warn("dictionary has changed");
key = *bufp ? vstring_str(unescape(keybuf, mystrtok(&bufp, " ="))) : 0;
value = mystrtok(&bufp, " =");
- if (strcmp(cmd, "del") == 0 && key && !value) {
+ if (strcmp(cmd, "verbose") == 0 && !key) {
+ msg_verbose++;
+ } else if (strcmp(cmd, "del") == 0 && key && !value) {
if (dict_del(dict, key))
vstream_printf("%s: not found\n", key);
else
projects / thirdparty / postfix.git / commitdiff
