From: Timo Sirainen Date: Fri, 23 Jul 2010 19:37:35 +0000 (+0100) Subject: doveadm: Added client/server architecture support for running mail commands. X-Git-Tag: 2.0.rc4~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=71df09024cea5f2faa93da3bb9513ee96ba6bf22;p=thirdparty%2Fdovecot%2Fcore.git doveadm: Added client/server architecture support for running mail commands. This is done when doveadm_worker_count is non-zero. --- diff --git a/.hgignore b/.hgignore index 9c39244fa8..336146189a 100644 --- a/.hgignore +++ b/.hgignore @@ -65,6 +65,7 @@ src/director/director src/director/director-test src/dns/dns-client src/doveadm/doveadm +src/doveadm/doveadm-server src/dsync/dsync src/imap-login/imap-login src/imap/imap diff --git a/doc/example-config/dovecot.conf b/doc/example-config/dovecot.conf index 2734d438ed..3a7707f610 100644 --- a/doc/example-config/dovecot.conf +++ b/doc/example-config/dovecot.conf @@ -56,6 +56,12 @@ # a problem if the upgrade is e.g. because of a security fix). #shutdown_clients = yes +# If non-zero, run mail commands via this many connections to doveadm server, +# instead of running them directly in the same process. +#doveadm_worker_count = 0 +# UNIX socket or host:port used for connecting to doveadm server +#doveadm_socket_path = doveadm-server + ## ## Dictionary server settings ## diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am index bcce0fae21..a834d7e15b 100644 --- a/src/doveadm/Makefile.am +++ b/src/doveadm/Makefile.am @@ -1,6 +1,7 @@ doveadm_moduledir = $(moduledir)/doveadm bin_PROGRAMS = doveadm +pkglibexec_PROGRAMS = doveadm-server AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -33,18 +34,42 @@ cmd_pw_libs = \ libs = \ $(LIBDOVECOT_STORAGE) \ - $(cmd_pw_libs) \ $(unused_objects) doveadm_LDADD = \ $(libs) $(AUTH_LIBS) \ + $(cmd_pw_libs) \ $(LIBDOVECOT) \ $(MODULE_LIBS) doveadm_DEPENDENCIES = \ + $(libs) \ + $(cmd_pw_libs) \ + $(LIBDOVECOT_DEPS) + +doveadm_server_LDADD = \ + $(libs) \ + $(LIBDOVECOT) \ + $(MODULE_LIBS) +doveadm_server_DEPENDENCIES = \ $(libs) \ $(LIBDOVECOT_DEPS) +common = \ + doveadm-mail.c \ + doveadm-mail-altmove.c \ + doveadm-mail-expunge.c \ + doveadm-mail-fetch.c \ + doveadm-mail-iter.c \ + doveadm-mail-mailbox.c \ + doveadm-mail-mailbox-status.c \ + doveadm-mail-list-iter.c \ + doveadm-mail-search.c \ + doveadm-print.c \ + doveadm-settings.c \ + doveadm-util.c + doveadm_SOURCES = \ + $(common) \ doveadm.c \ doveadm-auth.c \ doveadm-director.c \ @@ -56,28 +81,26 @@ doveadm_SOURCES = \ doveadm-kick.c \ doveadm-log.c \ doveadm-master.c \ - doveadm-mail.c \ - doveadm-mail-altmove.c \ - doveadm-mail-expunge.c \ - doveadm-mail-fetch.c \ - doveadm-mail-iter.c \ - doveadm-mail-mailbox.c \ - doveadm-mail-mailbox-status.c \ - doveadm-mail-list-iter.c \ - doveadm-mail-search.c \ + doveadm-mail-server.c \ doveadm-mutf7.c \ doveadm-penalty.c \ - doveadm-print.c \ doveadm-print-flow.c \ doveadm-print-pager.c \ doveadm-print-tab.c \ doveadm-print-table.c \ doveadm-pw.c \ - doveadm-settings.c \ - doveadm-util.c \ - doveadm-who.c + doveadm-who.c \ + server-connection.c + +doveadm_server_SOURCES = \ + $(common) \ + client-connection.c \ + doveadm-print-server.c \ + main.c noinst_HEADERS = \ + client-connection.h \ + server-connection.h \ doveadm.h \ doveadm-dump.h \ doveadm-mail.h \ @@ -85,6 +108,7 @@ noinst_HEADERS = \ doveadm-mail-list-iter.h \ doveadm-print.h \ doveadm-print-private.h \ + doveadm-server.h \ doveadm-settings.h \ doveadm-util.h \ doveadm-who.h diff --git a/src/doveadm/client-connection.c b/src/doveadm/client-connection.c new file mode 100644 index 0000000000..af5fe54c24 --- /dev/null +++ b/src/doveadm/client-connection.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "strescape.h" +#include "master-service.h" +#include "mail-storage-service.h" +#include "doveadm-util.h" +#include "doveadm-server.h" +#include "doveadm-mail.h" +#include "doveadm-print.h" +#include "client-connection.h" + +#include + +#define MAX_INBUF_SIZE 1024 + +struct client_connection { + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + + unsigned int handshaked:1; + unsigned int authenticated:1; +}; + +static bool doveadm_mail_cmd_server(const char *cmd_name, const char *username, + int argc, char *argv[]) +{ + enum mail_storage_service_flags service_flags = + MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + struct doveadm_mail_cmd_context *ctx; + const struct doveadm_mail_cmd *cmd; + const char *getopt_args; + bool add_username_header; + int c; + + cmd = doveadm_mail_cmd_find(cmd_name); + if (cmd == NULL) { + i_error("doveadm: Client sent unknown command: %s", cmd_name); + return FALSE; + } + + if (doveadm_debug) + service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; + + ctx = doveadm_mail_cmd_init(cmd); + getopt_args = t_strconcat("Au:", ctx->getopt_args, NULL); + while ((c = getopt(argc, argv, getopt_args)) > 0) { + switch (c) { + case 'A': + add_username_header = TRUE; + break; + case 'u': + if (strchr(optarg, '*') != NULL || + strchr(optarg, '?') != NULL) + add_username_header = TRUE; + break; + default: + if ((ctx->v.parse_arg == NULL || + !ctx->v.parse_arg(ctx, c))) { + i_error("doveadm %s: " + "Client sent unknown parameter: %c", + cmd->name, c); + ctx->v.deinit(ctx); + return FALSE; + } + } + } + + argv += optind; + if (argv[0] != NULL && cmd->usage_args == NULL) { + i_error("doveadm %s: Client sent unknown parameter: %s", + cmd->name, argv[0]); + ctx->v.deinit(ctx); + return FALSE; + } + + if (doveadm_print_is_initialized() && add_username_header) { + doveadm_print_header("username", "Username", + DOVEADM_PRINT_HEADER_FLAG_STICKY | + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print_sticky("username", username); + } + + ctx->v.init(ctx, (const void *)argv); + doveadm_mail_single_user(ctx, username, service_flags); + ctx->v.deinit(ctx); + doveadm_print_flush(); + return !ctx->failed; +} + +static bool client_handle_command(struct client_connection *conn, char **args) +{ + const char *flags, *username, *cmd_name; + unsigned int argc; + bool ret; + + for (argc = 0; args[argc] != NULL; argc++) + args[argc] = str_tabunescape(args[argc]); + + if (argc < 3) { + i_error("doveadm client: No command given"); + return FALSE; + } + flags = args[0]; + username = args[1]; + cmd_name = args[2]; + args += 3; + argc -= 3; + + doveadm_debug = FALSE; + doveadm_verbose = FALSE; + + for (; *flags != '\0'; flags++) { + switch (*flags) { + case 'D': + doveadm_debug = TRUE; + doveadm_verbose = TRUE; + break; + case 'v': + doveadm_verbose = TRUE; + break; + default: + i_error("doveadm client: Unknown flag: %c", *flags); + return FALSE; + } + } + + o_stream_cork(conn->output); + ret = doveadm_mail_cmd_server(cmd_name, username, argc, args); + if (ret) + o_stream_send(conn->output, "\n+\n", 3); + else + o_stream_send(conn->output, "\n-\n", 3); + o_stream_uncork(conn->output); + + /* flush the output and disconnect */ + net_set_nonblock(conn->fd, FALSE); + (void)o_stream_flush(conn->output); + return FALSE; +} + +static bool +client_connection_authenticate(struct client_connection *conn ATTR_UNUSED) +{ + i_fatal("Authentication not supported yet"); + return FALSE; +} + +static void client_connection_input(struct client_connection *conn) +{ + const char *line; + bool ret = TRUE; + + if (!conn->handshaked) { + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof || conn->input->stream_errno != 0) + client_connection_destroy(&conn); + return; + } + + if (!version_string_verify(line, "doveadm-server", + DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR)) { + i_error("doveadm client not compatible with this server " + "(mixed old and new binaries?)"); + client_connection_destroy(&conn); + return; + } + conn->handshaked = TRUE; + } + if (!conn->authenticated) { + if (!client_connection_authenticate(conn)) + return; + } + + while ((line = i_stream_read_next_line(conn->input)) != NULL && ret) { + T_BEGIN { + char **args; + + args = p_strsplit(pool_datastack_create(), line, "\t"); + ret = client_handle_command(conn, args); + } T_END; + } + if (conn->input->eof || conn->input->stream_errno != 0 || !ret) + client_connection_destroy(&conn); +} + +struct client_connection *client_connection_create(int fd, int listen_fd) +{ + struct client_connection *conn; + struct stat st; + const char *listen_path; + + conn = i_new(struct client_connection, 1); + conn->fd = fd; + conn->io = io_add(fd, IO_READ, client_connection_input, conn); + conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); + conn->output = o_stream_create_fd(fd, (size_t)-1, FALSE); + + /* we'll have to do this with stat(), because at least in Linux + fstat() always returns mode as 0777 */ + if (net_getunixname(listen_fd, &listen_path) == 0 && + stat(listen_path, &st) == 0 && S_ISSOCK(st.st_mode) && + (st.st_mode & 0777) == 0600 && st.st_uid == geteuid()) { + /* no need for client to authenticate */ + conn->authenticated = TRUE; + o_stream_send(conn->output, "+\n", 2); + } else { + o_stream_send(conn->output, "-\n", 2); + } + return conn; +} + +void client_connection_destroy(struct client_connection **_conn) +{ + struct client_connection *conn = *_conn; + + *_conn = NULL; + + i_stream_destroy(&conn->input); + o_stream_destroy(&conn->output); + io_remove(&conn->io); + if (close(conn->fd) < 0) + i_error("close(client) failed: %m"); + i_free(conn); + + doveadm_client = NULL; + master_service_client_connection_destroyed(master_service); +} + +struct ostream *client_connection_get_output(struct client_connection *conn) +{ + return conn->output; +} diff --git a/src/doveadm/client-connection.h b/src/doveadm/client-connection.h new file mode 100644 index 0000000000..1eb8beb1ef --- /dev/null +++ b/src/doveadm/client-connection.h @@ -0,0 +1,9 @@ +#ifndef CLIENT_CONNECTION_H +#define CLIENT_CONNECTION_H + +struct client_connection *client_connection_create(int fd, int listen_fd); +void client_connection_destroy(struct client_connection **conn); + +struct ostream *client_connection_get_output(struct client_connection *conn); + +#endif diff --git a/src/doveadm/doveadm-mail-server.c b/src/doveadm/doveadm-mail-server.c new file mode 100644 index 0000000000..768813274f --- /dev/null +++ b/src/doveadm/doveadm-mail-server.c @@ -0,0 +1,269 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "ioloop.h" +#include "master-service.h" +#include "mail-storage.h" +#include "mail-storage-service.h" +#include "server-connection.h" +#include "doveadm-settings.h" +#include "doveadm-print.h" +#include "doveadm-server.h" +#include "doveadm-mail.h" + +#define DOVEADM_SERVER_CONNECTIONS_MAX 4 +#define DOVEADM_SERVER_QUEUE_MAX 16 + +#define DOVEADM_MAIL_SERVER_FAILED() \ + (internal_failure || master_service_is_killed(master_service)) + +static struct hash_table *servers; +static pool_t server_pool; +static struct doveadm_mail_cmd_context *cmd_ctx; +static bool internal_failure = FALSE; + +static void doveadm_mail_server_handle(struct server_connection *conn, + const char *username); + +static struct doveadm_server *doveadm_server_get(const char *name) +{ + struct doveadm_server *server; + char *dup_name; + + if (servers == NULL) { + server_pool = pool_alloconly_create("doveadm servers", 1024*16); + servers = hash_table_create(default_pool, server_pool, 0, + str_hash, + (hash_cmp_callback_t *)strcmp); + } + server = hash_table_lookup(servers, name); + if (server == NULL) { + server = p_new(server_pool, struct doveadm_server, 1); + server->name = dup_name = p_strdup(server_pool, name); + p_array_init(&server->connections, server_pool, + doveadm_settings->doveadm_worker_count); + p_array_init(&server->queue, server_pool, + DOVEADM_SERVER_QUEUE_MAX); + hash_table_insert(servers, dup_name, server); + } + return server; +} + +static struct server_connection * +doveadm_server_find_unused_conn(struct doveadm_server *server) +{ + struct server_connection *const *connp; + + array_foreach(&server->connections, connp) { + if (server_connection_is_idle(*connp)) + return *connp; + } + return NULL; +} + +static bool doveadm_server_have_used_connections(struct doveadm_server *server) +{ + struct server_connection *const *connp; + + array_foreach(&server->connections, connp) { + if (!server_connection_is_idle(*connp)) + return TRUE; + } + return FALSE; +} + +static void doveadm_cmd_callback(enum server_cmd_reply reply, void *context) +{ + struct server_connection *conn = context; + struct doveadm_server *server; + + if (reply == SERVER_CMD_REPLY_INTERNAL_FAILURE) { + internal_failure = TRUE; + master_service_stop(master_service); + return; + } + + if (reply != SERVER_CMD_REPLY_OK) + cmd_ctx->failed = TRUE; + + server = server_connection_get_server(conn); + if (array_count(&server->queue) > 0) { + char *const *usernamep = array_idx(&server->queue, 0); + char *username = *usernamep; + + conn = doveadm_server_find_unused_conn(server); + if (conn != NULL) { + array_delete(&server->queue, 0, 1); + doveadm_mail_server_handle(conn, username); + i_free(username); + } + } + + master_service_stop(master_service); +} + +static void doveadm_mail_server_handle(struct server_connection *conn, + const char *username) +{ + string_t *cmd; + unsigned int i; + + /* [] */ + cmd = t_str_new(256); + if (doveadm_debug) + str_append_c(cmd, 'D'); + else if (doveadm_verbose) + str_append_c(cmd, 'v'); + str_append_c(cmd, '\t'); + + str_tabescape_write(cmd, username); + for (i = 0; cmd_ctx->args[i] != NULL; i++) { + str_append_c(cmd, '\t'); + str_tabescape_write(cmd, cmd_ctx->args[i]); + } + str_append_c(cmd, '\n'); + server_connection_cmd(conn, str_c(cmd), doveadm_cmd_callback, conn); +} + +static void doveadm_server_flush_one(struct doveadm_server *server) +{ + unsigned int count = array_count(&server->queue); + + do { + master_service_run(master_service, NULL); + } while (array_count(&server->queue) == count && + doveadm_server_have_used_connections(server) && + !DOVEADM_MAIL_SERVER_FAILED()); +} + +static const char *userdb_field_find(const char *const *fields, const char *key) +{ + unsigned int i, len = strlen(key); + + if (fields == NULL) + return NULL; + + for (i = 0; fields[i] != NULL; i++) { + if (strncmp(fields[i], key, len) == 0) { + if (fields[i][len] == '\0') + return ""; + if (fields[i][len] == '=') + return fields[i]+len+1; + } + } + return NULL; +} + +int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx, + struct mail_storage_service_user *user) +{ + const struct mail_storage_service_input *input; + struct doveadm_server *server; + struct server_connection *conn; + const char *host; + char *username_dup; + void **sets; + + i_assert(cmd_ctx == ctx || cmd_ctx == NULL); + cmd_ctx = ctx; + + /* server sends the sticky headers for each row as well, + so undo any sticks we might have added already */ + doveadm_print_unstick_headers(); + + input = mail_storage_service_user_get_input(user); + sets = mail_storage_service_user_get_set(user); + + if (userdb_field_find(input->userdb_fields, "proxy") != NULL) { + host = userdb_field_find(input->userdb_fields, "host"); + if (host == NULL) { + i_error("user %s: Proxy is missing destination host", + input->username); + return 0; + } + } else { + host = doveadm_settings->doveadm_socket_path; + } + + server = doveadm_server_get(host); + conn = doveadm_server_find_unused_conn(server); + if (conn != NULL) + doveadm_mail_server_handle(conn, input->username); + else if (array_count(&server->connections) < + doveadm_settings->doveadm_worker_count) { + conn = server_connection_create(server); + doveadm_mail_server_handle(conn, input->username); + } else { + if (array_count(&server->queue) >= DOVEADM_SERVER_QUEUE_MAX) + doveadm_server_flush_one(server); + + username_dup = i_strdup(input->username); + array_append(&server->queue, &username_dup, 1); + } + return DOVEADM_MAIL_SERVER_FAILED() ? -1 : 0; +} + +static struct doveadm_server *doveadm_server_find_used(void) +{ + struct hash_iterate_context *iter; + struct doveadm_server *ret = NULL; + void *key, *value; + + iter = hash_table_iterate_init(servers); + while (hash_table_iterate(iter, &key, &value)) { + struct doveadm_server *server = value; + + if (doveadm_server_have_used_connections(server)) { + ret = server; + break; + } + } + hash_table_iterate_deinit(&iter); + return ret; +} + +static void doveadm_servers_destroy_all_connections(void) +{ + struct hash_iterate_context *iter; + void *key, *value; + + iter = hash_table_iterate_init(servers); + while (hash_table_iterate(iter, &key, &value)) { + struct doveadm_server *server = value; + + while (array_count(&server->connections) > 0) { + struct server_connection *const *connp, *conn; + + connp = array_idx(&server->connections, 0); + conn = *connp; + server_connection_destroy(&conn); + } + } + hash_table_iterate_deinit(&iter); +} + +void doveadm_mail_server_flush(void) +{ + struct doveadm_server *server; + + if (servers == NULL) + return; + + while ((server = doveadm_server_find_used()) != NULL && + !DOVEADM_MAIL_SERVER_FAILED()) + doveadm_server_flush_one(server); + + doveadm_servers_destroy_all_connections(); + if (master_service_is_killed(master_service)) + i_error("Aborted"); + if (DOVEADM_MAIL_SERVER_FAILED()) + cmd_ctx->failed = TRUE; + + hash_table_destroy(&servers); + pool_unref(&server_pool); + cmd_ctx = NULL; +} diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c index 8c0a84ac43..b275dd3878 100644 --- a/src/doveadm/doveadm-mail.c +++ b/src/doveadm/doveadm-mail.c @@ -202,6 +202,15 @@ doveadm_mail_next_user(struct doveadm_mail_cmd_context *ctx, return ret; } + if (doveadm_settings->doveadm_worker_count > 0 && !doveadm_server) { + /* execute this command via doveadm server */ + T_BEGIN { + ret = doveadm_mail_server_user(ctx, service_user); + mail_storage_service_user_free(&service_user); + } T_END; + return ret < 0 ? -1 : 1; + } + ret = mail_storage_service_next(ctx->storage_service, service_user, &ctx->cur_mail_user); if (ret < 0) { @@ -311,6 +320,7 @@ doveadm_mail_all_users(struct doveadm_mail_cmd_context *ctx, if (ret < 0) i_error("Failed to iterate through some users"); mail_storage_service_deinit(&ctx->storage_service); + doveadm_mail_server_flush(); } static void @@ -337,6 +347,7 @@ doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd) struct doveadm_mail_cmd_context *ctx; ctx = cmd->alloc(); + ctx->cmd = cmd; if (ctx->v.init == NULL) ctx->v.init = doveadm_mail_cmd_init_noop; if (ctx->v.get_next_user == NULL) @@ -362,6 +373,8 @@ doveadm_mail_cmd(const struct doveadm_mail_cmd *cmd, int argc, char *argv[]) service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; ctx = doveadm_mail_cmd_init(cmd); + ctx->args = (const void *)argv; + getopt_args = t_strconcat("Au:", ctx->getopt_args, NULL); username = getenv("USER"); wildcard_user = NULL; diff --git a/src/doveadm/doveadm-mail.h b/src/doveadm/doveadm-mail.h index e62d253ec0..a6f82bcc80 100644 --- a/src/doveadm/doveadm-mail.h +++ b/src/doveadm/doveadm-mail.h @@ -35,6 +35,9 @@ union doveadm_mail_cmd_module_context { struct doveadm_mail_cmd_context { pool_t pool; + const struct doveadm_mail_cmd *cmd; + const char *const *args; + const char *getopt_args; struct mail_storage_service_ctx *storage_service; /* search args aren't set for all mail commands */ @@ -78,6 +81,9 @@ doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd); void doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, const char *username, enum mail_storage_service_flags service_flags); +int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx, + struct mail_storage_service_user *user); +void doveadm_mail_server_flush(void); int doveadm_mailbox_find_and_sync(struct mail_user *user, const char *mailbox, struct mailbox **box_r); diff --git a/src/doveadm/doveadm-print-server.c b/src/doveadm/doveadm-print-server.c new file mode 100644 index 0000000000..b6172b8c37 --- /dev/null +++ b/src/doveadm/doveadm-print-server.c @@ -0,0 +1,82 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "ostream.h" +#include "client-connection.h" +#include "doveadm-server.h" +#include "doveadm-print-private.h" + +struct doveadm_print_server_context { + unsigned int header_idx, header_count; + + string_t *str; +}; + +static struct doveadm_print_server_context ctx; + +static void doveadm_print_server_flush(void); + +static void doveadm_print_server_init(void) +{ + ctx.str = str_new(default_pool, 256); +} + +static void doveadm_print_server_deinit(void) +{ + str_free(&ctx.str); +} + +static void +doveadm_print_server_header(const struct doveadm_print_header *hdr ATTR_UNUSED) +{ + /* no need to transfer these. the client should already know what + it's getting */ + ctx.header_count++; +} + +static void doveadm_print_server_print(const char *value) +{ + str_tabescape_write(ctx.str, value); + str_append_c(ctx.str, '\t'); + + if (++ctx.header_idx == ctx.header_count) { + ctx.header_idx = 0; + doveadm_print_server_flush(); + } +} + +static void +doveadm_print_server_print_stream(const unsigned char *value, size_t size) +{ + if (size == 0) { + doveadm_print_server_print(""); + return; + } + T_BEGIN { + str_tabescape_write(ctx.str, t_strndup(value, size)); + } T_END; + + if (str_len(ctx.str) >= IO_BLOCK_SIZE) + doveadm_print_server_flush(); +} + +static void doveadm_print_server_flush(void) +{ + o_stream_send(client_connection_get_output(doveadm_client), + str_data(ctx.str), str_len(ctx.str)); + str_truncate(ctx.str, 0); +} + +struct doveadm_print_vfuncs doveadm_print_server_vfuncs = { + DOVEADM_PRINT_TYPE_SERVER, + + doveadm_print_server_init, + doveadm_print_server_deinit, + doveadm_print_server_header, + doveadm_print_server_print, + doveadm_print_server_print_stream, + doveadm_print_server_flush +}; diff --git a/src/doveadm/doveadm-print.c b/src/doveadm/doveadm-print.c index 30843fcd5b..9dd0925f36 100644 --- a/src/doveadm/doveadm-print.c +++ b/src/doveadm/doveadm-print.c @@ -113,6 +113,14 @@ void doveadm_print_flush(void) fflush(stdout); } +void doveadm_print_unstick_headers(void) +{ + struct doveadm_print_header_context *hdr; + + array_foreach_modifiable(&ctx->headers, hdr) + hdr->sticky = FALSE; +} + void doveadm_print_init(const char *name) { pool_t pool; diff --git a/src/doveadm/doveadm-print.h b/src/doveadm/doveadm-print.h index c4caef02e2..191697f962 100644 --- a/src/doveadm/doveadm-print.h +++ b/src/doveadm/doveadm-print.h @@ -23,6 +23,7 @@ void doveadm_print_num(uintmax_t value); void doveadm_print_stream(const void *value, size_t size); void doveadm_print_sticky(const char *key, const char *value); void doveadm_print_flush(void); +void doveadm_print_unstick_headers(void); void doveadm_print_init(const char *name); void doveadm_print_deinit(void); diff --git a/src/doveadm/doveadm-server.h b/src/doveadm/doveadm-server.h new file mode 100644 index 0000000000..9dc8b0c6e6 --- /dev/null +++ b/src/doveadm/doveadm-server.h @@ -0,0 +1,16 @@ +#ifndef DOVEADM_SERVER_H +#define DOVEADM_SERVER_H + +#define DOVEADM_PRINT_TYPE_SERVER "server" + +extern struct client_connection *doveadm_client; +extern struct doveadm_print_vfuncs doveadm_print_server_vfuncs; + +struct doveadm_server { + const char *name; + + ARRAY_DEFINE(connections, struct server_connection *); + ARRAY_TYPE(string) queue; +}; + +#endif diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c index 9cc2845680..a0d30dd0ef 100644 --- a/src/doveadm/doveadm-settings.c +++ b/src/doveadm/doveadm-settings.c @@ -1,10 +1,52 @@ /* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "buffer.h" #include "settings-parser.h" +#include "service-settings.h" #include "mail-storage-settings.h" #include "doveadm-settings.h" +static bool doveadm_settings_check(void *_set, pool_t pool, const char **error_r); + +/* */ +static struct file_listener_settings doveadm_unix_listeners_array[] = { + { "doveadm-server", 0600, "", "" } +}; +static struct file_listener_settings *doveadm_unix_listeners[] = { + &doveadm_unix_listeners_array[0] +}; +static buffer_t doveadm_unix_listeners_buf = { + doveadm_unix_listeners, sizeof(doveadm_unix_listeners), { 0, } +}; +/* */ + +struct service_settings doveadm_service_settings = { + .name = "doveadm", + .protocol = "", + .type = "", + .executable = "doveadm-server", + .user = "", + .group = "", + .privileged_group = "", + .extra_groups = "", + .chroot = "", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 0, + .client_limit = 1, + .service_count = 1, + .idle_kill = 0, + .vsz_limit = -1U, + + .unix_listeners = { { &doveadm_unix_listeners_buf, + sizeof(doveadm_unix_listeners[0]) } }, + .fifo_listeners = ARRAY_INIT, + .inet_listeners = ARRAY_INIT +}; + #undef DEF #define DEF(type, name) \ { type, #name, offsetof(struct doveadm_settings, name), NULL } @@ -13,6 +55,9 @@ static const struct setting_define doveadm_setting_defines[] = { DEF(SET_STR, base_dir), DEF(SET_STR, mail_plugins), DEF(SET_STR, mail_plugin_dir), + DEF(SET_STR, doveadm_socket_path), + DEF(SET_UINT, doveadm_worker_count), + { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL }, SETTING_DEFINE_LIST_END @@ -22,6 +67,8 @@ const struct doveadm_settings doveadm_default_settings = { .base_dir = PKG_RUNDIR, .mail_plugins = "", .mail_plugin_dir = MODULEDIR, + .doveadm_socket_path = "doveadm-server", + .doveadm_worker_count = 0, .plugin_envs = ARRAY_INIT }; @@ -40,7 +87,29 @@ const struct setting_parser_info doveadm_setting_parser_info = { .struct_size = sizeof(struct doveadm_settings), .parent_offset = (size_t)-1, + .check_func = doveadm_settings_check, .dependencies = doveadm_setting_dependencies }; const struct doveadm_settings *doveadm_settings; + +static void +fix_base_path(struct doveadm_settings *set, pool_t pool, const char **str) +{ + if (*str != NULL && **str != '\0' && **str != '/') + *str = p_strconcat(pool, set->base_dir, "/", *str, NULL); +} + +/* */ +static bool doveadm_settings_check(void *_set ATTR_UNUSED, + pool_t pool ATTR_UNUSED, + const char **error_r ATTR_UNUSED) +{ +#ifndef CONFIG_BINARY + struct doveadm_settings *set = _set; + + fix_base_path(set, pool, &set->doveadm_socket_path); +#endif + return TRUE; +} +/* */ diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h index e01c99b3d8..07ec3276bb 100644 --- a/src/doveadm/doveadm-settings.h +++ b/src/doveadm/doveadm-settings.h @@ -5,6 +5,8 @@ struct doveadm_settings { const char *base_dir; const char *mail_plugins; const char *mail_plugin_dir; + const char *doveadm_socket_path; + unsigned int doveadm_worker_count; ARRAY_DEFINE(plugin_envs, const char *); }; diff --git a/src/doveadm/doveadm-util.c b/src/doveadm/doveadm-util.c index 60f3baf1af..c78e8a7f73 100644 --- a/src/doveadm/doveadm-util.c +++ b/src/doveadm/doveadm-util.c @@ -13,7 +13,7 @@ #include #include -bool doveadm_verbose = FALSE, doveadm_debug = FALSE; +bool doveadm_verbose = FALSE, doveadm_debug = FALSE, doveadm_server = FALSE; static struct module *modules = NULL; void doveadm_load_modules(void) diff --git a/src/doveadm/doveadm-util.h b/src/doveadm/doveadm-util.h index f66c3ffdd5..2214e500c0 100644 --- a/src/doveadm/doveadm-util.h +++ b/src/doveadm/doveadm-util.h @@ -3,7 +3,7 @@ #define DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR 1 -extern bool doveadm_verbose, doveadm_debug; +extern bool doveadm_verbose, doveadm_debug, doveadm_server; const char *unixdate2str(time_t timestamp); const char *doveadm_plugin_getenv(const char *name); diff --git a/src/doveadm/main.c b/src/doveadm/main.c new file mode 100644 index 0000000000..72b5145504 --- /dev/null +++ b/src/doveadm/main.c @@ -0,0 +1,101 @@ +/* Copyright (c) 2005-2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "restrict-access.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "settings-parser.h" +#include "client-connection.h" +#include "doveadm-settings.h" +#include "doveadm-mail.h" +#include "doveadm-print-private.h" +#include "doveadm-server.h" + +const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[] = { + &doveadm_print_server_vfuncs, + NULL +}; + +struct client_connection *doveadm_client; + +int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED, + struct mail_storage_service_user *user ATTR_UNUSED) +{ + /* this function should be called only by doveadm client code */ + i_unreached(); +} +void doveadm_mail_server_flush(void) +{ +} + +static void doveadm_die(void) +{ + /* do nothing. doveadm connections should be over soon. */ +} + +static void client_connected(struct master_service_connection *conn) +{ + if (doveadm_client != NULL) { + i_error("doveadm server can handle only a single client"); + return; + } + + master_service_client_connection_accept(conn); + doveadm_client = client_connection_create(conn->fd, conn->listen_fd); +} + +static void main_preinit(void) +{ + restrict_access_by_env(NULL, FALSE); + restrict_access_allow_coredumps(TRUE); +} + +static void main_init(void) +{ + doveadm_server = TRUE; + doveadm_settings = master_service_settings_get_others(master_service)[0]; + doveadm_settings = settings_dup(&doveadm_setting_parser_info, + doveadm_settings, + pool_datastack_create()); + + doveadm_mail_init(); + doveadm_load_modules(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); +} + +static void main_deinit(void) +{ + if (doveadm_client != NULL) + client_connection_destroy(&doveadm_client); + doveadm_mail_deinit(); + doveadm_unload_modules(); +} + +int main(int argc, char *argv[]) +{ + const struct setting_parser_info *set_roots[] = { + &doveadm_setting_parser_info, + NULL + }; + const char *error; + + master_service = master_service_init("doveadm", 0, &argc, &argv, NULL); + if (master_getopt(master_service) > 0) + return FATAL_DEFAULT; + + if (master_service_settings_read_simple(master_service, set_roots, + &error) < 0) + i_fatal("Error reading configuration: %s", error); + + master_service_init_log(master_service, "doveadm: "); + main_preinit(); + master_service_init_finish(master_service); + master_service_set_die_callback(master_service, doveadm_die); + + main_init(); + master_service_run(master_service, client_connected); + + main_deinit(); + master_service_deinit(&master_service); + return 0; +} diff --git a/src/doveadm/server-connection.c b/src/doveadm/server-connection.c new file mode 100644 index 0000000000..88ac714bef --- /dev/null +++ b/src/doveadm/server-connection.c @@ -0,0 +1,290 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "network.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "doveadm-print.h" +#include "doveadm-util.h" +#include "doveadm-server.h" +#include "server-connection.h" + +#include + +#define MAX_INBUF_SIZE (1024*32) + +enum server_reply_state { + SERVER_REPLY_STATE_DONE = 0, + SERVER_REPLY_STATE_PRINT, + SERVER_REPLY_STATE_RET +}; + +struct server_connection { + struct doveadm_server *server; + + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + + server_cmd_callback_t *callback; + void *context; + + enum server_reply_state state; + + unsigned int handshaked:1; + unsigned int authenticated:1; + unsigned int streaming:1; +}; + +static struct server_connection *printing_conn = NULL; + +static void server_connection_input(struct server_connection *conn); + +static void print_connection_released(void) +{ + struct doveadm_server *server = printing_conn->server; + struct server_connection *const *conns; + unsigned int i, count; + + printing_conn = NULL; + + conns = array_get(&server->connections, &count); + for (i = 0; i < count; i++) { + if (conns[i]->io != NULL) + continue; + + conns[i]->io = io_add(conns[i]->fd, IO_READ, + server_connection_input, conns[i]); + server_connection_input(conns[i]); + if (printing_conn != NULL) + break; + } +} + +static void +server_connection_callback(struct server_connection *conn, + enum server_cmd_reply reply) +{ + server_cmd_callback_t *callback = conn->callback; + + conn->callback = NULL; + callback(reply, conn->context); +} + +static void stream_data(string_t *str, const unsigned char *data, size_t size) +{ + const char *text; + + str_truncate(str, 0); + str_append_n(str, data, size); + text = str_tabunescape(str_c_modifiable(str)); + doveadm_print_stream(text, strlen(text)); +} + +static void server_flush_field(struct server_connection *conn, string_t *str, + const unsigned char *data, size_t size) +{ + if (conn->streaming) { + conn->streaming = FALSE; + stream_data(str, data, size); + doveadm_print_stream("", 0); + } else { + const char *text; + + str_truncate(str, 0); + str_append_n(str, data, size); + text = str_tabunescape(str_c_modifiable(str)); + doveadm_print(text); + } +} + +static void +server_handle_input(struct server_connection *conn, + const unsigned char *data, size_t size) +{ + string_t *str; + size_t i, start; + + if (printing_conn == conn) { + /* continue printing */ + } else if (printing_conn == NULL) { + printing_conn = conn; + } else { + /* someone else is printing. don't continue until it + goes away */ + io_remove(&conn->io); + return; + } + + if (data[size-1] == '\001') { + /* last character is an escape */ + size--; + } + + str = t_str_new(128); + for (i = start = 0; i < size; i++) { + if (data[i] == '\n') { + if (i != start) + i_error("doveadm server sent broken input"); + conn->state = SERVER_REPLY_STATE_RET; + i_stream_skip(conn->input, i + 1); + + print_connection_released(); + return; + } + if (data[i] == '\t') { + server_flush_field(conn, str, data + start, i - start); + start = i + 1; + } + } + if (start != size) { + conn->streaming = TRUE; + stream_data(str, data + start, size - start); + } + i_stream_skip(conn->input, size); +} + +static void +server_connection_authenticate(struct server_connection *conn ATTR_UNUSED) +{ + i_fatal("Authentication not supported yet"); +} + +static void server_connection_input(struct server_connection *conn) +{ + const unsigned char *data; + size_t size; + const char *line; + + if (!conn->handshaked) { + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof || conn->input->stream_errno != 0) + server_connection_destroy(&conn); + return; + } + + conn->handshaked = TRUE; + if (strcmp(line, "+") == 0) + conn->authenticated = TRUE; + else if (strcmp(line, "-") == 0) + server_connection_authenticate(conn); + else { + i_error("doveadm server sent invalid handshake: %s", + line); + server_connection_destroy(&conn); + return; + } + } + + if (i_stream_read(conn->input) == -1) { + /* disconnected */ + server_connection_destroy(&conn); + return; + } + data = i_stream_get_data(conn->input, &size); + if (size == 0) + return; + + switch (conn->state) { + case SERVER_REPLY_STATE_DONE: + i_error("doveadm server sent unexpected input"); + server_connection_destroy(&conn); + return; + case SERVER_REPLY_STATE_PRINT: + server_handle_input(conn, data, size); + if (conn->state != SERVER_REPLY_STATE_RET) + break; + /* fall through */ + data = i_stream_get_data(conn->input, &size); + case SERVER_REPLY_STATE_RET: + if (size < 2) + return; + if (data[0] == '+' && data[1] == '\n') + server_connection_callback(conn, SERVER_CMD_REPLY_OK); + else if (data[0] == '-' && data[1] == '\n') + server_connection_callback(conn, SERVER_CMD_REPLY_FAIL); + else { + i_error("doveadm server sent broken input"); + server_connection_destroy(&conn); + return; + } + break; + } +} + +struct server_connection * +server_connection_create(struct doveadm_server *server) +{ +#define DOVEADM_SERVER_HANDSHAKE "VERSION\tdoveadm-server\t1\t0\n" + struct server_connection *conn; + + conn = i_new(struct server_connection, 1); + conn->server = server; + conn->fd = doveadm_connect(server->name); + net_set_nonblock(conn->fd, TRUE); + conn->io = io_add(conn->fd, IO_READ, server_connection_input, conn); + conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE, FALSE); + conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); + conn->state = SERVER_REPLY_STATE_DONE; + o_stream_send_str(conn->output, DOVEADM_SERVER_HANDSHAKE); + + array_append(&conn->server->connections, &conn, 1); + return conn; +} + +void server_connection_destroy(struct server_connection **_conn) +{ + struct server_connection *conn = *_conn; + struct server_connection *const *conns; + unsigned int i, count; + + *_conn = NULL; + + conns = array_get(&conn->server->connections, &count); + for (i = 0; i < count; i++) { + if (conns[i] == conn) { + array_delete(&conn->server->connections, i, 1); + break; + } + } + + if (conn->callback != NULL) { + server_connection_callback(conn, + SERVER_CMD_REPLY_INTERNAL_FAILURE); + } + if (printing_conn == conn) + print_connection_released(); + + i_stream_destroy(&conn->input); + o_stream_destroy(&conn->output); + if (conn->io != NULL) + io_remove(&conn->io); + if (close(conn->fd) < 0) + i_error("close(server) failed: %m"); + i_free(conn); +} + +struct doveadm_server * +server_connection_get_server(struct server_connection *conn) +{ + return conn->server; +} + +void server_connection_cmd(struct server_connection *conn, const char *line, + server_cmd_callback_t *callback, void *context) +{ + conn->state = SERVER_REPLY_STATE_PRINT; + o_stream_send_str(conn->output, line); + conn->callback = callback; + conn->context = context; +} + +bool server_connection_is_idle(struct server_connection *conn) +{ + return conn->callback == NULL; +} diff --git a/src/doveadm/server-connection.h b/src/doveadm/server-connection.h new file mode 100644 index 0000000000..4e00a367c3 --- /dev/null +++ b/src/doveadm/server-connection.h @@ -0,0 +1,27 @@ +#ifndef SERVER_CONNECTION_H +#define SERVER_CONNECTION_H + +enum server_cmd_reply { + SERVER_CMD_REPLY_INTERNAL_FAILURE, + SERVER_CMD_REPLY_FAIL, + SERVER_CMD_REPLY_OK +}; + +struct doveadm_server; + +typedef void server_cmd_callback_t(enum server_cmd_reply reply, void *context); + +struct server_connection * +server_connection_create(struct doveadm_server *server); +void server_connection_destroy(struct server_connection **conn); + +/* Return the server given to create() */ +struct doveadm_server * +server_connection_get_server(struct server_connection *conn); + +void server_connection_cmd(struct server_connection *conn, const char *line, + server_cmd_callback_t *callback, void *context); +/* Returns TRUE if no command is being processed */ +bool server_connection_is_idle(struct server_connection *conn); + +#endif diff --git a/src/lib-master/master-service-private.h b/src/lib-master/master-service-private.h index be87e44a70..b152301b4f 100644 --- a/src/lib-master/master-service-private.h +++ b/src/lib-master/master-service-private.h @@ -53,6 +53,7 @@ struct master_service { const struct master_service_settings *set; struct setting_parser_context *set_parser; + unsigned int killed:1; unsigned int stopping:1; unsigned int keep_environment:1; unsigned int log_directly:1; diff --git a/src/lib-master/master-service.c b/src/lib-master/master-service.c index 49e6814d35..38f64a1c65 100644 --- a/src/lib-master/master-service.c +++ b/src/lib-master/master-service.c @@ -69,6 +69,7 @@ static void sig_die(const siginfo_t *si, void *context) return; } + service->killed = TRUE; io_loop_stop(service->ioloop); } @@ -530,6 +531,11 @@ void master_service_stop_new_connections(struct master_service *service) master_login_stop(service->login); } +bool master_service_is_killed(struct master_service *service) +{ + return service->killed; +} + void master_service_anvil_send(struct master_service *service, const char *cmd) { ssize_t ret; diff --git a/src/lib-master/master-service.h b/src/lib-master/master-service.h index 95fd38c0f2..8308450493 100644 --- a/src/lib-master/master-service.h +++ b/src/lib-master/master-service.h @@ -110,6 +110,8 @@ void master_service_stop(struct master_service *service); /* Stop once we're done serving existing new connections, but don't accept any new ones. */ void master_service_stop_new_connections(struct master_service *service); +/* Returns TRUE if we've received a SIGINT/SIGTERM and we've decided to stop. */ +bool master_service_is_killed(struct master_service *service); /* Send command to anvil process, if we have fd to it. */ void master_service_anvil_send(struct master_service *service, const char *cmd); diff --git a/src/plugins/expire/doveadm-expire.c b/src/plugins/expire/doveadm-expire.c index 9cd865eb97..ce3c9dc309 100644 --- a/src/plugins/expire/doveadm-expire.c +++ b/src/plugins/expire/doveadm-expire.c @@ -8,6 +8,7 @@ #include "imap-match.h" #include "expire-set.h" #include "mail-search.h" +#include "doveadm-settings.h" #include "doveadm-mail.h" #define DOVEADM_EXPIRE_MAIL_CMD_CONTEXT(obj) \