From: Timo Sirainen Date: Mon, 27 Dec 2021 15:03:37 +0000 (+0200) Subject: anvil: Add admin-client API with connection pooling X-Git-Tag: 2.4.0~4497 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=2a73706d787b35d4a04b8df29dd65775d712b962;p=thirdparty%2Fdovecot%2Fcore.git anvil: Add admin-client API with connection pooling --- diff --git a/src/anvil/Makefile.am b/src/anvil/Makefile.am index 0b146c3692..01369987b7 100644 --- a/src/anvil/Makefile.am +++ b/src/anvil/Makefile.am @@ -18,12 +18,16 @@ anvil_DEPENDENCIES = $(LIBDOVECOT_DEPS) anvil_SOURCES = \ main.c \ + admin-client.c \ + admin-client-pool.c \ anvil-connection.c \ anvil-settings.c \ connect-limit.c \ penalty.c noinst_HEADERS = \ + admin-client.h \ + admin-client-pool.h \ anvil-connection.h \ common.h \ connect-limit.h \ diff --git a/src/anvil/admin-client-pool.c b/src/anvil/admin-client-pool.c new file mode 100644 index 0000000000..88551487dd --- /dev/null +++ b/src/anvil/admin-client-pool.c @@ -0,0 +1,130 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "admin-client-pool.h" + +struct admin_client_cmd_context { + struct admin_client_ref *client; + admin_client_callback_t *callback; + void *context; +}; + +struct admin_client_ref { + int refcount; + pid_t pid; + struct admin_client *client; + struct admin_client_pool *pool; +}; + +struct admin_client_pool { + char *base_dir; + unsigned int max_connections; + unsigned int idle_connection_count; + ARRAY(struct admin_client_ref *) clients; +}; + +struct admin_client_pool * +admin_client_pool_init(const char *base_dir, unsigned int max_connections) +{ + struct admin_client_pool *pool; + + pool = i_new(struct admin_client_pool, 1); + pool->base_dir = i_strdup(base_dir); + pool->max_connections = max_connections; + i_array_init(&pool->clients, 8); + return pool; + +} + +void admin_client_pool_deinit(struct admin_client_pool **_pool) +{ + struct admin_client_pool *pool = *_pool; + struct admin_client_ref *client; + + array_foreach_elem(&pool->clients, client) { + i_assert(client->refcount == 0); + admin_client_deinit(&client->client); + } + array_free(&pool->clients); + i_free(pool->base_dir); + i_free(pool); +} + +static void admin_client_pool_cleanup(struct admin_client_pool *pool) +{ + struct admin_client_ref *const *clients; + unsigned int i, count; + + /* see if we need to destroy the connection to keep it under the max + connections limit. */ + clients = array_get(&pool->clients, &count); + for (i = count; i > 0; i--) { + if (array_count(&pool->clients) <= pool->max_connections) + break; + if (pool->idle_connection_count == 0) + break; + if (clients[i-1]->refcount == 0) { + admin_client_deinit(&clients[i-1]->client); + array_delete(&pool->clients, i-1, 1); + } + } +} + +static struct admin_client_ref * +admin_client_pool_get(struct admin_client_pool *pool, + const char *service, pid_t pid) +{ + struct admin_client_ref *client; + + array_foreach_elem(&pool->clients, client) { + if (client->pid == pid) { + if (client->refcount++ == 0) { + i_assert(pool->idle_connection_count > 0); + pool->idle_connection_count--; + } + return client; + } + } + + client = i_new(struct admin_client_ref, 1); + client->refcount = 1; + client->pid = pid; + client->client = admin_client_init(pool->base_dir, service, pid); + client->pool = pool; + array_push_back(&pool->clients, &client); + admin_client_pool_cleanup(pool); + return client; +} + +static void +admin_client_pool_cmd_callback(const char *reply, const char *error, + struct admin_client_cmd_context *cmd_ctx) +{ + struct admin_client_ref *client = cmd_ctx->client; + + i_assert(client->refcount > 0); + + cmd_ctx->callback(reply, error, cmd_ctx->context); + i_free(cmd_ctx); + + if (--client->refcount > 0) + return; + client->pool->idle_connection_count++; + admin_client_pool_cleanup(client->pool); +} + +void admin_client_pool_send_cmd(struct admin_client_pool *pool, + const char *service, pid_t pid, const char *cmd, + admin_client_callback_t *callback, + void *context) +{ + struct admin_client_cmd_context *cmd_ctx; + + cmd_ctx = i_new(struct admin_client_cmd_context, 1); + cmd_ctx->client = admin_client_pool_get(pool, service, pid); + cmd_ctx->callback = callback; + cmd_ctx->context = context; + admin_client_send_cmd(cmd_ctx->client->client, cmd, + admin_client_pool_cmd_callback, cmd_ctx); +} diff --git a/src/anvil/admin-client-pool.h b/src/anvil/admin-client-pool.h new file mode 100644 index 0000000000..68e69926a6 --- /dev/null +++ b/src/anvil/admin-client-pool.h @@ -0,0 +1,15 @@ +#ifndef ADMIN_CLIENT_POOL_H +#define ADMIN_CLIENT_POOL_H + +#include "admin-client.h" + +struct admin_client_pool * +admin_client_pool_init(const char *base_dir, unsigned int max_connections); +void admin_client_pool_deinit(struct admin_client_pool **pool); + +void admin_client_pool_send_cmd(struct admin_client_pool *pool, + const char *service, pid_t pid, const char *cmd, + admin_client_callback_t *callback, + void *context); + +#endif diff --git a/src/anvil/admin-client.c b/src/anvil/admin-client.c new file mode 100644 index 0000000000..e757ae082d --- /dev/null +++ b/src/anvil/admin-client.c @@ -0,0 +1,155 @@ +/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "connection.h" +#include "ostream.h" +#include "admin-client.h" + +struct admin_client_command { + char *cmdline; + admin_client_callback_t *callback; + void *context; +}; + +struct admin_client { + struct connection conn; + struct timeout *to_failed; + ARRAY(struct admin_client_command) commands; +}; + +static struct connection_list *admin_clients; + +struct admin_client * +admin_client_init(const char *base_dir, const char *service, pid_t pid) +{ + struct admin_client *client; + const char *path; + + path = t_strdup_printf("%s/srv.%s/%ld", + base_dir, service, (long)pid); + + client = i_new(struct admin_client, 1); + connection_init_client_unix(admin_clients, &client->conn, path); + i_array_init(&client->commands, 8); + return client; +} + +static void admin_client_reply(struct admin_client *client, + const char *reply, const char *error) +{ + struct admin_client_command *cmd; + + cmd = array_idx_modifiable(&client->commands, 0); + cmd->callback(reply, error, cmd->context); + i_free(cmd->cmdline); + array_pop_front(&client->commands); +} + +static void +admin_client_fail_commands(struct admin_client *client, const char *error) +{ + timeout_remove(&client->to_failed); + while (array_count(&client->commands) > 0) + admin_client_reply(client, NULL, error); +} + +static void admin_client_destroy(struct connection *conn) +{ + struct admin_client *client = + container_of(conn, struct admin_client, conn); + + admin_client_fail_commands(client, connection_disconnect_reason(conn)); +} + +void admin_client_deinit(struct admin_client **_client) +{ + struct admin_client *client = *_client; + struct admin_client_command *cmd; + + *_client = NULL; + array_foreach_modifiable(&client->commands, cmd) + i_free(cmd->cmdline); + array_free(&client->commands); + connection_deinit(&client->conn); + i_free(client); +} + +static void admin_client_connect_failed_callback(struct admin_client *client) +{ + admin_client_fail_commands(client, "Failed to connect to admin socket"); +} + +#undef admin_client_send_cmd +void admin_client_send_cmd(struct admin_client *client, const char *cmdline, + admin_client_callback_t *callback, void *context) +{ + struct admin_client_command *cmd; + + cmd = array_append_space(&client->commands); + cmd->cmdline = i_strdup(cmdline); + cmd->callback = callback; + cmd->context = context; + + if (client->conn.disconnected) { + if (connection_client_connect(&client->conn) < 0) { + e_error(client->conn.event, + "net_connect_unix(%s) failed: %m", + client->conn.base_name); + if (client->to_failed == NULL) { + client->to_failed = timeout_add_short(0, + admin_client_connect_failed_callback, client); + } + return; + } + } + const struct const_iovec iov[] = { + { cmdline, strlen(cmdline) }, + { "\n", 1 } + }; + o_stream_nsendv(client->conn.output, iov, N_ELEMENTS(iov)); +} + +static int +admin_client_input_line(struct connection *conn, const char *line) +{ + struct admin_client *client = + container_of(conn, struct admin_client, conn); + + if (!conn->version_received) + return connection_input_line_default(conn, line); + + if (array_count(&client->commands) == 0) { + e_error(conn->event, "Unexpected input: %s", line); + return -1; + } + admin_client_reply(client, line, NULL); + return 1; +} + +static const struct connection_settings admin_client_set = { + .service_name_in = "master-admin-server", + .service_name_out = "master-admin-client", + .major_version = 1, + .minor_version = 0, + + .input_max_size = SIZE_MAX, + .output_max_size = SIZE_MAX, + .client = TRUE, +}; + +static const struct connection_vfuncs admin_client_vfuncs = { + .destroy = admin_client_destroy, + .input_line = admin_client_input_line, +}; + +void admin_clients_init(void) +{ + admin_clients = connection_list_init(&admin_client_set, + &admin_client_vfuncs); +} + +void admin_clients_deinit(void) +{ + connection_list_deinit(&admin_clients); +} diff --git a/src/anvil/admin-client.h b/src/anvil/admin-client.h new file mode 100644 index 0000000000..877972af3d --- /dev/null +++ b/src/anvil/admin-client.h @@ -0,0 +1,24 @@ +#ifndef ADMIN_CLIENT_H +#define ADMIN_CLIENT_H + +/* Error is set and reply=NULL on internal errors. */ +typedef void +admin_client_callback_t(const char *reply, const char *error, void *context); + +struct admin_client * +admin_client_init(const char *base_dir, const char *service, pid_t pid); +void admin_client_deinit(struct admin_client **client); + +void admin_client_send_cmd(struct admin_client *client, const char *cmdline, + admin_client_callback_t *callback, void *context); +#define admin_client_send_cmd(client, cmd, callback, context) \ + admin_client_send_cmd(client, cmd, \ + (admin_client_callback_t *)callback, \ + TRUE ? context : CALLBACK_TYPECHECK(callback, \ + void (*)(const char *, const char *, \ + typeof(context)))) + +void admin_clients_init(void); +void admin_clients_deinit(void); + +#endif diff --git a/src/anvil/main.c b/src/anvil/main.c index 275345103f..c49602c32f 100644 --- a/src/anvil/main.c +++ b/src/anvil/main.c @@ -9,16 +9,21 @@ #include "master-service.h" #include "master-service-settings.h" #include "master-interface.h" +#include "admin-client-pool.h" #include "connect-limit.h" #include "penalty.h" #include "anvil-connection.h" #include +#define ANVIL_CLIENT_POOL_MAX_CONNECTIONS 100 + struct connect_limit *connect_limit; struct penalty *penalty; bool anvil_restarted; + static struct io *log_fdpass_io; +static struct admin_client_pool *admin_pool; static void client_connected(struct master_service_connection *conn) { @@ -52,11 +57,17 @@ log_fdpass_input(void *context ATTR_UNUSED) static void main_init(void) { + const struct master_service_settings *set = + master_service_settings_get(master_service); + /* delay dying until all of our clients are gone */ master_service_set_die_with_master(master_service, FALSE); anvil_restarted = getenv("ANVIL_RESTARTED") != NULL; anvil_connections_init(); + admin_clients_init(); + admin_pool = admin_client_pool_init(set->base_dir, + ANVIL_CLIENT_POOL_MAX_CONNECTIONS); connect_limit = connect_limit_init(); penalty = penalty_init(); log_fdpass_io = io_add(MASTER_ANVIL_LOG_FDPASS_FD, IO_READ, @@ -68,6 +79,8 @@ static void main_deinit(void) io_remove(&log_fdpass_io); penalty_deinit(&penalty); connect_limit_deinit(&connect_limit); + admin_client_pool_deinit(&admin_pool); + admin_clients_deinit(); anvil_connections_deinit(); }