auth: Throttle SQL auth worker process creation if they can't connect to database.

diff --git a/src/auth/auth-settings.c b/src/auth/auth-settings.c
index 56f683be933c8e61db060f033dd2672461701508..546d10ab664b2f37f73416712b43b04ad055ba3f 100644 (file)
--- a/src/auth/auth-settings.c
+++ b/src/auth/auth-settings.c
@@ -282,6 +282,11 @@ static bool auth_settings_check(void *_set, pool_t pool,
        if (set->debug)
                set->verbose = TRUE;
 
+       if (set->worker_max_count == 0) {
+               *error_r = "auth_worker_max_count must be above zero";
+               return FALSE;
+       }
+
        if (set->cache_size > 0 && set->cache_size < 1024) {
                /* probably a configuration error.
                   older versions used megabyte numbers */

diff --git a/src/auth/auth-worker-client.c b/src/auth/auth-worker-client.c
index f6019370e2cc9bd0c505a1b5da073dc00906d439..ed6239306f083e68779a500337b9395881311bfa 100644 (file)
--- a/src/auth/auth-worker-client.c
+++ b/src/auth/auth-worker-client.c
@@ -8,6 +8,7 @@
 #include "ostream.h"
 #include "hex-binary.h"
 #include "str.h"
+#include "process-title.h"
 #include "master-service.h"
 #include "auth-request.h"
 #include "auth-worker-client.h"
@@ -30,6 +31,7 @@ struct auth_worker_client {
 
        unsigned int version_received:1;
        unsigned int dbhash_received:1;
+       unsigned int error_sent:1;
 };
 
 struct auth_worker_list_context {
@@ -40,10 +42,23 @@ struct auth_worker_list_context {
 };
 
 struct auth_worker_client *auth_worker_client;
+static bool auth_worker_client_error = FALSE;
 
 static void auth_worker_input(struct auth_worker_client *client);
 static int auth_worker_output(struct auth_worker_client *client);
 
+void auth_worker_refresh_proctitle(const char *state)
+{
+       if (!global_auth_settings->verbose_proctitle || !worker)
+               return;
+
+       if (auth_worker_client_error)
+               state = "error";
+       else if (auth_worker_client == NULL)
+               state = "waiting for connection";
+       process_title_set(t_strdup_printf("worker: %s", state));
+}
+
 static void
 auth_worker_client_check_throttle(struct auth_worker_client *client)
 {
@@ -673,6 +688,8 @@ auth_worker_client_create(struct auth *auth, int fd)
        auth_worker_refresh_proctitle(CLIENT_STATE_HANDSHAKE);
 
        auth_worker_client = client;
+       if (auth_worker_client_error)
+               auth_worker_client_send_error();
        return client;
 }
 
@@ -694,8 +711,8 @@ void auth_worker_client_destroy(struct auth_worker_client **_client)
        client->fd = -1;
        auth_worker_client_unref(&client);
 
-       auth_worker_refresh_proctitle(NULL);
        auth_worker_client = NULL;
+       auth_worker_refresh_proctitle("");
        master_service_client_connection_destroyed(master_service);
 }
 
@@ -712,3 +729,25 @@ void auth_worker_client_unref(struct auth_worker_client **_client)
        o_stream_unref(&client->output);
        i_free(client);
 }
+
+void auth_worker_client_send_error(void)
+{
+       auth_worker_client_error = TRUE;
+       if (auth_worker_client != NULL &&
+           !auth_worker_client->error_sent) {
+               o_stream_send_str(auth_worker_client->output, "ERROR\n");
+               auth_worker_client->error_sent = TRUE;
+       }
+       auth_worker_refresh_proctitle("");
+}
+
+void auth_worker_client_send_success(void)
+{
+       auth_worker_client_error = FALSE;
+       if (auth_worker_client != NULL &&
+           auth_worker_client->error_sent) {
+               o_stream_send_str(auth_worker_client->output, "SUCCESS\n");
+               auth_worker_client->error_sent = FALSE;
+       }
+       auth_worker_refresh_proctitle(CLIENT_STATE_IDLE);
+}

diff --git a/src/auth/auth-worker-client.h b/src/auth/auth-worker-client.h
index e3a7e9f5b6cf8837bfd2160bea65723705970355..108dc8f5e7577b35430b098541119405620c5979 100644 (file)
--- a/src/auth/auth-worker-client.h
+++ b/src/auth/auth-worker-client.h
@@ -11,6 +11,9 @@ struct auth_worker_client *auth_worker_client_create(struct auth *auth, int fd);
 void auth_worker_client_destroy(struct auth_worker_client **client);
 void auth_worker_client_unref(struct auth_worker_client **client);
 
+void auth_worker_client_send_error(void);
+void auth_worker_client_send_success(void);
+
 const char *auth_worker_client_get_state(struct auth_worker_client *client);
 
 #endif

diff --git a/src/auth/auth-worker-server.c b/src/auth/auth-worker-server.c
index b8e6fd8dfeae7e4d043886e083fae5e32cac706f..1a9545088d86a9f10896803390056c86d9c39002 100644 (file)
--- a/src/auth/auth-worker-server.c
+++ b/src/auth/auth-worker-server.c
@@ -41,14 +41,16 @@ struct auth_worker_connection {
        struct auth_worker_request *request;
        unsigned int id_counter;
 
+       unsigned int received_error:1;
        unsigned int shutdown:1;
 };
 
 static ARRAY_DEFINE(connections, struct auth_worker_connection *) = ARRAY_INIT;
-static unsigned int idle_count;
+static unsigned int idle_count = 0, auth_workers_with_errors = 0;
 static ARRAY_DEFINE(worker_request_array, struct auth_worker_request *);
 static struct aqueue *worker_request_queue;
 static time_t auth_worker_last_warn;
+static unsigned int auth_workers_throttle_count;
 
 static const char *worker_socket_path;
 
@@ -150,7 +152,7 @@ static struct auth_worker_connection *auth_worker_create(void)
        struct auth_worker_connection *conn;
        int fd;
 
-       if (array_count(&connections) >= global_auth_settings->worker_max_count)
+       if (array_count(&connections) >= auth_workers_throttle_count)
                return NULL;
 
        fd = net_connect_unix_with_retries(worker_socket_path, 5000);
@@ -189,6 +191,12 @@ static void auth_worker_destroy(struct auth_worker_connection **_conn,
 
        *_conn = NULL;
 
+       if (conn->received_error) {
+               i_assert(auth_workers_with_errors > 0);
+               i_assert(auth_workers_with_errors <= array_count(&connections));
+               auth_workers_with_errors--;
+       }
+
        array_foreach(&connections, conns) {
                if (*conns == conn) {
                        idx = array_foreach_idx(&connections, conns);
@@ -260,6 +268,51 @@ static void auth_worker_request_handle(struct auth_worker_connection *conn,
                io_remove(&conn->io);
 }
 
+static bool auth_worker_error(struct auth_worker_connection *conn)
+{
+       if (conn->received_error)
+               return TRUE;
+       conn->received_error = TRUE;
+       auth_workers_with_errors++;
+       i_assert(auth_workers_with_errors <= array_count(&connections));
+
+       if (auth_workers_with_errors == 1) {
+               /* this is the only failing auth worker connection.
+                  don't create new ones until this one sends SUCCESS. */
+               auth_workers_throttle_count = array_count(&connections);
+               return TRUE;
+       }
+
+       /* too many auth workers, reduce them */
+       i_assert(array_count(&connections) > 1);
+       if (auth_workers_throttle_count >= array_count(&connections))
+               auth_workers_throttle_count = array_count(&connections)-1;
+       else if (auth_workers_throttle_count > 1)
+               auth_workers_throttle_count--;
+       auth_worker_destroy(&conn, "Internal auth worker failure", FALSE);
+       return FALSE;
+}
+
+static void auth_worker_success(struct auth_worker_connection *conn)
+{
+       unsigned int max_count = global_auth_settings->worker_max_count;
+
+       if (!conn->received_error)
+               return;
+
+       i_assert(auth_workers_with_errors > 0);
+       i_assert(auth_workers_with_errors <= array_count(&connections));
+       auth_workers_with_errors--;
+
+       if (auth_workers_with_errors == 0) {
+               /* all workers are succeeding now, set the limit back to
+                  original. */
+               auth_workers_throttle_count = max_count;
+       } else if (auth_workers_throttle_count < max_count)
+               auth_workers_throttle_count++;
+       conn->received_error = FALSE;
+}
+
 static void worker_input(struct auth_worker_connection *conn)
 {
        const char *line, *id_str;
@@ -286,6 +339,15 @@ static void worker_input(struct auth_worker_connection *conn)
                        conn->shutdown = TRUE;
                        continue;
                }
+               if (strcmp(line, "ERROR") == 0) {
+                       if (!auth_worker_error(conn))
+                               return;
+                       continue;
+               }
+               if (strcmp(line, "SUCCESS") == 0) {
+                       auth_worker_success(conn);
+                       continue;
+               }
                id_str = line;
                line = strchr(line, '\t');
                if (line == NULL ||
@@ -358,6 +420,8 @@ void auth_worker_server_resume_input(struct auth_worker_connection *conn)
 void auth_worker_server_init(void)
 {
        worker_socket_path = "auth-worker";
+       auth_workers_throttle_count = global_auth_settings->worker_max_count;
+       i_assert(auth_workers_throttle_count > 0);
 
        i_array_init(&worker_request_array, 128);
        worker_request_queue = aqueue_init(&worker_request_array.arr);

diff --git a/src/auth/db-sql.c b/src/auth/db-sql.c
index 3f0d2366e8a2cc55facb5a27465f03eb9ca0d31a..eaff498873fb480e9281cf4e2d3a14e6f73467ed 100644 (file)
--- a/src/auth/db-sql.c
+++ b/src/auth/db-sql.c
@@ -6,6 +6,7 @@
 
 #include "settings.h"
 #include "auth-request.h"
+#include "auth-worker-client.h"
 #include "db-sql.h"
 
 #include <stddef.h>
@@ -129,6 +130,25 @@ void db_sql_unref(struct sql_connection **_conn)
        pool_unref(&conn->pool);
 }
 
+void db_sql_connect(struct sql_connection *conn)
+{
+       if (sql_connect(conn->db) < 0 && worker) {
+               /* auth worker's sql connection failed. we can't do anything
+                  useful until the connection works. there's no point in
+                  having tons of worker processes all logging failures,
+                  so tell the auth master to stop creating new workers (and
+                  maybe close old ones). this handling is especially useful if
+                  we reach the max. number of connections for sql server. */
+               auth_worker_client_send_error();
+       }
+}
+
+void db_sql_success(struct sql_connection *conn ATTR_UNUSED)
+{
+       if (worker)
+               auth_worker_client_send_success();
+}
+
 void db_sql_check_userdb_warning(struct sql_connection *conn)
 {
        if (worker || conn->userdb_used || conn->set.userdb_warning_disable)

diff --git a/src/auth/db-sql.h b/src/auth/db-sql.h
index 8f9ff7adc279a0a0f0314ca002d8f22dda441be9..3bd6a1a0fa86a30f9a0bd2752947ff648d820850 100644 (file)
--- a/src/auth/db-sql.h
+++ b/src/auth/db-sql.h
@@ -34,6 +34,9 @@ struct sql_connection {
 struct sql_connection *db_sql_init(const char *config_path, bool userdb);
 void db_sql_unref(struct sql_connection **conn);
 
+void db_sql_connect(struct sql_connection *conn);
+void db_sql_success(struct sql_connection *conn);
+
 void db_sql_check_userdb_warning(struct sql_connection *conn);
 
 #endif

diff --git a/src/auth/main.c b/src/auth/main.c
index 355a9851e1d1bb817a482695d9da232b99a03b66..ee298dc06fc048473c789250e47a647a94bca121 100644 (file)
--- a/src/auth/main.c
+++ b/src/auth/main.c
@@ -71,16 +71,6 @@ void auth_refresh_proctitle(void)
                auth_request_state_count[AUTH_REQUEST_STATE_USERDB]));
 }
 
-void auth_worker_refresh_proctitle(const char *state)
-{
-       if (!global_auth_settings->verbose_proctitle || !worker)
-               return;
-
-       if (state == NULL)
-               state = "waiting for connection";
-       process_title_set(t_strdup_printf("worker: %s", state));
-}
-
 static const char *const *read_global_settings(void)
 {
        struct master_service_settings_output set_output;
@@ -235,6 +225,10 @@ static void main_init(void)
        lib_signals_ignore(SIGHUP, TRUE);
        lib_signals_ignore(SIGUSR2, TRUE);
 
+       /* set proctitles before init()s, since they may set them to error */
+       auth_refresh_proctitle();
+       auth_worker_refresh_proctitle(NULL);
+
        child_wait_init();
        auth_worker_server_init();
        auths_init();
@@ -248,8 +242,6 @@ static void main_init(void)
                /* caching is handled only by the main auth process */
                passdb_cache_init(global_auth_settings);
        }
-       auth_refresh_proctitle();
-       auth_worker_refresh_proctitle(NULL);
 }
 
 static void main_deinit(void)

diff --git a/src/auth/passdb-sql.c b/src/auth/passdb-sql.c
index 060b59cc635b223f409af9d1b846930c3d492b76..e0ef3da0ad30004e4712c5d29a349c6d8066e73a 100644 (file)
--- a/src/auth/passdb-sql.c
+++ b/src/auth/passdb-sql.c
@@ -66,6 +66,8 @@ static void sql_query_callback(struct sql_result *result,
        password = NULL;
 
        ret = sql_result_next_row(result);
+       if (ret >= 0)
+               db_sql_success(module->conn);
        if (ret < 0) {
                if (!module->conn->default_password_query) {
                        auth_request_log_error(auth_request, "sql",
@@ -269,7 +271,7 @@ static void passdb_sql_init(struct passdb_module *_module)
        module->module.blocking = (flags & SQL_DB_FLAG_BLOCKING) != 0;
 
        if (!module->module.blocking || worker)
-               sql_connect(module->conn->db);
+               db_sql_connect(module->conn);
        db_sql_check_userdb_warning(module->conn);
 }
 

diff --git a/src/auth/userdb-sql.c b/src/auth/userdb-sql.c
index 07e6160a12292549cff06290f2e632520f15ed2c..a84f9fee9ec7f614041f7ab252feb14f09bdcffd 100644 (file)
--- a/src/auth/userdb-sql.c
+++ b/src/auth/userdb-sql.c
@@ -67,6 +67,8 @@ static void sql_query_callback(struct sql_result *sql_result,
        int ret;
 
        ret = sql_result_next_row(sql_result);
+       if (ret >= 0)
+               db_sql_success(module->conn);
        if (ret < 0) {
                if (!module->conn->default_user_query) {
                        auth_request_log_error(auth_request, "sql",
@@ -212,6 +214,8 @@ static void userdb_sql_iterate_next(struct userdb_iterate_context *_ctx)
        }
 
        ret = sql_result_next_row(ctx->result);
+       if (ret >= 0)
+               db_sql_success(module->conn);
        if (ret > 0) {
                if (userdb_sql_iterate_get_user(ctx, &user) < 0)
                        i_error("sql: Iterate query didn't return 'user' field");
@@ -278,7 +282,7 @@ static void userdb_sql_init(struct userdb_module *_module)
        _module->blocking = (flags & SQL_DB_FLAG_BLOCKING) != 0;
 
        if (!_module->blocking || worker)
-               sql_connect(module->conn->db);
+               db_sql_connect(module->conn);
 }
 
 static void userdb_sql_deinit(struct userdb_module *_module)