director: Throttle user move/kill failure errors after 100/sec.

If it happened to a lot of users, they can take a lot of space in logs.

diff --git a/src/director/director.c b/src/director/director.c
index 1e42902f2cc04f7497c553fc2b6d639d13d49c24..9b5be3f31808f582684eba77e5f6d91b4563f7e2 100644 (file)
--- a/src/director/director.c
+++ b/src/director/director.c
@@ -5,6 +5,7 @@
 #include "array.h"
 #include "str.h"
 #include "strescape.h"
+#include "log-throttle.h"
 #include "ipc-client.h"
 #include "user-directory.h"
 #include "mail-host.h"
@@ -24,6 +25,14 @@
 
 bool director_debug;
 
+static struct log_throttle *user_move_throttle;
+static struct log_throttle *user_kill_fail_throttle;
+
+static const struct log_throttle_settings director_log_throttle_settings = {
+       .throttle_at_max_per_interval = 100,
+       .unthrottle_at_max_per_interval = 2,
+};
+
 static bool director_is_self_ip_set(struct director *dir)
 {
        if (net_ip_compare(&dir->self_ip, &net_ip4_any))
@@ -740,6 +749,12 @@ director_finish_user_kill(struct director *dir, struct user *user, bool self)
        }
 }
 
+static void director_user_kill_fail_throttled(unsigned int new_events_count,
+                                             void *context ATTR_UNUSED)
+{
+       i_error("Failed to kill %u users' connections", new_events_count);
+}
+
 static void director_kill_user_callback(enum ipc_client_cmd_state state,
                                        const char *data, void *context)
 {
@@ -756,8 +771,10 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
        case IPC_CLIENT_CMD_STATE_OK:
                break;
        case IPC_CLIENT_CMD_STATE_ERROR:
-               i_error("Failed to kill user %u connections: %s",
-                       ctx->username_hash, data);
+               if (log_throttle_accept(user_kill_fail_throttle)) {
+                       i_error("Failed to kill user %u connections: %s",
+                               ctx->username_hash, data);
+               }
                /* we can't really do anything but continue anyway */
                break;
        }
@@ -775,12 +792,21 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
        i_free(ctx);
 }
 
+static void director_user_move_throttled(unsigned int new_events_count,
+                                        void *context ATTR_UNUSED)
+{
+       i_error("%u users' move timed out, their state may now be inconsistent",
+               new_events_count);
+}
+
 static void director_user_move_timeout(struct user *user)
 {
        i_assert(user->kill_state != USER_KILL_STATE_DELAY);
 
-       i_error("Finishing user %u move timed out, "
-               "its state may now be inconsistent", user->username_hash);
+       if (log_throttle_accept(user_move_throttle)) {
+               i_error("Finishing user %u move timed out, "
+                       "its state may now be inconsistent", user->username_hash);
+       }
 
        user->kill_state = USER_KILL_STATE_NONE;
        timeout_remove(&user->to_move);
@@ -1063,3 +1089,19 @@ void dir_debug(const char *fmt, ...)
        } T_END;
        va_end(args);
 }
+
+void directors_init(void)
+{
+       user_move_throttle =
+               log_throttle_init(&director_log_throttle_settings,
+                                 director_user_move_throttled, NULL);
+       user_kill_fail_throttle =
+               log_throttle_init(&director_log_throttle_settings,
+                                 director_user_kill_fail_throttled, NULL);
+}
+
+void directors_deinit(void)
+{
+       log_throttle_deinit(&user_move_throttle);
+       log_throttle_deinit(&user_kill_fail_throttle);
+}

diff --git a/src/director/director.h b/src/director/director.h
index ef3e6cb6eb8124b6bd0964a0f87429da58c6c006..0403423bed0a84d80c817dcb4d802a7525d9db76 100644 (file)
--- a/src/director/director.h
+++ b/src/director/director.h
@@ -175,6 +175,9 @@ void director_update_send_version(struct director *dir,
 
 int director_connect_host(struct director *dir, struct director_host *host);
 
+void directors_init(void);
+void directors_deinit(void);
+
 void dir_debug(const char *fmt, ...) ATTR_FORMAT(1, 2);
 
 #endif

diff --git a/src/director/main.c b/src/director/main.c
index 6bdd605817a0d7f3287b11442cf709e466f53b6d..b7f560dfe7d87fa5895f89493b9ff419314b3777 100644 (file)
--- a/src/director/main.c
+++ b/src/director/main.c
@@ -253,6 +253,7 @@ static void main_preinit(void)
                        "(for standalone keep director_servers empty)");
        }
 
+       directors_init();
        director = director_init(set, &listen_ip, listen_port,
                                 director_state_changed);
        director_host_add_from_string(director, set->director_servers);
@@ -273,6 +274,7 @@ static void main_deinit(void)
        if (notify_conn != NULL)
                notify_connection_deinit(&notify_conn);
        director_deinit(&director);
+       directors_deinit();
        doveadm_connections_deinit();
        login_connections_deinit();
        auth_connections_deinit();