director: Throttle user move/kill failure errors after 100/sec.

If it happened to a lot of users, they can take a lot of space in logs.

diff --git a/src/director/director.c b/src/director/director.c
index 11089b5cf70315f2d34669b6fa2a0a7b270cdb5a..f7f6c637047b4738c681c88460a43acf36bc65c3 100644 (file)
--- a/src/director/director.c
+++ b/src/director/director.c
@@ -5,6 +5,7 @@
 #include "array.h"
 #include "str.h"
 #include "strescape.h"
+#include "log-throttle.h"
 #include "ipc-client.h"
 #include "user-directory.h"
 #include "mail-host.h"
@@ -24,6 +25,14 @@
 
 bool director_debug;
 
+static struct log_throttle *user_move_throttle;
+static struct log_throttle *user_kill_fail_throttle;
+
+static const struct log_throttle_settings director_log_throttle_settings = {
+       .throttle_at_max_per_interval = 100,
+       .unthrottle_at_max_per_interval = 2,
+};
+
 static bool director_is_self_ip_set(struct director *dir)
 {
        struct ip_addr ip;
@@ -744,6 +753,12 @@ director_finish_user_kill(struct director *dir, struct user *user, bool self)
        }
 }
 
+static void director_user_kill_fail_throttled(unsigned int new_events_count,
+                                             void *context ATTR_UNUSED)
+{
+       i_error("Failed to kill %u users' connections", new_events_count);
+}
+
 static void director_kill_user_callback(enum ipc_client_cmd_state state,
                                        const char *data, void *context)
 {
@@ -760,8 +775,10 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
        case IPC_CLIENT_CMD_STATE_OK:
                break;
        case IPC_CLIENT_CMD_STATE_ERROR:
-               i_error("Failed to kill user %u connections: %s",
-                       ctx->username_hash, data);
+               if (log_throttle_accept(user_kill_fail_throttle)) {
+                       i_error("Failed to kill user %u connections: %s",
+                               ctx->username_hash, data);
+               }
                /* we can't really do anything but continue anyway */
                break;
        }
@@ -779,12 +796,21 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
        i_free(ctx);
 }
 
+static void director_user_move_throttled(unsigned int new_events_count,
+                                        void *context ATTR_UNUSED)
+{
+       i_error("%u users' move timed out, their state may now be inconsistent",
+               new_events_count);
+}
+
 static void director_user_move_timeout(struct user *user)
 {
        i_assert(user->kill_state != USER_KILL_STATE_DELAY);
 
-       i_error("Finishing user %u move timed out, "
-               "its state may now be inconsistent", user->username_hash);
+       if (log_throttle_accept(user_move_throttle)) {
+               i_error("Finishing user %u move timed out, "
+                       "its state may now be inconsistent", user->username_hash);
+       }
 
        user->kill_state = USER_KILL_STATE_NONE;
        timeout_remove(&user->to_move);
@@ -1067,3 +1093,19 @@ void dir_debug(const char *fmt, ...)
        } T_END;
        va_end(args);
 }
+
+void directors_init(void)
+{
+       user_move_throttle =
+               log_throttle_init(&director_log_throttle_settings,
+                                 director_user_move_throttled, NULL);
+       user_kill_fail_throttle =
+               log_throttle_init(&director_log_throttle_settings,
+                                 director_user_kill_fail_throttled, NULL);
+}
+
+void directors_deinit(void)
+{
+       log_throttle_deinit(&user_move_throttle);
+       log_throttle_deinit(&user_kill_fail_throttle);
+}

diff --git a/src/director/director.h b/src/director/director.h
index a2e5c5469fcff9ca0555421ca1da1fc37893627e..42e225bdbc30084bd153f3144a100e8877afc68f 100644 (file)
--- a/src/director/director.h
+++ b/src/director/director.h
@@ -175,6 +175,9 @@ void director_update_send_version(struct director *dir,
 
 int director_connect_host(struct director *dir, struct director_host *host);
 
+void directors_init(void);
+void directors_deinit(void);
+
 void dir_debug(const char *fmt, ...) ATTR_FORMAT(1, 2);
 
 #endif

diff --git a/src/director/main.c b/src/director/main.c
index 34995dbc7d98874836cd9601113502faa0fea235..243f7098e0b6c5b708ff231baaea1b23e2ac0a39 100644 (file)
--- a/src/director/main.c
+++ b/src/director/main.c
@@ -258,6 +258,7 @@ static void main_preinit(void)
                        "(for standalone keep director_servers empty)");
        }
 
+       directors_init();
        director = director_init(set, &listen_ip, listen_port,
                                 director_state_changed);
        director_host_add_from_string(director, set->director_servers);
@@ -278,6 +279,7 @@ static void main_deinit(void)
        if (notify_conn != NULL)
                notify_connection_deinit(&notify_conn);
        director_deinit(&director);
+       directors_deinit();
        doveadm_connections_deinit();
        login_connections_deinit();
        auth_connections_deinit();