From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 2 Aug 2016 20:25:57 +0000 (+0300)
Subject: director: Throttle user move/kill failure errors after 100/sec.
X-Git-Tag: 2.2.26~418
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=588a9f849e3acd286117332dc0829f2fa32d400b;p=thirdparty%2Fdovecot%2Fcore.git

director: Throttle user move/kill failure errors after 100/sec.

If it happened to a lot of users, they can take a lot of space in logs.
---

diff --git a/src/director/director.c b/src/director/director.c
index 11089b5cf7..f7f6c63704 100644
--- a/src/director/director.c
+++ b/src/director/director.c
@@ -5,6 +5,7 @@
 #include "array.h"
 #include "str.h"
 #include "strescape.h"
+#include "log-throttle.h"
 #include "ipc-client.h"
 #include "user-directory.h"
 #include "mail-host.h"
@@ -24,6 +25,14 @@
 
 bool director_debug;
 
+static struct log_throttle *user_move_throttle;
+static struct log_throttle *user_kill_fail_throttle;
+
+static const struct log_throttle_settings director_log_throttle_settings = {
+	.throttle_at_max_per_interval = 100,
+	.unthrottle_at_max_per_interval = 2,
+};
+
 static bool director_is_self_ip_set(struct director *dir)
 {
 	struct ip_addr ip;
@@ -744,6 +753,12 @@ director_finish_user_kill(struct director *dir, struct user *user, bool self)
 	}
 }
 
+static void director_user_kill_fail_throttled(unsigned int new_events_count,
+					      void *context ATTR_UNUSED)
+{
+	i_error("Failed to kill %u users' connections", new_events_count);
+}
+
 static void director_kill_user_callback(enum ipc_client_cmd_state state,
 					const char *data, void *context)
 {
@@ -760,8 +775,10 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
 	case IPC_CLIENT_CMD_STATE_OK:
 		break;
 	case IPC_CLIENT_CMD_STATE_ERROR:
-		i_error("Failed to kill user %u connections: %s",
-			ctx->username_hash, data);
+		if (log_throttle_accept(user_kill_fail_throttle)) {
+			i_error("Failed to kill user %u connections: %s",
+				ctx->username_hash, data);
+		}
 		/* we can't really do anything but continue anyway */
 		break;
 	}
@@ -779,12 +796,21 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state,
 	i_free(ctx);
 }
 
+static void director_user_move_throttled(unsigned int new_events_count,
+					 void *context ATTR_UNUSED)
+{
+	i_error("%u users' move timed out, their state may now be inconsistent",
+		new_events_count);
+}
+
 static void director_user_move_timeout(struct user *user)
 {
 	i_assert(user->kill_state != USER_KILL_STATE_DELAY);
 
-	i_error("Finishing user %u move timed out, "
-		"its state may now be inconsistent", user->username_hash);
+	if (log_throttle_accept(user_move_throttle)) {
+		i_error("Finishing user %u move timed out, "
+			"its state may now be inconsistent", user->username_hash);
+	}
 
 	user->kill_state = USER_KILL_STATE_NONE;
 	timeout_remove(&user->to_move);
@@ -1067,3 +1093,19 @@ void dir_debug(const char *fmt, ...)
 	} T_END;
 	va_end(args);
 }
+
+void directors_init(void)
+{
+	user_move_throttle =
+		log_throttle_init(&director_log_throttle_settings,
+				  director_user_move_throttled, NULL);
+	user_kill_fail_throttle =
+		log_throttle_init(&director_log_throttle_settings,
+				  director_user_kill_fail_throttled, NULL);
+}
+
+void directors_deinit(void)
+{
+	log_throttle_deinit(&user_move_throttle);
+	log_throttle_deinit(&user_kill_fail_throttle);
+}
diff --git a/src/director/director.h b/src/director/director.h
index a2e5c5469f..42e225bdbc 100644
--- a/src/director/director.h
+++ b/src/director/director.h
@@ -175,6 +175,9 @@ void director_update_send_version(struct director *dir,
 
 int director_connect_host(struct director *dir, struct director_host *host);
 
+void directors_init(void);
+void directors_deinit(void);
+
 void dir_debug(const char *fmt, ...) ATTR_FORMAT(1, 2);
 
 #endif
diff --git a/src/director/main.c b/src/director/main.c
index 34995dbc7d..243f7098e0 100644
--- a/src/director/main.c
+++ b/src/director/main.c
@@ -258,6 +258,7 @@ static void main_preinit(void)
 			"(for standalone keep director_servers empty)");
 	}
 
+	directors_init();
 	director = director_init(set, &listen_ip, listen_port,
 				 director_state_changed);
 	director_host_add_from_string(director, set->director_servers);
@@ -278,6 +279,7 @@ static void main_deinit(void)
 	if (notify_conn != NULL)
 		notify_connection_deinit(&notify_conn);
 	director_deinit(&director);
+	directors_deinit();
 	doveadm_connections_deinit();
 	login_connections_deinit();
 	auth_connections_deinit();