From: Vsevolod Stakhov Date: Thu, 30 Jul 2026 16:11:01 +0000 (+0100) Subject: [Fix] controller: rate-limit auth failures per source X-Git-Url: http://git.ipfire.org/gitweb/index.cgi?a=commitdiff_plain;ds=sidebyside;p=thirdparty%2Frspamd.git [Fix] controller: rate-limit auth failures per source Verifying a controller password runs a deliberately expensive KDF (catena costs ~25ms, and it is what `rspamadm pw` emits by default) synchronously in the controller's event loop. The password cache keys on the plaintext that succeeded, so a stream of distinct wrong candidates falls through to the KDF every time. With `count = 1` for the controller worker, roughly 40 wrong passwords per second are enough to saturate it and stall every other request it serves. Bound the damage with a per-source leaky bucket of failed attempts, checked before the password logic so a throttled source costs no CPU: sources over budget get 429 in well under a millisecond instead of tens of milliseconds. A successful authentication clears the bucket so a mistyped password does not leave an operator throttled, and trusted sources (secure_ip, unix socket) return before the check and are never throttled. Throttled requests do not add penalty, so a shared source cannot be pushed into a permanent lockout. Configurable via `max_auth_failures` (default 10, 0 disables) and `auth_failure_window` (default 60s). Also skip the second KDF when `password` and `enable_password` are configured to the very same hash: the enable check repeats the normal one verbatim, which doubled the cost of every wrong password. Note this bounds a single source, not a distributed one; keys are the real TCP peer, so behind a trusted reverse proxy all clients share one bucket. --- diff --git a/src/controller.c b/src/controller.c index f5fd483458..95433709a8 100644 --- a/src/controller.c +++ b/src/controller.c @@ -17,6 +17,7 @@ #include "libserver/dynamic_cfg.h" #include "libserver/cfg_file_private.h" #include "libutil/rrd.h" +#include "libutil/hash.h" #include "libserver/maps/map.h" #include "libserver/maps/map_helpers.h" #include "libserver/maps/map_private.h" @@ -41,6 +42,15 @@ /* 60 seconds for worker's IO */ #define DEFAULT_WORKER_IO_TIMEOUT 60000 +/* + * Authentication failures allowed from a single source before it is throttled, + * and the time it takes for a full bucket to drain again. + */ +#define DEFAULT_MAX_AUTH_FAILURES 10 +#define DEFAULT_AUTH_FAILURE_WINDOW 60.0 +/* Bounds the memory used by the throttling state */ +#define AUTH_FAILURES_CACHE_SIZE 1024 + /* HTTP paths */ #define PATH_AUTH "/auth" #define PATH_SYMBOLS "/symbols" @@ -148,6 +158,14 @@ struct rspamd_controller_worker_ctx { /* Cached versions of the passwords */ rspamd_ftok_t cached_password; rspamd_ftok_t cached_enable_password; + /* + * Leaky buckets of failed authentication attempts, keyed by source address. + * Password verification runs a deliberately expensive KDF in this worker's + * event loop, so unauthenticated attempts must be bounded per source. + */ + rspamd_lru_hash_t *auth_failures; + unsigned int max_auth_failures; + double auth_failure_window; /* HTTP server */ struct rspamd_http_context *http_ctx; struct rspamd_http_connection_router *http; @@ -519,6 +537,120 @@ rspamd_controller_check_forwarded(struct rspamd_controller_session *session, return ret; } +/* + * A leaky bucket of failed authentication attempts for a single source address. + * `penalty` is decayed lazily on access, so no timer is needed to drain it. + */ +struct rspamd_controller_auth_bucket { + double penalty; + ev_tstamp last_update; +}; + +/* + * Returns the bucket's penalty decayed to `now`. A full bucket drains in + * exactly `auth_failure_window` seconds. + */ +static double +rspamd_controller_auth_decay(struct rspamd_controller_worker_ctx *ctx, + struct rspamd_controller_auth_bucket *bucket, + ev_tstamp now) +{ + double leaked, elapsed; + + elapsed = now - bucket->last_update; + + if (elapsed <= 0) { + /* Clock went backwards, do not gift the source any credit */ + return bucket->penalty; + } + + leaked = elapsed * ((double) ctx->max_auth_failures / ctx->auth_failure_window); + + return MAX(0.0, bucket->penalty - leaked); +} + +/* + * Checks whether a source address is still allowed to attempt authentication. + * This must be called *before* any KDF invocation: verifying a password is + * deliberately expensive (tens of milliseconds), and the controller runs a + * single event loop, so an unthrottled stream of wrong passwords from one + * source is enough to stall every other request the worker serves. + */ +static gboolean +rspamd_controller_auth_is_allowed(struct rspamd_controller_worker_ctx *ctx, + const rspamd_inet_addr_t *from_addr) +{ + struct rspamd_controller_auth_bucket *bucket; + ev_tstamp now; + + if (ctx->auth_failures == NULL || ctx->max_auth_failures == 0) { + /* Throttling is disabled */ + return TRUE; + } + + now = ev_now(ctx->event_loop); + bucket = rspamd_lru_hash_lookup(ctx->auth_failures, from_addr, (time_t) now); + + if (bucket == NULL) { + return TRUE; + } + + return rspamd_controller_auth_decay(ctx, bucket, now) < (double) ctx->max_auth_failures; +} + +/* + * Accounts a failed authentication attempt against its source address. + */ +static void +rspamd_controller_auth_failure(struct rspamd_controller_worker_ctx *ctx, + const rspamd_inet_addr_t *from_addr) +{ + struct rspamd_controller_auth_bucket *bucket; + ev_tstamp now; + + if (ctx->auth_failures == NULL || ctx->max_auth_failures == 0) { + return; + } + + now = ev_now(ctx->event_loop); + bucket = rspamd_lru_hash_lookup(ctx->auth_failures, from_addr, (time_t) now); + + if (bucket != NULL) { + bucket->penalty = rspamd_controller_auth_decay(ctx, bucket, now) + 1.0; + bucket->last_update = now; + } + else { + bucket = g_malloc0(sizeof(*bucket)); + bucket->penalty = 1.0; + bucket->last_update = now; + /* + * The TTL only reclaims idle entries: a bucket always decays to zero + * within `auth_failure_window` anyway, so expiry can never be laxer + * than the decay itself. + */ + rspamd_lru_hash_insert(ctx->auth_failures, + rspamd_inet_address_copy(from_addr, NULL), + bucket, + (time_t) now, + (unsigned int) ctx->auth_failure_window); + } +} + +/* + * Clears the penalty of a source that has just authenticated successfully, so + * that an operator who mistyped a password a few times is not left throttled. + */ +static void +rspamd_controller_auth_success(struct rspamd_controller_worker_ctx *ctx, + const rspamd_inet_addr_t *from_addr) +{ + if (ctx->auth_failures == NULL || ctx->max_auth_failures == 0) { + return; + } + + rspamd_lru_hash_remove(ctx->auth_failures, from_addr); +} + /* Check for password if it is required by configuration */ static gboolean rspamd_controller_check_password(struct rspamd_http_connection_entry *entry, @@ -533,6 +665,13 @@ rspamd_controller_check_password(struct rspamd_http_connection_entry *entry, gboolean check_normal = FALSE, check_enable = FALSE, ret = TRUE, use_enable = FALSE; const struct rspamd_controller_pbkdf *pbkdf = NULL; + /* + * Both passwords configured to the very same hash: the enable check would + * repeat the normal one verbatim, so its (expensive) KDF can be skipped and + * its verdict reused. + */ + const gboolean same_pw = (ctx->password != NULL && ctx->enable_password != NULL && + strcmp(ctx->password, ctx->enable_password) == 0); /* Fail-safety */ session->is_read_only = TRUE; @@ -572,6 +711,18 @@ rspamd_controller_check_password(struct rspamd_http_connection_entry *entry, } } + /* + * Everything below can reach the KDF, so throttle sources that keep + * failing before spending any CPU on them. + */ + if (!rspamd_controller_auth_is_allowed(ctx, session->from_addr)) { + msg_info_session("throttling authentication: too many failures; source ip: %s", + rspamd_inet_address_to_string_pretty(session->from_addr)); + rspamd_controller_send_error(entry, 429, "Too many authentication failures"); + + return FALSE; + } + /* Password logic */ password = rspamd_http_message_find_header(msg, "Password"); @@ -677,6 +828,10 @@ rspamd_controller_check_password(struct rspamd_http_connection_entry *entry, /* We have passed password check and no enable password is specified */ session->is_read_only = FALSE; } + else if (same_pw) { + /* Identical to the check we have just passed */ + check_enable = TRUE; + } else { /* * Even if we have passed normal password check, we don't really @@ -703,7 +858,13 @@ rspamd_controller_check_password(struct rspamd_http_connection_entry *entry, } } - if ((!check_normal && !check_enable) && ctx->enable_password != NULL) { + /* + * `same_pw` implies the normal check above already ran against + * this very hash and rejected the password, so repeating it here + * would only burn another KDF for the same verdict. + */ + if ((!check_normal && !check_enable) && ctx->enable_password != NULL && + !same_pw) { check = ctx->enable_password; if (!rspamd_is_encrypted_password(check, &pbkdf)) { @@ -741,8 +902,12 @@ end: } if (!ret) { + rspamd_controller_auth_failure(ctx, session->from_addr); rspamd_controller_send_error(entry, 401, "Unauthorized"); } + else { + rspamd_controller_auth_success(ctx, session->from_addr); + } return ret; } @@ -3824,6 +3989,8 @@ init_controller_worker(struct rspamd_config *cfg) ctx->magic = rspamd_controller_ctx_magic; ctx->timeout = DEFAULT_WORKER_IO_TIMEOUT; ctx->task_timeout = NAN; + ctx->max_auth_failures = DEFAULT_MAX_AUTH_FAILURES; + ctx->auth_failure_window = DEFAULT_AUTH_FAILURE_WINDOW; rspamd_rcl_register_worker_option(cfg, type, @@ -3919,6 +4086,28 @@ init_controller_worker(struct rspamd_config *cfg) RSPAMD_CL_FLAG_TIME_FLOAT, "Maximum task processing time, default: 8.0 seconds"); + rspamd_rcl_register_worker_option(cfg, + type, + "max_auth_failures", + rspamd_rcl_parse_struct_integer, + ctx, + G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, + max_auth_failures), + RSPAMD_CL_FLAG_INT_32 | RSPAMD_CL_FLAG_UINT, + "Authentication failures allowed from one source before it is " + "throttled, 0 disables throttling, default: 10"); + + rspamd_rcl_register_worker_option(cfg, + type, + "auth_failure_window", + rspamd_rcl_parse_struct_time, + ctx, + G_STRUCT_OFFSET(struct rspamd_controller_worker_ctx, + auth_failure_window), + RSPAMD_CL_FLAG_TIME_FLOAT, + "Time for a source to regain all of its authentication " + "attempts, default: 60 seconds"); + return ctx; } @@ -4209,6 +4398,24 @@ start_controller_worker(struct rspamd_worker *worker) ctx->task_timeout = rspamd_worker_check_and_adjust_timeout(ctx->cfg, ctx->task_timeout); + if (ctx->max_auth_failures > 0) { + if (!(ctx->auth_failure_window > 0)) { + msg_warn_ctx("invalid auth_failure_window: %.1f, using the default %.1f " + "seconds instead", + ctx->auth_failure_window, (double) DEFAULT_AUTH_FAILURE_WINDOW); + ctx->auth_failure_window = DEFAULT_AUTH_FAILURE_WINDOW; + } + + ctx->auth_failures = rspamd_lru_hash_new_full(AUTH_FAILURES_CACHE_SIZE, + (GDestroyNotify) rspamd_inet_address_free, + g_free, + rspamd_inet_address_hash, + rspamd_inet_address_equal); + } + else { + msg_info_ctx("authentication failure throttling is disabled"); + } + if (ctx->secure_ip != NULL) { rspamd_config_radix_from_ucl(ctx->cfg, ctx->secure_ip, "Allow unauthenticated requests from these addresses", @@ -4427,6 +4634,10 @@ start_controller_worker(struct rspamd_worker *worker) munmap(m, ctx->cached_enable_password.len); } + if (ctx->auth_failures != NULL) { + rspamd_lru_hash_destroy(ctx->auth_failures); + } + g_hash_table_unref(ctx->plugins); g_hash_table_unref(ctx->custom_commands); diff --git a/test/functional/cases/103_password.robot b/test/functional/cases/103_password.robot index 2237a6a31d..18c9300a86 100644 --- a/test/functional/cases/103_password.robot +++ b/test/functional/cases/103_password.robot @@ -63,6 +63,35 @@ PASSWORD - ENABLE EQUAL TO NORMAL WRONG PASSWORD ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -P nq2 stat Should Be Equal As Integers ${result.rc} 1 +PASSWORD - FAILURES ARE THROTTLED + [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD} + # Verifying a password runs an expensive KDF, so a source that keeps failing + # must stop being served before it can stall the controller's event loop + ${allowed} = Set Variable ${0} + ${code} = Set Variable ${0} + FOR ${i} IN RANGE 30 + ${code} = Controller Auth Status ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} wrong${i} + Exit For Loop If ${code} == 429 + Should Be Equal As Integers ${code} 401 + ${allowed} = Evaluate ${allowed} + 1 + END + Should Be Equal As Integers ${code} 429 + Should Be True ${allowed} >= 10 + +PASSWORD - THROTTLING RESET BY A SUCCESSFUL AUTH + [Setup] Password Setup ${RSPAMD_CATENA_PASSWORD} ${RSPAMD_ENABLE_CATENA_PASSWORD} + FOR ${i} IN RANGE 5 + ${code} = Controller Auth Status ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} wrong${i} + Should Be Equal As Integers ${code} 401 + END + ${code} = Controller Auth Status ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} nq1 + Should Be Equal As Integers ${code} 200 + # The budget is replenished, so these still get a verdict rather than a 429 + FOR ${i} IN RANGE 10 + ${code} = Controller Auth Status ${RSPAMD_LOCAL_ADDR} ${RSPAMD_PORT_CONTROLLER} other${i} + Should Be Equal As Integers ${code} 401 + END + *** Keywords *** Password Setup [Arguments] ${RSPAMD_PASSWORD} ${RSPAMD_ENABLE_PASSWORD} diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index 7912e69cc2..da7da1c1c7 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -822,6 +822,23 @@ def ping_rspamd(addr, port): return str(urlopen("http://%s:%s/ping" % (addr, port)).read()) +def controller_auth_status(addr, port, password): + """Sends a controller /auth request and returns its HTTP status code. + + Unlike rspamc, this distinguishes a rejected password (401) from a + throttled source (429). + + Example: + | ${code} = | Controller Auth Status | ${RSPAMD_LOCAL_ADDR} | ${RSPAMD_PORT_CONTROLLER} | q1 | + """ + conn = http.client.HTTPConnection(addr, int(port), timeout=10) + try: + conn.request("GET", "/auth", headers={"Password": password}) + return conn.getresponse().status + finally: + conn.close() + + def redis_check(addr, port): """Attempts to open a TCP connection to specified address:port