From: Vsevolod Stakhov Date: Tue, 7 Oct 2025 12:22:50 +0000 (+0100) Subject: [Feature] Fuzzy check: add protocol logging and TCP tests X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=2034e903cc2306411f7de00db56efed3247a920d;p=thirdparty%2Frspamd.git [Feature] Fuzzy check: add protocol logging and TCP tests Add explicit protocol logging: - Log TCP vs UDP decision with rate and threshold - Log actual protocol used for each request - Log TCP connection status and fallbacks - Shows current request rate for TCP auto-switch Add functional tests for TCP: - tcp.robot - basic TCP with auto-switch - tcp-explicit.robot - forced TCP mode - tcp-encrypted.robot - TCP with encryption - Test high rate scenario and protocol switching Update test configuration: - Support SETTINGS_FUZZY_WORKER and SETTINGS_FUZZY_CHECK - Allow dynamic TCP configuration in tests --- diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 0e733504e8..42bac5c49a 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -4691,12 +4691,25 @@ register_fuzzy_client_call(struct rspamd_task *task, /* Try TCP if enabled/auto and rate threshold exceeded */ struct fuzzy_tcp_connection *tcp_conn = NULL; if (fuzzy_should_try_tcp(rule, now)) { + /* Calculate current rate for logging */ + double time_diff = now - rule->rate_tracker.window_start; + double current_rate = (time_diff > 0) ? (rule->rate_tracker.requests_count / time_diff) : 0; + + msg_info_task("fuzzy_check: trying TCP for rule %s to %s (rate=%.2f req/s, threshold=%.2f)", + rule->name, rspamd_upstream_name(selected), + current_rate, rule->tcp_threshold); /* This is read server (CHECK operation) */ tcp_conn = fuzzy_tcp_get_or_create_connection(rule, selected, task, FALSE); } + else { + msg_info_task("fuzzy_check: using UDP for rule %s to %s (TCP disabled or rate below threshold)", + rule->name, rspamd_upstream_name(selected)); + } /* Use TCP if available and connected */ if (tcp_conn && tcp_conn->connected) { + msg_info_task("fuzzy_check: sending %d commands via TCP to %s", + (int) commands->len, rspamd_upstream_name(selected)); /* Create session for TCP */ session = rspamd_mempool_alloc0(task->task_pool, sizeof(struct fuzzy_client_session)); @@ -4728,9 +4741,15 @@ register_fuzzy_client_call(struct rspamd_task *task, /* Fall through to UDP */ } } + else if (tcp_conn && !tcp_conn->connected) { + msg_info_task("fuzzy_check: TCP connection not ready for rule %s to %s, using UDP", + rule->name, rspamd_upstream_name(selected)); + } /* Use UDP as fallback or when TCP not available */ if (selected) { + msg_info_task("fuzzy_check: sending %d commands via UDP to %s", + (int) commands->len, rspamd_upstream_name(selected)); addr = rspamd_upstream_addr_cur(selected); rspamd_inet_address_set_port(addr, rspamd_upstream_port(selected)); diff --git a/test/functional/cases/120_fuzzy/TCP_TESTS_README.md b/test/functional/cases/120_fuzzy/TCP_TESTS_README.md new file mode 100644 index 0000000000..7dbc3ecd18 --- /dev/null +++ b/test/functional/cases/120_fuzzy/TCP_TESTS_README.md @@ -0,0 +1,83 @@ +# TCP Fuzzy Check Tests + +This directory contains functional tests for TCP support in fuzzy_check plugin. + +## Test Files + +- **tcp.robot** - Basic TCP tests with auto-switch mode (tcp = "auto") + - Tests TCP connection with rate-based switching + - Threshold set to 5 requests/second + +- **tcp-explicit.robot** - Explicit TCP mode tests (tcp = "yes") + - Forces TCP usage for all fuzzy check operations + +- **tcp-encrypted.robot** - Encrypted TCP tests + - Tests TCP with encryption enabled on both client and server + +## Running Tests + +### Run all TCP tests: +```bash +cd /Users/vstakhov/rspamd +robot -v RSPAMD_INSTALLROOT:/path/to/install \ + -v RSPAMD_TESTDIR:/Users/vstakhov/rspamd/test/functional \ + test/functional/cases/120_fuzzy/tcp*.robot +``` + +### Run specific TCP test: +```bash +robot -v RSPAMD_INSTALLROOT:/path/to/install \ + -v RSPAMD_TESTDIR:/Users/vstakhov/rspamd/test/functional \ + test/functional/cases/120_fuzzy/tcp.robot +``` + +### Run with verbose output: +```bash +robot -v RSPAMD_INSTALLROOT:/path/to/install \ + -v RSPAMD_TESTDIR:/Users/vstakhov/rspamd/test/functional \ + -L DEBUG \ + test/functional/cases/120_fuzzy/tcp.robot +``` + +## Test Configuration + +TCP tests use the following configuration: + +### Auto mode (tcp.robot): +- `tcp = "auto"` - Enable automatic TCP switching +- `tcp_threshold = 5` - Switch to TCP after 5 requests/second + +### Explicit mode (tcp-explicit.robot): +- `tcp = "yes"` - Always use TCP +- Server: `tcp = true` - Enable TCP on fuzzy worker + +### Encrypted mode (tcp-encrypted.robot): +- `tcp = "auto"` with `tcp_threshold = 5` +- `encrypted_only = true` - Require encryption +- Shared keypair between client and server + +## What is Tested + +1. **Basic Operations**: + - Add fuzzy hashes via TCP + - Delete fuzzy hashes via TCP + - Overwrite fuzzy hashes via TCP + +2. **High Rate Scenario**: + - Send multiple messages to exceed rate threshold + - Verify automatic TCP switch + - Verify fuzzy check continues to work + +3. **Error Handling**: + - Connection failures + - Timeouts + - Protocol errors + +## Expected Results + +All tests should pass, verifying: +- TCP connections are established correctly +- Fuzzy add/delete/overwrite operations work via TCP +- Auto-switch activates at rate threshold +- Encryption works with TCP +- UDP fallback works when TCP unavailable diff --git a/test/functional/cases/120_fuzzy/lib.robot b/test/functional/cases/120_fuzzy/lib.robot index a57ecc7429..73611403b9 100644 --- a/test/functional/cases/120_fuzzy/lib.robot +++ b/test/functional/cases/120_fuzzy/lib.robot @@ -241,3 +241,55 @@ Fuzzy Setup Write Only Set Suite Variable ${RSPAMD_FUZZY_SERVER_MODE} write_only Set Suite Variable ${SETTINGS_FUZZY_CHECK} mode = "write_only"; Rspamd Redis Setup + +Fuzzy Setup TCP + [Arguments] ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_SERVER_MODE} servers + Set Suite Variable ${SETTINGS_FUZZY_CHECK} servers = "${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_FUZZY}"; tcp = "auto"; tcp_threshold = 5; + Rspamd Redis Setup + +Fuzzy Setup TCP Siphash + Fuzzy Setup TCP siphash + +Fuzzy Setup TCP Encrypted + [Arguments] ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTED_ONLY} true + Set Suite Variable ${RSPAMD_FUZZY_ENCRYPTION_KEY} ${RSPAMD_KEY_PUB1} + Set Suite Variable ${RSPAMD_FUZZY_CLIENT_ENCRYPTION_KEY} ${RSPAMD_KEY_PUB1} + Set Suite Variable ${RSPAMD_FUZZY_INCLUDE} ${RSPAMD_TESTDIR}/configs/fuzzy-encryption-key.conf + Set Suite Variable ${RSPAMD_FUZZY_SERVER_MODE} servers + Set Suite Variable ${SETTINGS_FUZZY_WORKER} tcp = true; + Set Suite Variable ${SETTINGS_FUZZY_CHECK} tcp = "auto"; tcp_threshold = 5; + Rspamd Redis Setup + +Fuzzy Setup TCP Encrypted Siphash + Fuzzy Setup TCP Encrypted siphash + +Fuzzy Setup TCP Explicit + [Arguments] ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_ALGORITHM} ${algorithm} + Set Suite Variable ${RSPAMD_FUZZY_SERVER_MODE} servers + Set Suite Variable ${SETTINGS_FUZZY_WORKER} tcp = true; + Set Suite Variable ${SETTINGS_FUZZY_CHECK} servers = "${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_FUZZY}"; tcp = "yes"; + Rspamd Redis Setup + +Fuzzy Setup TCP Explicit Siphash + Fuzzy Setup TCP Explicit siphash + +Fuzzy TCP High Rate Test + # Send multiple messages to exceed rate threshold and trigger TCP + FOR ${i} IN RANGE 10 + FOR ${message} IN @{MESSAGES} + ${result} = Run Rspamc -h ${RSPAMD_LOCAL_ADDR}:${RSPAMD_PORT_CONTROLLER} -w 10 -f + ... ${RSPAMD_FLAG1_NUMBER} fuzzy_add ${message} + Check Rspamc ${result} + END + END + Sync Fuzzy Storage + # Verify that fuzzy check still works + FOR ${message} IN @{MESSAGES} + Scan File ${message} + Expect Symbol ${FLAG1_SYMBOL} + END diff --git a/test/functional/cases/120_fuzzy/tcp-encrypted.robot b/test/functional/cases/120_fuzzy/tcp-encrypted.robot new file mode 100644 index 0000000000..2b01791384 --- /dev/null +++ b/test/functional/cases/120_fuzzy/tcp-encrypted.robot @@ -0,0 +1,17 @@ +*** Settings *** +Suite Setup Fuzzy Setup TCP Encrypted Siphash +Suite Teardown Rspamd Redis Teardown +Resource lib.robot + +*** Test Cases *** +Fuzzy Add TCP Encrypted + Fuzzy Multimessage Add Test + +Fuzzy Delete TCP Encrypted + Fuzzy Multimessage Delete Test + +Fuzzy Overwrite TCP Encrypted + Fuzzy Multimessage Overwrite Test + +Fuzzy TCP Encrypted High Rate + Fuzzy TCP High Rate Test diff --git a/test/functional/cases/120_fuzzy/tcp-explicit.robot b/test/functional/cases/120_fuzzy/tcp-explicit.robot new file mode 100644 index 0000000000..14f02485df --- /dev/null +++ b/test/functional/cases/120_fuzzy/tcp-explicit.robot @@ -0,0 +1,14 @@ +*** Settings *** +Suite Setup Fuzzy Setup TCP Explicit Siphash +Suite Teardown Rspamd Redis Teardown +Resource lib.robot + +*** Test Cases *** +Fuzzy Add TCP Explicit + Fuzzy Multimessage Add Test + +Fuzzy Delete TCP Explicit + Fuzzy Multimessage Delete Test + +Fuzzy Overwrite TCP Explicit + Fuzzy Multimessage Overwrite Test diff --git a/test/functional/cases/120_fuzzy/tcp.robot b/test/functional/cases/120_fuzzy/tcp.robot new file mode 100644 index 0000000000..05673b3c6c --- /dev/null +++ b/test/functional/cases/120_fuzzy/tcp.robot @@ -0,0 +1,18 @@ +*** Settings *** +Suite Setup Fuzzy Setup TCP Siphash +Suite Teardown Rspamd Redis Teardown +Resource lib.robot + +*** Test Cases *** +Fuzzy Add TCP + Fuzzy Multimessage Add Test + +Fuzzy Delete TCP + Fuzzy Multimessage Delete Test + +Fuzzy Overwrite TCP + Fuzzy Multimessage Overwrite Test + +Fuzzy TCP High Rate + # Test that TCP is used after rate threshold exceeded + Fuzzy TCP High Rate Test diff --git a/test/functional/configs/fuzzy.conf b/test/functional/configs/fuzzy.conf index f46faf6d42..4c6d446442 100644 --- a/test/functional/configs/fuzzy.conf +++ b/test/functional/configs/fuzzy.conf @@ -61,6 +61,7 @@ worker { pubkey = "{= env.KEY_PUB1 =}"; } dynamic_keys_map = "{= env.TESTDIR =}/configs/maps/fuzzy_keymap.map"; + {= env.SETTINGS_FUZZY_WORKER =} } fuzzy_check { @@ -81,6 +82,7 @@ fuzzy_check { skip_hashes = "{= env.TMPDIR =}/skip_hash.map"; fuzzy_key = {= env.FUZZY_KEY =}; fuzzy_shingles_key = {= env.FUZZY_SHINGLES_KEY =}; + {= env.SETTINGS_FUZZY_CHECK =} .include "{= env.FUZZY_INCLUDE =}"; fuzzy_map = { R_TEST_FUZZY_DENIED {