From: Vsevolod Stakhov Date: Sat, 14 Feb 2026 22:47:30 +0000 (+0000) Subject: [Test] Add functional tests for structured metadata exporter X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b4aa241c71fee12b1d066df7a468d35c16f98aa4;p=thirdparty%2Frspamd.git [Test] Add functional tests for structured metadata exporter --- diff --git a/.github/workflows/ci_rspamd.yml b/.github/workflows/ci_rspamd.yml index c8a1b82dac..3c795b4b87 100644 --- a/.github/workflows/ci_rspamd.yml +++ b/.github/workflows/ci_rspamd.yml @@ -78,6 +78,10 @@ jobs: run: | sudo mv /usr/bin/miltertest /usr/bin/miltertest.is.broken.on.fedora || true + - name: Install Python dependencies for functional tests + run: | + pip install --break-system-packages msgpack redis + - name: Run functional tests run: | cd ${GITHUB_WORKSPACE}/build diff --git a/test/functional/cases/560_metadata_exporter_structured.robot b/test/functional/cases/560_metadata_exporter_structured.robot new file mode 100644 index 0000000000..fe07dc1f26 --- /dev/null +++ b/test/functional/cases/560_metadata_exporter_structured.robot @@ -0,0 +1,80 @@ +*** Settings *** +Test Setup Metadata Exporter Structured Setup +Test Teardown Metadata Exporter Structured Teardown +Library Process +Library ${RSPAMD_TESTDIR}/lib/rspamd.py +Resource ${RSPAMD_TESTDIR}/lib/rspamd.robot +Variables ${RSPAMD_TESTDIR}/lib/vars.py + +*** Variables *** +${CONFIG} ${RSPAMD_TESTDIR}/configs/metadata_exporter_structured.conf +${MESSAGE} ${RSPAMD_TESTDIR}/messages/spam_message.eml +${ATTACHMENT_MESSAGE} ${RSPAMD_TESTDIR}/messages/zip.eml +${RSPAMD_LUA_SCRIPT} ${RSPAMD_TESTDIR}/lua/metadata_exporter_structured.lua +${RSPAMD_SCOPE} Suite +${RSPAMD_URL_TLD} ${RSPAMD_TESTDIR}/../lua/unit/test_tld.dat +${REDIS_SCOPE} Suite + +*** Test Cases *** +Structured export to Redis stream - UUID v7 and metadata + [Documentation] Export structured metadata to Redis stream, decode msgpack and verify UUID v7 format + # Scan message - triggers default selector + Scan File ${MESSAGE} + ... Settings={symbols_enabled = []} + + # Wait for async export to complete + Sleep 1s + + # Read and decode msgpack from Redis stream + ${data} = Redis Stream Read Msgpack ${RSPAMD_REDIS_ADDR} ${RSPAMD_REDIS_PORT} test:structured + Log ${data} + + # Validate required fields and UUID v7 format + Validate Structured Metadata ${data} uuid,ip,score,action + +Structured export with zstd compression + [Documentation] Export with zstd compression on content fields + Scan File ${MESSAGE} + ... Settings={symbols_enabled = []} + + # Wait for async export + Sleep 1s + + # Read from zstd stream + ${data} = Redis Stream Read Msgpack ${RSPAMD_REDIS_ADDR} ${RSPAMD_REDIS_PORT} test:structured_zstd + Log ${data} + + # Validate required fields + Validate Structured Metadata ${data} uuid,ip,score + + # Verify zstd compression markers are set + ${count} = Validate Zstd Compressed Fields ${data} + Log Compressed fields count: ${count} + +Attachment with detected MIME type + [Documentation] Scan message with attachment and verify content_type in export + Scan File ${ATTACHMENT_MESSAGE} + ... Settings={symbols_enabled = []} + + # Wait for async export + Sleep 1s + + # Read from stream + ${data} = Redis Stream Read Msgpack ${RSPAMD_REDIS_ADDR} ${RSPAMD_REDIS_PORT} test:structured + Log ${data} + + # Validate required fields + Validate Structured Metadata ${data} uuid,ip,score + + # Verify attachments have content_type + ${count} = Validate Attachments Have Content Type ${data} + Should Be True ${count} >= 1 msg=Expected at least 1 attachment with content_type + +*** Keywords *** +Metadata Exporter Structured Setup + Run Redis + Rspamd Setup + +Metadata Exporter Structured Teardown + Rspamd Teardown + Redis Teardown diff --git a/test/functional/configs/metadata_exporter_structured.conf b/test/functional/configs/metadata_exporter_structured.conf new file mode 100644 index 0000000000..33c10885bc --- /dev/null +++ b/test/functional/configs/metadata_exporter_structured.conf @@ -0,0 +1,28 @@ +# Config for metadata_exporter structured formatter functional tests +.include(duplicate=append,priority=0) "{= env.TESTDIR =}/configs/plugins.conf" + +lua = "{= env.LUA_SCRIPT =}"; +redis { + servers = "{= env.REDIS_ADDR =}:{= env.REDIS_PORT =}"; +} + +# Configure metadata_exporter with structured formatter and redis_stream backend +metadata_exporter { + rules { + STRUCTURED_EXPORT { + backend = "redis_stream"; + formatter = "structured"; + stream_key = "test:structured"; + max_len = 100; + selector = "default"; + } + STRUCTURED_EXPORT_ZSTD { + backend = "redis_stream"; + formatter = "structured"; + zstd_compress = true; + stream_key = "test:structured_zstd"; + max_len = 100; + selector = "default"; + } + } +} diff --git a/test/functional/lib/rspamd.py b/test/functional/lib/rspamd.py index 1302e059ab..f692089313 100644 --- a/test/functional/lib/rspamd.py +++ b/test/functional/lib/rspamd.py @@ -700,3 +700,130 @@ def collect_lua_coverage(): def file_exists(file): return os.path.isfile(file) + + +def redis_stream_read_msgpack(host, port, stream_key): + """Read latest entry from Redis stream and decode msgpack data. + + Returns decoded dict with metadata fields. + + Example: + | ${data} = | Redis Stream Read Msgpack | ${RSPAMD_REDIS_ADDR} | ${RSPAMD_REDIS_PORT} | test:structured | + """ + try: + import redis + except ImportError: + raise Exception("redis module not installed - run: pip install redis") + + try: + import msgpack + except ImportError: + raise Exception("msgpack module not installed - run: pip install msgpack") + + r = redis.Redis(host=host, port=int(port), decode_responses=False) + + # Read from stream + entries = r.xrange(stream_key, count=1) + if not entries: + raise Exception(f"No data in stream {stream_key}") + + # Get the first entry's data field + entry_id, fields = entries[0] + if b'data' not in fields: + raise Exception(f"No data field in stream entry, keys: {list(fields.keys())}") + + msgpack_data = fields[b'data'] + + # Decode msgpack with raw=True to preserve bytes, then convert what we can + decoded = msgpack.unpackb(msgpack_data, raw=True) + + # Convert bytes keys to strings for easier access + def convert_keys(obj): + if isinstance(obj, dict): + return {k.decode('utf-8') if isinstance(k, bytes) else k: convert_keys(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [convert_keys(item) for item in obj] + elif isinstance(obj, bytes): + # Try to decode as UTF-8, otherwise keep as bytes + try: + return obj.decode('utf-8') + except UnicodeDecodeError: + return obj + return obj + + return convert_keys(decoded) + + +def validate_structured_metadata(data, expected_fields=None): + """Validate structured metadata export format. + + Checks that required fields exist and UUID v7 has correct format. + + Example: + | Validate Structured Metadata | ${data} | uuid,ip,score,action | + """ + import re + + if expected_fields is None: + expected_fields = 'uuid,ip,score,action' + + errors = [] + + for field in expected_fields.split(','): + field = field.strip() + if field not in data: + errors.append(f"Missing field: {field}") + + # Validate UUID v7 format if present + if 'uuid' in data: + uuid = data['uuid'] + # UUID v7: xxxxxxxx-xxxx-7xxx-xxxx-xxxxxxxxxxxx + if not re.match(r'^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$', str(uuid)): + errors.append(f"Invalid UUID v7 format: {uuid}") + + if errors: + raise Exception("Validation errors: " + "; ".join(errors)) + + return True + + +def validate_zstd_compressed_fields(data): + """Validate that zstd compression markers are set correctly. + + Returns count of compressed fields found. + + Example: + | ${count} = | Validate Zstd Compressed Fields | ${data} | + """ + count = 0 + + # Check text_compressed flag + if data.get('text_compressed'): + count += 1 + + # Check attachments + for att in data.get('attachments', []): + if att.get('content_compressed'): + count += 1 + + # Check images + for img in data.get('images', []): + if img.get('content_compressed'): + count += 1 + + return count + + +def validate_attachments_have_content_type(data): + """Validate that attachments have content_type field. + + Returns count of attachments with content_type. + + Example: + | ${count} = | Validate Attachments Have Content Type | ${data} | + """ + count = 0 + for att in data.get('attachments', []): + if 'content_type' in att and att['content_type']: + count += 1 + return count diff --git a/test/functional/lua/metadata_exporter_structured.lua b/test/functional/lua/metadata_exporter_structured.lua new file mode 100644 index 0000000000..88c105a02e --- /dev/null +++ b/test/functional/lua/metadata_exporter_structured.lua @@ -0,0 +1,5 @@ +-- Lua helper for metadata_exporter structured formatter functional tests +-- This file is loaded by the test config + +-- No additional symbols needed - the metadata_exporter plugin handles everything +-- This file exists just to satisfy the lua = config option