From: Mikhail Galanin Date: Tue, 21 Aug 2018 14:55:50 +0000 (+0100) Subject: [Test] Added test for clickhouse migration vNONE -> v2 X-Git-Tag: 1.8.0~224^2 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2433%2Fhead;p=thirdparty%2Frspamd.git [Test] Added test for clickhouse migration vNONE -> v2 --- diff --git a/.circleci/config.yml b/.circleci/config.yml index 223ae76cf0..1df9ddc7f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,10 +86,14 @@ jobs: - attach_workspace: at: *workspace_root + - run: echo 'deb http://repo.yandex.ru/clickhouse/deb/stable/ main/' | sudo tee /etc/apt/sources.list.d/clickhouse.list + - run: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E0C56BD4 # optional, clickhouse key + - run: sudo apt-get update -qq || true - run: sudo apt-get install -qq libluajit-5.1-dev libpcre3-dev luarocks opendkim-tools python-pip redis-server + - run: sudo apt-get install clickhouse-server - - run: sudo pip install demjson psutil robotframework + - run: sudo pip install demjson psutil robotframework requests - run: sudo luarocks install luacheck - run: cd ../build diff --git a/test/functional/cases/210_clickhouse/001_migration.robot b/test/functional/cases/210_clickhouse/001_migration.robot new file mode 100644 index 0000000000..81cfa9dcae --- /dev/null +++ b/test/functional/cases/210_clickhouse/001_migration.robot @@ -0,0 +1,68 @@ +*** Settings *** +Documentation Checks if rspamd is able to upgrade migration schema from v0 (very initial) to v2 +Variables ${TESTDIR}/lib/vars.py +Library ${TESTDIR}/lib/rspamd.py +Library clickhouse.py +Resource ${TESTDIR}/lib/rspamd.robot + +Suite Setup Clickhouse Setup +Suite Teardown Clickhosue Teardown + +*** Variables *** +${CONFIG} ${TESTDIR}/configs/clickhouse.conf +${RSPAMD_SCOPE} Suite + +*** Test Cases *** +Migration + Upload new schema ${TESTDIR}/data/initial_schema/schema.sql + Insert data rspamd ${TESTDIR}/data/initial_schema/data.rspamd.sql + Insert data rspamd_asn ${TESTDIR}/data/initial_schema/data.rspamd_asn.sql + Insert data rspamd_emails ${TESTDIR}/data/initial_schema/data.rspamd_emails.sql + Insert data rspamd_urls ${TESTDIR}/data/initial_schema/data.rspamd_urls.sql + Insert data rspamd_attachments ${TESTDIR}/data/initial_schema/data.rspamd_attachments.sql + Insert data rspamd_symbols ${TESTDIR}/data/initial_schema/data.rspamd_symbols.sql + + Prepare rspamd + + Sleep 1 #TODO: replace this check with waiting until migration finishes + + Column should exist rspamd Symbols.Scores + Column should exist rspamd Attachments.Digest + Column should exist rspamd Symbols.Scores + Schema version should be 2 + + +*** Keywords *** +Clickhouse Setup + ${TMPDIR} = Make Temporary Directory + Set Global Variable ${TMPDIR} + Set Directory Ownership ${TMPDIR} ${RSPAMD_USER} ${RSPAMD_GROUP} + ${template} = Get File ${TESTDIR}/configs/clickhouse-config.xml + ${config} = Replace Variables ${template} + Create File ${TMPDIR}/clickhouse-config.xml ${config} + Copy File ${TESTDIR}/configs/clickhouse-users.xml ${TMPDIR}/users.xml + Create Directory ${TMPDIR}/metadata + Create Directory ${TMPDIR}/metadata/default + Create Directory ${TMPDIR}/data/default + ${result} = Run Process clickhouse-server --daemon --config-file\=${TMPDIR}/clickhouse-config.xml --pid-file\=${TMPDIR}/clickhouse.pid + Run Keyword If ${result.rc} != 0 Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Wait Until Keyword Succeeds 5 sec 1 sec Check Pidfile ${TMPDIR}/clickhouse.pid timeout=5 sec + Set Suite Variable ${TMPDIR} ${TMPDIR} + + + +Clickhosue Teardown + # Sleep 30 + ${clickhouse_pid} = Get File ${TMPDIR}/clickhouse.pid + Shutdown Process With Children ${clickhouse_pid} + Simple Teardown + + +Prepare rspamd + &{d} = Run Rspamd CONFIG=${TESTDIR}/configs/clickhouse.conf TMPDIR=${TMPDIR} + ${keys} = Get Dictionary Keys ${d} + : FOR ${i} IN @{keys} + \ Run Keyword If '${RSPAMD_SCOPE}' == 'Suite' Set Suite Variable ${${i}} &{d}[${i}] + \ ... ELSE IF '${RSPAMD_SCOPE}' == 'Test' Set Test Variable ${${i}} &{d}[${i}] + \ ... ELSE Fail 'RSPAMD_SCOPE must be Test or Suite' diff --git a/test/functional/cases/210_clickhouse/clickhouse.py b/test/functional/cases/210_clickhouse/clickhouse.py new file mode 100644 index 0000000000..069589fed4 --- /dev/null +++ b/test/functional/cases/210_clickhouse/clickhouse.py @@ -0,0 +1,67 @@ +import requests +import json +from robot.libraries.BuiltIn import BuiltIn +from robot.api import logger + +__client = None + + +class Client: + def __init__(self): + self.port = 18123 + + def get_query_string(self): + return "http://localhost:%d/?default_format=JSONEachRow" % (self.port) + + def execute(self, sql): + r = requests.post(self.get_query_string(), sql) + if r.status_code != 200: + raise Exception("Clickhouse request failed: " + r.content) + return r + + def query(self, sql): + r = self.execute(sql) + + # logger.console("decoding " + r.content) + # [logger.console("decoding " + _) for _ in r.content.strip().split("\n")] + response = [json.loads(_) for _ in r.content.strip().split("\n")] + return response + + +def client(): + global __client + if __client is None: + __client = Client() + return __client + + +def upload_new_schema(schema_file): + with open(schema_file, 'r') as content_file: + content = content_file.read() + + queries = content.split(";") + for q in queries: + if q.strip() == "": + continue + client().execute(q) # throws exception on error + + +def insert_data(table_name, filename): + with open(filename, 'r') as content_file: + content = content_file.read() + + client().execute("insert into %s format Values %s;" % (table_name, content)) # throws exception on error + + +def column_should_exist(table_name, column_name): + sql = "select hasColumnInTable('default', '%s', '%s') as is_exist" % (table_name, column_name) + r = client().query(sql) + if r[0]['is_exist'] != 1: + raise Exception("Failed asseting that column '%s' exists in table 'default'.'%s'" % (column_name, table_name)) + + +def schema_version_should_be(version): + sql = "select max(Version) as version from rspamd_version" + r = client().query(sql) + if r[0]['version'] != 2: + raise Exception("Failed asseting that schema version is '%d'" % version) \ No newline at end of file diff --git a/test/functional/configs/clickhouse-config.xml b/test/functional/configs/clickhouse-config.xml new file mode 100644 index 0000000000..10530a580b --- /dev/null +++ b/test/functional/configs/clickhouse-config.xml @@ -0,0 +1,342 @@ + + + + + trace + ${TMPDIR}/clickhouse-server.log + ${TMPDIR}/clickhouse-server.err.log + 1000M + 10 + + + + 18123 + 19000 + + + + + + + + + 19009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + 256 + + + 8589934592 + + + 5368709120 + + + + ${TMPDIR}/ + + + ${TMPDIR} + + + ${TMPDIR} + + + ${TMPDIR}/users.xml + + + default + + + + + + default + + + + + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/test/functional/configs/clickhouse-users.xml b/test/functional/configs/clickhouse-users.xml new file mode 100644 index 0000000000..6f746baf2a --- /dev/null +++ b/test/functional/configs/clickhouse-users.xml @@ -0,0 +1,109 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + ::1 + 127.0.0.1 + + readonly + default + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/test/functional/configs/clickhouse.conf b/test/functional/configs/clickhouse.conf new file mode 100644 index 0000000000..2e1b8d4663 --- /dev/null +++ b/test/functional/configs/clickhouse.conf @@ -0,0 +1,51 @@ +options = { + filters = ["spf", "dkim", "regexp"] + pidfile = "${TMPDIR}/rspamd.pid" + lua_path = "${INSTALLROOT}/share/rspamd/lib/?.lua" + dns { + nameserver = ["8.8.8.8", "8.8.4.4"]; + retransmits = 10; + timeout = 2s; + fake_records = [{ # ed25519 + name = "test._domainkey.example.com"; + type = txt; + replies = ["k=ed25519; p=yi50DjK5O9pqbFpNHklsv9lqaS0ArSYu02qp1S0DW1Y="]; + }]; + } +} +clickhouse { + # Push update when 1000 records are collected (1000 if unset) + limit = 1; + # IP:port of Clickhouse server + server = "localhost:18123"; + allow_local = true; +} +logging = { + type = "file", + level = "debug" + filename = "${TMPDIR}/rspamd.log" +} +metric = { + name = "default", + actions = { + reject = 100500, + } + unknown_weight = 1 +} +worker { + type = normal + bind_socket = ${LOCAL_ADDR}:${PORT_NORMAL} + count = 1 + task_timeout = 60s; +} +worker { + type = controller + bind_socket = ${LOCAL_ADDR}:${PORT_CONTROLLER} + count = 1 + secure_ip = ["127.0.0.1", "::1"]; + stats_path = "${TMPDIR}/stats.ucl" +} +modules { + path = "${TESTDIR}/../../src/plugins/lua/" +} +lua = "${INSTALLROOT}/share/rspamd/rules/rspamd.lua" diff --git a/test/functional/data/initial_schema/data.rspamd.sql b/test/functional/data/initial_schema/data.rspamd.sql new file mode 100644 index 0000000000..3c03c9881e --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd.sql @@ -0,0 +1,56 @@ +('2018-08-21','2018-08-21 11:24:36','xent.com','evergo.net','127.0.0.0',1.19,0,3248,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','johnhall','','','friends of rohit khare ','162bc579fb3145dc3c65669328fa39cb'), +('2018-08-21','2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2890,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c772a14916e765a4593939a5991bb859'), +('2018-08-21','2018-08-21 11:24:46','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,975,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','83cfbad1dbcb7234843ff01b810ea2c0'), +('2018-08-21','2018-08-21 11:24:46','xent.com','cse.ucsc.edu','127.0.0.0',0.69,0,3225,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','elias','','','friends of rohit khare ','9fa1aa06c0afe91f1aad86a128c475f6'), +('2018-08-21','2018-08-21 11:24:15','example.com','example.com','127.0.0.0',0.9,0,1312,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b3e5e01c86a8fd8a9bc5a66d033cc32e'), +('2018-08-21','2018-08-21 11:24:15','example.sourceforge.net','users.sourceforge.net','127.0.0.0',0.69,0,4198,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-commits-admin','yyyyason','','','','b8c4054a876b3e6593b0c7ea427045ba'), +('2018-08-21','2018-08-21 11:24:15','freshrpms.net','camperquake.de','127.0.0.0',1.69,0,3663,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ralf','','','freshrpms rpm discussion list ','bb4fcb9417cb59838b2c83c4b4342ffa'), +('2018-08-21','2018-08-21 11:24:15','freshrpms.net','rpmforge.net','127.0.0.0',1.69,0,3934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list ','2b1b347f25159ca1ea8d4a9ee5187ba8'), +('2018-08-21','2018-08-21 11:24:15','linux.ie','steorn.com','127.0.0.0',1.19,0,2934,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','carlos.luna','','','irish linux users\' group ','c95d8ffd4f70a91328eeec15c452ab6b'), +('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1078,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','5a46062bde8cce224ea90aa134e10da1'), +('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2088,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','cf0d79f76e51a3e6c5bcfdfdac094685'), +('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','2ab3790853d8bf1a90623b6d788abeb0'), +('2018-08-21','2018-08-21 11:24:15','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1124,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','47362127e1870aa9aaf3eef8ae5c749b'), +('2018-08-21','2018-08-21 11:24:15','xent.com','mithral.com','127.0.0.0',0.69,0,3546,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','beberg','','','friends of rohit khare ','f4d74c809ba48b987fd6271baf82c12f'), +('2018-08-21','2018-08-21 11:24:20','baesystems.com','baesystems.com','127.0.0.0',0.9,0,3341,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','robert.chambers','robert.chambers','','','','835aaf36a75a56ce93dfa3badaeb92eb'), +('2018-08-21','2018-08-21 11:24:20','example.sourceforge.net','telus.net','127.0.0.0',1.69,0,3852,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','cfortune','','','talk about spamassassin ','8e799460b63b7919b4c4587dadd7b063'), +('2018-08-21','2018-08-21 11:24:20','freshrpms.net','egwn.net','127.0.0.0',1.69,0,5074,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list ','3950ac78c11ec35d55cb56c265d9474a'), +('2018-08-21','2018-08-21 11:24:20','freshrpms.net','fluid.com','127.0.0.0',0.69,0,3712,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','tengel','','','freshrpms rpm discussion list ','61936d1c00246dc8410391c9e5f03f5a'), +('2018-08-21','2018-08-21 11:24:20','linux.ie','eircom.net','127.0.0.0',1.3,0,2763,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','ilug-admin','cout','','','irish linux users\' group ','d1e819de70503d5ebfa49b6708703942'), +('2018-08-21','2018-08-21 11:24:20','petting-zoo.net','','127.0.0.0',1.8900000000000001,0,2538,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','0xdeadbeef-request','','','','','c6046b370d90b6b13a4788557e77cf24'), +('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','hotmail.com','127.0.0.0',0.89,0,4135,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-56028-1034088521-zzzz=example.com','skitster','','','','c22813bc4c29a3f5fa0dd1cbd598c7c8'), +('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5651,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=example.com','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'), +('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','earthlink.net','127.0.0.0',1.3900000000000001,0,2885,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-60166-1038789202-yyyy=spamassassin.taint.org','billjac','','','','5a914e03141f3f7099e50be362c007c2'), +('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','srv0.ems.ed.ac.uk','127.0.0.0',1.3900000000000001,0,5706,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-53809-1031308404-zzzz=spamassassin.taint.org','martin','','','','a10e11dcdc3d527ade0f5e85bc583f78'), +('2018-08-21','2018-08-21 11:24:20','returns.groups.yahoo.com','bestirishmusic.com','127.0.0.0',1.3900000000000001,0,3709,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','sentto-2242572-55915-1033992114-zzzz=spamassassin.taint.org','webmaster','','','','5e3597c0f14b241c915c161c462c4804'), +('2018-08-21','2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,2168,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','472774888e942d317fecc91724b304f8'), +('2018-08-21','2018-08-21 11:24:20','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1125,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','54b07928d68e59fb370ea89f2efdaabd'), +('2018-08-21','2018-08-21 11:24:20','xent.com','ianbell.com','127.0.0.0',0.69,0,15012,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork','','','friends of rohit khare ','64216f07ed5995242c78fe27b537c30b'), +('2018-08-21','2018-08-21 11:24:20','xent.com','techmonkeys.net','127.0.0.0',0.69,0,4501,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare ','a96c963bc4d5297e3ebdf4f2ea27927f'), +('2018-08-21','2018-08-21 11:24:20','xent.com','hotmail.com','127.0.0.0',1.69,0,2606,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','fork_list','','','friends of rohit khare ','9d743f10d435a607f9179d9429eb0ea1'), +('2018-08-21','2018-08-21 11:24:20','xent.com','vertexdev.com','127.0.0.0',1.19,0,3224,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','jeff','','','friends of rohit khare ','5008d567ff90facc36d94be316d4e0fe'), +('2018-08-21','2018-08-21 11:24:20','xent.com','panix.com','127.0.0.0',2.69,0,2366,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','lgonze','','','friends of rohit khare ','154b831310873a800b92169b7e049a8f'), +('2018-08-21','2018-08-21 11:24:20','xent.com','magnesium.net','127.0.0.0',0.69,0,6039,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare ','0fcc0fdd0995b10de6f07585354e2941'), +('2018-08-21','2018-08-21 11:24:20','xent.com','canada.com','127.0.0.0',0.69,0,3685,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare ','c0796f14caaf4e02ca683309021f6812'), +('2018-08-21','2018-08-21 11:24:25','freshrpms.net','egwn.net','127.0.0.0',1.69,0,3500,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','matthias','','','freshrpms rpm discussion list ','9848f3ace4d8b54ed52061cf0330d64f'), +('2018-08-21','2018-08-21 11:24:25','xent.com','endeavors.com','127.0.0.0',0.69,0,3561,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','gbolcer','','','friends of rohit khare ','a5109e184c129e5521438be5375e7804'), +('2018-08-21','2018-08-21 11:24:26','comcast.net','','undefined',6.9,0,4682,'unknown','unknown','unknown','unknown','unknown','unknown',0,'add header','tim.one','','','','','9367812a3c0f279aa3dbf691e9d9ec11'), +('2018-08-21','2018-08-21 11:24:26','example.sourceforge.net','pobox.com','127.0.0.0',1.19,0,4167,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','razor-users-admin','wstearns','','','','8dd6255d0616f90f16ec8cfbba496143'), +('2018-08-21','2018-08-21 11:24:26','xent.com','canada.com','127.0.0.0',0.69,0,3031,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare ','e5ccbf749700f93ece4a41907d81593b'), +('2018-08-21','2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,2154,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','27e6cbfca0423d5c2aa6fa6731670699'), +('2018-08-21','2018-08-21 11:24:36','example.com','example.com','127.0.0.0',0.9,0,1630,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','b35cf34eb7186872f7c035a7bd450e92'), +('2018-08-21','2018-08-21 11:24:36','example.com','panasas.com','127.0.0.0',0.69,0,4910,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','exmh-users-admin','welch','','','discussion list for exmh users ','7987eec136f5391445df5bfbe64d4b33'), +('2018-08-21','2018-08-21 11:24:36','example.sourceforge.net','gmx.net','127.0.0.0',1.69,0,5667,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','spamassassin-talk-admin','msquadrat.nospamplease','','','talk about spamassassin ','2c6645233690c3bd7677fc5a6b637000'), +('2018-08-21','2018-08-21 11:24:36','freshrpms.net','bellsouth.net','127.0.0.0',0.69,0,4817,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','lance_tt','','','freshrpms rpm discussion list ','7201979c9c64847048d2e20512a6b5d9'), +('2018-08-21','2018-08-21 11:24:36','freshrpms.net','ckloiber.com','127.0.0.0',0.69,0,3161,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rpm-zzzlist-admin','ckloiber','','','freshrpms rpm discussion list ','52fee32a1ec50eb0d0dab4dada87c497'), +('2018-08-21','2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1474,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','e3a16ef77e2ee48767c499104f67270b'), +('2018-08-21','2018-08-21 11:24:36','spamassassin.taint.org','spamassassin.taint.org','127.0.0.0',0.9,0,1053,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','c3a8d98433975fefb64f3c059d6046f0'), +('2018-08-21','2018-08-21 11:24:36','xent.com','techmonkeys.net','127.0.0.0',0.69,0,2218,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','cdale','','','friends of rohit khare ','672fa037061f621085fbe997d236f374'), +('2018-08-21','2018-08-21 11:24:36','xent.com','','127.0.0.0',0.69,0,5136,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','','','','friends of rohit khare ','3b86bc8189eba5d6ed9b633ce2995c6d'), +('2018-08-21','2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,3388,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare ','03190fd537aa97e9be7d4a75fed57d36'), +('2018-08-21','2018-08-21 11:24:36','xent.com','slack.net','127.0.0.0',0.69,0,2554,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','tomwhore','','','friends of rohit khare ','2a1f34828b8a6a9750a1a09bc305983d'), +('2018-08-21','2018-08-21 11:24:36','xent.com','permafrost.net','127.0.0.0',0.69,0,4069,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','owen','','','friends of rohit khare ','9167f411217b18af953884010508dc2f'), +('2018-08-21','2018-08-21 11:24:36','xent.com','magnesium.net','127.0.0.0',0.69,0,2547,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','bitbitch','','','friends of rohit khare ','8d5d3caf91661a5b36f857a3010c4a48'), +('2018-08-21','2018-08-21 11:24:36','xent.com','canada.com','127.0.0.0',0.69,0,2806,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','garym','','','friends of rohit khare ','236f08f521364e701ad8dc43afde4fc4'), +('2018-08-21','2018-08-21 11:24:46','example.com','example.com','127.0.0.0',0.9,0,1639,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','rssfeeds','rssfeeds','','','','a8f132bbb47ea2b4310e550d9617023f'), +('2018-08-21','2018-08-21 11:24:46','xent.com','hotmail.com','127.0.0.0',1.69,0,3376,'unknown','unknown','unknown','unknown','unknown','unknown',0,'no action','fork-admin','deafbox','','','friends of rohit khare ','8d7f99415bb42f744c9d06ccf8d5ccbf') \ No newline at end of file diff --git a/test/functional/data/initial_schema/data.rspamd_asn.sql b/test/functional/data/initial_schema/data.rspamd_asn.sql new file mode 100644 index 0000000000..6cef209241 --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd_asn.sql @@ -0,0 +1,66 @@ +('2018-08-21','43607711d98b62ce6f3d13013ce7241f','--','--','--'), +('2018-08-21','4ddf3d1d31558d038d9e7bba5a862494','--','--','--'), +('2018-08-21','8db807c0b535792ea4e2d9b7f69b113c','--','--','--'), +('2018-08-21','905721f001ce446b2ad23d6833716bcb','--','--','--'), +('2018-08-21','8d7f99415bb42f744c9d06ccf8d5ccbf','--','--','--'), +('2018-08-21','a8f132bbb47ea2b4310e550d9617023f','--','--','--'), +('2018-08-21','03190fd537aa97e9be7d4a75fed57d36','--','--','--'), +('2018-08-21','0fcc0fdd0995b10de6f07585354e2941','--','--','--'), +('2018-08-21','154b831310873a800b92169b7e049a8f','--','--','--'), +('2018-08-21','162bc579fb3145dc3c65669328fa39cb','--','--','--'), +('2018-08-21','236f08f521364e701ad8dc43afde4fc4','--','--','--'), +('2018-08-21','27e6cbfca0423d5c2aa6fa6731670699','--','--','--'), +('2018-08-21','28ebd9e7e8a4ea4d54f18c02305382a8','--','--','--'), +('2018-08-21','2a1f34828b8a6a9750a1a09bc305983d','--','--','--'), +('2018-08-21','2ab3790853d8bf1a90623b6d788abeb0','--','--','--'), +('2018-08-21','2b1b347f25159ca1ea8d4a9ee5187ba8','--','--','--'), +('2018-08-21','2c6645233690c3bd7677fc5a6b637000','--','--','--'), +('2018-08-21','3950ac78c11ec35d55cb56c265d9474a','--','--','--'), +('2018-08-21','3b86bc8189eba5d6ed9b633ce2995c6d','--','--','--'), +('2018-08-21','472774888e942d317fecc91724b304f8','--','--','--'), +('2018-08-21','47362127e1870aa9aaf3eef8ae5c749b','--','--','--'), +('2018-08-21','5008d567ff90facc36d94be316d4e0fe','--','--','--'), +('2018-08-21','51cd92041e96327f9fc502ee19990c04','--','--','--'), +('2018-08-21','52fee32a1ec50eb0d0dab4dada87c497','--','--','--'), +('2018-08-21','54b07928d68e59fb370ea89f2efdaabd','--','--','--'), +('2018-08-21','5a46062bde8cce224ea90aa134e10da1','--','--','--'), +('2018-08-21','5a914e03141f3f7099e50be362c007c2','--','--','--'), +('2018-08-21','5e3597c0f14b241c915c161c462c4804','--','--','--'), +('2018-08-21','61936d1c00246dc8410391c9e5f03f5a','--','--','--'), +('2018-08-21','64216f07ed5995242c78fe27b537c30b','--','--','--'), +('2018-08-21','672fa037061f621085fbe997d236f374','--','--','--'), +('2018-08-21','69afbde36dc807ef7190ad549f94e697','--','--','--'), +('2018-08-21','7201979c9c64847048d2e20512a6b5d9','--','--','--'), +('2018-08-21','7987eec136f5391445df5bfbe64d4b33','--','--','--'), +('2018-08-21','835aaf36a75a56ce93dfa3badaeb92eb','--','--','--'), +('2018-08-21','8d5d3caf91661a5b36f857a3010c4a48','--','--','--'), +('2018-08-21','8dd6255d0616f90f16ec8cfbba496143','--','--','--'), +('2018-08-21','8e799460b63b7919b4c4587dadd7b063','--','--','--'), +('2018-08-21','9367812a3c0f279aa3dbf691e9d9ec11','--','--','--'), +('2018-08-21','9848f3ace4d8b54ed52061cf0330d64f','--','--','--'), +('2018-08-21','9d743f10d435a607f9179d9429eb0ea1','--','--','--'), +('2018-08-21','a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'), +('2018-08-21','a10e11dcdc3d527ade0f5e85bc583f78','--','--','--'), +('2018-08-21','a5109e184c129e5521438be5375e7804','--','--','--'), +('2018-08-21','a96c963bc4d5297e3ebdf4f2ea27927f','--','--','--'), +('2018-08-21','b35cf34eb7186872f7c035a7bd450e92','--','--','--'), +('2018-08-21','b3e5e01c86a8fd8a9bc5a66d033cc32e','--','--','--'), +('2018-08-21','b8c4054a876b3e6593b0c7ea427045ba','--','--','--'), +('2018-08-21','bb4fcb9417cb59838b2c83c4b4342ffa','--','--','--'), +('2018-08-21','c0796f14caaf4e02ca683309021f6812','--','--','--'), +('2018-08-21','c22813bc4c29a3f5fa0dd1cbd598c7c8','--','--','--'), +('2018-08-21','c4bb938afcc5d6af78c0ea66145bbc2d','--','--','--'), +('2018-08-21','c6046b370d90b6b13a4788557e77cf24','--','--','--'), +('2018-08-21','c772a14916e765a4593939a5991bb859','--','--','--'), +('2018-08-21','c95d8ffd4f70a91328eeec15c452ab6b','--','--','--'), +('2018-08-21','cd6dea756b83f81172d07a1e0f5b08a9','--','--','--'), +('2018-08-21','cf0d79f76e51a3e6c5bcfdfdac094685','--','--','--'), +('2018-08-21','d0bbb7fbc8d4e31471a25456fcfaff64','--','--','--'), +('2018-08-21','d10e5a118896958c447eaa343e3d2db8','--','--','--'), +('2018-08-21','d1e819de70503d5ebfa49b6708703942','--','--','--'), +('2018-08-21','d9ffc9704faf5057b1394c507153c277','--','--','--'), +('2018-08-21','e3a16ef77e2ee48767c499104f67270b','--','--','--'), +('2018-08-21','e5ccbf749700f93ece4a41907d81593b','--','--','--'), +('2018-08-21','f4d74c809ba48b987fd6271baf82c12f','--','--','--'), +('2018-08-21','faf1b3f92d8aa05c97fbc212f11afc54','--','--','--'), +('2018-08-21','feae0c04282d5e27705366d3211bc69e','--','--','--') \ No newline at end of file diff --git a/test/functional/data/initial_schema/data.rspamd_attachments.sql b/test/functional/data/initial_schema/data.rspamd_attachments.sql new file mode 100644 index 0000000000..5e68591ccf --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd_attachments.sql @@ -0,0 +1,10 @@ +('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['fsouoxtjvb.xls'], ['application/vnd.ms-excel'], [39424], ['d66d8dfcab696065']), +('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['eybxf.gif','llfoe.gif'], ['image/gif','image/gif'], [6558,67], ['d3e6d83395091599','abfae96a0fc709e9']), +('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['fake.xls'], ['application/vnd.ms-excel'], [361472], ['e0791f7961db0e00']), +('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['dsn_status','header'], ['message/delivery-status','text/rfc822-headers'], [451,650], ['4e5127839ad4af96','ee1db81d572b606c']), +('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['vdxcpjg.xls'], ['application/vnd.ms-excel'], [39424], ['163b098ba7aee8f9']), +('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']), +('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']), +('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']), +('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']), +('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['photo399.jpg'], ['image/jpeg'], [22495], ['7fa378464a00d645']) \ No newline at end of file diff --git a/test/functional/data/initial_schema/data.rspamd_emails.sql b/test/functional/data/initial_schema/data.rspamd_emails.sql new file mode 100644 index 0000000000..7c55026bbf --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd_emails.sql @@ -0,0 +1,6 @@ +('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['fake1@mail.ru']), +('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['fake2@mail.ru']), +('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['info_stop_mailer@yahoo.co.jp']), +('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['info_stop_mailer@yahoo.co.jp']), +('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['info_stop_mailer@yahoo.co.jp']), +('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['fake3@g-mail.com','fake4@g-mail.com']) \ No newline at end of file diff --git a/test/functional/data/initial_schema/data.rspamd_symbols.sql b/test/functional/data/initial_schema/data.rspamd_symbols.sql new file mode 100644 index 0000000000..32617972de --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd_symbols.sql @@ -0,0 +1,10 @@ +('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','has_x_prio_one','asn','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,0.1,3,0.2,0,0,0,0,0,0,0], ['','','','','fakexx@qq.com','','','','1','1','qq.com','','68','','1','qq.com','','','1','asn:4134, ipnet:49.64.0.0/11, country:cn','']), +('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:201725, ipnet:176.97.248.0/21, country:pl','']), +('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','subject_needs_encoding','to_dn_none','from_needs_encoding','rcpt_count_one','many_invisible_parts','freemail_envrcpt','mime_base64_text','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,1,0,1,0,0.05,0,0.1,0.2,0,0,0,0,0,0,0], ['','','','','fakeq3@qq.com','','','','','','1','1','qq.com','','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','5','']), +('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake@mailto.gq','','','asn:28187, ipnet:189.89.208.0/22, country:br','']), +('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake3@wowmail.cf','','','asn:52935, ipnet:191.5.108.0/23, country:br','']), +('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','missing_mime_version','to_match_envrcpt_all','mime_good','to_dn_none','freemail_envrcpt','rambler_uribl','has_x_prio_three','rcpt_count_seven','dbl_abuse','rcvd_count_one','freemail_to','fsl_single_url','rcvd_no_tls_last','invuri','asn','freemail_cc'], [0,0,0,0,2,0,-0.1,0,0,0,0,0,6.5,0,0,0.1,0,0,0,0], ['','','suportdesk.com.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','','text/plain','','bellsouth.net,aol.com,gmx.de,hotmail.com,gmail.com,me.com','suportdesk.com','3','7','suportdesk.com.dbl.uri.fslupdate.com','1','bellsouth.net','','','suportdesk.com.invuri.uri.fslupdate.com','asn:10429, ipnet:189.56.0.0/15, country:br','aol.com']), +('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','to_match_envrcpt_all','freemail_envrcpt','r_bad_cte_7bit','to_dn_none','subject_needs_encoding','rcpt_count_one','many_invisible_parts','has_x_prio_three','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0.01,0,0,0,0,3.5,0,1,0,0.05,0,0.2,0,0,0,0,0,0], ['','','','','fake5@qq.com','','qq.com','7bit','','','1','1','3','','1','qq.com','','','asn:4134, ipnet:113.120.0.0/13, country:cn','']), +('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['arc_na','rcvd_via_smtp_auth','zetascan_uri','from_has_dn','to_match_envrcpt_all','sem_uribl','to_dn_none','rambler_uribl','many_invisible_parts','mime_base64_text','has_x_prio_three','rcpt_count_twelve','mime_html_only','rcvd_count_one','uribl_black','rcvd_no_tls_last','from_eq_envfrom','mid_rhs_not_fqdn','invuri','asn','dbl_spam','uribl_sbl','forged_outlook_html'], [0,0,0,0,0,3.5,0,0,0.7,0.1,0,0,0.2,0,7.5,0,0,0.5,0,0,6.5,6.5,5], ['','','gdguojian.net.e93be9862ef26e5fccb23696d6707ffc.dblack.api.zetascan.com','','','gdguojian.net.uribl.spameatingmonkey.net','','gdguojian.net','8','','3','21','','1','gdguojian.net.multi.uri.fslupdate.com','','','','gdguojian.net.invuri.uri.fslupdate.com','asn:4134, ipnet:182.96.0.0/12, country:cn','gdguojian.net.dbl.uri.fslupdate.com','gdguojian.net','']), +('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['arc_na','rcvd_via_smtp_auth','xm_ua_no_version','from_has_dn','spamtrap','missing_mime_version','to_match_envrcpt_all','to_dn_none','rcpt_count_one','many_invisible_parts','freemail_envrcpt','subj_all_caps','mime_html_only','rcvd_count_one','freemail_to','rcvd_no_tls_last','from_eq_envfrom','asn','has_x_prio_five','mid_rhs_match_from'], [0,0,0.01,0,0,2,0,0,0,0.05,0,3,0.2,0,0,0,0,0,0,0], ['','','','','fake9@qq.com','','','','1','1','qq.com','74','','1','qq.com','','','asn:4134, ipnet:182.32.0.0/12, country:cn','5','']), +('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['arc_na','rcvd_via_smtp_auth','from_has_dn','to_match_envrcpt_all','mv_case','mime_good','to_dn_none','rcpt_count_one','rcvd_count_one','spamtrap','rcvd_no_tls_last','from_eq_envfrom','asn','mid_rhs_match_from'], [0,0,0,0,0.5,-0.1,0,0,0,0,0,0,0,0], ['','','','','','text/plain','','1','1','fake10@mailto.gq','','','asn:262812, ipnet:200.66.120.0/23, country:br','']) \ No newline at end of file diff --git a/test/functional/data/initial_schema/data.rspamd_urls.sql b/test/functional/data/initial_schema/data.rspamd_urls.sql new file mode 100644 index 0000000000..a9e20c3ef6 --- /dev/null +++ b/test/functional/data/initial_schema/data.rspamd_urls.sql @@ -0,0 +1,10 @@ +('2018-08-08', 'd1e819de70503d5ebfa49b6708703942', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '5e3597c0f14b241c915c161c462c4804', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '8dd6255d0616f90f16ec8cfbba496143', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '8d7f99415bb42f744c9d06ccf8d5ccbf', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '2b1b347f25159ca1ea8d4a9ee5187ba8', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '2ab3790853d8bf1a90623b6d788abeb0', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '0fcc0fdd0995b10de6f07585354e2941', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '7987eec136f5391445df5bfbe64d4b33', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '2c6645233690c3bd7677fc5a6b637000', ['gdguojian.net'], ['http://gdguojian.net/']), +('2018-08-08', '52fee32a1ec50eb0d0dab4dada87c497', ['gdguojian.net'], ['http://gdguojian.net/']) \ No newline at end of file diff --git a/test/functional/data/initial_schema/schema.sql b/test/functional/data/initial_schema/schema.sql new file mode 100644 index 0000000000..7fa8cb7c8e --- /dev/null +++ b/test/functional/data/initial_schema/schema.sql @@ -0,0 +1,70 @@ +CREATE TABLE IF NOT EXISTS rspamd +( + Date Date, + TS DateTime, + From String, + MimeFrom String, + IP String, + Score Float64, + NRcpt UInt8, + Size UInt32, + IsWhitelist Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('blacklist' = 0, 'whitelist' = 1, 'unknown' = 2)), + IsBayes Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)), + IsFuzzy Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('whitelist' = 0, 'deny' = 1, 'unknown' = 2)), + IsFann Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('ham' = 0, 'spam' = 1, 'unknown' = 2)), + IsDkim Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)), + IsDmarc Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2) DEFAULT CAST('unknown' AS Enum8('reject' = 0, 'allow' = 1, 'unknown' = 2)), + NUrls Int32, + Action Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5) DEFAULT CAST('no action' AS Enum8('reject' = 0, 'rewrite subject' = 1, 'add header' = 2, 'greylist' = 3, 'no action' = 4, 'soft reject' = 5)), + FromUser String, + MimeUser String, + RcptUser String, + RcptDomain String, + ListId String, + Digest FixedString(32) +) ENGINE = MergeTree(Date, (TS, From), 8192); + + +CREATE TABLE IF NOT EXISTS rspamd_attachments ( + Date Date, + Digest FixedString(32), + `Attachments.FileName` Array(String), + `Attachments.ContentType` Array(String), + `Attachments.Length` Array(UInt32), + `Attachments.Digest` Array(FixedString(16)) +) ENGINE = MergeTree(Date, Digest, 8192); + + +CREATE TABLE IF NOT EXISTS rspamd_urls ( + Date Date, + Digest FixedString(32), + `Urls.Tld` Array(String), + `Urls.Url` Array(String) +) ENGINE = MergeTree(Date, Digest, 8192); + + +CREATE TABLE IF NOT EXISTS rspamd_emails ( + Date Date, + Digest FixedString(32), + Emails Array(String) +) ENGINE = MergeTree(Date, Digest, 8192); + + +CREATE TABLE IF NOT EXISTS rspamd_asn ( + Date Date, + Digest FixedString(32), + ASN String, + Country FixedString(2), + IPNet String +) ENGINE = MergeTree(Date, Digest, 8192); + + +CREATE TABLE IF NOT EXISTS rspamd_symbols ( + Date Date, + Digest FixedString(32), + `Symbols.Names` Array(String), + `Symbols.Scores` Array(Float64), + `Symbols.Options` Array(String) +) ENGINE = MergeTree(Date, Digest, 8192); + + diff --git a/test/functional/lib/rspamd.robot b/test/functional/lib/rspamd.robot index ec2fc5e84c..45c7c10574 100644 --- a/test/functional/lib/rspamd.robot +++ b/test/functional/lib/rspamd.robot @@ -10,8 +10,8 @@ Check Controller Errors Log @{result}[1] Check Pidfile - [Arguments] ${pidfile} - Wait Until Created ${pidfile} + [Arguments] ${pidfile} ${timeout}=1 min + Wait Until Created ${pidfile} timeout=${timeout} ${size} = Get File Size ${pidfile} Should Not Be Equal As Integers ${size} 0 @@ -70,7 +70,7 @@ Generic Teardown [Arguments] @{ports} Run Keyword If '${CONTROLLER_ERRORS}' == 'True' Check Controller Errors Shutdown Process With Children ${RSPAMD_PID} - Save Run Results ${TMPDIR} rspamd.log redis.log rspamd.conf + Save Run Results ${TMPDIR} rspamd.log redis.log rspamd.conf clickhouse-server.log clickhouse-server.err.log clickhouse-config.xml Cleanup Temporary Directory ${TMPDIR} Log Logs