integration test - executes suricata-update

And checks exit codes and outputs. Can be run
as a script, or as a tox setup where it will
be run under multiple Python versions.

remove integration test from unit tests.

diff --git a/Makefile b/Makefile
index 0dd08b0374edde2fe888f08a85810456f0b6d1e8..65d93f3c649011c57acb5f1762033e202600b808 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -8,13 +8,18 @@ build:
 install:
        python setup.py install
 
-test:
+tox:
        @if ! which tox 2>&1 > /dev/null; then \
                echo "error: tox required to run tests"; \
                exit 1; \
        fi
+
+test: tox
        @tox
 
+integration-test: tox
+       @tox -c tox-integration.ini
+
 clean:
        find . -name \*.pyc -print0 | xargs -0 rm -f
        find . -name \*~ -print0 | xargs -0 rm -f

diff --git a/tests/empty b/tests/empty
new file mode 100644 (file)
index 0000000..3c0e2c9
--- /dev/null
+++ b/tests/empty
@@ -0,0 +1 @@
+# An empty configuration for test purposes.

diff --git a/tests/index.yaml b/tests/index.yaml
new file mode 100644 (file)
index 0000000..f515a0b
--- /dev/null
+++ b/tests/index.yaml
@@ -0,0 +1,51 @@
+# This is a version 1 formatted index.
+version: 1
+
+sources:
+
+  # Proofpoint/Emerging Threats Open ruleset.
+  et/open:
+    vendor: Proofpoint
+    license: MIT
+    summary: Emerging Threats Open Ruleset
+    url: https://rules.emergingthreats.net/open/suricata-%(__version__)s/emerging.rules.tar.gz
+
+  # Proofpoint/Emerging Threats Pro ruleset.
+  et/pro:
+    summary: Emerging Threats Pro Ruleset
+    description: |
+      Proofpoint ET Pro is a timely and accurate rule set for detecting and blocking advanced threats
+    vendor: Proofpoint
+    license: Commercial
+    url: https://rules.emergingthreatspro.com/%(secret-code)s/suricata-%(__version__)s/etpro.rules.tar.gz
+    subscribe-url: https://www.proofpoint.com/us/threat-insight/et-pro-ruleset
+    parameters:
+      secret-code:
+        prompt: Emerging Threats Pro access code
+    replaces:
+      - et/open
+
+  # The OISF Traffic ID ruleset.
+  oisf/trafficid:
+    vendor: OISF
+    summary: Suricata Traffic ID ruleset
+    license: MIT
+    url: https://raw.githubusercontent.com/jasonish/suricata-trafficid/master/rules/traffic-id.rules
+    support-url: https://redmine.openinfosecfoundation.org/
+    min-version: 4.0.0
+
+  ptresearch/attackdetection:
+    vendor: Positive Technologies
+    summary: Positive Technologies Attack Detection Team ruleset
+    description: |
+      The Attack Detection Team searches for new vulnerabilities and 0-days, reproduces it and creates PoC exploits to understand how these security flaws work and how related attacks can be detected on the network layer. Additionally, we are interested in malware and hackers’ TTPs, so we develop Suricata rules for detecting all sorts of such activities.
+    url: https://raw.githubusercontent.com/ptresearch/AttackDetection/master/pt.rules.tar.gz
+    license: Custom
+    license-url: https://raw.githubusercontent.com/ptresearch/AttackDetection/master/LICENSE
+
+  # SSBL FP blacklist ruleset.
+  sslbl/ssl-fp-blacklist:
+    summary: Abuse.ch SSL Blacklist
+    vendor: Abuse.ch
+    license: Non-Commercial
+    url: https://sslbl.abuse.ch/blacklist/sslblacklist.rules

diff --git a/tests/integration_tests.py b/tests/integration_tests.py
new file mode 100755 (executable)
index 0000000..1772fd0
--- /dev/null
+++ b/tests/integration_tests.py
@@ -0,0 +1,83 @@
+import sys
+import os
+import subprocess
+import shutil
+
+DATA_DIR = "./tests/tmp"
+
+def run(args):
+    subprocess.check_call(args)
+
+def delete(path):
+    if os.path.isdir(path):
+        shutil.rmtree(path)
+    else:
+        os.unlink(path)
+
+print("Python executable: %s" % sys.executable)
+print("Current directory: %s" % os.getcwd())
+
+# Override the default source index URL to avoid hitting the network.
+os.environ["SOURCE_INDEX_URL"] = "file://%s/tests/index.yaml" % (
+    os.getcwd())
+
+os.environ["ETOPEN_URL"] = "file://%s/tests/emerging.rules.tar.gz" % (
+    os.getcwd())
+
+if os.path.exists(DATA_DIR):
+    delete(DATA_DIR)
+
+common_args = [
+    "./bin/suricata-update",
+    "-D", DATA_DIR,
+    "-c" "./tests/empty",
+]
+
+common_update_args = [
+    "--no-test",
+    "--no-reload",
+    "--disable-conf", "./tests/empty",
+    "--enable-conf", "./tests/empty",
+    "--drop-conf", "./tests/empty",
+    "--modify-conf", "./tests/empty",
+]
+
+# Default run with data directory.
+run(common_args + common_update_args)
+assert(os.path.exists(DATA_DIR))
+assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache")))
+assert(os.path.exists(os.path.join(DATA_DIR, "rules", "suricata.rules")))
+
+# Still a default run, but set --output to an alternate location."
+run(common_args + common_update_args + ["--output", "./tests/tmp/_rules"])
+assert(os.path.exists(os.path.join(DATA_DIR, "_rules")))
+
+# Update sources.
+run(common_args + ["update-sources"])
+assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache", "index.yaml")))
+
+# Now delete the index and run lists-sources to see if it downloads
+# the index.
+delete(os.path.join(DATA_DIR, "update", "cache", "index.yaml"))
+run(common_args + ["list-sources"])
+assert(os.path.exists(os.path.join(DATA_DIR, "update", "cache", "index.yaml")))
+
+# Enable a source.
+run(common_args + ["enable-source", "oisf/trafficid"])
+assert(os.path.exists(
+    os.path.join(DATA_DIR, "update", "sources", "oisf-trafficid.yaml")))
+
+# Disable the source.
+run(common_args + ["disable-source", "oisf/trafficid"])
+assert(not os.path.exists(
+    os.path.join(
+        DATA_DIR, "update", "sources", "oisf-trafficid.yaml")))
+assert(os.path.exists(
+    os.path.join(
+        DATA_DIR, "update", "sources", "oisf-trafficid.yaml.disabled")))
+
+# Remove the source.
+run(common_args + ["remove-source", "oisf/trafficid"])
+assert(not os.path.exists(
+    os.path.join(
+        DATA_DIR, "update", "sources", "oisf-trafficid.yaml.disabled")))

diff --git a/tests/test_main.py b/tests/test_main.py
index 1ed6ffb25b06d894e57b81fe9ad28f23db8c8f55..874f518359c1b8f2abb0ca0509e8635751888f5c 100644 (file)
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -17,36 +17,13 @@
 
 from __future__ import print_function
 
-import sys
 import os
 import unittest
-import shlex
-import re
-import subprocess
-import shutil
 
 import suricata.update.rule
 from suricata.update import main
 import suricata.update.extract
 
-def has_python2():
-    r = subprocess.call(
-        ["python2", "--version"],
-        stderr=open("/dev/null", "wb"),
-        stdout=open("/dev/null", "wb"))
-    if r == 0:
-        return True
-    return False
-
-def has_python3():
-    r = subprocess.call(
-        ["python3", "--version"],
-        stderr=open("/dev/null", "wb"),
-        stdout=open("/dev/null", "wb"))
-    if r == 0:
-        return True
-    return False
-
 class TestRulecat(unittest.TestCase):
 
     def test_extract_tar(self):
@@ -72,49 +49,6 @@ class TestRulecat(unittest.TestCase):
             "tests/emerging-current_events.rules")
         self.assertIsNone(files)
 
-    def test_run(self):
-        old_path = os.getcwd()
-        try:
-            os.chdir(os.path.dirname(os.path.realpath(__file__)))
-            if os.path.exists("./tmp"):
-                shutil.rmtree("tmp")
-            os.makedirs("./tmp/rules")
-            subprocess.check_call(
-                ["/usr/bin/env", sys.executable,
-                 "../bin/suricata-update",
-                 "-D", "./tmp",
-                 "-v",
-                 "-c", "./update.yaml",
-                 "--url",
-                 "file://%s/emerging.rules.tar.gz" % (
-                     os.getcwd()),
-                 "--local", "./rule-with-unicode.rules",
-                 "--force",
-                 "--output", "./tmp/rules/",
-                 "--yaml-fragment", "./tmp/suricata-rules.yaml",
-                 "--sid-msg-map", "./tmp/sid-msg.map",
-                 "--sid-msg-map-2", "./tmp/sid-msg-v2.map",
-                 "--no-test",
-                 "--reload-command", "true",
-                ],
-                env={
-                    "PATH": os.getenv("PATH"),
-                },
-                stdout=open("./tmp/stdout", "wb"),
-                stderr=open("./tmp/stderr", "wb"),
-            )
-            shutil.rmtree("tmp")
-        except:
-            if os.path.exists("./tmp/stdout"):
-                print("STDOUT")
-                print(open("./tmp/stdout").read())
-            if os.path.exists("./tmp/stderr"):
-                print("STDERR")
-                print(open("./tmp/stderr").read())
-            raise
-        finally:
-            os.chdir(old_path)
-
 class TestFetch(unittest.TestCase):
 
     def test_check_checksum(self):

diff --git a/tox-integration.ini b/tox-integration.ini
new file mode 100644 (file)
index 0000000..238766d
--- /dev/null
+++ b/tox-integration.ini
@@ -0,0 +1,13 @@
+# Tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py27, py34, py35, py36
+
+[testenv]
+commands = python ./tests/integration_tests.py
+deps =
+    pytest
+    pyyaml