From: Jason Ish Date: Thu, 22 Mar 2018 21:35:21 +0000 (-0600) Subject: bundle the index; for when it can't be downloaded X-Git-Tag: 1.0.0rc1~14 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e5b1fe25117e9143b405248cc8420dc1eb436516;p=thirdparty%2Fsuricata-update.git bundle the index; for when it can't be downloaded Bundle a copy of the index and fall back to it when the index can't be downloaded. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 989e906..3a95974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## unreleased - Python 3 fixes. +- Bundle a copy of the index which can be used if download source for + the index is not available, and no index was previously + downloaded. Warnings will be issued. ## 1.0.0b1 - 2018-01-19 - Various fixes for Python 3. diff --git a/Makefile b/Makefile index f65902a..577b562 100644 --- a/Makefile +++ b/Makefile @@ -45,3 +45,6 @@ sdist: sdist-upload: python setup.py sdist upload + +update-index: + python -m suricata.update.data.update diff --git a/setup.py b/setup.py index 2367f75..227fed3 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup( "suricata.update.configs", "suricata.update.compat", "suricata.update.compat.argparse", + "suricata.update.data", ], url="https://github.com/OISF/suricata-update", license="GPLv2", diff --git a/suricata/update/commands/enablesource.py b/suricata/update/commands/enablesource.py index 6d92c1a..9600618 100644 --- a/suricata/update/commands/enablesource.py +++ b/suricata/update/commands/enablesource.py @@ -51,10 +51,8 @@ def enable_source(): return 0 if not os.path.exists(sources.get_index_filename()): - logger.warning( - "Source index does not exist, " - "try running suricata-update update-sources.") - return 1 + logger.warning("Source index does not exist, will use bundled one.") + logger.warning("Please run suricata-update update-sources.") source_index = sources.load_source_index(config) diff --git a/suricata/update/commands/listsources.py b/suricata/update/commands/listsources.py index 26a8540..230a99e 100644 --- a/suricata/update/commands/listsources.py +++ b/suricata/update/commands/listsources.py @@ -22,6 +22,7 @@ from suricata.update import config from suricata.update import sources from suricata.update import util from suricata.update.commands.updatesources import update_sources +from suricata.update import exceptions logger = logging.getLogger() @@ -31,7 +32,10 @@ def register(parser): def list_sources(): if not sources.source_index_exists(config): logger.info("No source index found, running update-sources") - update_sources() + try: + update_sources() + except exceptions.ApplicationError as err: + logger.warning("%s: will use bundled index.", err) index = sources.load_source_index(config) for name, source in index.get_sources().items(): print("%s: %s" % (util.bright_cyan("Name"), util.bright_magenta(name))) diff --git a/suricata/update/data/__init__.py b/suricata/update/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/suricata/update/data/index.py b/suricata/update/data/index.py new file mode 100644 index 0000000..9a1cbe5 --- /dev/null +++ b/suricata/update/data/index.py @@ -0,0 +1 @@ +index = {'sources': {'oisf/trafficid': {'vendor': 'OISF', 'license': 'MIT', 'url': 'https://raw.githubusercontent.com/jasonish/suricata-trafficid/master/rules/traffic-id.rules', 'min-version': '4.0.0', 'support-url': 'https://redmine.openinfosecfoundation.org/', 'summary': 'Suricata Traffic ID ruleset'}, 'ptresearch/attackdetection': {'vendor': 'Positive Technologies', 'description': u'The Attack Detection Team searches for new vulnerabilities and 0-days, reproduces it and creates PoC exploits to understand how these security flaws work and how related attacks can be detected on the network layer. Additionally, we are interested in malware and hackers\u2019 TTPs, so we develop Suricata rules for detecting all sorts of such activities.\n', 'license': 'Custom', 'url': 'https://raw.githubusercontent.com/ptresearch/AttackDetection/master/pt.rules.tar.gz', 'license-url': 'https://raw.githubusercontent.com/ptresearch/AttackDetection/master/LICENSE', 'summary': 'Positive Technologies Attack Detection Team ruleset'}, 'sslbl/ssl-fp-blacklist': {'url': 'https://sslbl.abuse.ch/blacklist/sslblacklist.rules', 'vendor': 'Abuse.ch', 'license': 'Non-Commercial', 'summary': 'Abuse.ch SSL Blacklist'}, 'et/open': {'url': 'https://rules.emergingthreats.net/open/suricata-%(__version__)s/emerging.rules.tar.gz', 'vendor': 'Proofpoint', 'license': 'MIT', 'summary': 'Emerging Threats Open Ruleset'}, 'scwx/security': {'vendor': 'Secureworks', 'description': 'Broad ruleset composed of malware rules and other security-related countermeasures, and curated by the Secureworks Counter Threat Unit research team.\n', 'license': 'Commercial', 'url': 'https://ws.secureworks.com/ti/ruleset/%(secret-code)s/Suricata_suricata-security_latest.tgz', 'summary': 'Secureworks suricata-security ruleset.', 'min-version': '2.0.9', 'subscribe-url': 'https://www.secureworks.com/contact/ (Please reference CTU Countermeasures)', 'parameters': {'secret-code': {'prompt': 'Secureworks Threat Intelligence Authentication Token'}}}, 'scwx/malware': {'vendor': 'Secureworks', 'description': 'High-fidelity, high-priority ruleset composed mainly of malware-related countermeasures and curated by the Secureworks Counter Threat Unit research team.\n', 'license': 'Commercial', 'url': 'https://ws.secureworks.com/ti/ruleset/%(secret-code)s/Suricata_suricata-malware_latest.tgz', 'summary': 'Secureworks suricata-malware ruleset.', 'min-version': '2.0.9', 'subscribe-url': 'https://www.secureworks.com/contact/ (Please reference CTU Countermeasures)', 'parameters': {'secret-code': {'prompt': 'Secureworks Threat Intelligence Authentication Token'}}}, 'et/pro': {'replaces': ['et/open'], 'vendor': 'Proofpoint', 'description': 'Proofpoint ET Pro is a timely and accurate rule set for detecting and blocking advanced threats\n', 'license': 'Commercial', 'url': 'https://rules.emergingthreatspro.com/%(secret-code)s/suricata-%(__version__)s/etpro.rules.tar.gz', 'summary': 'Emerging Threats Pro Ruleset', 'subscribe-url': 'https://www.proofpoint.com/us/threat-insight/et-pro-ruleset', 'parameters': {'secret-code': {'prompt': 'Emerging Threats Pro access code'}}}}, 'version': 1} \ No newline at end of file diff --git a/suricata/update/data/update.py b/suricata/update/data/update.py new file mode 100644 index 0000000..f4edb11 --- /dev/null +++ b/suricata/update/data/update.py @@ -0,0 +1,37 @@ +# Copyright (C) 2018 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +import os.path + +try: + from urllib2 import urlopen +except: + from urllib.request import urlopen + +import yaml + +def embed_index(): + """Embed a copy of the index as a Python source file. We can't use a + datafile yet as there is no easy way to do with distutils.""" + dist_filename = os.path.join(os.path.dirname(__file__), "index.py") + url = "https://raw.githubusercontent.com/jasonish/suricata-intel-index/master/index.yaml" + response = urlopen(url) + index = yaml.safe_load(response.read()) + with open(dist_filename, "w") as fileobj: + fileobj.write("index = %s" % (str(index))) + +if __name__ == "__main__": + embed_index() diff --git a/suricata/update/main.py b/suricata/update/main.py index e4fff0a..984dd21 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -921,12 +921,11 @@ def load_sources(suricata_version): # If we have new sources, we also need to load the index. if enabled_sources: - index_filename = os.path.join( - config.get_cache_dir(), sources.SOURCE_INDEX_FILENAME) - if os.path.exists(index_filename): - index = sources.Index(index_filename) - else: - index = None + index_filename = sources.get_index_filename() + if not os.path.exists(index_filename): + logger.warning("No index exists, will use bundled index.") + logger.warning("Please run suricata-update update-sources.") + index = sources.Index(index_filename) for (name, source) in enabled_sources.items(): params = source["params"] if "params" in source else {} diff --git a/suricata/update/sources.py b/suricata/update/sources.py index 9368d6c..d786063 100644 --- a/suricata/update/sources.py +++ b/suricata/update/sources.py @@ -27,6 +27,7 @@ from suricata.update import config from suricata.update import net from suricata.update import util from suricata.update import loghandler +from suricata.update.data.index import index as bundled_index logger = logging.getLogger() @@ -96,11 +97,14 @@ class Index: def __init__(self, filename): self.filename = filename self.index = {} - self.reload() - - def reload(self): - index = yaml.safe_load(open(self.filename, "rb")) - self.index = index + self.load() + + def load(self): + if os.path.exists(self.filename): + index = yaml.safe_load(open(self.filename, "rb")) + self.index = index + else: + self.index = bundled_index def resolve_url(self, name, params={}): if not name in self.index["sources"]: