From: Jason Ish Date: Wed, 29 Nov 2017 12:33:55 +0000 (-0600) Subject: new commands: add-source, list-sources, list-enabled-sources X-Git-Tag: 1.0.0a1~35 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0ee8f2cd6781f4ff196a168a5efeea9c5ed33240;p=thirdparty%2Fsuricata-update.git new commands: add-source, list-sources, list-enabled-sources --- diff --git a/doc/index.rst b/doc/index.rst index 199484a..f30453a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -145,16 +145,14 @@ Options .. option:: --etopen - Download the ET open ruleset. This is the default if ``--url`` or - ``--etpro`` are not provided. + Download the ET/Open ruleset. - If one of ``etpro`` or ``--url`` is also specified, this option - will at the ET open URL to the list of remote ruleset to be - downloaded. + This is the default action of no ``--url`` options are provided or + no sources are configured. -.. option:: --etpro= - - Download the ET pro ruleset using the provided code. + Use this option to enable the ET/Open ruleset in addition to any + URLs provided on the command line or sources provided in the + configuration. .. option:: -q, --quiet diff --git a/setup.py b/setup.py index 9ae634e..e51d8ff 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ setup( packages=[ "suricata", "suricata.update", + "suricata.update.commands", "suricata.update.configs", "suricata.update.compat", "suricata.update.compat.argparse", diff --git a/suricata/update/commands/__init__.py b/suricata/update/commands/__init__.py new file mode 100644 index 0000000..78af72d --- /dev/null +++ b/suricata/update/commands/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from suricata.update.commands import listenabledsources +from suricata.update.commands import addsource +from suricata.update.commands import listsources diff --git a/suricata/update/commands/addsource.py b/suricata/update/commands/addsource.py new file mode 100644 index 0000000..cdff4d1 --- /dev/null +++ b/suricata/update/commands/addsource.py @@ -0,0 +1,54 @@ +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from __future__ import print_function + +import logging + +from suricata.update import sources + +logger = logging.getLogger() + +def register(parser): + parser.add_argument("--name", metavar="", help="Name of source") + parser.add_argument("--url", metavar="", help="Source URL") + parser.set_defaults(func=add_source) + +def add_source(config): + args = config.args + + if args.name: + name = args.name + else: + while True: + name = raw_input("Name of source: ").strip() + if name: + break + + if sources.source_name_exists(name): + logger.error("A source with name %s already exists.", name) + return 1 + + if args.url: + url = args.url + else: + while True: + url = raw_input("URL: ").strip() + if url: + break + + source_config = sources.SourceConfiguration(name, url=url) + sources.save_source_config(source_config) diff --git a/suricata/update/commands/listenabledsources.py b/suricata/update/commands/listenabledsources.py new file mode 100644 index 0000000..e6c9931 --- /dev/null +++ b/suricata/update/commands/listenabledsources.py @@ -0,0 +1,57 @@ +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from __future__ import print_function + +import logging + +from suricata.update import sources + +logger = logging.getLogger() + +def register(parser): + parser.set_defaults(func=list_enabled_sources) + +def list_enabled_sources(config): + + found = False + + # First list sources from the main config. + config_sources = config.get("sources") + if config_sources: + found = True + print("From %s:" % (config.filename)) + for source in config_sources: + print(" - %s" % (source)) + + # And local files. + local = config.get("local") + if local: + found = True + print("Local files/directories:") + for filename in local: + print(" - %s" % (filename)) + + enabled_sources = sources.get_enabled_sources() + if enabled_sources: + found = True + print("Enabled sources:") + for source in enabled_sources.values(): + print(" - %s" % (source["source"])) + + # If no enabled sources were found, log it. + if not found: + logger.warning("No enabled sources.") diff --git a/suricata/update/commands/listsources.py b/suricata/update/commands/listsources.py new file mode 100644 index 0000000..2472d8a --- /dev/null +++ b/suricata/update/commands/listsources.py @@ -0,0 +1,41 @@ +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from __future__ import print_function + +import logging + +from suricata.update import sources + +logger = logging.getLogger() + +def register(parser): + parser.set_defaults(func=list_sources) + +def list_sources(config): + if not sources.source_index_exists(config): + logger.warning( + "Source index does not exist, please run: " + "suricata-update update-sources") + return 1 + index = sources.load_source_index(config) + for name, source in index.get_sources().items(): + print("Name: %s" % (name)) + print(" Vendor: %s" % (source["vendor"])) + print(" Summary: %s" % (source["summary"])) + print(" License: %s" % (source["license"])) + if "tags" in source: + print(" Tags: %s" % ", ".join(source["tags"])) diff --git a/suricata/update/configs/update.yaml b/suricata/update/configs/update.yaml index 3382056..1448012 100644 --- a/suricata/update/configs/update.yaml +++ b/suricata/update/configs/update.yaml @@ -38,12 +38,12 @@ ignore: # Remote rule sources. sources: # Emerging Threats Open - - type: etopen + - source: etopen # Emerging Threats Pro - - type: etpro + - source: etpro code: xxxxx # A URL - - type: url + - source: url url: https://sslbl.abuse.ch/blacklist/sslblacklist.rules # A list of local rule sources. Each entry can be a rule file, a diff --git a/suricata/update/main.py b/suricata/update/main.py index 97fdd01..2905f66 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -50,6 +50,7 @@ from suricata.update import configs from suricata.update import extract from suricata.update import util from suricata.update import sources +from suricata.update import commands # Initialize logging, use colour if on a tty. if len(logging.root.handlers) == 0 and os.isatty(sys.stderr.fileno()): @@ -63,18 +64,7 @@ else: logger = logging.getLogger() # If Suricata is not found, default to this version. -DEFAULT_SURICATA_VERSION = "4.0" - -# Template URL for Emerging Threats Pro rules. -ET_PRO_URL = ("https://rules.emergingthreatspro.com/" - "%(code)s/" - "suricata%(version)s/" - "etpro.rules.tar.gz") - -# Template URL for Emerging Threats Open rules. -ET_OPEN_URL = ("https://rules.emergingthreats.net/open/" - "suricata%(version)s/" - "emerging.rules.tar.gz") +DEFAULT_SURICATA_VERSION = "4.0.0" # The default filename to use for the output rule file. This is a # single file concatenating all input rule files together. @@ -758,19 +748,12 @@ class FileTracker: return True return False -def resolve_etpro_url(etpro, suricata_version): - mappings = { - "code": etpro, - "version": "", - } - - mappings["version"] = "-%d.%d.%d" % (suricata_version.major, - suricata_version.minor, - suricata_version.patch) - - return ET_PRO_URL % mappings - def resolve_etopen_url(suricata_version): + # Template URL for Emerging Threats Open rules. + template_url = ("https://rules.emergingthreats.net/open/" + "suricata%(version)s/" + "emerging.rules.tar.gz") + mappings = { "version": "", } @@ -779,7 +762,7 @@ def resolve_etopen_url(suricata_version): suricata_version.minor, suricata_version.patch) - return ET_OPEN_URL % mappings + return template_url % mappings def ignore_file(ignore_files, filename): if not ignore_files: @@ -947,6 +930,15 @@ def load_sources(config, suricata_version): # Get the new style sources. enabled_sources = sources.get_enabled_sources() + # Convert the Suricata version to a version string. + version_string = "%d.%d.%d" % ( + suricata_version.major, suricata_version.minor, + suricata_version.patch) + + # Construct the URL replacement parameters that are internal to + # suricata-update. + internal_params = {"__version__": version_string} + # If we have new sources, we also need to load the index. if enabled_sources: index_filename = os.path.join( @@ -957,49 +949,43 @@ def load_sources(config, suricata_version): ("Run suricata-update update-sources"))) index = sources.Index(index_filename) - version_string = "%d.%d.%d" % ( - suricata_version.major, suricata_version.minor, - suricata_version.patch) - for (name, source) in enabled_sources.items(): - if "params" in source and source["params"]: - params = source["params"] + params = source["params"] if "params" in source else {} + params.update(internal_params) + if "url" in source: + # No need to go off to the index. + url = source["url"] % params else: - params = {} - params["__version__"] = version_string - url = index.resolve_url(name, params) + url = index.resolve_url(name, params) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) if config.get("sources"): for source in config.get("sources"): - if not "type" in source: - logger.error("Source is missing a type: %s", str(source)) + source_name = None + if "source" in source : + source_name = source["source"] + else: + logger.error("Source is missing the \"source\" field.") continue - if source["type"] == "url": + + if source_name == "url": urls.append(source["url"]) - elif source["type"] == "etopen": + elif source_name == "etopen": urls.append(resolve_etopen_url(suricata_version)) - elif source["type"] == "etpro": - if "code" in source: - code = source["code"] - else: - code = config.get("etpro") - suricata.update.loghandler.add_secret(code, "code") - if not code: - logger.error("ET-Pro source specified without code: %s", - str(source)) - else: - urls.append(resolve_etpro_url(code, suricata_version)) else: - logger.error("Unknown source type: %s", source["type"]) - - # If --etpro, add it. - if config.get("etpro"): - urls.append(resolve_etpro_url(config.get("etpro"), suricata_version)) + logger.error( + "Unknown source: %s; " + "try running suricata-update update-sources", + source["source"]) + + # If no URLs, default to ET/Open. + if not urls: + logger.info("No sources configured, will use Emerging Threats Open") + urls.append(resolve_etopen_url(suricata_version)) - # If --etopen or urls is still empty, add ET Open. - if config.get("etopen") or not urls: + # If --etopen is on the command line, make sure its added. + if config.get("etopen"): urls.append(resolve_etopen_url(suricata_version)) # Converting the URLs to a set removed dupes. @@ -1033,7 +1019,7 @@ def main(): sys.argv.insert(1, "update") # Support the Python argparse style of configuration file. - parser = argparse.ArgumentParser(fromfile_prefix_chars="@", add_help=False) + parser = argparse.ArgumentParser() # Arguments that are common to all sub-commands. common_parser = argparse.ArgumentParser(add_help=False) @@ -1049,7 +1035,7 @@ def main(): "-o", "--output", metavar="", dest="output", default="/var/lib/suricata/rules", help="Directory to write rules to") - subparsers = parser.add_subparsers(dest="subcommand") + subparsers = parser.add_subparsers(dest="subcommand", metavar="") # The "update" (default) sub-command parser. update_parser = subparsers.add_parser( @@ -1099,8 +1085,6 @@ def main(): update_parser.add_argument("--dump-sample-configs", action="store_true", default=False, help="Dump sample config files to current directory") - update_parser.add_argument("--etpro", metavar="", - help="Use ET-Pro rules with provided ET-Pro code") update_parser.add_argument("--etopen", action="store_true", help="Use ET-Open rules (default)") update_parser.add_argument("--reload-command", metavar="", @@ -1130,9 +1114,6 @@ def main(): help=argparse.SUPPRESS) update_parser.add_argument("--drop", default=False, help=argparse.SUPPRESS) - list_sources_parser = subparsers.add_parser( - "list-sources", parents=[common_parser]) - disable_source_parser = subparsers.add_parser( "disable-source", parents=[common_parser]) disable_source_parser.add_argument("name") @@ -1149,6 +1130,13 @@ def main(): update_sources_parser = subparsers.add_parser( "update-sources", parents=[common_parser]) + commands.listsources.register(subparsers.add_parser( + "list-sources", parents=[common_parser])) + commands.listenabledsources.register(subparsers.add_parser( + "list-enabled-sources", parents=[common_parser])) + commands.addsource.register(subparsers.add_parser( + "add-source", parents=[common_parser])) + args = parser.parse_args() # Error out if any reserved/unimplemented arguments were set. @@ -1182,14 +1170,14 @@ def main(): if args.subcommand: if args.subcommand == "update-sources": return sources.update_sources(config) - elif args.subcommand == "list-sources": - return sources.list_sources(config) elif args.subcommand == "enable-source": return sources.enable_source(config) elif args.subcommand == "disable-source": return sources.disable_source(config) elif args.subcommand == "remove-source": return sources.remove_source(config) + elif hasattr(args, "func"): + return args.func(config) elif args.subcommand != "update": logger.error("Unknown command: %s", args.subcommand) return 1 @@ -1209,6 +1197,8 @@ def main(): enable-source Enable a source from the index disable-source Disable an enabled source remove-source Remove an enabled or disabled source + list-enabled-sources List all enabled sources + add-source Add a new source by URL """) return 0 @@ -1291,7 +1281,8 @@ def main(): os.makedirs(config.get_cache_dir(), mode=0o770) except Exception as err: logger.warning( - "Cache directory does exist and could not be created. /var/tmp will be used instead.") + "Cache directory does not exist and could not be created. " + "/var/tmp will be used instead.") config.set_cache_dir("/var/tmp") files = load_sources(config, suricata_version) diff --git a/suricata/update/sources.py b/suricata/update/sources.py index 98eebed..bb8e9ed 100644 --- a/suricata/update/sources.py +++ b/suricata/update/sources.py @@ -36,18 +36,57 @@ ENABLED_SOURCE_DIRECTORY = "/var/lib/suricata/update/sources" def get_index_filename(config): return os.path.join(config.get_cache_dir(), SOURCE_INDEX_FILENAME) +def get_enabled_source_filename(name): + return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % ( + safe_filename(name))) + +def get_disabled_source_filename(name): + return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % ( + safe_filename(name))) + +def source_name_exists(name): + """Return True if a source already exists with name.""" + if os.path.exists(get_enabled_source_filename(name)) or \ + os.path.exists(get_disabled_source_filename(name)): + return True + return False + +def source_index_exists(config): + """Return True if the source index file exists.""" + return os.path.exists(get_index_filename(config)) + +def save_source_config(source_config): + with open(get_enabled_source_filename(source_config.name), "wb") as fileobj: + fileobj.write(yaml.safe_dump( + source_config.dict(), default_flow_style=False)) + +class SourceConfiguration: + + def __init__(self, name, url=None, params={}): + self.name = name + self.url = url + self.params = params + + def dict(self): + d = { + "source": self.name, + } + if self.url: + d["url"] = self.url + if self.params: + d["params"] = self.params + return d + class Index: def __init__(self, filename): self.filename = filename - self.index = {} self.reload() def reload(self): - if os.path.exists(self.filename): - index = yaml.load(open(self.filename)) - self.index = index + index = yaml.load(open(self.filename, "rb")) + self.index = index def resolve_url(self, name, params={}): if not name in self.index["sources"]: @@ -58,6 +97,12 @@ class Index: except KeyError as err: raise Exception("Missing URL parameter: %s" % (str(err.args[0]))) + def get_sources(self): + return self.index["sources"] + +def load_source_index(config): + return Index(get_index_filename(config)) + def get_enabled_sources(): """Return a map of enabled sources, keyed by name.""" if not os.path.exists(ENABLED_SOURCE_DIRECTORY): @@ -93,6 +138,13 @@ def update_sources(config): net.get(get_source_index_url(config), fileobj) except Exception as err: raise Exception("Failed to download index: %s: %s" % (url, err)) + if not os.path.exists(config.get_cache_dir()): + try: + os.makedirs(config.get_cache_dir()) + except Exception as err: + logger.error("Failed to create directory %s: %s", + config.get_cache_dir(), err) + return 1 with open(source_cache_filename, "w") as outobj: outobj.write(fileobj.getvalue()) logger.debug("Saved %s", source_cache_filename) @@ -105,17 +157,6 @@ def load_sources(config): return index["sources"] return {} -def list_sources(config): - sources = load_sources(config) - if not sources: - logger.error("No sources exist. Try running update-sources.") - return - for name, source in sources.items(): - print("Name: %s" % (name)) - print(" Vendor: %s" % (source["vendor"])) - print(" Description: %s" % (source["description"])) - print(" License: %s" % (source["license"])) - def enable_source(config): name = config.args.name @@ -153,6 +194,11 @@ def enable_source(config): opts[key] = val source = sources[config.args.name] + + if "subscribe-url" in source: + print("The source %s requires a subscription. Subscribe here:" % (name)) + print(" %s" % source["subscribe-url"]) + params = {} if "parameters" in source: for param in source["parameters"]: @@ -160,14 +206,13 @@ def enable_source(config): params[param] = opts[param] else: prompt = source["parameters"][param]["prompt"] - r = raw_input("%s (%s): " % (prompt, param)) + while True: + r = raw_input("%s (%s): " % (prompt, param)) + r = r.strip() + if r: + break params[param] = r.strip() - new_source = { - "source": name, - } - if params: - new_source["params"] = params - new_sources = [new_source] + new_source = SourceConfiguration(name, params=params).dict() if not os.path.exists(ENABLED_SOURCE_DIRECTORY): try: @@ -215,14 +260,6 @@ def remove_source(config): logger.warning("Source %s does not exist.", name) return 1 -def get_enabled_source_filename(name): - return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % ( - safe_filename(name))) - -def get_disabled_source_filename(name): - return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % ( - safe_filename(name))) - def safe_filename(name): """Utility function to make a source short-name safe as a filename."""