From: Jason Ish Date: Tue, 28 Nov 2017 04:21:52 +0000 (-0600) Subject: update-sources: new command to download source index X-Git-Tag: 1.0.0a1~37 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0b961528c60d86d154bdcbd67369ebc4d1a8bcac;p=thirdparty%2Fsuricata-update.git update-sources: new command to download source index --- diff --git a/suricata/update/main.py b/suricata/update/main.py index 219bdaf..b1984d4 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -49,6 +49,7 @@ import suricata.update.loghandler from suricata.update import configs from suricata.update import extract from suricata.update import util +from suricata.update import sources # Initialize logging, use colour if on a tty. if len(logging.root.handlers) == 0 and os.isatty(sys.stderr.fileno()): @@ -807,7 +808,7 @@ class Config: self.args = args self.config = {} self.config.update(self.DEFAULTS) - + self.filename = self.DEFAULT_LOCATIONS[0] self.cache_dir = None def load(self): @@ -865,6 +866,13 @@ class Config: def set_cache_dir(self, directory): self.cache_dir = directory + def save_new_source(self, source): + config = yaml.load(open(self.filename)) + if not "sources" in config: + config["sources"] = [] + config["sources"].append(source) + print(yaml.dump(config, default_flow_style=False)) + def test_suricata(config, suricata_path): if not suricata_path: logger.info("No suricata application binary found, skipping test.") @@ -993,85 +1001,118 @@ def main(): suricata_path = suricata.update.engine.get_path() + # If no command given, default to the "update" command. + if len(sys.argv) == 1 or sys.argv[1].startswith("-"): + sys.argv.insert(1, "update") + # Support the Python argparse style of configuration file. - parser = argparse.ArgumentParser(fromfile_prefix_chars="@") - - parser.add_argument("-v", "--verbose", action="store_true", default=False, - help="Be more verbose") - parser.add_argument("-c", "--config", metavar="", - help="Configuration file") - parser.add_argument("-o", "--output", metavar="", - dest="output", default="/var/lib/suricata/rules", - help="Directory to write rules to") - parser.add_argument("--suricata", metavar="", - help="Path to Suricata program") - parser.add_argument("--suricata-version", metavar="", - help="Override Suricata version") - parser.add_argument("-f", "--force", action="store_true", default=False, - help="Force operations that might otherwise be skipped") - parser.add_argument("--yaml-fragment", metavar="", - help="Output YAML fragment for rule inclusion") - parser.add_argument("--url", metavar="", action="append", - default=[], - help="URL to use instead of auto-generating one (can be specified multiple times)") - parser.add_argument("--local", metavar="", action="append", - default=[], - help="Local rule files or directories (can be specified multiple times)") - parser.add_argument("--sid-msg-map", metavar="", - help="Generate a sid-msg.map file") - parser.add_argument("--sid-msg-map-2", metavar="", - help="Generate a v2 sid-msg.map file") - - parser.add_argument("--disable-conf", metavar="", - help="Filename of rule disable filters") - parser.add_argument("--enable-conf", metavar="", - help="Filename of rule enable filters") - parser.add_argument("--modify-conf", metavar="", - help="Filename of rule modification filters") - parser.add_argument("--drop-conf", metavar="", - help="Filename of drop rules filters") - - parser.add_argument("--ignore", metavar="", action="append", - default=[], - help="Filenames to ignore (can be specified multiple times; default: *deleted.rules)") - parser.add_argument("--no-ignore", action="store_true", default=False, - help="Disables the ignore option.") - - parser.add_argument("--threshold-in", metavar="", - help="Filename of rule thresholding configuration") - parser.add_argument("--threshold-out", metavar="", - help="Output of processed threshold configuration") - - parser.add_argument("--dump-sample-configs", action="store_true", - default=False, - help="Dump sample config files to current directory") - parser.add_argument("--etpro", metavar="", - help="Use ET-Pro rules with provided ET-Pro code") - parser.add_argument("--etopen", action="store_true", - help="Use ET-Open rules (default)") - parser.add_argument("-q", "--quiet", action="store_true", default=False, - help="Be quiet, warning and error messages only") - parser.add_argument("--reload-command", metavar="", - help="Command to run after update if modified") - parser.add_argument("--no-reload", action="store_true", default=False, - help="Disable reload") - parser.add_argument("-T", "--test-command", metavar="", - help="Command to test Suricata configuration") - parser.add_argument("--no-test", action="store_true", default=False, - help="Disable testing rules with Suricata") - parser.add_argument("-V", "--version", action="store_true", default=False, - help="Display version") - - parser.add_argument("--no-merge", action="store_true", default=False, - help="Do not merge the rules into a single file") + parser = argparse.ArgumentParser(fromfile_prefix_chars="@", add_help=False) + + # Arguments that are common to all sub-commands. + common_parser = argparse.ArgumentParser(add_help=False) + common_parser.add_argument( + "-c", "--config", metavar="", help="Configuration file") + common_parser.add_argument( + "-v", "--verbose", action="store_true", default=False, + help="Be more verbose") + common_parser.add_argument( + "-q", "--quiet", action="store_true", default=False, + help="Be quiet, warning and error messages only") + common_parser.add_argument( + "-o", "--output", metavar="", dest="output", + default="/var/lib/suricata/rules", help="Directory to write rules to") + + subparsers = parser.add_subparsers(dest="subcommand") + + # The "update" (default) sub-command parser. + update_parser = subparsers.add_parser( + "update", add_help=False, parents=[common_parser]) + + update_parser.add_argument("--suricata", metavar="", + help="Path to Suricata program") + update_parser.add_argument("--suricata-version", metavar="", + help="Override Suricata version") + update_parser.add_argument("-f", "--force", action="store_true", + default=False, + help="Force operations that might otherwise be skipped") + update_parser.add_argument("--yaml-fragment", metavar="", + help="Output YAML fragment for rule inclusion") + update_parser.add_argument("--url", metavar="", action="append", + default=[], + help="URL to use instead of auto-generating one (can be specified multiple times)") + update_parser.add_argument("--local", metavar="", action="append", + default=[], + help="Local rule files or directories (can be specified multiple times)") + update_parser.add_argument("--sid-msg-map", metavar="", + help="Generate a sid-msg.map file") + update_parser.add_argument("--sid-msg-map-2", metavar="", + help="Generate a v2 sid-msg.map file") + + update_parser.add_argument("--disable-conf", metavar="", + help="Filename of rule disable filters") + update_parser.add_argument("--enable-conf", metavar="", + help="Filename of rule enable filters") + update_parser.add_argument("--modify-conf", metavar="", + help="Filename of rule modification filters") + update_parser.add_argument("--drop-conf", metavar="", + help="Filename of drop rules filters") + + update_parser.add_argument("--ignore", metavar="", action="append", + default=[], + help="Filenames to ignore (can be specified multiple times; default: *deleted.rules)") + update_parser.add_argument("--no-ignore", action="store_true", + default=False, + help="Disables the ignore option.") + + update_parser.add_argument("--threshold-in", metavar="", + help="Filename of rule thresholding configuration") + update_parser.add_argument("--threshold-out", metavar="", + help="Output of processed threshold configuration") + + update_parser.add_argument("--dump-sample-configs", action="store_true", + default=False, + help="Dump sample config files to current directory") + update_parser.add_argument("--etpro", metavar="", + help="Use ET-Pro rules with provided ET-Pro code") + update_parser.add_argument("--etopen", action="store_true", + help="Use ET-Open rules (default)") + update_parser.add_argument("--reload-command", metavar="", + help="Command to run after update if modified") + update_parser.add_argument("--no-reload", action="store_true", default=False, + help="Disable reload") + update_parser.add_argument("-T", "--test-command", metavar="", + help="Command to test Suricata configuration") + update_parser.add_argument("--no-test", action="store_true", default=False, + help="Disable testing rules with Suricata") + update_parser.add_argument("-V", "--version", action="store_true", default=False, + help="Display version") + + update_parser.add_argument("--no-merge", action="store_true", default=False, + help="Do not merge the rules into a single file") + update_parser.add_argument("-h", "--help", action="store_true") + # The Python 2.7 argparse module does prefix matching which can be # undesirable. Reserve some names here that would match existing # options to prevent prefix matching. - parser.add_argument("--disable", default=False, help=argparse.SUPPRESS) - parser.add_argument("--enable", default=False, help=argparse.SUPPRESS) - parser.add_argument("--modify", default=False, help=argparse.SUPPRESS) - parser.add_argument("--drop", default=False, help=argparse.SUPPRESS) + update_parser.add_argument("--disable", default=False, + help=argparse.SUPPRESS) + update_parser.add_argument("--enable", default=False, + help=argparse.SUPPRESS) + update_parser.add_argument("--modify", default=False, + help=argparse.SUPPRESS) + update_parser.add_argument("--drop", default=False, help=argparse.SUPPRESS) + + list_sources_parser = subparsers.add_parser( + "list-sources", parents=[common_parser]) + + enable_source_parser = subparsers.add_parser( + "enable-source", parents=[common_parser]) + enable_source_parser.add_argument("name") + enable_source_parser.add_argument("params", nargs="*", metavar="param=val") + + update_sources_parser = subparsers.add_parser( + "update-sources", parents=[common_parser]) args = parser.parse_args() @@ -1083,14 +1124,10 @@ def main(): "drop", ] for arg in unimplemented_args: - if getattr(args, arg): + if hasattr(args, arg) and getattr(args, arg): logger.error("--%s not implemented", arg) return 1 - if args.version: - print("suricata-update version %s" % suricata.update.version) - return 0 - if args.verbose: logger.setLevel(logging.DEBUG) if args.quiet: @@ -1100,9 +1137,6 @@ def main(): suricata.update.version, sys.version.replace("\n", "- "))) - if args.dump_sample_configs: - return dump_sample_configs() - config = Config(args) try: config.load() @@ -1110,9 +1144,35 @@ def main(): logger.error("Failed to load configuration: %s" % (err)) return 1 + if args.subcommand: + if args.subcommand == "update-sources": + return sources.update_sources(config) + elif args.subcommand == "list-sources": + return sources.list_sources(config) + elif args.subcommand == "enable-source": + return sources.enable_source(config) + elif args.subcommand != "update": + logger.error("Unknown command: %s", args.command) + return 1 + + if args.dump_sample_configs: + return dump_sample_configs() + + if args.version: + print("suricata-update version %s" % suricata.update.version) + return 0 + + if args.help: + print(update_parser.format_help()) + print("""other commands: + update-sources + list-sources + enable-source +""") + return 0 + # If --no-ignore was provided, make sure args.ignore is # empty. Otherwise if no ignores are provided, set a sane default. - if args.no_ignore: config.set("ignore", []) elif not config.get("ignore"): diff --git a/suricata/update/sources.py b/suricata/update/sources.py new file mode 100644 index 0000000..47105c1 --- /dev/null +++ b/suricata/update/sources.py @@ -0,0 +1,102 @@ +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from __future__ import print_function + +import os +import logging +import io +import argparse + +import yaml + +from suricata.update import net +from suricata.update import util + +logger = logging.getLogger() + +DEFAULT_SOURCE_INDEX_URL = "https://raw.githubusercontent.com/jasonish/suricata-intel-index/master/index.yaml" +SOURCE_INDEX_FILENAME = "index.yaml" + +def get_source_index_url(config): + if os.getenv("SOURCE_INDEX_URL"): + return os.getenv("SOURCE_INDEX_URL") + return DEFAULT_SOURCE_INDEX_URL + +def update_sources(config): + source_cache_filename = os.path.join( + config.get_cache_dir(), SOURCE_INDEX_FILENAME) + source_templates = {} + with io.BytesIO() as fileobj: + try: + url = get_source_index_url(config) + logger.debug("Downloading %s", url) + net.get(get_source_index_url(config), fileobj) + except Exception as err: + raise Exception("Failed to download index: %s: %s" % (url, err)) + with open(source_cache_filename, "w") as outobj: + outobj.write(fileobj.getvalue()) + logger.debug("Saved %s", source_cache_filename) + +def load_sources(config): + sources_cache_filename = os.path.join( + config.get_cache_dir(), SOURCE_INDEX_FILENAME) + if os.path.exists(sources_cache_filename): + index = yaml.load(open(sources_cache_filename).read()) + return index["sources"] + return {} + +def list_sources(config): + sources = load_sources(config) + if not sources: + logger.error("No sources exist. Try running update-sources.") + return + for name, source in sources.items(): + print("Name: %s" % (name)) + print(" Vendor: %s" % (source["vendor"])) + print(" Description: %s" % (source["description"])) + print(" License: %s" % (source["license"])) + +def enable_source(config): + name = config.args.name + sources = load_sources(config) + if not config.args.name in sources: + logger.error("Unknown source: %s", config.args.name) + return 1 + + # Parse key=val options. + opts = {} + for opt in config.args.params: + key, val = opt.params("=", 1) + opts[key] = val + + source = sources[config.args.name] + params = {} + if "parameters" in source: + for param in source["parameters"]: + if param in opts: + params[param] = opts[param] + else: + prompt = source["parameters"][param]["prompt"] + r = raw_input("%s (%s): " % (prompt, param)) + params[param] = r.strip() + new_source = { + "source": name, + } + if params: + new_source["params"] = params + new_sources = [new_source] + config.save_new_source(new_source) diff --git a/suricata/update/util.py b/suricata/update/util.py index 1d510b0..1913553 100644 --- a/suricata/update/util.py +++ b/suricata/update/util.py @@ -21,6 +21,7 @@ import hashlib import tempfile import atexit import shutil +import zipfile def md5_hexdigest(filename): """ Compute the MD5 checksum for the contents of the provided filename. @@ -37,3 +38,39 @@ def mktempdir(delete_on_exit=True): if delete_on_exit: atexit.register(shutil.rmtree, tmpdir, ignore_errors=True) return tmpdir + +class ZipArchiveReader: + + def __init__(self, zipfile): + self.zipfile = zipfile + self.names = self.zipfile.namelist() + + def __iter__(self): + return self + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.zipfile.close() + + def next(self): + if self.names: + name = self.names.pop(0) + if name.endswith("/"): + # Is a directory, ignore + return self.next() + return name + raise StopIteration + + def open(self, name): + return self.zipfile.open(name) + + def read(self, name): + return self.zipfile.read(name) + + @classmethod + def from_fileobj(cls, fileobj): + zf = zipfile.ZipFile(fileobj) + return cls(zf) +