.. option:: --etopen
- Download the ET open ruleset. This is the default if ``--url`` or
- ``--etpro`` are not provided.
+ Download the ET/Open ruleset.
- If one of ``etpro`` or ``--url`` is also specified, this option
- will at the ET open URL to the list of remote ruleset to be
- downloaded.
+ This is the default action of no ``--url`` options are provided or
+ no sources are configured.
-.. option:: --etpro=<code>
-
- Download the ET pro ruleset using the provided code.
+ Use this option to enable the ET/Open ruleset in addition to any
+ URLs provided on the command line or sources provided in the
+ configuration.
.. option:: -q, --quiet
packages=[
"suricata",
"suricata.update",
+ "suricata.update.commands",
"suricata.update.configs",
"suricata.update.compat",
"suricata.update.compat.argparse",
--- /dev/null
+# Copyright (C) 2017 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+from suricata.update.commands import listenabledsources
+from suricata.update.commands import addsource
+from suricata.update.commands import listsources
--- /dev/null
+# Copyright (C) 2017 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+from __future__ import print_function
+
+import logging
+
+from suricata.update import sources
+
+logger = logging.getLogger()
+
+def register(parser):
+ parser.add_argument("--name", metavar="<name>", help="Name of source")
+ parser.add_argument("--url", metavar="<url>", help="Source URL")
+ parser.set_defaults(func=add_source)
+
+def add_source(config):
+ args = config.args
+
+ if args.name:
+ name = args.name
+ else:
+ while True:
+ name = raw_input("Name of source: ").strip()
+ if name:
+ break
+
+ if sources.source_name_exists(name):
+ logger.error("A source with name %s already exists.", name)
+ return 1
+
+ if args.url:
+ url = args.url
+ else:
+ while True:
+ url = raw_input("URL: ").strip()
+ if url:
+ break
+
+ source_config = sources.SourceConfiguration(name, url=url)
+ sources.save_source_config(source_config)
--- /dev/null
+# Copyright (C) 2017 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+from __future__ import print_function
+
+import logging
+
+from suricata.update import sources
+
+logger = logging.getLogger()
+
+def register(parser):
+ parser.set_defaults(func=list_enabled_sources)
+
+def list_enabled_sources(config):
+
+ found = False
+
+ # First list sources from the main config.
+ config_sources = config.get("sources")
+ if config_sources:
+ found = True
+ print("From %s:" % (config.filename))
+ for source in config_sources:
+ print(" - %s" % (source))
+
+ # And local files.
+ local = config.get("local")
+ if local:
+ found = True
+ print("Local files/directories:")
+ for filename in local:
+ print(" - %s" % (filename))
+
+ enabled_sources = sources.get_enabled_sources()
+ if enabled_sources:
+ found = True
+ print("Enabled sources:")
+ for source in enabled_sources.values():
+ print(" - %s" % (source["source"]))
+
+ # If no enabled sources were found, log it.
+ if not found:
+ logger.warning("No enabled sources.")
--- /dev/null
+# Copyright (C) 2017 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+from __future__ import print_function
+
+import logging
+
+from suricata.update import sources
+
+logger = logging.getLogger()
+
+def register(parser):
+ parser.set_defaults(func=list_sources)
+
+def list_sources(config):
+ if not sources.source_index_exists(config):
+ logger.warning(
+ "Source index does not exist, please run: "
+ "suricata-update update-sources")
+ return 1
+ index = sources.load_source_index(config)
+ for name, source in index.get_sources().items():
+ print("Name: %s" % (name))
+ print(" Vendor: %s" % (source["vendor"]))
+ print(" Summary: %s" % (source["summary"]))
+ print(" License: %s" % (source["license"]))
+ if "tags" in source:
+ print(" Tags: %s" % ", ".join(source["tags"]))
# Remote rule sources.
sources:
# Emerging Threats Open
- - type: etopen
+ - source: etopen
# Emerging Threats Pro
- - type: etpro
+ - source: etpro
code: xxxxx
# A URL
- - type: url
+ - source: url
url: https://sslbl.abuse.ch/blacklist/sslblacklist.rules
# A list of local rule sources. Each entry can be a rule file, a
from suricata.update import extract
from suricata.update import util
from suricata.update import sources
+from suricata.update import commands
# Initialize logging, use colour if on a tty.
if len(logging.root.handlers) == 0 and os.isatty(sys.stderr.fileno()):
logger = logging.getLogger()
# If Suricata is not found, default to this version.
-DEFAULT_SURICATA_VERSION = "4.0"
-
-# Template URL for Emerging Threats Pro rules.
-ET_PRO_URL = ("https://rules.emergingthreatspro.com/"
- "%(code)s/"
- "suricata%(version)s/"
- "etpro.rules.tar.gz")
-
-# Template URL for Emerging Threats Open rules.
-ET_OPEN_URL = ("https://rules.emergingthreats.net/open/"
- "suricata%(version)s/"
- "emerging.rules.tar.gz")
+DEFAULT_SURICATA_VERSION = "4.0.0"
# The default filename to use for the output rule file. This is a
# single file concatenating all input rule files together.
return True
return False
-def resolve_etpro_url(etpro, suricata_version):
- mappings = {
- "code": etpro,
- "version": "",
- }
-
- mappings["version"] = "-%d.%d.%d" % (suricata_version.major,
- suricata_version.minor,
- suricata_version.patch)
-
- return ET_PRO_URL % mappings
-
def resolve_etopen_url(suricata_version):
+ # Template URL for Emerging Threats Open rules.
+ template_url = ("https://rules.emergingthreats.net/open/"
+ "suricata%(version)s/"
+ "emerging.rules.tar.gz")
+
mappings = {
"version": "",
}
suricata_version.minor,
suricata_version.patch)
- return ET_OPEN_URL % mappings
+ return template_url % mappings
def ignore_file(ignore_files, filename):
if not ignore_files:
# Get the new style sources.
enabled_sources = sources.get_enabled_sources()
+ # Convert the Suricata version to a version string.
+ version_string = "%d.%d.%d" % (
+ suricata_version.major, suricata_version.minor,
+ suricata_version.patch)
+
+ # Construct the URL replacement parameters that are internal to
+ # suricata-update.
+ internal_params = {"__version__": version_string}
+
# If we have new sources, we also need to load the index.
if enabled_sources:
index_filename = os.path.join(
("Run suricata-update update-sources")))
index = sources.Index(index_filename)
- version_string = "%d.%d.%d" % (
- suricata_version.major, suricata_version.minor,
- suricata_version.patch)
-
for (name, source) in enabled_sources.items():
- if "params" in source and source["params"]:
- params = source["params"]
+ params = source["params"] if "params" in source else {}
+ params.update(internal_params)
+ if "url" in source:
+ # No need to go off to the index.
+ url = source["url"] % params
else:
- params = {}
- params["__version__"] = version_string
- url = index.resolve_url(name, params)
+ url = index.resolve_url(name, params)
logger.debug("Resolved source %s to URL %s.", name, url)
urls.append(url)
if config.get("sources"):
for source in config.get("sources"):
- if not "type" in source:
- logger.error("Source is missing a type: %s", str(source))
+ source_name = None
+ if "source" in source :
+ source_name = source["source"]
+ else:
+ logger.error("Source is missing the \"source\" field.")
continue
- if source["type"] == "url":
+
+ if source_name == "url":
urls.append(source["url"])
- elif source["type"] == "etopen":
+ elif source_name == "etopen":
urls.append(resolve_etopen_url(suricata_version))
- elif source["type"] == "etpro":
- if "code" in source:
- code = source["code"]
- else:
- code = config.get("etpro")
- suricata.update.loghandler.add_secret(code, "code")
- if not code:
- logger.error("ET-Pro source specified without code: %s",
- str(source))
- else:
- urls.append(resolve_etpro_url(code, suricata_version))
else:
- logger.error("Unknown source type: %s", source["type"])
-
- # If --etpro, add it.
- if config.get("etpro"):
- urls.append(resolve_etpro_url(config.get("etpro"), suricata_version))
+ logger.error(
+ "Unknown source: %s; "
+ "try running suricata-update update-sources",
+ source["source"])
+
+ # If no URLs, default to ET/Open.
+ if not urls:
+ logger.info("No sources configured, will use Emerging Threats Open")
+ urls.append(resolve_etopen_url(suricata_version))
- # If --etopen or urls is still empty, add ET Open.
- if config.get("etopen") or not urls:
+ # If --etopen is on the command line, make sure its added.
+ if config.get("etopen"):
urls.append(resolve_etopen_url(suricata_version))
# Converting the URLs to a set removed dupes.
sys.argv.insert(1, "update")
# Support the Python argparse style of configuration file.
- parser = argparse.ArgumentParser(fromfile_prefix_chars="@", add_help=False)
+ parser = argparse.ArgumentParser()
# Arguments that are common to all sub-commands.
common_parser = argparse.ArgumentParser(add_help=False)
"-o", "--output", metavar="<directory>", dest="output",
default="/var/lib/suricata/rules", help="Directory to write rules to")
- subparsers = parser.add_subparsers(dest="subcommand")
+ subparsers = parser.add_subparsers(dest="subcommand", metavar="<command>")
# The "update" (default) sub-command parser.
update_parser = subparsers.add_parser(
update_parser.add_argument("--dump-sample-configs", action="store_true",
default=False,
help="Dump sample config files to current directory")
- update_parser.add_argument("--etpro", metavar="<etpro-code>",
- help="Use ET-Pro rules with provided ET-Pro code")
update_parser.add_argument("--etopen", action="store_true",
help="Use ET-Open rules (default)")
update_parser.add_argument("--reload-command", metavar="<command>",
help=argparse.SUPPRESS)
update_parser.add_argument("--drop", default=False, help=argparse.SUPPRESS)
- list_sources_parser = subparsers.add_parser(
- "list-sources", parents=[common_parser])
-
disable_source_parser = subparsers.add_parser(
"disable-source", parents=[common_parser])
disable_source_parser.add_argument("name")
update_sources_parser = subparsers.add_parser(
"update-sources", parents=[common_parser])
+ commands.listsources.register(subparsers.add_parser(
+ "list-sources", parents=[common_parser]))
+ commands.listenabledsources.register(subparsers.add_parser(
+ "list-enabled-sources", parents=[common_parser]))
+ commands.addsource.register(subparsers.add_parser(
+ "add-source", parents=[common_parser]))
+
args = parser.parse_args()
# Error out if any reserved/unimplemented arguments were set.
if args.subcommand:
if args.subcommand == "update-sources":
return sources.update_sources(config)
- elif args.subcommand == "list-sources":
- return sources.list_sources(config)
elif args.subcommand == "enable-source":
return sources.enable_source(config)
elif args.subcommand == "disable-source":
return sources.disable_source(config)
elif args.subcommand == "remove-source":
return sources.remove_source(config)
+ elif hasattr(args, "func"):
+ return args.func(config)
elif args.subcommand != "update":
logger.error("Unknown command: %s", args.subcommand)
return 1
enable-source Enable a source from the index
disable-source Disable an enabled source
remove-source Remove an enabled or disabled source
+ list-enabled-sources List all enabled sources
+ add-source Add a new source by URL
""")
return 0
os.makedirs(config.get_cache_dir(), mode=0o770)
except Exception as err:
logger.warning(
- "Cache directory does exist and could not be created. /var/tmp will be used instead.")
+ "Cache directory does not exist and could not be created. "
+ "/var/tmp will be used instead.")
config.set_cache_dir("/var/tmp")
files = load_sources(config, suricata_version)
def get_index_filename(config):
return os.path.join(config.get_cache_dir(), SOURCE_INDEX_FILENAME)
+def get_enabled_source_filename(name):
+ return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (
+ safe_filename(name)))
+
+def get_disabled_source_filename(name):
+ return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % (
+ safe_filename(name)))
+
+def source_name_exists(name):
+ """Return True if a source already exists with name."""
+ if os.path.exists(get_enabled_source_filename(name)) or \
+ os.path.exists(get_disabled_source_filename(name)):
+ return True
+ return False
+
+def source_index_exists(config):
+ """Return True if the source index file exists."""
+ return os.path.exists(get_index_filename(config))
+
+def save_source_config(source_config):
+ with open(get_enabled_source_filename(source_config.name), "wb") as fileobj:
+ fileobj.write(yaml.safe_dump(
+ source_config.dict(), default_flow_style=False))
+
+class SourceConfiguration:
+
+ def __init__(self, name, url=None, params={}):
+ self.name = name
+ self.url = url
+ self.params = params
+
+ def dict(self):
+ d = {
+ "source": self.name,
+ }
+ if self.url:
+ d["url"] = self.url
+ if self.params:
+ d["params"] = self.params
+ return d
+
class Index:
def __init__(self, filename):
self.filename = filename
-
self.index = {}
self.reload()
def reload(self):
- if os.path.exists(self.filename):
- index = yaml.load(open(self.filename))
- self.index = index
+ index = yaml.load(open(self.filename, "rb"))
+ self.index = index
def resolve_url(self, name, params={}):
if not name in self.index["sources"]:
except KeyError as err:
raise Exception("Missing URL parameter: %s" % (str(err.args[0])))
+ def get_sources(self):
+ return self.index["sources"]
+
+def load_source_index(config):
+ return Index(get_index_filename(config))
+
def get_enabled_sources():
"""Return a map of enabled sources, keyed by name."""
if not os.path.exists(ENABLED_SOURCE_DIRECTORY):
net.get(get_source_index_url(config), fileobj)
except Exception as err:
raise Exception("Failed to download index: %s: %s" % (url, err))
+ if not os.path.exists(config.get_cache_dir()):
+ try:
+ os.makedirs(config.get_cache_dir())
+ except Exception as err:
+ logger.error("Failed to create directory %s: %s",
+ config.get_cache_dir(), err)
+ return 1
with open(source_cache_filename, "w") as outobj:
outobj.write(fileobj.getvalue())
logger.debug("Saved %s", source_cache_filename)
return index["sources"]
return {}
-def list_sources(config):
- sources = load_sources(config)
- if not sources:
- logger.error("No sources exist. Try running update-sources.")
- return
- for name, source in sources.items():
- print("Name: %s" % (name))
- print(" Vendor: %s" % (source["vendor"]))
- print(" Description: %s" % (source["description"]))
- print(" License: %s" % (source["license"]))
-
def enable_source(config):
name = config.args.name
opts[key] = val
source = sources[config.args.name]
+
+ if "subscribe-url" in source:
+ print("The source %s requires a subscription. Subscribe here:" % (name))
+ print(" %s" % source["subscribe-url"])
+
params = {}
if "parameters" in source:
for param in source["parameters"]:
params[param] = opts[param]
else:
prompt = source["parameters"][param]["prompt"]
- r = raw_input("%s (%s): " % (prompt, param))
+ while True:
+ r = raw_input("%s (%s): " % (prompt, param))
+ r = r.strip()
+ if r:
+ break
params[param] = r.strip()
- new_source = {
- "source": name,
- }
- if params:
- new_source["params"] = params
- new_sources = [new_source]
+ new_source = SourceConfiguration(name, params=params).dict()
if not os.path.exists(ENABLED_SOURCE_DIRECTORY):
try:
logger.warning("Source %s does not exist.", name)
return 1
-def get_enabled_source_filename(name):
- return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (
- safe_filename(name)))
-
-def get_disabled_source_filename(name):
- return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % (
- safe_filename(name)))
-
def safe_filename(name):
"""Utility function to make a source short-name safe as a
filename."""
projects / thirdparty / suricata-update.git / commitdiff
