sources: resolve urls from index

diff --git a/suricata/update/main.py b/suricata/update/main.py
index b1984d46670c2fe2768d03bfad58db75e4eecc92..97fdd01e58a40b77886b0d5c347de00780b5a98d 100644 (file)
--- a/suricata/update/main.py
+++ b/suricata/update/main.py
@@ -944,6 +944,33 @@ def load_sources(config, suricata_version):
         for url in config.args.url:
             urls.append(url)
 
+    # Get the new style sources.
+    enabled_sources = sources.get_enabled_sources()
+
+    # If we have new sources, we also need to load the index.
+    if enabled_sources:
+        index_filename = os.path.join(
+            config.get_cache_dir(), sources.SOURCE_INDEX_FILENAME)
+        if not os.path.exists(index_filename):
+            raise Exception(
+                ("Source index is required but does not exist.",
+                 ("Run suricata-update update-sources")))
+        index = sources.Index(index_filename)
+
+        version_string = "%d.%d.%d" % (
+            suricata_version.major, suricata_version.minor,
+            suricata_version.patch)
+
+        for (name, source) in enabled_sources.items():
+            if "params" in source and source["params"]:
+                params = source["params"]
+            else:
+                params = {}
+            params["__version__"] = version_string
+            url = index.resolve_url(name, params)
+            logger.debug("Resolved source %s to URL %s.", name, url)
+            urls.append(url)
+
     if config.get("sources"):
         for source in config.get("sources"):
             if not "type" in source:
@@ -1106,6 +1133,14 @@ def main():
     list_sources_parser = subparsers.add_parser(
         "list-sources", parents=[common_parser])
 
+    disable_source_parser = subparsers.add_parser(
+        "disable-source", parents=[common_parser])
+    disable_source_parser.add_argument("name")
+
+    remove_source_parser = subparsers.add_parser(
+        "remove-source", help="Remove a source", parents=[common_parser])
+    remove_source_parser.add_argument("name")
+
     enable_source_parser = subparsers.add_parser(
         "enable-source", parents=[common_parser])
     enable_source_parser.add_argument("name")
@@ -1151,8 +1186,12 @@ def main():
             return sources.list_sources(config)
         elif args.subcommand == "enable-source":
             return sources.enable_source(config)
+        elif args.subcommand == "disable-source":
+            return sources.disable_source(config)
+        elif args.subcommand == "remove-source":
+            return sources.remove_source(config)
         elif args.subcommand != "update":
-            logger.error("Unknown command: %s", args.command)
+            logger.error("Unknown command: %s", args.subcommand)
             return 1
 
     if args.dump_sample_configs:
@@ -1165,9 +1204,11 @@ def main():
     if args.help:
         print(update_parser.format_help())
         print("""other commands:
-    update-sources
-    list-sources
-    enable-source
+    update-sources             Update the source index
+    list-sources               List available sources
+    enable-source              Enable a source from the index
+    disable-source             Disable an enabled source
+    remove-source              Remove an enabled or disabled source
 """)
         return 0
 

diff --git a/suricata/update/sources.py b/suricata/update/sources.py
index 47105c127fe13aeb3e3ab7d6af9984ee23d43abd..98eebed2ea683e3528325ede3f56b9dff8f9c869 100644 (file)
--- a/suricata/update/sources.py
+++ b/suricata/update/sources.py
@@ -25,11 +25,57 @@ import yaml
 
 from suricata.update import net
 from suricata.update import util
+from suricata.update import loghandler
 
 logger = logging.getLogger()
 
 DEFAULT_SOURCE_INDEX_URL = "https://raw.githubusercontent.com/jasonish/suricata-intel-index/master/index.yaml"
 SOURCE_INDEX_FILENAME = "index.yaml"
+ENABLED_SOURCE_DIRECTORY = "/var/lib/suricata/update/sources"
+
+def get_index_filename(config):
+    return os.path.join(config.get_cache_dir(), SOURCE_INDEX_FILENAME)
+
+class Index:
+
+    def __init__(self, filename):
+        self.filename = filename
+
+        self.index = {}
+        self.reload()
+
+    def reload(self):
+        if os.path.exists(self.filename):
+            index = yaml.load(open(self.filename))
+            self.index = index
+
+    def resolve_url(self, name, params={}):
+        if not name in self.index["sources"]:
+            raise Exception("Source name not in index: %s" % (name))
+        source = self.index["sources"][name]
+        try:
+            return source["url"] % params
+        except KeyError as err:
+            raise Exception("Missing URL parameter: %s" % (str(err.args[0])))
+
+def get_enabled_sources():
+    """Return a map of enabled sources, keyed by name."""
+    if not os.path.exists(ENABLED_SOURCE_DIRECTORY):
+        return {}
+    sources = {}
+    for dirpath, dirnames, filenames in os.walk(ENABLED_SOURCE_DIRECTORY):
+        for filename in filenames:
+            if filename.endswith(".yaml"):
+                path = os.path.join(dirpath, filename)
+                source = yaml.load(open(path, "rb"))
+                sources[source["source"]] = source
+
+                if "params" in source:
+                    for param in source["params"]:
+                        if param.startswith("secret"):
+                            loghandler.add_secret(source["params"][param], param)
+
+    return sources
 
 def get_source_index_url(config):
     if os.getenv("SOURCE_INDEX_URL"):
@@ -72,6 +118,29 @@ def list_sources(config):
 
 def enable_source(config):
     name = config.args.name
+
+    # Check if source is already enabled.
+    enabled_source_filename = os.path.join(
+        ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (safe_filename(name)))
+    if os.path.exists(enabled_source_filename):
+        logger.error("The source %s is already enabled.", name)
+        return 1
+
+    # First check if this source was previous disabled and then just
+    # re-enable it.
+    disabled_source_filename = os.path.join(
+        ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % (safe_filename(name)))
+    if os.path.exists(disabled_source_filename):
+        logger.info("Re-enabling previous disabled source for %s.", name)
+        os.rename(disabled_source_filename, enabled_source_filename)
+        return 0
+
+    if not os.path.exists(get_index_filename(config)):
+        logger.warning(
+            "Source index does not exist, "
+            "try running suricata-update update-sources.")
+        return 1
+
     sources = load_sources(config)
     if not config.args.name in sources:
         logger.error("Unknown source: %s", config.args.name)
@@ -79,8 +148,8 @@ def enable_source(config):
 
     # Parse key=val options.
     opts = {}
-    for opt in config.args.params:
-        key, val = opt.params("=", 1)
+    for param in config.args.params:
+        key, val = param.split("=", 1)
         opts[key] = val
 
     source = sources[config.args.name]
@@ -99,4 +168,64 @@ def enable_source(config):
     if params:
         new_source["params"] = params
     new_sources = [new_source]
-    config.save_new_source(new_source)
+
+    if not os.path.exists(ENABLED_SOURCE_DIRECTORY):
+        try:
+            logger.info("Creating directory %s", ENABLED_SOURCE_DIRECTORY)
+            os.makedirs(ENABLED_SOURCE_DIRECTORY)
+        except Exception as err:
+            logger.error("Failed to create directory %s: %s",
+                         ENABLED_SOURCE_DIRECTORY, err)
+            return 1
+
+    filename = os.path.join(
+        ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (safe_filename(name)))
+    logger.info("Writing %s", filename)
+    with open(filename, "w") as fileobj:
+        fileobj.write(yaml.dump(new_source, default_flow_style=False))
+
+def disable_source(config):
+    name = config.args.name
+    filename = os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (
+        safe_filename(name)))
+    if not os.path.exists(filename):
+        logger.debug("Filename %s does not exist.", filename)
+        logger.warning("Source %s is not enabled.", name)
+        return 1
+    logger.debug("Renaming %s to %s.disabled.", filename, filename)
+    os.rename(filename, "%s.disabled" % (filename))
+
+def remove_source(config):
+    name = config.args.name
+
+    enabled_source_filename = get_enabled_source_filename(name)
+    if os.path.exists(enabled_source_filename):
+        logger.debug("Deleting file %s.", enabled_source_filename)
+        os.remove(enabled_source_filename)
+        logger.info("Source %s removed, previously enabled.", name)
+        return 0
+
+    disabled_source_filename = get_disabled_source_filename(name)
+    if os.path.exists(disabled_source_filename):
+        logger.debug("Deleting file %s.", disabled_source_filename)
+        os.remove(disabled_source_filename)
+        logger.info("Source %s removed, previously disabled.", name)
+        return 0
+    
+    logger.warning("Source %s does not exist.", name)
+    return 1
+
+def get_enabled_source_filename(name):
+    return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml" % (
+        safe_filename(name)))
+
+def get_disabled_source_filename(name):
+    return os.path.join(ENABLED_SOURCE_DIRECTORY, "%s.yaml.disabled" % (
+        safe_filename(name)))
+
+def safe_filename(name):
+    """Utility function to make a source short-name safe as a
+    filename."""
+    name = name.replace("/", "-")
+    return name
+