Add no-checksum option

Added a `--no-checksum` option to the add-source command and an optional
"checksum" in SourceConfiguration class for add-sources and
enabled-sources.
Also, a check is added to skip downloading the checksum URL if the
source is configured  with checksum false.

Redmine issue:
    https://redmine.openinfosecfoundation.org/issues/3100

diff --git a/doc/add-source.rst b/doc/add-source.rst
index 22471a894afef6443ed7da2e016bc2791bcaaddf..e5b2a532748df5930f0cff48bf5ecc31ad0d1877 100644 (file)
--- a/doc/add-source.rst
+++ b/doc/add-source.rst
@@ -25,6 +25,10 @@ Options
 
      add-source --http-header "X-API-Key: 1234"
 
+.. option:: --no-checksum
+
+   Skips downloading the checksum URL for the rule source.
+
 Common Options
 ==============
 

diff --git a/suricata/update/commands/addsource.py b/suricata/update/commands/addsource.py
index 42d4c638cc47a8c07a93bc30cb0006f16f9e58fa..a87095c0224b50be74f6907624af941201757813 100644 (file)
--- a/suricata/update/commands/addsource.py
+++ b/suricata/update/commands/addsource.py
@@ -28,14 +28,18 @@ except:
 
 logger = logging.getLogger()
 
+
 def register(parser):
     parser.add_argument("name", metavar="<name>", nargs="?",
                         help="Name of source")
     parser.add_argument("url", metavar="<url>", nargs="?", help="Source URL")
     parser.add_argument("--http-header", metavar="<http-header>",
                         help="Additional HTTP header to add to requests")
+    parser.add_argument("--no-checksum", action="store_false",
+                        help="Skips downloading the checksum URL")
     parser.set_defaults(func=add_source)
 
+
 def add_source():
     args = config.args()
 
@@ -59,7 +63,10 @@ def add_source():
             if url:
                 break
 
+    checksum = args.no_checksum
+
     header = args.http_header if args.http_header else None
 
-    source_config = sources.SourceConfiguration(name, header=header, url=url)
+    source_config = sources.SourceConfiguration(
+        name, header=header, url=url, checksum=checksum)
     sources.save_source_config(source_config)

diff --git a/suricata/update/commands/enablesource.py b/suricata/update/commands/enablesource.py
index 121138542f0d24df5927b358b5ba812359b6ed7d..45a859878d37371c276201d273960fa3e857df24 100644 (file)
--- a/suricata/update/commands/enablesource.py
+++ b/suricata/update/commands/enablesource.py
@@ -103,7 +103,13 @@ def enable_source():
                         break
                 params[param] = r.strip()
 
-    new_source = sources.SourceConfiguration(name, params=params)
+    if "checksum" in source:
+        checksum = source["checksum"]
+    else:
+        checksum = source.get("checksum", True)
+
+    new_source = sources.SourceConfiguration(
+        name, params=params, checksum=checksum)
 
     # If the source directory does not exist, create it. Also create
     # the default rule-source of et/open, unless the source being

diff --git a/suricata/update/main.py b/suricata/update/main.py
index 6ac980579db1eea59c3585899219cbe16dee74e2..3afe8fd17286913822adf6fda6b1358feb200450 100644 (file)
--- a/suricata/update/main.py
+++ b/suricata/update/main.py
@@ -354,7 +354,8 @@ class Fetch:
 
     def fetch(self, url):
         net_arg = url
-        url = url[0] if isinstance(url, tuple) else url
+        checksum = url[2]
+        url = url[0]
         tmp_filename = self.get_tmp_filename(url)
         if config.args().offline:
             if config.args().force:
@@ -372,9 +373,11 @@ class Fetch:
                     "Last download less than 15 minutes ago. Not downloading %s.",
                     url)
                 return self.extract_files(tmp_filename)
-            if self.check_checksum(tmp_filename, url):
-                logger.info("Remote checksum has not changed. Not fetching.")
-                return self.extract_files(tmp_filename)
+            if checksum:
+                if self.check_checksum(tmp_filename, url):
+                    logger.info("Remote checksum has not changed. "
+                                "Not fetching.")
+                    return self.extract_files(tmp_filename)
         if not os.path.exists(config.get_cache_dir()):
             os.makedirs(config.get_cache_dir(), mode=0o770)
         logger.info("Fetching %s." % (url))
@@ -956,10 +959,13 @@ def load_sources(suricata_version):
 
     urls = []
 
+    http_header = None
+    checksum = True
+
     # Add any URLs added with the --url command line parameter.
     if config.args().url:
         for url in config.args().url:
-            urls.append(url)
+            urls.append((url, http_header, checksum))
 
     # Get the new style sources.
     enabled_sources = sources.get_enabled_sources()
@@ -991,14 +997,22 @@ def load_sources(suricata_version):
             params.update(internal_params)
             if "url" in source:
                 # No need to go off to the index.
-                url = (source["url"] % params, source.get("http-header"))
+                http_header = source.get("http_header")
+                checksum = source.get("checksum")
+                url = (source["url"] % params, http_header, checksum)
                 logger.debug("Resolved source %s to URL %s.", name, url[0])
             else:
                 if not index:
                     raise exceptions.ApplicationError(
                         "Source index is required for source %s; "
                         "run suricata-update update-sources" % (source["source"]))
-                url = index.resolve_url(name, params)
+                source_config = index.get_source_by_name(name)
+                try:
+                    checksum = source_config["checksum"]
+                except:
+                    checksum = True
+                url = (index.resolve_url(name, params), http_header,
+                       checksum)
                 logger.debug("Resolved source %s to URL %s.", name, url)
             urls.append(url)
 
@@ -1007,7 +1021,7 @@ def load_sources(suricata_version):
             if type(url) not in [type("")]:
                 raise exceptions.InvalidConfigurationError(
                     "Invalid datatype for source URL: %s" % (str(url)))
-            url = url % internal_params
+            url = (url % internal_params, http_header, checksum)
             logger.debug("Adding source %s.", url)
             urls.append(url)
 
@@ -1016,7 +1030,8 @@ def load_sources(suricata_version):
     if config.get("etopen") or not urls:
         if not config.args().offline and not urls:
             logger.info("No sources configured, will use Emerging Threats Open")
-        urls.append(sources.get_etopen_url(internal_params))
+        urls.append((sources.get_etopen_url(internal_params), http_header,
+                     checksum))
 
     # Converting the URLs to a set removed dupes.
     urls = set(urls)

diff --git a/suricata/update/sources.py b/suricata/update/sources.py
index bf7b64bf03a43fe71482cc0e2314d4e12171cba4..e87cb4a9e686efb0f8bf1999b87d5bf505d127d9 100644 (file)
--- a/suricata/update/sources.py
+++ b/suricata/update/sources.py
@@ -77,11 +77,13 @@ def save_source_config(source_config):
 
 class SourceConfiguration:
 
-    def __init__(self, name, header=None, url=None, params={}):
+    def __init__(self, name, header=None, url=None,
+                 params={}, checksum=True):
         self.name = name
         self.url = url
         self.params = params
         self.header = header
+        self.checksum = checksum
 
     def dict(self):
         d = {
@@ -93,6 +95,8 @@ class SourceConfiguration:
             d["params"] = self.params
         if self.header:
             d["http-header"] = self.header
+        if self.checksum:
+            d["checksum"] = self.checksum
         return d
 
 class Index: