From f36c71bb9549791ed7d721221407d5cbd13c5169 Mon Sep 17 00:00:00 2001 From: Vagisha Gupta Date: Mon, 2 Sep 2019 13:24:12 +0530 Subject: [PATCH] Add no-checksum option Added a `--no-checksum` option to the add-source command and an optional "checksum" in SourceConfiguration class for add-sources and enabled-sources. Also, a check is added to skip downloading the checksum URL if the source is configured with checksum false. Redmine issue: https://redmine.openinfosecfoundation.org/issues/3100 --- doc/add-source.rst | 4 +++ suricata/update/commands/addsource.py | 9 ++++++- suricata/update/commands/enablesource.py | 8 +++++- suricata/update/main.py | 33 +++++++++++++++++------- suricata/update/sources.py | 6 ++++- 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/doc/add-source.rst b/doc/add-source.rst index 22471a8..e5b2a53 100644 --- a/doc/add-source.rst +++ b/doc/add-source.rst @@ -25,6 +25,10 @@ Options add-source --http-header "X-API-Key: 1234" +.. option:: --no-checksum + + Skips downloading the checksum URL for the rule source. + Common Options ============== diff --git a/suricata/update/commands/addsource.py b/suricata/update/commands/addsource.py index 42d4c63..a87095c 100644 --- a/suricata/update/commands/addsource.py +++ b/suricata/update/commands/addsource.py @@ -28,14 +28,18 @@ except: logger = logging.getLogger() + def register(parser): parser.add_argument("name", metavar="", nargs="?", help="Name of source") parser.add_argument("url", metavar="", nargs="?", help="Source URL") parser.add_argument("--http-header", metavar="", help="Additional HTTP header to add to requests") + parser.add_argument("--no-checksum", action="store_false", + help="Skips downloading the checksum URL") parser.set_defaults(func=add_source) + def add_source(): args = config.args() @@ -59,7 +63,10 @@ def add_source(): if url: break + checksum = args.no_checksum + header = args.http_header if args.http_header else None - source_config = sources.SourceConfiguration(name, header=header, url=url) + source_config = sources.SourceConfiguration( + name, header=header, url=url, checksum=checksum) sources.save_source_config(source_config) diff --git a/suricata/update/commands/enablesource.py b/suricata/update/commands/enablesource.py index 1211385..45a8598 100644 --- a/suricata/update/commands/enablesource.py +++ b/suricata/update/commands/enablesource.py @@ -103,7 +103,13 @@ def enable_source(): break params[param] = r.strip() - new_source = sources.SourceConfiguration(name, params=params) + if "checksum" in source: + checksum = source["checksum"] + else: + checksum = source.get("checksum", True) + + new_source = sources.SourceConfiguration( + name, params=params, checksum=checksum) # If the source directory does not exist, create it. Also create # the default rule-source of et/open, unless the source being diff --git a/suricata/update/main.py b/suricata/update/main.py index 6ac9805..3afe8fd 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -354,7 +354,8 @@ class Fetch: def fetch(self, url): net_arg = url - url = url[0] if isinstance(url, tuple) else url + checksum = url[2] + url = url[0] tmp_filename = self.get_tmp_filename(url) if config.args().offline: if config.args().force: @@ -372,9 +373,11 @@ class Fetch: "Last download less than 15 minutes ago. Not downloading %s.", url) return self.extract_files(tmp_filename) - if self.check_checksum(tmp_filename, url): - logger.info("Remote checksum has not changed. Not fetching.") - return self.extract_files(tmp_filename) + if checksum: + if self.check_checksum(tmp_filename, url): + logger.info("Remote checksum has not changed. " + "Not fetching.") + return self.extract_files(tmp_filename) if not os.path.exists(config.get_cache_dir()): os.makedirs(config.get_cache_dir(), mode=0o770) logger.info("Fetching %s." % (url)) @@ -956,10 +959,13 @@ def load_sources(suricata_version): urls = [] + http_header = None + checksum = True + # Add any URLs added with the --url command line parameter. if config.args().url: for url in config.args().url: - urls.append(url) + urls.append((url, http_header, checksum)) # Get the new style sources. enabled_sources = sources.get_enabled_sources() @@ -991,14 +997,22 @@ def load_sources(suricata_version): params.update(internal_params) if "url" in source: # No need to go off to the index. - url = (source["url"] % params, source.get("http-header")) + http_header = source.get("http_header") + checksum = source.get("checksum") + url = (source["url"] % params, http_header, checksum) logger.debug("Resolved source %s to URL %s.", name, url[0]) else: if not index: raise exceptions.ApplicationError( "Source index is required for source %s; " "run suricata-update update-sources" % (source["source"])) - url = index.resolve_url(name, params) + source_config = index.get_source_by_name(name) + try: + checksum = source_config["checksum"] + except: + checksum = True + url = (index.resolve_url(name, params), http_header, + checksum) logger.debug("Resolved source %s to URL %s.", name, url) urls.append(url) @@ -1007,7 +1021,7 @@ def load_sources(suricata_version): if type(url) not in [type("")]: raise exceptions.InvalidConfigurationError( "Invalid datatype for source URL: %s" % (str(url))) - url = url % internal_params + url = (url % internal_params, http_header, checksum) logger.debug("Adding source %s.", url) urls.append(url) @@ -1016,7 +1030,8 @@ def load_sources(suricata_version): if config.get("etopen") or not urls: if not config.args().offline and not urls: logger.info("No sources configured, will use Emerging Threats Open") - urls.append(sources.get_etopen_url(internal_params)) + urls.append((sources.get_etopen_url(internal_params), http_header, + checksum)) # Converting the URLs to a set removed dupes. urls = set(urls) diff --git a/suricata/update/sources.py b/suricata/update/sources.py index bf7b64b..e87cb4a 100644 --- a/suricata/update/sources.py +++ b/suricata/update/sources.py @@ -77,11 +77,13 @@ def save_source_config(source_config): class SourceConfiguration: - def __init__(self, name, header=None, url=None, params={}): + def __init__(self, name, header=None, url=None, + params={}, checksum=True): self.name = name self.url = url self.params = params self.header = header + self.checksum = checksum def dict(self): d = { @@ -93,6 +95,8 @@ class SourceConfiguration: d["params"] = self.params if self.header: d["http-header"] = self.header + if self.checksum: + d["checksum"] = self.checksum return d class Index: -- 2.47.3