Add summary for updated sources

`suricata-update` when used with its option `update-sources` did not
give any information about what changed and in which source. Add the
logs for any change in the sources, namely, addition, removal or change
in the content.
The log output now looks like:
```
$ ./bin/suricata-update update-sources
31/10/2018 -- 00:03:52 - <Info> -- Loading /etc/suricata/update.yaml
31/10/2018 -- 00:03:52 - <Info> -- Using data-directory /var/lib/suricata.
31/10/2018 -- 00:03:52 - <Info> -- Using Suricata configuration /etc/suricata/suricata.yaml
31/10/2018 -- 00:03:52 - <Info> -- Using /etc/suricata/rules for Suricata provided rules.
31/10/2018 -- 00:03:52 - <Info> -- Found Suricata version 4.1.0-dev at /usr/sbin/suricata.
31/10/2018 -- 00:03:52 - <Info> -- Downloading https://www.openinfosecfoundation.org/rules/index.yaml
31/10/2018 -- 00:03:53 - <Info> -- Source et/open was added
31/10/2018 -- 00:03:53 - <Info> -- Source empty/something was removed
31/10/2018 -- 00:03:53 - <Info> -- Source et/pro was changed
31/10/2018 -- 00:03:53 - <Info> -- Saved /var/lib/suricata/update/cache/index.yaml
```

Closes redmine ticket #2472.

diff --git a/suricata/update/commands/updatesources.py b/suricata/update/commands/updatesources.py
index 7f6bfedef4e76e3175b4cb0b41ef98df2506b53f..9e22e735c9075f7f9332b71310284c1499217e1a 100644 (file)
--- a/suricata/update/commands/updatesources.py
+++ b/suricata/update/commands/updatesources.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 import os
 import logging
 import io
+import yaml
 
 from suricata.update import config
 from suricata.update import sources
@@ -27,11 +28,36 @@ from suricata.update import exceptions
 
 logger = logging.getLogger()
 
+
 def register(parser):
     parser.set_defaults(func=update_sources)
 
+
+def compare_sources(initial_content, final_content):
+    if initial_content == final_content:
+        logger.info("No change in sources")
+        return
+    initial_sources = initial_content.get("sources")
+    final_sources = final_content.get("sources")
+    added_sources = {source: final_sources[source]
+                     for source in final_sources if source not in initial_sources}
+    removed_sources = {source: initial_sources[source]
+                       for source in initial_sources if source not in final_sources}
+    if added_sources:
+        for source in added_sources:
+            logger.info("Source %s was added", source)
+    if removed_sources:
+        for source in removed_sources:
+            logger.info("Source %s was removed", source)
+    for source in set(initial_sources) & set(final_sources):
+        if initial_sources[source] != final_sources[source]:
+            logger.info("Source %s was changed", source)
+
+
 def update_sources():
     local_index_filename = sources.get_index_filename()
+    with open(local_index_filename) as stream:
+        initial_content = yaml.safe_load(stream)
     with io.BytesIO() as fileobj:
         url = sources.get_source_index_url()
         logger.info("Downloading %s", url)
@@ -49,4 +75,7 @@ def update_sources():
                 return 1
         with open(local_index_filename, "wb") as outobj:
             outobj.write(fileobj.getvalue())
+        with open(local_index_filename) as stream:
+            final_content = yaml.safe_load(stream)
+        compare_sources(initial_content, final_content)
         logger.info("Saved %s", local_index_filename)