reporter: Remove all data older than 5 years from the database

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/reporter.conf.in b/src/reporter.conf.in
index ba16d21cfe59f4379835c53e3dc8064bb0c81c46..5943006e4f93e4fa0d45fd83262ce61764d96e1c 100644 (file)
--- a/src/reporter.conf.in
+++ b/src/reporter.conf.in
@@ -19,6 +19,10 @@
 ; The path to the database
 ;database = @suricatalogdir@/reporter.db
 
+[database]
+; Retain all events for this long (in days)
+;retention = 1825
+
 [syslog]
 ; Enable sending any alerts to syslog in the human-reable fast.log format
 ;enabled = true

diff --git a/src/suricata-reporter.in b/src/suricata-reporter.in
index c6b91bccd0c6db293ec41cfb55fb8544b773b5c3..1e302b237dcd5baf3af131cf340f78575fdad07b 100644 (file)
--- a/src/suricata-reporter.in
+++ b/src/suricata-reporter.in
@@ -79,6 +79,9 @@ class Reporter(object):
                # Create an events queue
                self.queue = queue.Queue(1024)
 
+               # Remember the last time the database was cleaned
+               self.last_cleanup_at = None
+
                # Keep references to our workers
                self.workers = []
 
@@ -307,6 +310,10 @@ class Worker(threading.Thread):
 
                        # If there was nothing in the queue, we will try again
                        except queue.Empty:
+                               # We have time to cleanup the database
+                               self.cleanup()
+
+                               # Nothing else to do in this iteration...
                                continue
 
                        # Parse the event
@@ -340,6 +347,49 @@ class Worker(threading.Thread):
 
                log.debug("Worker %s terminated" % self.native_id)
 
+       _cleanup = threading.Lock()
+
+       def cleanup(self):
+               """
+                       Cleanup the database
+               """
+               now = datetime.datetime.utcnow()
+
+               # Cleanup the database if it has never been cleaned up
+               if self.reporter.last_cleanup_at is None:
+                       pass
+
+               # Cleanup the database if the last cleanup has been
+               elif self.reporter.last_cleanup_at + datetime.timedelta(hours=6) <= now:
+                       pass
+
+               # Otherwise we won't cleanup the database
+               else:
+                       return
+
+               # Acquire the lock so this will only run once
+               if self._cleanup.acquire(blocking=False):
+                       try:
+                               log.debug("Cleaning up the database...")
+
+                               # Determine the retention time
+                               retention_days = datetime.timedelta(
+                                       days = self.config.getint("database", "retention", fallback=365 * 5)
+                               )
+
+                               # Save when we performed this last
+                               self.reporter.last_cleanup_at = now
+
+                               # Remove everything
+                               self.db.execute(
+                                       "DELETE FROM alerts WHERE timestamp <= ?",
+                                       (now - retention_days,),
+                               )
+
+                       # Release the lock
+                       finally:
+                               self._cleanup.release()
+
        def process(self, event):
                """
                        Called whenever we have received an event