exporters: Check if an export actually needs an update

This allows us to export only if we actually have any changes which will
take a lot of load away from the server.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/dbl/exporters.py b/src/dbl/exporters.py
index 9e580b869f4941a0b637e78e768e74a4c1434e49..a02af4ccdc2a293d8fa3ccbd04847056642100c1 100644 (file)
--- a/src/dbl/exporters.py
+++ b/src/dbl/exporters.py
@@ -22,6 +22,7 @@ import abc
 import base64
 import datetime
 import io
+import logging
 import os
 import pathlib
 import tarfile
@@ -30,6 +31,9 @@ import tempfile
 from . import util
 from .i18n import _
 
+# Setup logging
+log = logging.getLogger(__name__)
+
 class Exporter(abc.ABC):
        """
                Abstract class to define an exporter
@@ -669,7 +673,7 @@ class DirectoryExporter(MultiExporter):
                # Store the root
                self.root = pathlib.Path(root)
 
-       async def __call__(self):
+       async def __call__(self, force=False):
                # Ensure the root directory exists
                try:
                        self.root.mkdir()
@@ -681,21 +685,26 @@ class DirectoryExporter(MultiExporter):
                        # For MultiExporters, we will have to export everything at once
                        if issubclass(exporter, MultiExporter):
                                e = exporter(self.backend, self.lists)
-                               await self.export(e, name)
+                               await self.export(e, name, force=force)
 
                        # For regular exporters, we will have to export each list at a time
                        else:
                                for list in self.lists:
                                        e = exporter(self.backend, list)
-                                       await self.export(e, name, list=list)
+                                       await self.export(e, name, list=list, force=force)
 
-       async def export(self, exporter, name, **kwargs):
+       async def export(self, exporter, name, force=False, **kwargs):
                """
                        This function takes an exporter instance and runs it
                """
                # Make the path
                path = self._make_path(name, **kwargs)
 
+               # Skip the export if we don't need it
+               if not force and os.path.getmtime(path) >= exporter.exported_at.timestamp():
+                       log.debug("Skipping export because the file seems to be recent enough")
+                       return
+
                # Ensure the parent directory exists
                try:
                        path.parent.mkdir()

diff --git a/src/scripts/dbl.in b/src/scripts/dbl.in
index 1a9b4a6c471823bb21d38c2a8701a8f4a1cc86c3..0d28f9ab98af7a79bdfaa9d3793b7582bd63c4cc 100644 (file)
--- a/src/scripts/dbl.in
+++ b/src/scripts/dbl.in
@@ -120,6 +120,7 @@ class CLI(object):
                # export-all
                export = subparsers.add_parser("export-all", help=_("Export all lists"))
                export.add_argument("directory", help=_("Output Directory"))
+               export.add_argument("--force", action="store_true", help=_("Force an export"))
                export.set_defaults(func=self.__export_all)
 
                # add-source
@@ -407,7 +408,7 @@ class CLI(object):
                # Launch the DirectoryExporter
                exporter = dbl.exporters.DirectoryExporter(backend,
                        root=args.directory, lists=[l async for l in backend.lists])
-               await exporter()
+               await exporter(force=args.force)
 
        async def __add_source(self, backend, args):
                """