repositories: Make updates smarter

Repositories won't be (re-)written more than once at a time. Instead we
will mark that another update was requested and execute it as soon as
the first one finished.

That way, multiple updates will only cause the repository to be
rewritten twice.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/builds.py b/src/buildservice/builds.py
index faae08cc2aeba34b0d030713d8da2764076804eb..25752b85cb1b0c45b17b8fd75a1c4e6c9b010d20 100644 (file)
--- a/src/buildservice/builds.py
+++ b/src/buildservice/builds.py
@@ -776,7 +776,7 @@ class Build(base.DataObject):
                        This method should be called if the repositories should be updated
                """
                for repo in self.repos:
-                       await repo.has_changed()
+                       await repo.changed()
 
        ## Bugs
 

diff --git a/src/buildservice/repository.py b/src/buildservice/repository.py
index 347da7dafc15a739d899b980958ac978720f999a..1d60956c98393bf0a7c556b7522a22239c5c50a1 100644 (file)
--- a/src/buildservice/repository.py
+++ b/src/buildservice/repository.py
@@ -443,7 +443,7 @@ class Repository(base.DataObject):
                # XXX TODO
 
                # Update the repository (in the background)
-               await self.has_changed()
+               await self.changed()
 
        async def remove_build(self, build, user=None):
                """
@@ -469,7 +469,7 @@ class Repository(base.DataObject):
                        pass
 
                # Update the repository (in the background)
-               await self.has_changed()
+               await self.changed()
 
        # Sources
 
@@ -825,7 +825,23 @@ class Repository(base.DataObject):
 
                return self.backend.pakfire(distro=self.distro, repos=[self], **kwargs)
 
-       async def has_changed(self):
+       # Has Changed?
+
+       __has_changed = {}
+
+       @property
+       def has_changed(self):
+               """
+                       This is set when the repository has been changed and needs an update
+               """
+               try:
+                       event = self.__has_changed[self.id]
+               except KeyError:
+                       event = self.__has_changed[self.id] = asyncio.Event()
+
+               return event
+
+       async def changed(self):
                """
                        Convenience function that should be called when a package has been
                        added/removed from this repository.
@@ -833,12 +849,45 @@ class Repository(base.DataObject):
                # Run the update process at the next convenient moment
                self.backend.run_task(self.update)
 
+       __locks = {}
+
+       @property
+       def lock(self):
+               """
+                       Locks the repository when it is being rewritten
+               """
+               try:
+                       lock = self.__locks[self.id]
+               except KeyError:
+                       lock = self.__locks[self.id] = asyncio.Lock()
+
+               return lock
+
        async def update(self):
                """
                        Called to perform an update of this repository
                """
-               # Write/rewrite the repository
-               await self.write()
+               # If the repository is currently being rewritten,
+               # we will mark it as changed and return.
+               if self.lock.locked():
+                       log.debug("%s is currently being updated" % self)
+
+                       # Set flag
+                       self.has_changed.set()
+                       return
+
+               # Otherwise, try to acquire the lock
+               async with self.lock:
+                       # Clear the "has changed" flag
+                       self.has_changed.clear()
+
+                       # Write/rewrite the repository
+                       await self.write()
+
+               # Check if the repository has been changed in the meantime
+               # and if so, call ourselves again
+               if self.has_changed.is_set():
+                       await self.update()
 
                # Relaunch any pending jobs
                await self.relaunch_pending_jobs()
@@ -862,23 +911,11 @@ class Repository(base.DataObject):
 
        # Write repository
 
-       __locks = {}
-
-       @property
-       def __lock(self):
-               try:
-                       lock = self.__locks[self.id]
-               except KeyError:
-                       lock = self.__locks[self.id] = asyncio.Lock()
-
-               return lock
-
        async def write(self):
                """
                        Called to write/rewrite/update the repository metadata
                """
-               with self.__lock:
-                       return await asyncio.to_thread(self._write)
+               return await asyncio.to_thread(self._write)
 
        def _write(self):
                log.info("Writing repository %s..." % self)