sources: Create dist jobs for all packages on the initial commit import

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py
index 158e40d5ef226e995f88a42c0a88729e68531019..966517ae18bd6657ec5bcda9b6307986963e88dd 100644 (file)
--- a/src/buildservice/sources.py
+++ b/src/buildservice/sources.py
@@ -278,10 +278,12 @@ class Source(base.DataObject):
 
        # Commits
 
-       async def _create_commit(self, revision):
+       async def _create_commit(self, revision, initial_commit=False):
                """
                        Imports the commit with the given revision
                """
+               log.info("%s: Creating commit %s" % (self, revision))
+
                # Fetch the author's name and email address
                author = await self.git.show_attribute(revision, r"%an <%ae>")
 
@@ -326,14 +328,21 @@ class Source(base.DataObject):
                        source=self,
                )
 
+               # If we are processing the initial commit, we get a list of all files in the tree
+               if initial_commit:
+                       changed_files = [("A", f) for f in await self.git.ls_tree(revision, "*/*.nm")]
+
                # Find changed files
-               changed_files = await self.git.changed_files(revision, filter="*/*.nm")
+               else:
+                       changed_files = await self.git.changed_files(revision, filter="*/*.nm")
 
                # Create jobs for each file
                for status, changed_file in changed_files:
                        # Find the package name
                        name = os.path.dirname(changed_file)
 
+                       log.debug("%s: %s: Processing '%s' (status = %s)" % (self, revision, name, status))
+
                        # Check that the file part matches
                        if not changed_file.endswith("/%s.nm" % name):
                                raise ValueError("Invalid package name")
@@ -417,17 +426,24 @@ class Source(base.DataObject):
                else:
                        await self.git.clone()
 
-               # Determine which commits there are to process
-               revisions = await self.git.revisions(self.revision, self.branch)
-
                with self.db.transaction():
+                       # Did we already import something?
+                       initial_commit = not self.revision
+
+                       # Determine which commits there are to process
+                       revisions = await self.git.revisions(self.revision, self.branch)
+
+                       # Import all revisions
                        for revision in revisions:
                                # Import the commit
-                               await self._create_commit(revision)
+                               await self._create_commit(revision, initial_commit=initial_commit)
 
                                # Store the updated revision
                                self._set_attribute("revision", revision)
 
+                               # Only the first revision would the initial commit
+                               initial_commit = False
+
                        # Store when we fetched
                        self._set_attribute_now("last_fetched_at")
 
@@ -603,6 +619,9 @@ class Commit(base.DataObject):
                """
                        Creates a new job
                """
+               log.info("%s: %s: Created '%s' job for '%s'" \
+                       % (self.source, self.revision, action, name))
+
                job = self.backend.sources._get_job("""
                        INSERT INTO
                                source_commit_jobs
@@ -812,7 +831,7 @@ class Job(base.DataObject):
                                self.source.repo, package, group=self.commit.builds)
 
                        # Add any watchers
-                       for user in commit.get_watchers():
+                       for user in self.commit.get_watchers():
                                build.add_watcher(user)
 
                        # Return the build
@@ -931,7 +950,7 @@ class Git(object):
                        Returns a list with all revisions between start and end
                """
                if start is None:
-                       return await self.show_ref(end)
+                       return [await self.show_ref(end)]
 
                # Fetch the hashes of all revisions
                revisions = await self.command("log", "--format=%H",
@@ -979,6 +998,37 @@ class Git(object):
 
                return changed_files
 
+       async def ls_tree(self, revision, filter=None):
+               """
+                       Returns the paths of all files in the given tree
+               """
+               files = []
+
+               output = await self.command(
+                       "ls-tree",
+
+                       # Give us the filenames only
+                       "--name-only",
+
+                       # Recurse into sub-trees
+                       "-r",
+
+                       # Revision
+                       revision,
+
+                       # Ask for the output to be returned
+                       return_output=True,
+               )
+
+               for filename in output.splitlines():
+                       # Filter out anything we don't want
+                       if filter and not fnmatch.fnmatch(filename, filter):
+                               continue
+
+                       files.append(filename)
+
+               return files
+
        async def checkout(self, revision, path):
                """
                        Creates a working directory at the revision in path