From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 20 Jul 2023 10:15:05 +0000 (+0000)
Subject: sources: Create dist jobs for all packages on the initial commit import
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0ecffff1dd83058f3d99c8a6eb21be69fa5e2864;p=pbs.git

sources: Create dist jobs for all packages on the initial commit import

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py
index 158e40d5..966517ae 100644
--- a/src/buildservice/sources.py
+++ b/src/buildservice/sources.py
@@ -278,10 +278,12 @@ class Source(base.DataObject):
 
 	# Commits
 
-	async def _create_commit(self, revision):
+	async def _create_commit(self, revision, initial_commit=False):
 		"""
 			Imports the commit with the given revision
 		"""
+		log.info("%s: Creating commit %s" % (self, revision))
+
 		# Fetch the author's name and email address
 		author = await self.git.show_attribute(revision, r"%an <%ae>")
 
@@ -326,14 +328,21 @@ class Source(base.DataObject):
 			source=self,
 		)
 
+		# If we are processing the initial commit, we get a list of all files in the tree
+		if initial_commit:
+			changed_files = [("A", f) for f in await self.git.ls_tree(revision, "*/*.nm")]
+
 		# Find changed files
-		changed_files = await self.git.changed_files(revision, filter="*/*.nm")
+		else:
+			changed_files = await self.git.changed_files(revision, filter="*/*.nm")
 
 		# Create jobs for each file
 		for status, changed_file in changed_files:
 			# Find the package name
 			name = os.path.dirname(changed_file)
 
+			log.debug("%s: %s: Processing '%s' (status = %s)" % (self, revision, name, status))
+
 			# Check that the file part matches
 			if not changed_file.endswith("/%s.nm" % name):
 				raise ValueError("Invalid package name")
@@ -417,17 +426,24 @@ class Source(base.DataObject):
 		else:
 			await self.git.clone()
 
-		# Determine which commits there are to process
-		revisions = await self.git.revisions(self.revision, self.branch)
-
 		with self.db.transaction():
+			# Did we already import something?
+			initial_commit = not self.revision
+
+			# Determine which commits there are to process
+			revisions = await self.git.revisions(self.revision, self.branch)
+
+			# Import all revisions
 			for revision in revisions:
 				# Import the commit
-				await self._create_commit(revision)
+				await self._create_commit(revision, initial_commit=initial_commit)
 
 				# Store the updated revision
 				self._set_attribute("revision", revision)
 
+				# Only the first revision would the initial commit
+				initial_commit = False
+
 			# Store when we fetched
 			self._set_attribute_now("last_fetched_at")
 
@@ -603,6 +619,9 @@ class Commit(base.DataObject):
 		"""
 			Creates a new job
 		"""
+		log.info("%s: %s: Created '%s' job for '%s'" \
+			% (self.source, self.revision, action, name))
+
 		job = self.backend.sources._get_job("""
 			INSERT INTO
 				source_commit_jobs
@@ -812,7 +831,7 @@ class Job(base.DataObject):
 				self.source.repo, package, group=self.commit.builds)
 
 			# Add any watchers
-			for user in commit.get_watchers():
+			for user in self.commit.get_watchers():
 				build.add_watcher(user)
 
 			# Return the build
@@ -931,7 +950,7 @@ class Git(object):
 			Returns a list with all revisions between start and end
 		"""
 		if start is None:
-			return await self.show_ref(end)
+			return [await self.show_ref(end)]
 
 		# Fetch the hashes of all revisions
 		revisions = await self.command("log", "--format=%H",
@@ -979,6 +998,37 @@ class Git(object):
 
 		return changed_files
 
+	async def ls_tree(self, revision, filter=None):
+		"""
+			Returns the paths of all files in the given tree
+		"""
+		files = []
+
+		output = await self.command(
+			"ls-tree",
+
+			# Give us the filenames only
+			"--name-only",
+
+			# Recurse into sub-trees
+			"-r",
+
+			# Revision
+			revision,
+
+			# Ask for the output to be returned
+			return_output=True,
+		)
+
+		for filename in output.splitlines():
+			# Filter out anything we don't want
+			if filter and not fnmatch.fnmatch(filename, filter):
+				continue
+
+			files.append(filename)
+
+		return files
+
 	async def checkout(self, revision, path):
 		"""
 			Creates a working directory at the revision in path