]>
git.ipfire.org Git - pbs.git/blob - src/buildservice/sources.py
16 from .constants
import *
17 from .decorators
import *
30 class Sources(base
.Object
):
31 def _get_source(self
, query
, *args
):
32 res
= self
.db
.get(query
, *args
)
35 return Source(self
.backend
, res
.id, data
=res
)
37 def _get_sources(self
, query
, *args
):
38 res
= self
.db
.query(query
, *args
)
41 yield Source(self
.backend
, row
.id, data
=row
)
43 def _get_commit(self
, query
, *args
):
44 res
= self
.db
.get(query
, *args
)
47 return Commit(self
.backend
, res
.id, data
=res
)
49 def _get_commits(self
, query
, *args
):
50 res
= self
.db
.query(query
, *args
)
53 yield Commit(self
.backend
, row
.id, data
=row
)
56 return self
._get
_sources
("SELECT * FROM sources")
58 def get_by_id(self
, id):
59 return self
._get
_source
("SELECT * FROM sources \
62 def get_by_distro(self
, distro
):
63 return self
._get
_sources
("SELECT * FROM sources \
64 WHERE distro_id = %s", distro
.id)
66 def update_revision(self
, source_id
, revision
):
67 query
= "UPDATE sources SET revision = %s WHERE id = %s"
69 return self
.db
.execute(query
, revision
, source_id
)
71 def get_commit_by_id(self
, commit_id
):
72 commit
= self
.db
.get("SELECT id FROM sources_commits WHERE id = %s", commit_id
)
75 return Commit(self
.pakfire
, commit
.id)
79 with git
.Repo(self
.backend
, source
, mode
="mirror") as repo
:
80 # Fetch the latest updates
83 # Import all new revisions
84 repo
.import_revisions()
87 # Walk through all source repositories
89 # Get access to the git repo
90 with git
.Repo(self
.pakfire
, source
) as repo
:
91 # Walk through all pending commits
92 for commit
in source
.pending_commits
:
93 commit
.state
= "running"
95 logging
.debug("Processing commit %s: %s" % (commit
.revision
, commit
.subject
))
97 # Navigate to the right revision.
98 repo
.checkout(commit
.revision
)
100 # Get all changed makefiles.
104 for file in repo
.changed_files(commit
.revision
):
105 # Don't care about files that are not a makefile.
106 if not file.endswith(".%s" % MAKEFILE_EXTENSION
):
109 if os
.path
.exists(file):
110 updated_files
.append(file)
112 deleted_files
.append(file)
115 # Create a temporary directory where to put all the files
116 # that are generated here.
117 pkg_dir
= tempfile
.mkdtemp()
120 config
= pakfire
.config
.Config(["general.conf",])
121 config
.parse(source
.distro
.get_config())
123 p
= pakfire
.PakfireServer(config
=config
)
126 for file in updated_files
:
128 pkg_file
= p
.dist(file, pkg_dir
)
129 pkgs
.append(pkg_file
)
133 # Import all packages in one swoop.
135 with self
.db
.transaction():
136 build
= self
.backend
.builds
.create_from_source_package(pkg
,
137 source
.distro
, commit
=commit
, type="release")
139 # Import any testers from the commit message
140 for tester
in commit
.testers
:
145 commit
.state
= "failed"
150 if os
.path
.exists(pkg_dir
):
151 shutil
.rmtree(pkg_dir
)
153 for file in deleted_files
:
154 # Determine the name of the package.
155 name
= os
.path
.basename(file)
156 name
= name
[:len(MAKEFILE_EXTENSION
) + 1]
158 source
.distro
.delete_package(name
)
161 commit
.state
= "finished"
164 class Commit(base
.DataObject
):
165 table
= "sources_commits"
169 return self
.data
.revision
173 return self
.backend
.sources
.get_by_id(self
.data
.source_id
)
178 A shortcut to the distribution this commit
181 return self
.source
.distro
183 def set_state(self
, state
):
184 self
._set
_attribute
("state", state
)
186 state
= property(lambda s
: s
.data
.state
, set_state
)
190 return self
.backend
.users
.find_maintainer(self
.data
.author
) or self
.data
.author
194 return self
.backend
.users
.find_maintainer(self
.data
.committer
) or self
.data
.committer
198 return self
.data
.subject
.strip()
202 return self
.data
.body
.strip()
207 Returns the message without any Git tags
210 r
= re
.compile("^(%s):" % "|".join(VALID_TAGS
), re
.IGNORECASE
)
213 for line
in self
.body
.splitlines():
214 # Find lines that start with a known Git tag
220 # If all lines are empty lines, we send back an empty message
221 if all((l
== "" for l
in message
)):
224 # We will now break the message into paragraphs
225 paragraphs
= re
.split("\n\n+", "\n".join(message
))
229 for paragraph
in paragraphs
:
230 # Remove all line breaks that are not following a colon
231 # and where the next line does not start with a star.
232 paragraph
= re
.sub("(?<=\:)\n(?=[\*\s])", " ", paragraph
)
234 message
.append(paragraph
)
236 return "\n\n".join(message
)
239 def message_full(self
):
240 message
= self
.subject
243 message
+= "\n\n%s" % self
.message
247 def get_tag(self
, tag
):
249 Returns a list of the values of this Git tag
251 if not tag
in VALID_TAGS
:
252 raise ValueError("Unknown tag: %s" % tag
)
255 r
= re
.compile("^%s: (.*)$" % tag
, re
.IGNORECASE
)
258 for line
in self
.body
.splitlines():
259 # Skip all empty lines
263 # Check if line matches the regex
266 values
.append(m
.group(1))
271 def contributors(self
):
277 for tag
in ("Acked-by", "Cc", "Reported-by", "Reviewed-by", "Signed-off-by", "Suggested-by", "Tested-by"):
278 contributors
+= self
.get_tag(tag
)
280 # Get all user accounts that we know
281 users
= self
.backend
.users
.find_maintainers(contributors
)
283 # Add all email addresses where a user could not be found
284 for contributor
in contributors
[:]:
286 if user
.has_email_address(contributor
):
288 contributors
.remove(contributor
)
292 return sorted(contributors
+ users
)
298 for tag
in ("Acked-by", "Reviewed-by", "Signed-off-by", "Tested-by"):
299 users
+= self
.get_tag(tag
)
301 return self
.backend
.users
.find_maintainers(users
)
305 return self
.data
.date
309 return self
.backend
.packages
._get
_packages
("SELECT * FROM packages \
310 WHERE commit_id = %s", self
.id)
314 Removes all packages that have been created by this commit and
315 resets the state so it will be processed again.
317 # Remove all packages and corresponding builds.
318 for pkg
in self
.packages
:
319 # Check if there is a build associated with the package.
320 # If so, the whole build will be deleted.
325 # Delete the package.
331 # Reset the state to 'pending'.
332 self
.state
= "pending"
335 class Source(base
.DataObject
):
338 def __eq__(self
, other
):
339 return self
.id == other
.id
342 ret
= self
.db
.get("SELECT COUNT(*) AS len FROM sources_commits \
343 WHERE source_id = %s", self
.id)
347 def create_commit(self
, revision
, author
, committer
, subject
, body
, date
):
348 commit
= self
.backend
.sources
._get
_commit
("INSERT INTO sources_commits(source_id, \
349 revision, author, committer, subject, body, date) VALUES(%s, %s, %s, %s, %s, %s, %s) \
350 RETURNING *", self
.id, revision
, author
, committer
, subject
, body
, date
)
364 "targetpath" : self
.targetpath
,
365 "revision" : self
.revision
,
366 "branch" : self
.branch
,
371 return self
.data
.name
374 def identifier(self
):
375 return self
.data
.identifier
383 return self
.data
.gitweb
387 return self
.data
.revision
391 return self
.data
.branch
395 return self
.pakfire
.builds
.get_by_source(self
.id)
399 return self
.pakfire
.distros
.get_by_id(self
.data
.distro_id
)
402 def start_revision(self
):
403 return self
.data
.revision
406 def head_revision(self
):
407 return self
.backend
.sources
._get
_commit
("SELECT * FROM sources_commits \
408 WHERE source_id = %s ORDER BY id DESC LIMIT 1", self
.id)
410 def get_commits(self
, limit
=None, offset
=None):
411 return self
.backend
.sources
._get
_commits
("SELECT * FROM sources_commits \
412 WHERE source_id = %s ORDER BY id DESC LIMIT %s OFFSET %s", limit
, offset
)
414 def get_commit(self
, revision
):
415 commit
= self
.backend
.sources
._get
_commit
("SELECT * FROM sources_commits \
416 WHERE source_id = %s AND revision = %s", self
.id, revision
)
423 def pending_commits(self
):
424 return self
.backend
.sources
._get
_commits
("SELECT * FROM sources_commits \
425 WHERE state = %s ORDER BY imported_at", "pending")