]> git.ipfire.org Git - people/jschlag/pbs.git/blame - src/buildservice/builds.py
Add command to list packages in a repository
[people/jschlag/pbs.git] / src / buildservice / builds.py
CommitLineData
f6e6ff79
MT
1#!/usr/bin/python
2
3import datetime
4import hashlib
5import logging
6import os
7import re
8import shutil
9import uuid
10
11import pakfire
12import pakfire.config
13import pakfire.packages
14
2c909128
MT
15from . import base
16from . import builders
17from . import logs
18from . import packages
19from . import repository
20from . import updates
21from . import users
22
23from .constants import *
044a9c43 24from .decorators import *
f6e6ff79
MT
25
26def import_from_package(_pakfire, filename, distro=None, commit=None, type="release",
27 arches=None, check_for_duplicates=True, owner=None):
28
29 if distro is None:
30 distro = commit.source.distro
31
32 assert distro
33
34 # Open the package file to read some basic information.
35 pkg = pakfire.packages.open(None, None, filename)
36
37 if check_for_duplicates:
38 if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
39 logging.warning("Duplicate package detected: %s. Skipping." % pkg)
40 return
41
42 # Open the package and add it to the database.
43 pkg = packages.Package.open(_pakfire, filename)
44 logging.debug("Created new package: %s" % pkg)
45
46 # Associate the package to the processed commit.
47 if commit:
48 pkg.commit = commit
49
50 # Create a new build object from the package which
51 # is always a release build.
52 build = Build.create(_pakfire, pkg, type=type, owner=owner, distro=distro)
53 logging.debug("Created new build job: %s" % build)
54
55 # Create all automatic jobs.
56 build.create_autojobs(arches=arches)
57
58 return pkg, build
59
60
61class Builds(base.Object):
764b87d2
MT
62 def _get_build(self, query, *args):
63 res = self.db.get(query, *args)
64
65 if res:
66 return Build(self.backend, res.id, data=res)
67
68 def _get_builds(self, query, *args):
69 res = self.db.query(query, *args)
70
71 for row in res:
72 yield Build(self.backend, row.id, data=row)
73
eedc6432
MT
74 def get_by_id(self, id, data=None):
75 return Build(self.pakfire, id, data=data)
f6e6ff79
MT
76
77 def get_by_uuid(self, uuid):
78 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
79
80 if build:
81 return self.get_by_id(build.id)
82
83 def get_all(self, limit=50):
eedc6432 84 query = "SELECT * FROM builds ORDER BY time_created DESC"
f6e6ff79
MT
85
86 if limit:
87 query += " LIMIT %d" % limit
88
eedc6432 89 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
f6e6ff79 90
eedc6432 91 def get_by_user(self, user, type=None, public=None):
f6e6ff79
MT
92 args = []
93 conditions = []
94
95 if not type or type == "scratch":
96 # On scratch builds the user id equals the owner id.
97 conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
98 args.append(user.id)
99
100 elif not type or type == "release":
101 pass # TODO
102
103 if public is True:
104 conditions.append("public = 'Y'")
105 elif public is False:
106 conditions.append("public = 'N'")
107
eedc6432 108 query = "SELECT builds.* AS id FROM builds \
f6e6ff79
MT
109 JOIN packages ON builds.pkg_id = packages.id"
110
111 if conditions:
112 query += " WHERE %s" % " AND ".join(conditions)
113
eedc6432 114 query += " ORDER BY builds.time_created DESC"
f6e6ff79 115
eedc6432 116 builds = []
f6e6ff79 117 for build in self.db.query(query, *args):
eedc6432
MT
118 build = Build(self.pakfire, build.id, build)
119 builds.append(build)
120
121 return builds
f6e6ff79 122
a15d6139 123 def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
f6e6ff79
MT
124 args = [name,]
125 conditions = [
126 "packages.name = %s",
127 ]
128
129 if type:
130 conditions.append("builds.type = %s")
131 args.append(type)
132
133 or_conditions = []
134 if public is True:
135 or_conditions.append("public = 'Y'")
136 elif public is False:
137 or_conditions.append("public = 'N'")
138
139 if user and not user.is_admin():
140 or_conditions.append("builds.owner_id = %s")
141 args.append(user.id)
142
a15d6139 143 query = "SELECT builds.* AS id FROM builds \
f6e6ff79
MT
144 JOIN packages ON builds.pkg_id = packages.id"
145
146 if or_conditions:
147 conditions.append(" OR ".join(or_conditions))
148
149 if conditions:
150 query += " WHERE %s" % " AND ".join(conditions)
151
a15d6139
MT
152 if type == "release":
153 query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
154 elif type == "scratch":
155 query += " ORDER BY time_created DESC"
f6e6ff79 156
a15d6139
MT
157 if limit:
158 if offset:
159 query += " LIMIT %s,%s"
160 args.extend([offset, limit])
161 else:
162 query += " LIMIT %s"
163 args.append(limit)
164
165 return [Build(self.pakfire, b.id, b) for b in self.db.query(query, *args)]
f6e6ff79
MT
166
167 def get_latest_by_name(self, name, type=None, public=None):
2f45327a
MT
168 query = "\
169 SELECT * FROM builds \
170 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
171 WHERE builds_latest.package_name = %s"
f6e6ff79
MT
172 args = [name,]
173
2f45327a
MT
174 if type:
175 query += " AND builds_latest.build_type = %s"
176 args.append(type)
177
f6e6ff79 178 if public is True:
2f45327a
MT
179 query += " AND builds.public = %s"
180 args.append("Y")
f6e6ff79 181 elif public is False:
2f45327a
MT
182 query += " AND builds.public = %s"
183 args.append("N")
f6e6ff79 184
2f45327a
MT
185 # Get the last one only.
186 # Prefer release builds over scratch builds.
187 query += "\
188 ORDER BY \
189 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
190 builds.time_created DESC \
191 LIMIT 1"
f6e6ff79 192
2f45327a 193 res = self.db.get(query, *args)
f6e6ff79 194
2f45327a
MT
195 if res:
196 return Build(self.pakfire, res.id, res)
f6e6ff79 197
fd0e70ec
MT
198 def get_active_builds(self, name, public=None):
199 query = "\
aff0187d
MT
200 SELECT * FROM builds \
201 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
2f83864f
MT
202 WHERE builds_latest.package_name = %s AND builds.type = %s"
203 args = [name, "release"]
fd0e70ec
MT
204
205 if public is True:
206 query += " AND builds.public = %s"
207 args.append("Y")
208 elif public is False:
209 query += " AND builds.public = %s"
210 args.append("N")
211
fd0e70ec
MT
212 builds = []
213 for row in self.db.query(query, *args):
214 b = Build(self.pakfire, row.id, row)
215 builds.append(b)
216
217 # Sort the result. Lastest build first.
218 builds.sort(reverse=True)
219
220 return builds
221
f6e6ff79 222 def count(self):
966498de
MT
223 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
224 if builds:
225 return builds.count
f6e6ff79
MT
226
227 def needs_test(self, threshold, arch, limit=None, randomize=False):
228 query = "SELECT id FROM builds \
229 WHERE NOT EXISTS \
230 (SELECT * FROM jobs WHERE \
231 jobs.build_id = builds.id AND \
044a9c43 232 jobs.arch = %s AND \
f6e6ff79
MT
233 (jobs.state != 'finished' OR \
234 jobs.time_finished >= %s) \
235 ) \
236 AND EXISTS \
237 (SELECT * FROM jobs WHERE \
238 jobs.build_id = builds.id AND \
044a9c43 239 jobs.arch = %s AND \
f6e6ff79
MT
240 jobs.type = 'build' AND \
241 jobs.state = 'finished' AND \
242 jobs.time_finished < %s \
243 ) \
83be3106
MT
244 AND builds.type = 'release' \
245 AND (builds.state = 'stable' OR builds.state = 'testing')"
044a9c43 246 args = [arch, threshold, arch, threshold]
f6e6ff79
MT
247
248 if randomize:
249 query += " ORDER BY RAND()"
250
251 if limit:
252 query += " LIMIT %s"
253 args.append(limit)
254
255 return [Build(self.pakfire, b.id) for b in self.db.query(query, *args)]
256
257 def get_obsolete(self, repo=None):
258 """
259 Get all obsoleted builds.
260
261 If repo is True: which are in any repository.
262 If repo is some Repository object: which are in this repository.
263 """
264 args = []
265
266 if repo is None:
267 query = "SELECT id FROM builds WHERE state = 'obsolete'"
268
269 else:
270 query = "SELECT build_id AS id FROM repositories_builds \
271 JOIN builds ON builds.id = repositories_builds.build_id \
272 WHERE builds.state = 'obsolete'"
273
274 if repo and not repo is True:
275 query += " AND repositories_builds.repo_id = %s"
276 args.append(repo.id)
277
278 res = self.db.query(query, *args)
279
280 builds = []
281 for build in res:
282 build = Build(self.pakfire, build.id)
283 builds.append(build)
284
285 return builds
286
4b1e87c4
MT
287 def get_changelog(self, name, public=None, limit=5, offset=0):
288 query = "SELECT builds.* FROM builds \
289 JOIN packages ON builds.pkg_id = packages.id \
290 WHERE \
291 builds.type = %s \
292 AND \
293 packages.name = %s"
294 args = ["release", name,]
295
296 if public == True:
297 query += " AND builds.public = %s"
298 args.append("Y")
299 elif public == False:
300 query += " AND builds.public = %s"
301 args.append("N")
302
303 query += " ORDER BY builds.time_created DESC"
304
305 if limit:
306 if offset:
307 query += " LIMIT %s,%s"
308 args += [offset, limit]
309 else:
310 query += " LIMIT %s"
311 args.append(limit)
312
313 builds = []
314 for b in self.db.query(query, *args):
315 b = Build(self.pakfire, b.id, b)
316 builds.append(b)
317
318 builds.sort(reverse=True)
319
320 return builds
321
62c7e7cd
MT
322 def get_comments(self, limit=10, offset=None, user=None):
323 query = "SELECT * FROM builds_comments \
324 JOIN users ON builds_comments.user_id = users.id"
325 args = []
326
327 wheres = []
328 if user:
329 wheres.append("users.id = %s")
330 args.append(user.id)
331
332 if wheres:
333 query += " WHERE %s" % " AND ".join(wheres)
334
335 # Sort everything.
336 query += " ORDER BY time_created DESC"
337
338 # Limits.
339 if limit:
340 if offset:
341 query += " LIMIT %s,%s"
342 args.append(offset)
343 else:
344 query += " LIMIT %s"
345
346 args.append(limit)
347
348 comments = []
349 for comment in self.db.query(query, *args):
350 comment = logs.CommentLogEntry(self.pakfire, comment)
351 comments.append(comment)
352
353 return comments
354
a90bd9b0 355 def get_build_times_summary(self, name=None, job_type=None, arch=None):
bc293d03
MT
356 query = "\
357 SELECT \
358 builds_times.arch AS arch, \
359 MAX(duration) AS maximum, \
360 MIN(duration) AS minimum, \
361 AVG(duration) AS average, \
362 SUM(duration) AS sum, \
363 STDDEV_POP(duration) AS stddev \
364 FROM builds_times \
365 LEFT JOIN builds ON builds_times.build_id = builds.id \
366 LEFT JOIN packages ON builds.pkg_id = packages.id"
367
368 args = []
369 conditions = []
370
371 # Filter for name.
372 if name:
373 conditions.append("packages.name = %s")
374 args.append(name)
375
376 # Filter by job types.
a90bd9b0 377 if job_type:
bc293d03
MT
378 conditions.append("builds_times.job_type = %s")
379 args.append(job_type)
380
a90bd9b0
MT
381 # Filter by arch.
382 if arch:
383 conditions.append("builds_times.arch = %s")
384 args.append(arch)
385
bc293d03
MT
386 # Add conditions.
387 if conditions:
388 query += " WHERE %s" % " AND ".join(conditions)
389
390 # Grouping and sorting.
391 query += " GROUP BY arch ORDER BY arch DESC"
392
393 return self.db.query(query, *args)
394
a90bd9b0
MT
395 def get_build_times_by_arch(self, arch, **kwargs):
396 kwargs.update({
397 "arch" : arch,
398 })
399
400 build_times = self.get_build_times_summary(**kwargs)
401 if build_times:
402 return build_times[0]
403
f6e6ff79
MT
404
405class Build(base.Object):
734c61e0 406 def __init__(self, pakfire, id, data=None):
f6e6ff79
MT
407 base.Object.__init__(self, pakfire)
408
409 # ID of this build
410 self.id = id
411
412 # Cache data.
734c61e0 413 self._data = data
f6e6ff79
MT
414 self._jobs = None
415 self._jobs_test = None
416 self._depends_on = None
417 self._pkg = None
418 self._credits = None
419 self._owner = None
420 self._update = None
421 self._repo = None
422 self._distro = None
423
424 def __repr__(self):
425 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
426
427 def __cmp__(self, other):
428 assert self.pkg
429 assert other.pkg
430
431 return cmp(self.pkg, other.pkg)
432
764b87d2
MT
433 def __iter__(self):
434 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
435 WHERE build_id = %s", self.id)
436
437 return iter(sorted(jobs))
438
f6e6ff79
MT
439 @classmethod
440 def create(cls, pakfire, pkg, type="release", owner=None, distro=None, public=True):
441 assert type in ("release", "scratch", "test")
442 assert distro, "You need to specify the distribution of this build."
443
444 if public:
445 public = "Y"
446 else:
447 public = "N"
448
449 # Check if scratch build has an owner.
450 if type == "scratch" and not owner:
451 raise Exception, "Scratch builds require an owner"
452
453 # Set the default priority of this build.
454 if type == "release":
455 priority = 0
456
457 elif type == "scratch":
458 priority = 1
459
460 elif type == "test":
461 priority = -1
462
463 id = pakfire.db.execute("""
464 INSERT INTO builds(uuid, pkg_id, type, distro_id, time_created, public, priority)
465 VALUES(%s, %s, %s, %s, NOW(), %s, %s)""", "%s" % uuid.uuid4(), pkg.id,
466 type, distro.id, public, priority)
467
468 # Set the owner of this buildgroup.
469 if owner:
470 pakfire.db.execute("UPDATE builds SET owner_id = %s WHERE id = %s",
471 owner.id, id)
472
473 build = cls(pakfire, id)
474
475 # Log that the build has been created.
476 build.log("created", user=owner)
477
478 # Create directory where the files live.
479 if not os.path.exists(build.path):
480 os.makedirs(build.path)
481
482 # Move package file to the directory of the build.
483 source_path = os.path.join(build.path, "src")
484 build.pkg.move(source_path)
485
486 # Generate an update id.
487 build.generate_update_id()
488
489 # Obsolete all other builds with the same name to track updates.
490 build.obsolete_others()
491
492 # Search for possible bug IDs in the commit message.
493 build.search_for_bugs()
494
495 return build
496
497 def delete(self):
498 """
499 Deletes this build including all jobs, packages and the source
500 package.
501 """
502 # If the build is in a repository, we need to remove it.
503 if self.repo:
504 self.repo.rem_build(self)
505
506 for job in self.jobs + self.test_jobs:
507 job.delete()
508
509 if self.pkg:
510 self.pkg.delete()
511
512 # Delete everything related to this build.
513 self.__delete_bugs()
514 self.__delete_comments()
515 self.__delete_history()
516 self.__delete_watchers()
517
518 # Delete the build itself.
519 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
f6e6ff79
MT
520
521 def __delete_bugs(self):
522 """
523 Delete all associated bugs.
524 """
525 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
526
527 def __delete_comments(self):
528 """
529 Delete all comments.
530 """
531 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
532
533 def __delete_history(self):
534 """
535 Delete the repository history.
536 """
537 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
538
539 def __delete_watchers(self):
540 """
541 Delete all watchers.
542 """
543 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
544
545 def reset(self):
546 """
547 Resets the whole build so it can start again (as it has never
548 been started).
549 """
550 for job in self.jobs:
551 job.reset()
552
553 #self.__delete_bugs()
554 self.__delete_comments()
555 self.__delete_history()
556 self.__delete_watchers()
557
558 self.state = "building"
559
560 # XXX empty log
561
562 @property
563 def data(self):
564 """
565 Lazy fetching of data for this object.
566 """
567 if self._data is None:
966498de 568 self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id)
f6e6ff79
MT
569 assert self._data
570
571 return self._data
572
573 @property
574 def info(self):
575 """
576 A set of information that is sent to the XMLRPC client.
577 """
578 return { "uuid" : self.uuid }
579
580 def log(self, action, user=None, bug_id=None):
581 user_id = None
582 if user:
583 user_id = user.id
584
585 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
586 VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
587
588 @property
589 def uuid(self):
590 """
591 The UUID of this build.
592 """
593 return self.data.uuid
594
595 @property
596 def pkg(self):
597 """
598 Get package that is to be built in the build.
599 """
600 if self._pkg is None:
601 self._pkg = packages.Package(self.pakfire, self.data.pkg_id)
602
603 return self._pkg
604
605 @property
606 def name(self):
607 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
608
609 @property
610 def type(self):
611 """
612 The type of this build.
613 """
614 return self.data.type
615
616 @property
617 def owner_id(self):
618 """
619 The ID of the owner of this build.
620 """
621 return self.data.owner_id
622
623 @property
624 def owner(self):
625 """
626 The owner of this build.
627 """
628 if not self.owner_id:
629 return
630
631 if self._owner is None:
632 self._owner = self.pakfire.users.get_by_id(self.owner_id)
633 assert self._owner
634
635 return self._owner
636
637 @property
638 def distro_id(self):
639 return self.data.distro_id
640
641 @property
642 def distro(self):
643 if self._distro is None:
644 self._distro = self.pakfire.distros.get_by_id(self.distro_id)
645 assert self._distro
646
647 return self._distro
648
649 @property
650 def user(self):
651 if self.type == "scratch":
652 return self.owner
653
654 def get_depends_on(self):
655 if self.data.depends_on and self._depends_on is None:
656 self._depends_on = Build(self.pakfire, self.data.depends_on)
657
658 return self._depends_on
659
660 def set_depends_on(self, build):
661 self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s",
662 build.id, self.id)
f6e6ff79
MT
663
664 # Update cache.
665 self._depends_on = build
666 self._data["depends_on"] = build.id
667
668 depends_on = property(get_depends_on, set_depends_on)
669
670 @property
671 def created(self):
672 return self.data.time_created
673
eedc6432
MT
674 @property
675 def date(self):
676 return self.created.date()
677
f6e6ff79
MT
678 @property
679 def public(self):
680 """
681 Is this build public?
682 """
683 return self.data.public == "Y"
684
eedc6432
MT
685 @property
686 def size(self):
687 """
688 Returns the size on disk of this build.
689 """
690 s = 0
691
692 # Add the source package.
693 if self.pkg:
694 s += self.pkg.size
695
696 # Add all jobs.
697 s += sum((j.size for j in self.jobs))
698
699 return s
700
f6e6ff79
MT
701 #@property
702 #def state(self):
703 # # Cache all states.
704 # states = [j.state for j in self.jobs]
705 #
706 # target_state = "unknown"
707 #
708 # # If at least one job has failed, the whole build has failed.
709 # if "failed" in states:
710 # target_state = "failed"
711 #
712 # # It at least one of the jobs is still running, the whole
713 # # build is in running state.
714 # elif "running" in states:
715 # target_state = "running"
716 #
717 # # If all jobs are in the finished state, we turn into finished
718 # # state as well.
719 # elif all([s == "finished" for s in states]):
720 # target_state = "finished"
721 #
722 # return target_state
723
724 def auto_update_state(self):
725 """
726 Check if the state of this build can be updated and perform
727 the change if possible.
728 """
729 # Do not change the broken/obsolete state automatically.
730 if self.state in ("broken", "obsolete"):
731 return
732
733 if self.repo and self.repo.type == "stable":
734 self.update_state("stable")
735 return
736
737 # If any of the build jobs are finished, the build will be put in testing
738 # state.
739 for job in self.jobs:
740 if job.state == "finished":
741 self.update_state("testing")
742 break
743
744 def update_state(self, state, user=None, remove=False):
745 assert state in ("stable", "testing", "obsolete", "broken")
746
747 self.db.execute("UPDATE builds SET state = %s WHERE id = %s", state, self.id)
748
749 if self._data:
750 self._data["state"] = state
f6e6ff79
MT
751
752 # In broken state, the removal from the repository is forced and
753 # all jobs that are not finished yet will be aborted.
754 if state == "broken":
755 remove = True
756
757 for job in self.jobs:
758 if job.state in ("new", "pending", "running", "dependency_error"):
759 job.state = "aborted"
760
761 # If this build is in a repository, it will leave it.
762 if remove and self.repo:
763 self.repo.rem_build(self)
764
765 # If a release build is now in testing state, we put it into the
766 # first repository of the distribution.
767 elif self.type == "release" and state == "testing":
768 # If the build is not in a repository, yet and if there is
769 # a first repository, we put the build there.
770 if not self.repo and self.distro.first_repo:
771 self.distro.first_repo.add_build(self, user=user)
772
773 @property
774 def state(self):
775 return self.data.state
776
9fa1787c
MT
777 def is_broken(self):
778 return self.state == "broken"
779
f6e6ff79
MT
780 def obsolete_others(self):
781 if not self.type == "release":
782 return
783
784 for build in self.pakfire.builds.get_by_name(self.pkg.name, type="release"):
785 # Don't modify ourself.
786 if self.id == build.id:
787 continue
788
789 # Don't touch broken builds.
790 if build.state in ("obsolete", "broken"):
791 continue
792
793 # Obsolete the build.
794 build.update_state("obsolete")
795
796 def set_severity(self, severity):
797 self.db.execute("UPDATE builds SET severity = %s WHERE id = %s", state, self.id)
798
799 if self._data:
800 self._data["severity"] = severity
f6e6ff79
MT
801
802 def get_severity(self):
803 return self.data.severity
804
805 severity = property(get_severity, set_severity)
806
807 @property
808 def commit(self):
809 if self.pkg and self.pkg.commit:
810 return self.pkg.commit
811
812 def update_message(self, msg):
813 self.db.execute("UPDATE builds SET message = %s WHERE id = %s", msg, self.id)
814
815 if self._data:
816 self._data["message"] = msg
f6e6ff79
MT
817
818 def has_perm(self, user):
819 """
820 Check, if the given user has the right to perform administrative
821 operations on this build.
822 """
823 if user is None:
824 return False
825
826 if user.is_admin():
827 return True
828
829 # Check if the user is allowed to manage packages from the critical path.
830 if self.critical_path and not user.has_perm("manage_critical_path"):
831 return False
832
833 # Search for maintainers...
834
835 # Scratch builds.
836 if self.type == "scratch":
837 # The owner of a scratch build has the right to do anything with it.
838 if self.owner_id == user.id:
839 return True
840
841 # Release builds.
842 elif self.type == "release":
843 # The maintainer also is allowed to manage the build.
844 if self.pkg.maintainer == user:
845 return True
846
847 # Deny permission for all other cases.
848 return False
849
850 @property
851 def message(self):
852 message = ""
853
854 if self.data.message:
855 message = self.data.message
856
857 elif self.commit:
858 if self.commit.message:
859 message = "\n".join((self.commit.subject, self.commit.message))
860 else:
861 message = self.commit.subject
862
863 prefix = "%s: " % self.pkg.name
864 if message.startswith(prefix):
865 message = message[len(prefix):]
866
867 return message
868
869 def get_priority(self):
870 return self.data.priority
871
872 def set_priority(self, priority):
873 assert priority in (-2, -1, 0, 1, 2)
874
875 self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority,
876 self.id)
f6e6ff79
MT
877
878 if self._data:
879 self._data["priority"] = priority
880
881 priority = property(get_priority, set_priority)
882
883 @property
884 def path(self):
885 path = []
886 if self.type == "scratch":
887 path.append(BUILD_SCRATCH_DIR)
888 path.append(self.uuid)
889
890 elif self.type == "release":
891 path.append(BUILD_RELEASE_DIR)
892 path.append("%s/%s-%s-%s" % \
893 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
894
895 else:
896 raise Exception, "Unknown build type: %s" % self.type
897
898 return os.path.join(*path)
899
900 @property
901 def source_filename(self):
902 return os.path.basename(self.pkg.path)
903
904 @property
905 def download_prefix(self):
906 return "/".join((self.pakfire.settings.get("download_baseurl"), "packages"))
907
908 @property
909 def source_download(self):
910 return "/".join((self.download_prefix, self.pkg.path))
911
912 @property
913 def source_hash_sha512(self):
914 return self.pkg.hash_sha512
915
916 @property
917 def link(self):
918 # XXX maybe this should rather live in a uimodule.
919 # zlib-1.2.3-2.ip3 [src, i686, blah...]
920 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
921 (self.state, self.type, self.uuid, self.name)
922
923 s_jobs = []
924 for job in self.jobs:
925 s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
044a9c43 926 (job.state, job.type, job.uuid, job.arch))
f6e6ff79
MT
927
928 if s_jobs:
929 s += " [%s]" % ", ".join(s_jobs)
930
931 return s
932
933 @property
934 def supported_arches(self):
935 return self.pkg.supported_arches
936
937 @property
938 def critical_path(self):
939 return self.pkg.critical_path
940
941 def get_jobs(self, type=None):
942 """
943 Returns a list of jobs of this build.
944 """
945 return self.pakfire.jobs.get_by_build(self.id, self, type=type)
946
947 @property
948 def jobs(self):
949 """
950 Get a list of all build jobs that are in this build.
951 """
952 if self._jobs is None:
953 self._jobs = self.get_jobs(type="build")
954
955 return self._jobs
956
957 @property
958 def test_jobs(self):
959 if self._jobs_test is None:
960 self._jobs_test = self.get_jobs(type="test")
961
962 return self._jobs_test
963
964 @property
965 def all_jobs_finished(self):
966 ret = True
967
968 for job in self.jobs:
969 if not job.state == "finished":
970 ret = False
971 break
972
973 return ret
974
975 def create_autojobs(self, arches=None, type="build"):
976 jobs = []
977
978 # Arches may be passed to this function. If not we use all arches
979 # this package supports.
980 if arches is None:
981 arches = self.supported_arches
982
983 # Create a new job for every given archirecture.
984 for arch in self.pakfire.arches.expand(arches):
985 # Don't create jobs for src.
986 if arch.name == "src":
987 continue
988
989 job = self.add_job(arch, type=type)
990 jobs.append(job)
991
992 # Return all newly created jobs.
993 return jobs
994
995 def add_job(self, arch, type="build"):
996 job = Job.create(self.pakfire, self, arch, type=type)
997
998 # Add new job to cache.
999 if self._jobs:
1000 self._jobs.append(job)
1001
1002 return job
1003
1004 ## Update stuff
1005
1006 @property
1007 def update_id(self):
1008 if not self.type == "release":
1009 return
1010
1011 # Generate an update ID if none does exist, yet.
1012 self.generate_update_id()
1013
1014 s = [
1015 "%s" % self.distro.name.replace(" ", "").upper(),
1016 "%04d" % (self.data.update_year or 0),
1017 "%04d" % (self.data.update_num or 0),
1018 ]
1019
1020 return "-".join(s)
1021
1022 def generate_update_id(self):
1023 if not self.type == "release":
1024 return
1025
1026 if self.data.update_num:
1027 return
1028
1029 update = self.db.get("SELECT update_num AS num FROM builds \
1030 WHERE update_year = YEAR(NOW()) ORDER BY update_num DESC LIMIT 1")
1031
1032 if update:
1033 update_num = update.num + 1
1034 else:
1035 update_num = 1
1036
1037 self.db.execute("UPDATE builds SET update_year = YEAR(NOW()), update_num = %s \
1038 WHERE id = %s", update_num, self.id)
1039
1040 ## Comment stuff
1041
1042 def get_comments(self, limit=10, offset=0):
1043 query = "SELECT * FROM builds_comments \
1044 JOIN users ON builds_comments.user_id = users.id \
1045 WHERE build_id = %s ORDER BY time_created ASC"
1046
1047 comments = []
1048 for comment in self.db.query(query, self.id):
1049 comment = logs.CommentLogEntry(self.pakfire, comment)
1050 comments.append(comment)
1051
1052 return comments
1053
1054 def add_comment(self, user, text, credit):
1055 # Add the new comment to the database.
1056 id = self.db.execute("INSERT INTO \
1057 builds_comments(build_id, user_id, text, credit, time_created) \
1058 VALUES(%s, %s, %s, %s, NOW())",
1059 self.id, user.id, text, credit)
1060
1061 # Update the credit cache.
1062 if not self._credits is None:
1063 self._credits += credit
1064
1065 # Send the new comment to all watchers and stuff.
1066 self.send_comment_message(id)
1067
1068 # Return the ID of the newly created comment.
1069 return id
1070
1071 @property
1072 def score(self):
1073 # XXX UPDATE THIS
1074 if self._credits is None:
1075 # Get the sum of the credits from the database.
1076 query = self.db.get(
1077 "SELECT SUM(credit) as credits FROM builds_comments WHERE build_id = %s",
1078 self.id
1079 )
1080
1081 self._credits = query.credits or 0
1082
1083 return self._credits
1084
1085 @property
1086 def credits(self):
1087 # XXX COMPAT
1088 return self.score
1089
1090 def get_commenters(self):
1091 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
1092 JOIN users ON builds_comments.user_id = users.id \
1093 WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
1094 AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
1095
1096 return [users.User(self.pakfire, u.id) for u in users]
1097
1098 def send_comment_message(self, comment_id):
1099 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
1100 comment_id)
1101
1102 assert comment
1103 assert comment.build_id == self.id
1104
1105 # Get user who wrote the comment.
1106 user = self.pakfire.users.get_by_id(comment.user_id)
1107
1108 format = {
1109 "build_name" : self.name,
1110 "user_name" : user.realname,
1111 }
1112
1113 # XXX create beautiful message
1114
1115 self.pakfire.messages.send_to_all(self.message_recipients,
1116 N_("%(user_name)s commented on %(build_name)s"),
1117 comment.text, format)
1118
1119 ## Logging stuff
1120
1121 def get_log(self, comments=True, repo=True, limit=None):
1122 entries = []
1123
fd681905
MT
1124 # Created entry.
1125 created_entry = logs.CreatedLogEntry(self.pakfire, self)
1126 entries.append(created_entry)
1127
f6e6ff79
MT
1128 if comments:
1129 entries += self.get_comments(limit=limit)
1130
1131 if repo:
1132 entries += self.get_repo_moves(limit=limit)
1133
1134 # Sort all entries in chronological order.
1135 entries.sort()
1136
1137 if limit:
1138 entries = entries[:limit]
1139
1140 return entries
1141
1142 ## Watchers stuff
1143
1144 def get_watchers(self):
fe8e7f02 1145 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
f6e6ff79
MT
1146 JOIN users ON builds_watchers.user_id = users.id \
1147 WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1148 AND users.activated = 'Y' ORDER BY users.id", self.id)
1149
1150 return [users.User(self.pakfire, u.id) for u in query]
1151
1152 def add_watcher(self, user):
1153 # Don't add a user twice.
1154 if user in self.get_watchers():
1155 return
1156
1157 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1158 VALUES(%s, %s)", self.id, user.id)
1159
1160 @property
1161 def message_recipients(self):
1162 ret = []
1163
1164 for watcher in self.get_watchers():
1165 ret.append("%s <%s>" % (watcher.realname, watcher.email))
1166
1167 return ret
1168
1169 @property
1170 def update(self):
1171 if self._update is None:
1172 update = self.db.get("SELECT update_id AS id FROM updates_builds \
1173 WHERE build_id = %s", self.id)
1174
1175 if update:
1176 self._update = updates.Update(self.pakfire, update.id)
1177
1178 return self._update
1179
1180 @property
1181 def repo(self):
1182 if self._repo is None:
1183 repo = self.db.get("SELECT repo_id AS id FROM repositories_builds \
1184 WHERE build_id = %s", self.id)
1185
1186 if repo:
1187 self._repo = repository.Repository(self.pakfire, repo.id)
1188
1189 return self._repo
1190
1191 def get_repo_moves(self, limit=None):
1192 query = "SELECT * FROM repositories_history \
1193 WHERE build_id = %s ORDER BY time ASC"
1194
1195 actions = []
1196 for action in self.db.query(query, self.id):
1197 action = logs.RepositoryLogEntry(self.pakfire, action)
1198 actions.append(action)
1199
1200 return actions
1201
1202 @property
1203 def is_loose(self):
1204 if self.repo:
1205 return False
1206
1207 return True
1208
1209 @property
1210 def repo_time(self):
1211 repo = self.db.get("SELECT time_added FROM repositories_builds \
1212 WHERE build_id = %s", self.id)
1213
1214 if repo:
1215 return repo.time_added
1216
1217 def get_auto_move(self):
1218 return self.data.auto_move == "Y"
1219
1220 def set_auto_move(self, state):
1221 if state:
1222 state = "Y"
1223 else:
1224 state = "N"
1225
1226 self.db.execute("UPDATE builds SET auto_move = %s WHERE id = %s", self.id)
1227 if self._data:
1228 self._data["auto_move"] = state
1229
1230 auto_move = property(get_auto_move, set_auto_move)
1231
1232 @property
1233 def can_move_forward(self):
1234 if not self.repo:
1235 return False
1236
1237 # If there is no next repository, we cannot move anything.
d629da45 1238 if not self.repo.next:
f6e6ff79
MT
1239 return False
1240
1241 # If the needed amount of score is reached, we can move forward.
d629da45 1242 if self.score >= self.repo.next.score_needed:
f6e6ff79
MT
1243 return True
1244
1245 # If the repository does not require a minimal time,
1246 # we can move forward immediately.
1247 if not self.repo.time_min:
1248 return True
1249
1250 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1251 WHERE build_id = %s", self.id)
1252 duration = query.duration
1253
1254 if duration >= self.repo.time_min:
1255 return True
1256
1257 return False
1258
1259 ## Bugs
1260
1261 def get_bug_ids(self):
1262 query = self.db.query("SELECT bug_id FROM builds_bugs \
1263 WHERE build_id = %s", self.id)
1264
1265 return [b.bug_id for b in query]
1266
1267 def add_bug(self, bug_id, user=None, log=True):
1268 # Check if this bug is already in the list of bugs.
1269 if bug_id in self.get_bug_ids():
1270 return
1271
1272 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1273 VALUES(%s, %s)", self.id, bug_id)
1274
1275 # Log the event.
1276 if log:
1277 self.log("bug_added", user=user, bug_id=bug_id)
1278
1279 def rem_bug(self, bug_id, user=None, log=True):
1280 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1281 bug_id = %s", self.id, bug_id)
1282
1283 # Log the event.
1284 if log:
1285 self.log("bug_removed", user=user, bug_id=bug_id)
1286
1287 def search_for_bugs(self):
1288 if not self.commit:
1289 return
1290
1291 pattern = re.compile(r"(bug\s?|#)(\d+)")
1292
1293 for txt in (self.commit.subject, self.commit.message):
1294 for bug in re.finditer(pattern, txt):
1295 try:
1296 bugid = int(bug.group(2))
1297 except ValueError:
1298 continue
1299
1300 # Check if a bug with the given ID exists in BZ.
1301 bug = self.pakfire.bugzilla.get_bug(bugid)
1302 if not bug:
1303 continue
1304
1305 self.add_bug(bugid)
1306
1307 def get_bugs(self):
1308 bugs = []
1309 for bug_id in self.get_bug_ids():
1310 bug = self.pakfire.bugzilla.get_bug(bug_id)
1311 if not bug:
1312 continue
1313
1314 bugs.append(bug)
1315
1316 return bugs
1317
1318 def _update_bugs_helper(self, repo):
1319 """
1320 This function takes a new status and generates messages that
1321 are appended to all bugs.
1322 """
1323 try:
1324 kwargs = BUG_MESSAGES[repo.type].copy()
1325 except KeyError:
1326 return
1327
1328 baseurl = self.pakfire.settings.get("baseurl", "")
1329 args = {
1330 "build_url" : "%s/build/%s" % (baseurl, self.uuid),
1331 "distro_name" : self.distro.name,
1332 "package_name" : self.name,
1333 "repo_name" : repo.name,
1334 }
1335 kwargs["comment"] = kwargs["comment"] % args
1336
1337 self.update_bugs(**kwargs)
1338
1339 def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1340 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1341 VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1342
1343 def update_bugs(self, status, resolution=None, comment=None):
1344 # Update all bugs linked to this build.
1345 for bug_id in self.get_bug_ids():
1346 self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1347
1348
1349class Jobs(base.Object):
4b92e0a0
MT
1350 def _get_job(self, query, *args):
1351 res = self.db.get(query, *args)
1352
1353 if res:
1354 return Job(self.backend, res.id, data=res)
1355
fd43d5e1
MT
1356 def _get_jobs(self, query, *args):
1357 res = self.db.query(query, *args)
1358
1359 for row in res:
1360 yield Job(self.backend, row.id, data=row)
1361
4b92e0a0
MT
1362 def create(self, build, arch, type="build"):
1363 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1364 VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1365 job.log("created")
1366
1367 # Set cache for Build object.
1368 job.build = build
1369
1370 # Jobs are by default in state "new" and wait for being checked
1371 # for dependencies. Packages that do have no build dependencies
1372 # can directly be forwarded to "pending" state.
1373 if not job.pkg.requires:
1374 job.state = "pending"
1375
1376 return job
1377
6ea0393f
MT
1378 def get_by_id(self, id, data=None):
1379 return Job(self.pakfire, id, data)
f6e6ff79
MT
1380
1381 def get_by_uuid(self, uuid):
1382 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1383
1384 if job:
1385 return self.get_by_id(job.id)
1386
1387 def get_by_build(self, build_id, build=None, type=None):
1388 """
1389 Get all jobs in the specifies build.
1390 """
9fa1787c 1391 query = "SELECT * FROM jobs WHERE build_id = %s"
f6e6ff79
MT
1392 args = [build_id,]
1393
1394 if type:
1395 query += " AND type = %s"
1396 args.append(type)
1397
1398 # Get IDs of all builds in this group.
1399 jobs = []
1400 for job in self.db.query(query, *args):
9fa1787c 1401 job = Job(self.pakfire, job.id, job)
f6e6ff79
MT
1402
1403 # If the Build object was set, we set it so it won't be retrieved
1404 # from the database again.
1405 if build:
1406 job._build = build
1407
1408 jobs.append(job)
1409
1410 # Return sorted list of jobs.
1411 return sorted(jobs)
1412
163d9d8b
MT
1413 def get_active(self, host_id=None, builder=None, states=None):
1414 if builder:
1415 host_id = builder.id
f6e6ff79 1416
163d9d8b
MT
1417 if states is None:
1418 states = ["dispatching", "running", "uploading"]
f6e6ff79 1419
163d9d8b
MT
1420 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1421 args = states
f6e6ff79
MT
1422
1423 if host_id:
1424 query += " AND builder_id = %s" % host_id
1425
6e63ed49
MT
1426 query += " ORDER BY \
1427 CASE \
1428 WHEN jobs.state = 'running' THEN 0 \
1429 WHEN jobs.state = 'uploading' THEN 1 \
1430 WHEN jobs.state = 'dispatching' THEN 2 \
1431 WHEN jobs.state = 'pending' THEN 3 \
1432 WHEN jobs.state = 'new' THEN 4 \
1433 END, time_started ASC"
f6e6ff79 1434
163d9d8b 1435 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
f6e6ff79 1436
9177f86a 1437 def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
9fa1787c 1438 query = "SELECT * FROM jobs"
6e63ed49 1439 args = []
f6e6ff79 1440
6e63ed49 1441 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
9177f86a
MT
1442
1443 if arch:
044a9c43
MT
1444 where.append("arch = %s")
1445 args.append(arch)
9177f86a 1446
f6e6ff79 1447 if builder:
6e63ed49
MT
1448 where.append("builder_id = %s")
1449 args.append(builder.id)
1450
1451 if date:
6e63ed49 1452 try:
9177f86a 1453 year, month, day = date.split("-", 2)
6e63ed49
MT
1454 date = datetime.date(int(year), int(month), int(day))
1455 except ValueError:
1456 pass
6e63ed49 1457 else:
9177f86a
MT
1458 where.append("(DATE(time_created) = %s OR \
1459 DATE(time_started) = %s OR DATE(time_finished) = %s)")
1460 args += (date, date, date)
6e63ed49
MT
1461
1462 if age:
9779008c 1463 where.append("time_finished >= NOW() - '%s'::interval" % age)
f6e6ff79
MT
1464
1465 if where:
1466 query += " WHERE %s" % " AND ".join(where)
1467
6e63ed49
MT
1468 query += " ORDER BY time_finished DESC"
1469
1470 if limit:
1471 query += " LIMIT %s"
1472 args.append(limit)
f6e6ff79 1473
6e63ed49 1474 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
f6e6ff79
MT
1475
1476 def get_average_build_time(self):
1477 """
1478 Returns the average build time of all finished builds from the
1479 last 3 months.
1480 """
966498de
MT
1481 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1482 FROM jobs WHERE type = 'build' AND state = 'finished' AND \
9779008c 1483 time_finished >= NOW() - '3 months'::interval")
f6e6ff79 1484
966498de
MT
1485 if result:
1486 return result.average
f6e6ff79
MT
1487
1488 def count(self, *states):
966498de
MT
1489 query = "SELECT COUNT(*) AS count FROM jobs"
1490 args = []
f6e6ff79 1491
966498de
MT
1492 if states:
1493 query += " WHERE state IN %s"
1494 args.append(states)
f6e6ff79 1495
966498de
MT
1496 jobs = self.db.get(query, *args)
1497 if jobs:
1498 return jobs.count
f6e6ff79
MT
1499
1500
4b92e0a0
MT
1501class Job(base.DataObject):
1502 table = "jobs"
f6e6ff79
MT
1503
1504 def __str__(self):
1505 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1506
4b92e0a0
MT
1507 def __eq__(self, other):
1508 if isinstance(other, self.__class__):
1509 return self.id == other.id
f6e6ff79 1510
4b92e0a0
MT
1511 def __lt__(self, other):
1512 if isinstance(other, self.__class__):
1513 if (self.type, other.type) == ("build", "test"):
1514 return True
f6e6ff79 1515
4b92e0a0
MT
1516 if self.build == other.build:
1517 return self.arch < other.arch # XXX needs to use the arch prio
f6e6ff79 1518
4b92e0a0 1519 return self.time_created < other.time_created
f6e6ff79 1520
4b92e0a0
MT
1521 def __iter__(self):
1522 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1523 LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1524 WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
f6e6ff79 1525
4b92e0a0 1526 return iter(packages)
f6e6ff79 1527
4b92e0a0
MT
1528 def __len__(self):
1529 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1530 WHERE job_id = %s", self.id)
f6e6ff79 1531
4b92e0a0 1532 return res.len
f6e6ff79 1533
4b92e0a0
MT
1534 @property
1535 def distro(self):
1536 return self.build.distro
f6e6ff79
MT
1537
1538 def delete(self):
1539 self.__delete_buildroots()
1540 self.__delete_history()
1541 self.__delete_packages()
1542 self.__delete_logfiles()
1543
1544 # Delete the job itself.
1545 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
f6e6ff79
MT
1546
1547 def __delete_buildroots(self):
1548 """
1549 Removes all buildroots.
1550 """
1551 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1552
1553 def __delete_history(self):
1554 """
1555 Removes all references in the history to this build job.
1556 """
1557 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1558
1559 def __delete_packages(self):
1560 """
1561 Deletes all uploaded files from the job.
1562 """
1563 for pkg in self.packages:
1564 pkg.delete()
1565
1566 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1567
1568 def __delete_logfiles(self):
1569 for logfile in self.logfiles:
1570 self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1571
1572 def reset(self, user=None):
1573 self.__delete_buildroots()
1574 self.__delete_packages()
1575 self.__delete_history()
1576 self.__delete_logfiles()
1577
1578 self.state = "new"
1579 self.log("reset", user=user)
1580
f6e6ff79
MT
1581 ## Logging stuff
1582
1583 def log(self, action, user=None, state=None, builder=None, test_job=None):
1584 user_id = None
1585 if user:
1586 user_id = user.id
1587
1588 builder_id = None
1589 if builder:
1590 builder_id = builder.id
1591
1592 test_job_id = None
1593 if test_job:
1594 test_job_id = test_job.id
1595
1596 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1597 time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1598 self.id, action, state, user_id, builder_id, test_job_id)
1599
1600 def get_log(self, limit=None, offset=None, user=None):
1601 query = "SELECT * FROM jobs_history"
1602
1603 conditions = ["job_id = %s",]
1604 args = [self.id,]
1605
1606 if user:
1607 conditions.append("user_id = %s")
1608 args.append(user.id)
1609
1610 if conditions:
1611 query += " WHERE %s" % " AND ".join(conditions)
1612
1613 query += " ORDER BY time DESC"
1614
1615 if limit:
1616 if offset:
1617 query += " LIMIT %s,%s"
1618 args += [offset, limit,]
1619 else:
1620 query += " LIMIT %s"
1621 args += [limit,]
1622
1623 entries = []
1624 for entry in self.db.query(query, *args):
1625 entry = logs.JobLogEntry(self.pakfire, entry)
1626 entries.append(entry)
1627
1628 return entries
1629
1630 @property
1631 def uuid(self):
1632 return self.data.uuid
1633
1634 @property
1635 def type(self):
1636 return self.data.type
1637
1638 @property
1639 def build_id(self):
1640 return self.data.build_id
1641
4b92e0a0 1642 @lazy_property
f6e6ff79 1643 def build(self):
4b92e0a0 1644 return self.pakfire.builds.get_by_id(self.build_id)
f6e6ff79
MT
1645
1646 @property
1647 def related_jobs(self):
1648 ret = []
1649
1650 for job in self.build.jobs:
1651 if job == self:
1652 continue
1653
1654 ret.append(job)
1655
1656 return ret
1657
1658 @property
1659 def pkg(self):
1660 return self.build.pkg
1661
1662 @property
1663 def name(self):
044a9c43 1664 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
f6e6ff79 1665
eedc6432
MT
1666 @property
1667 def size(self):
1668 return sum((p.size for p in self.packages))
1669
ca3e24f8
MT
1670 @lazy_property
1671 def rank(self):
1672 """
1673 Returns the rank in the build queue
1674 """
1675 if not self.state == "pending":
1676 return
1677
1678 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1679
1680 if res:
1681 return res.rank
1682
a90bd9b0
MT
1683 def is_running(self):
1684 """
1685 Returns True if job is in a running state.
1686 """
1687 return self.state in ("pending", "dispatching", "running", "uploading")
1688
f6e6ff79
MT
1689 def get_state(self):
1690 return self.data.state
1691
1692 def set_state(self, state, user=None, log=True):
1693 # Nothing to do if the state remains.
1694 if not self.state == state:
1695 self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
f6e6ff79
MT
1696
1697 # Log the event.
1698 if log and not state == "new":
1699 self.log("state_change", state=state, user=user)
1700
1701 # Update cache.
1702 if self._data:
1703 self._data["state"] = state
1704
1705 # Always clear the message when the status is changed.
1706 self.update_message(None)
1707
1708 # Update some more informations.
1709 if state == "dispatching":
1710 # Set start time.
1711 self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1712 WHERE id = %s", self.id)
1713
1714 elif state == "pending":
1715 self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1716 time_finished = NULL WHERE id = %s", self.id)
1717
1718 elif state in ("aborted", "dependency_error", "finished", "failed"):
163d9d8b
MT
1719 # Set finish time and reset builder..
1720 self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
f6e6ff79
MT
1721
1722 # Send messages to the user.
1723 if state == "finished":
1724 self.send_finished_message()
1725
1726 elif state == "failed":
1727 # Remove all package files if a job is set to failed state.
1728 self.__delete_packages()
1729
1730 self.send_failed_message()
1731
1732 # Automatically update the state of the build (not on test builds).
1733 if self.type == "build":
1734 self.build.auto_update_state()
1735
1736 state = property(get_state, set_state)
1737
1738 @property
1739 def message(self):
1740 return self.data.message
1741
1742 def update_message(self, msg):
1743 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1744 msg, self.id)
f6e6ff79
MT
1745
1746 if self._data:
1747 self._data["message"] = msg
1748
f6e6ff79 1749 def get_builder(self):
4b92e0a0
MT
1750 if self.data.builder_id:
1751 return self.backend.builders.get_by_id(self.data.builder_id)
f6e6ff79
MT
1752
1753 def set_builder(self, builder, user=None):
1754 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1755 builder.id, self.id)
1756
1757 # Update cache.
1758 if self._data:
1759 self._data["builder_id"] = builder.id
f6e6ff79
MT
1760
1761 self._builder = builder
1762
1763 # Log the event.
1764 if user:
1765 self.log("builder_assigned", builder=builder, user=user)
1766
4b92e0a0 1767 builder = lazy_property(get_builder, set_builder)
f6e6ff79 1768
f6e6ff79
MT
1769 @property
1770 def arch(self):
044a9c43 1771 return self.data.arch
eb9d737f 1772
f6e6ff79
MT
1773 @property
1774 def duration(self):
1775 if not self.time_started:
1776 return 0
1777
1778 if self.time_finished:
1779 delta = self.time_finished - self.time_started
1780 else:
1781 delta = datetime.datetime.utcnow() - self.time_started
1782
1783 return delta.total_seconds()
1784
1785 @property
1786 def time_created(self):
1787 return self.data.time_created
1788
1789 @property
1790 def time_started(self):
1791 return self.data.time_started
1792
1793 @property
1794 def time_finished(self):
1795 return self.data.time_finished
1796
a90bd9b0
MT
1797 @property
1798 def expected_runtime(self):
1799 """
1800 Returns the estimated time and stddev, this job takes to finish.
1801 """
1802 # Get the average build time.
044a9c43 1803 build_times = self.pakfire.builds.get_build_times_by_arch(self.arch,
a90bd9b0
MT
1804 name=self.pkg.name)
1805
1806 # If there is no statistical data, we cannot estimate anything.
1807 if not build_times:
1808 return None, None
1809
1810 return build_times.average, build_times.stddev
1811
1812 @property
1813 def eta(self):
1814 expected_runtime, stddev = self.expected_runtime
1815
1816 if expected_runtime:
1817 return expected_runtime - int(self.duration), stddev
1818
f6e6ff79
MT
1819 @property
1820 def tries(self):
1821 return self.data.tries
1822
f6e6ff79 1823 def get_pkg_by_uuid(self, uuid):
4b92e0a0 1824 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
f6e6ff79
MT
1825 JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1826 WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1827 self.id, uuid)
1828
4b92e0a0
MT
1829 if pkg:
1830 pkg.job = self
1831 return pkg
f6e6ff79 1832
4b92e0a0 1833 @lazy_property
f6e6ff79 1834 def logfiles(self):
4b92e0a0 1835 logfiles = []
f6e6ff79 1836
4b92e0a0
MT
1837 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1838 log = logs.LogFile(self.pakfire, log.id)
1839 log._job = self
f6e6ff79 1840
4b92e0a0 1841 logfiles.append(log)
f6e6ff79 1842
4b92e0a0 1843 return logfiles
f6e6ff79
MT
1844
1845 def add_file(self, filename):
1846 """
1847 Add the specified file to this job.
1848
1849 The file is copied to the right directory by this function.
1850 """
1851 assert os.path.exists(filename)
1852
1853 if filename.endswith(".log"):
1854 self._add_file_log(filename)
1855
1856 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1857 # It is not allowed to upload packages on test builds.
1858 if self.type == "test":
1859 return
1860
1861 self._add_file_package(filename)
1862
1863 def _add_file_log(self, filename):
1864 """
1865 Attach a log file to this job.
1866 """
1867 target_dirname = os.path.join(self.build.path, "logs")
1868
1869 if self.type == "test":
1870 i = 1
1871 while True:
1872 target_filename = os.path.join(target_dirname,
044a9c43 1873 "test.%s.%s.%s.log" % (self.arch, i, self.tries))
f6e6ff79
MT
1874
1875 if os.path.exists(target_filename):
1876 i += 1
1877 else:
1878 break
1879 else:
1880 target_filename = os.path.join(target_dirname,
044a9c43 1881 "build.%s.%s.log" % (self.arch, self.tries))
f6e6ff79
MT
1882
1883 # Make sure the target directory exists.
1884 if not os.path.exists(target_dirname):
1885 os.makedirs(target_dirname)
1886
1887 # Calculate a SHA512 hash from that file.
1888 f = open(filename, "rb")
1889 h = hashlib.sha512()
1890 while True:
1891 buf = f.read(BUFFER_SIZE)
1892 if not buf:
1893 break
1894
1895 h.update(buf)
1896 f.close()
1897
1898 # Copy the file to the final location.
1899 shutil.copy2(filename, target_filename)
1900
1901 # Create an entry in the database.
1902 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1903 VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1904 os.path.getsize(target_filename), h.hexdigest())
1905
1906 def _add_file_package(self, filename):
1907 # Open package (creates entry in the database).
1908 pkg = packages.Package.open(self.pakfire, filename)
1909
1910 # Move package to the build directory.
044a9c43 1911 pkg.move(os.path.join(self.build.path, self.arch))
f6e6ff79
MT
1912
1913 # Attach the package to this job.
1914 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1915 self.id, pkg.id)
1916
1917 def get_aborted_state(self):
1918 return self.data.aborted_state
1919
1920 def set_aborted_state(self, state):
4b92e0a0 1921 self._set_attribute("aborted_state", state)
f6e6ff79
MT
1922
1923 aborted_state = property(get_aborted_state, set_aborted_state)
1924
1925 @property
1926 def message_recipients(self):
1927 l = []
1928
1929 # Add all people watching the build.
1930 l += self.build.message_recipients
1931
1932 # Add the package maintainer on release builds.
1933 if self.build.type == "release":
1934 maint = self.pkg.maintainer
1935
1936 if isinstance(maint, users.User):
1937 l.append("%s <%s>" % (maint.realname, maint.email))
1938 elif maint:
1939 l.append(maint)
1940
1941 # XXX add committer and commit author.
1942
1943 # Add the owner of the scratch build on scratch builds.
1944 elif self.build.type == "scratch" and self.build.user:
1945 l.append("%s <%s>" % \
1946 (self.build.user.realname, self.build.user.email))
1947
1948 return set(l)
1949
1950 def save_buildroot(self, pkgs):
1951 rows = []
1952
1953 for pkg_name, pkg_uuid in pkgs:
1954 rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1955
1956 # Cleanup old stuff first (for rebuilding packages).
1957 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1958 self.id, self.tries)
1959
1960 self.db.executemany("INSERT INTO \
1961 jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1962 VALUES(%s, %s, %s, %s)", rows)
1963
1964 def has_buildroot(self, tries=None):
1965 if tries is None:
1966 tries = self.tries
1967
1968 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
c2621706
MT
1969 WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1970 self.id, tries)
f6e6ff79
MT
1971
1972 if res:
1973 return res.num
1974
1975 return 0
1976
1977 def get_buildroot(self, tries=None):
1978 if tries is None:
1979 tries = self.tries
1980
1981 rows = self.db.query("SELECT * FROM jobs_buildroots \
1982 WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1983 ORDER BY pkg_name", self.id, tries)
1984
1985 pkgs = []
1986 for row in rows:
1987 # Search for this package in the packages table.
1988 pkg = self.pakfire.packages.get_by_uuid(row.pkg_uuid)
1989 pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1990
1991 return pkgs
1992
1993 def send_finished_message(self):
1994 # Send no finished mails for test jobs.
1995 if self.type == "test":
1996 return
1997
1998 logging.debug("Sending finished message for job %s to %s" % \
1999 (self.name, ", ".join(self.message_recipients)))
2000
2001 info = {
2002 "build_name" : self.name,
2003 "build_host" : self.builder.name,
2004 "build_uuid" : self.uuid,
2005 }
2006
2007 self.pakfire.messages.send_to_all(self.message_recipients,
2008 MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
2009
2010 def send_failed_message(self):
2011 logging.debug("Sending failed message for job %s to %s" % \
2012 (self.name, ", ".join(self.message_recipients)))
2013
2014 build_host = "--"
2015 if self.builder:
2016 build_host = self.builder.name
2017
2018 info = {
2019 "build_name" : self.name,
2020 "build_host" : build_host,
2021 "build_uuid" : self.uuid,
2022 }
2023
2024 self.pakfire.messages.send_to_all(self.message_recipients,
2025 MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
2026
2027 def set_start_time(self, start_time):
2028 if start_time is None:
2029 return
2030
2031 self.db.execute("UPDATE jobs SET start_not_before = NOW() + %s \
2032 WHERE id = %s LIMIT 1", start_time, self.id)
2033
2034 def schedule(self, type, start_time=None, user=None):
2035 assert type in ("rebuild", "test")
2036
2037 if type == "rebuild":
2038 if self.state == "finished":
2039 return
2040
2041 self.set_state("new", user=user, log=False)
2042 self.set_start_time(start_time)
2043
2044 # Log the event.
2045 self.log("schedule_rebuild", user=user)
2046
2047 elif type == "test":
2048 if not self.state == "finished":
2049 return
2050
2051 # Create a new job with same build and arch.
2052 job = self.create(self.pakfire, self.build, self.arch, type="test")
2053 job.set_start_time(start_time)
2054
2055 # Log the event.
2056 self.log("schedule_test_job", test_job=job, user=user)
2057
2058 return job
2059
2060 def schedule_test(self, start_not_before=None, user=None):
2061 # XXX to be removed
2062 return self.schedule("test", start_time=start_not_before, user=user)
2063
2064 def schedule_rebuild(self, start_not_before=None, user=None):
2065 # XXX to be removed
2066 return self.schedule("rebuild", start_time=start_not_before, user=user)
2067
2068 def get_build_repos(self):
2069 """
2070 Returns a list of all repositories that should be used when
2071 building this job.
2072 """
2073 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
2074 self.id)
2075
2076 if not repo_ids:
2077 return self.distro.get_build_repos()
2078
2079 repos = []
2080 for repo in self.distro.repositories:
2081 if repo.id in [r.id for r in repo_ids]:
2082 repos.append(repo)
2083
2084 return repos or self.distro.get_build_repos()
2085
2086 def get_repo_config(self):
2087 """
2088 Get repository configuration file that is sent to the builder.
2089 """
2090 confs = []
2091
2092 for repo in self.get_build_repos():
2093 confs.append(repo.get_conf())
2094
2095 return "\n\n".join(confs)
2096
2097 def get_config(self):
2098 """
2099 Get configuration file that is sent to the builder.
2100 """
2101 confs = []
2102
2103 # Add the distribution configuration.
2104 confs.append(self.distro.get_config())
2105
2106 # Then add all repositories for this build.
2107 confs.append(self.get_repo_config())
2108
2109 return "\n\n".join(confs)
2110
f6e6ff79
MT
2111 def resolvdep(self):
2112 config = pakfire.config.Config(files=["general.conf"])
2113 config.parse(self.get_config())
2114
2115 # The filename of the source file.
2116 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
2117 assert os.path.exists(filename), filename
2118
2119 # Create a new pakfire instance with the configuration for
2120 # this build.
044a9c43 2121 p = pakfire.PakfireServer(config=config, arch=self.arch)
f6e6ff79
MT
2122
2123 # Try to solve the build dependencies.
2124 try:
2125 solver = p.resolvdep(filename)
2126
2127 # Catch dependency errors and log the problem string.
2128 except DependencyError, e:
2129 self.state = "dependency_error"
2130 self.update_message(e)
2131
2132 else:
2133 # If the build dependencies can be resolved, we set the build in
2134 # pending state.
2135 if solver.status is True:
2136 if self.state in ("failed",):
2137 return
2138
2139 self.state = "pending"