src/buildservice/builds.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import hashlib
   5 import logging
   6 import os
   7 import re
   8 import shutil
   9 import uuid
  10
  11 import pakfire
  12 import pakfire.config
  13 import pakfire.packages
  14
  15 from . import arches
  16 from . import base
  17 from . import builders
  18 from . import logs
  19 from . import packages
  20 from . import repository
  21 from . import updates
  22 from . import users
  23
  24 log = logging.getLogger("builds")
  25 log.propagate = 1
  26
  27 from .constants import *
  28 from .decorators import *
  29
  30 class Builds(base.Object):
  31         def _get_build(self, query, *args):
  32                 res = self.db.get(query, *args)
  33
  34                 if res:
  35                         return Build(self.backend, res.id, data=res)
  36
  37         def _get_builds(self, query, *args):
  38                 res = self.db.query(query, *args)
  39
  40                 for row in res:
  41                         yield Build(self.backend, row.id, data=row)
  42
  43         def get_by_id(self, id, data=None):
  44                 return Build(self.backend, id, data=data)
  45
  46         def get_by_uuid(self, uuid):
  47                 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
  48
  49                 if build:
  50                         return self.get_by_id(build.id)
  51
  52         def get_all(self, limit=50):
  53                 query = "SELECT * FROM builds ORDER BY time_created DESC"
  54
  55                 if limit:
  56                         query += " LIMIT %d" % limit
  57
  58                 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
  59
  60         def get_by_user(self, user, type=None, public=None):
  61                 args = []
  62                 conditions = []
  63
  64                 if not type or type == "scratch":
  65                         # On scratch builds the user id equals the owner id.
  66                         conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
  67                         args.append(user.id)
  68
  69                 elif not type or type == "release":
  70                         pass # TODO
  71
  72                 if public is True:
  73                         conditions.append("public = 'Y'")
  74                 elif public is False:
  75                         conditions.append("public = 'N'")
  76
  77                 query = "SELECT builds.* AS id FROM builds \
  78                         JOIN packages ON builds.pkg_id = packages.id"
  79
  80                 if conditions:
  81                         query += " WHERE %s" % " AND ".join(conditions)
  82
  83                 query += " ORDER BY builds.time_created DESC"
  84
  85                 builds = []
  86                 for build in self.db.query(query, *args):
  87                         build = Build(self.backend, build.id, build)
  88                         builds.append(build)
  89
  90                 return builds
  91
  92         def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
  93                 args = [name,]
  94                 conditions = [
  95                         "packages.name = %s",
  96                 ]
  97
  98                 if type:
  99                         conditions.append("builds.type = %s")
 100                         args.append(type)
 101
 102                 or_conditions = []
 103                 if public is True:
 104                         or_conditions.append("public = 'Y'")
 105                 elif public is False:
 106                         or_conditions.append("public = 'N'")
 107
 108                 if user and not user.is_admin():
 109                         or_conditions.append("builds.owner_id = %s")
 110                         args.append(user.id)
 111
 112                 query = "SELECT builds.* AS id FROM builds \
 113                         JOIN packages ON builds.pkg_id = packages.id"
 114
 115                 if or_conditions:
 116                         conditions.append(" OR ".join(or_conditions))
 117
 118                 if conditions:
 119                         query += " WHERE %s" % " AND ".join(conditions)
 120
 121                 if type == "release":
 122                         query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
 123                 elif type == "scratch":
 124                         query += " ORDER BY time_created DESC"
 125
 126                 if limit:
 127                         if offset:
 128                                 query += " LIMIT %s,%s"
 129                                 args.extend([offset, limit])
 130                         else:
 131                                 query += " LIMIT %s"
 132                                 args.append(limit)
 133
 134                 return [Build(self.backend, b.id, b) for b in self.db.query(query, *args)]
 135
 136         def get_latest_by_name(self, name, type=None, public=None):
 137                 query = "\
 138                         SELECT * FROM builds \
 139                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 140                         WHERE builds_latest.package_name = %s"
 141                 args = [name,]
 142
 143                 if type:
 144                         query += " AND builds_latest.build_type = %s"
 145                         args.append(type)
 146
 147                 if public is True:
 148                         query += " AND builds.public = %s"
 149                         args.append("Y")
 150                 elif public is False:
 151                         query += " AND builds.public = %s"
 152                         args.append("N")
 153
 154                 # Get the last one only.
 155                 # Prefer release builds over scratch builds.
 156                 query += "\
 157                         ORDER BY \
 158                                 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
 159                                 builds.time_created DESC \
 160                         LIMIT 1"
 161
 162                 res = self.db.get(query, *args)
 163
 164                 if res:
 165                         return Build(self.backend, res.id, res)
 166
 167         def get_active_builds(self, name, public=None):
 168                 query = "\
 169                         SELECT * FROM builds \
 170                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 171                         WHERE builds_latest.package_name = %s AND builds.type = %s"
 172                 args = [name, "release"]
 173
 174                 if public is True:
 175                         query += " AND builds.public = %s"
 176                         args.append("Y")
 177                 elif public is False:
 178                         query += " AND builds.public = %s"
 179                         args.append("N")
 180
 181                 builds = []
 182                 for row in self.db.query(query, *args):
 183                         b = Build(self.backend, row.id, row)
 184                         builds.append(b)
 185
 186                 # Sort the result. Lastest build first.
 187                 builds.sort(reverse=True)
 188
 189                 return builds
 190
 191         def count(self):
 192                 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
 193                 if builds:
 194                         return builds.count
 195
 196         def get_obsolete(self, repo=None):
 197                 """
 198                         Get all obsoleted builds.
 199
 200                         If repo is True: which are in any repository.
 201                         If repo is some Repository object: which are in this repository.
 202                 """
 203                 args = []
 204
 205                 if repo is None:
 206                         query = "SELECT id FROM builds WHERE state = 'obsolete'"
 207
 208                 else:
 209                         query = "SELECT build_id AS id FROM repositories_builds \
 210                                 JOIN builds ON builds.id = repositories_builds.build_id \
 211                                 WHERE builds.state = 'obsolete'"
 212
 213                         if repo and not repo is True:
 214                                 query += " AND repositories_builds.repo_id = %s"
 215                                 args.append(repo.id)
 216
 217                 res = self.db.query(query, *args)
 218
 219                 builds = []
 220                 for build in res:
 221                         build = Build(self.backend, build.id)
 222                         builds.append(build)
 223
 224                 return builds
 225
 226         def create(self, pkg, type="release", owner=None, distro=None):
 227                 assert type in ("release", "scratch", "test")
 228                 assert distro, "You need to specify the distribution of this build."
 229
 230                 # Check if scratch build has an owner.
 231                 if type == "scratch" and not owner:
 232                         raise Exception, "Scratch builds require an owner"
 233
 234                 # Set the default priority of this build.
 235                 if type == "release":
 236                         priority = 0
 237
 238                 elif type == "scratch":
 239                         priority = 1
 240
 241                 elif type == "test":
 242                         priority = -1
 243
 244                 # Create build in database
 245                 build = self._get_build("INSERT INTO builds(uuid, pkg_id, type, distro_id, priority) \
 246                         VALUES(%s, %s, %s, %s, %s) RETURNING *", "%s" % uuid.uuid4(), pkg.id, type, distro.id, priority)
 247
 248                 # Set the owner of this build
 249                 if owner:
 250                         build.owner = owner
 251
 252                 # Log that the build has been created.
 253                 build.log("created", user=owner)
 254
 255                 # Create directory where the files live
 256                 if not os.path.exists(build.path):
 257                         os.makedirs(build.path)
 258
 259                 # Move package file to the directory of the build.
 260                 build.pkg.move(os.path.join(build.path, "src"))
 261
 262                 # Generate an update id.
 263                 build.generate_update_id()
 264
 265                 # Obsolete all other builds with the same name to track updates.
 266                 build.obsolete_others()
 267
 268                 # Search for possible bug IDs in the commit message.
 269                 build.search_for_bugs()
 270
 271                 return build
 272
 273         def create_from_source_package(self, filename, distro, commit=None, type="release",
 274                         arches=None, check_for_duplicates=True, owner=None):
 275                 assert distro
 276
 277                 # Open the package file to read some basic information.
 278                 pkg = pakfire.packages.open(None, None, filename)
 279
 280                 if check_for_duplicates:
 281                         if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
 282                                 log.warning("Duplicate package detected: %s. Skipping." % pkg)
 283                                 return
 284
 285                 # Open the package and add it to the database
 286                 pkg = self.backend.packages.create(filename)
 287
 288                 # Associate the package to the processed commit
 289                 if commit:
 290                         pkg.commit = commit
 291
 292                 # Create a new build object from the package
 293                 build = self.create(pkg, type=type, owner=owner, distro=distro)
 294
 295                 # Create all automatic jobs
 296                 build.create_autojobs(arches=arches)
 297
 298                 return build
 299
 300         def get_changelog(self, name, public=None, limit=5, offset=0):
 301                 query = "SELECT builds.* FROM builds \
 302                         JOIN packages ON builds.pkg_id = packages.id \
 303                         WHERE \
 304                                 builds.type = %s \
 305                         AND \
 306                                 packages.name = %s"
 307                 args = ["release", name,]
 308
 309                 if public == True:
 310                         query += " AND builds.public = %s"
 311                         args.append("Y")
 312                 elif public == False:
 313                         query += " AND builds.public = %s"
 314                         args.append("N")
 315
 316                 query += " ORDER BY builds.time_created DESC"
 317
 318                 if limit:
 319                         if offset:
 320                                 query += " LIMIT %s,%s"
 321                                 args += [offset, limit]
 322                         else:
 323                                 query += " LIMIT %s"
 324                                 args.append(limit)
 325
 326                 builds = []
 327                 for b in self.db.query(query, *args):
 328                         b = Build(self.backend, b.id, b)
 329                         builds.append(b)
 330
 331                 builds.sort(reverse=True)
 332
 333                 return builds
 334
 335         def get_comments(self, limit=10, offset=None, user=None):
 336                 query = "SELECT * FROM builds_comments \
 337                         JOIN users ON builds_comments.user_id = users.id"
 338                 args = []
 339
 340                 wheres = []
 341                 if user:
 342                         wheres.append("users.id = %s")
 343                         args.append(user.id)
 344
 345                 if wheres:
 346                         query += " WHERE %s" % " AND ".join(wheres)
 347
 348                 # Sort everything.
 349                 query += " ORDER BY time_created DESC"
 350
 351                 # Limits.
 352                 if limit:
 353                         if offset:
 354                                 query += " LIMIT %s,%s"
 355                                 args.append(offset)
 356                         else:
 357                                 query += " LIMIT %s"
 358
 359                         args.append(limit)
 360
 361                 comments = []
 362                 for comment in self.db.query(query, *args):
 363                         comment = logs.CommentLogEntry(self.backend, comment)
 364                         comments.append(comment)
 365
 366                 return comments
 367
 368         def get_build_times_summary(self, name=None, job_type=None, arch=None):
 369                 query = "\
 370                         SELECT \
 371                                 builds_times.arch AS arch, \
 372                                 MAX(duration) AS maximum, \
 373                                 MIN(duration) AS minimum, \
 374                                 AVG(duration) AS average, \
 375                                 SUM(duration) AS sum, \
 376                                 STDDEV_POP(duration) AS stddev \
 377                         FROM builds_times \
 378                                 LEFT JOIN builds ON builds_times.build_id = builds.id \
 379                                 LEFT JOIN packages ON builds.pkg_id = packages.id"
 380
 381                 args = []
 382                 conditions = []
 383
 384                 # Filter for name.
 385                 if name:
 386                         conditions.append("packages.name = %s")
 387                         args.append(name)
 388
 389                 # Filter by job types.
 390                 if job_type:
 391                         conditions.append("builds_times.job_type = %s")
 392                         args.append(job_type)
 393
 394                 # Filter by arch.
 395                 if arch:
 396                         conditions.append("builds_times.arch = %s")
 397                         args.append(arch)
 398
 399                 # Add conditions.
 400                 if conditions:
 401                         query += " WHERE %s" % " AND ".join(conditions)
 402
 403                 # Grouping and sorting.
 404                 query += " GROUP BY arch ORDER BY arch DESC"
 405
 406                 return self.db.query(query, *args)
 407
 408         def get_build_times_by_arch(self, arch, **kwargs):
 409                 kwargs.update({
 410                         "arch" : arch,
 411                 })
 412
 413                 build_times = self.get_build_times_summary(**kwargs)
 414                 if build_times:
 415                         return build_times[0]
 416
 417
 418 class Build(base.DataObject):
 419         table = "builds"
 420
 421         def __repr__(self):
 422                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
 423
 424         def __eq__(self, other):
 425                 if isinstance(other, self.__class__):
 426                         return self.id == other.id
 427
 428         def __lt__(self, other):
 429                 if isinstance(other, self.__class__):
 430                         return self.pkg < other.pkg
 431
 432         def __iter__(self):
 433                 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
 434                         WHERE build_id = %s", self.id)
 435
 436                 return iter(sorted(jobs))
 437
 438         def delete(self):
 439                 """
 440                         Deletes this build including all jobs, packages and the source
 441                         package.
 442                 """
 443                 # If the build is in a repository, we need to remove it.
 444                 if self.repo:
 445                         self.repo.rem_build(self)
 446
 447                 for job in self.jobs + self.test_jobs:
 448                         job.delete()
 449
 450                 if self.pkg:
 451                         self.pkg.delete()
 452
 453                 # Delete everything related to this build.
 454                 self.__delete_bugs()
 455                 self.__delete_comments()
 456                 self.__delete_history()
 457                 self.__delete_watchers()
 458
 459                 # Delete the build itself.
 460                 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
 461
 462         def __delete_bugs(self):
 463                 """
 464                         Delete all associated bugs.
 465                 """
 466                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
 467
 468         def __delete_comments(self):
 469                 """
 470                         Delete all comments.
 471                 """
 472                 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
 473
 474         def __delete_history(self):
 475                 """
 476                         Delete the repository history.
 477                 """
 478                 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
 479
 480         def __delete_watchers(self):
 481                 """
 482                         Delete all watchers.
 483                 """
 484                 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
 485
 486         def reset(self):
 487                 """
 488                         Resets the whole build so it can start again (as it has never
 489                         been started).
 490                 """
 491                 for job in self.jobs:
 492                         job.reset()
 493
 494                 #self.__delete_bugs()
 495                 self.__delete_comments()
 496                 self.__delete_history()
 497                 self.__delete_watchers()
 498
 499                 self.state = "building"
 500
 501                 # XXX empty log
 502
 503         @property
 504         def info(self):
 505                 """
 506                         A set of information that is sent to the XMLRPC client.
 507                 """
 508                 return { "uuid" : self.uuid }
 509
 510         def log(self, action, user=None, bug_id=None):
 511                 user_id = None
 512                 if user:
 513                         user_id = user.id
 514
 515                 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
 516                         VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
 517
 518         @property
 519         def uuid(self):
 520                 """
 521                         The UUID of this build.
 522                 """
 523                 return self.data.uuid
 524
 525         @lazy_property
 526         def pkg(self):
 527                 """
 528                         Get package that is to be built in the build.
 529                 """
 530                 return self.backend.packages.get_by_id(self.data.pkg_id)
 531
 532         @property
 533         def name(self):
 534                 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
 535
 536         @property
 537         def type(self):
 538                 """
 539                         The type of this build.
 540                 """
 541                 return self.data.type
 542
 543         def get_owner(self):
 544                 """
 545                         The owner of this build.
 546                 """
 547                 if self.data.owner_id:
 548                         return self.backend.users.get_by_id(self.data.owner_id)
 549
 550         def set_owner(self, owner):
 551                 if owner:
 552                         self._set_attribute("owner_id", owner.id)
 553                 else:
 554                         self._set_attribute("owner_id", None)
 555
 556         owner = lazy_property(get_owner, set_owner)
 557
 558         @lazy_property
 559         def distro(self):
 560                 return self.backend.distros.get_by_id(self.data.distro_id)
 561
 562         @property
 563         def user(self):
 564                 if self.type == "scratch":
 565                         return self.owner
 566
 567         def get_depends_on(self):
 568                 if self.data.depends_on:
 569                         return self.backend.builds.get_by_id(self.data.depends_on)
 570
 571         def set_depends_on(self, build):
 572                 self._set_attribute("depends_on", build.id)
 573
 574         depends_on = lazy_property(get_depends_on, set_depends_on)
 575
 576         @property
 577         def created(self):
 578                 return self.data.time_created
 579
 580         @property
 581         def date(self):
 582                 return self.created.date()
 583
 584         @property
 585         def public(self):
 586                 """
 587                         Is this build public?
 588                 """
 589                 return self.data.public
 590
 591         @lazy_property
 592         def size(self):
 593                 """
 594                         Returns the size on disk of this build.
 595                 """
 596                 s = 0
 597
 598                 # Add the source package.
 599                 if self.pkg:
 600                         s += self.pkg.size
 601
 602                 # Add all jobs.
 603                 s += sum((j.size for j in self.jobs))
 604
 605                 return s
 606
 607         def auto_update_state(self):
 608                 """
 609                         Check if the state of this build can be updated and perform
 610                         the change if possible.
 611                 """
 612                 # Do not change the broken/obsolete state automatically.
 613                 if self.state in ("broken", "obsolete"):
 614                         return
 615
 616                 if self.repo and self.repo.type == "stable":
 617                         self.update_state("stable")
 618                         return
 619
 620                 # If any of the build jobs are finished, the build will be put in testing
 621                 # state.
 622                 for job in self.jobs:
 623                         if job.state == "finished":
 624                                 self.update_state("testing")
 625                                 break
 626
 627         def update_state(self, state, user=None, remove=False):
 628                 assert state in ("stable", "testing", "obsolete", "broken")
 629
 630                 self._set_attribute("state", state)
 631
 632                 # In broken state, the removal from the repository is forced and
 633                 # all jobs that are not finished yet will be aborted.
 634                 if state == "broken":
 635                         remove = True
 636
 637                         for job in self.jobs:
 638                                 if job.state in ("new", "pending", "running", "dependency_error"):
 639                                         job.state = "aborted"
 640
 641                 # If this build is in a repository, it will leave it.
 642                 if remove and self.repo:
 643                                 self.repo.rem_build(self)
 644
 645                 # If a release build is now in testing state, we put it into the
 646                 # first repository of the distribution.
 647                 elif self.type == "release" and state == "testing":
 648                         # If the build is not in a repository, yet and if there is
 649                         # a first repository, we put the build there.
 650                         if not self.repo and self.distro.first_repo:
 651                                 self.distro.first_repo.add_build(self, user=user)
 652
 653         @property
 654         def state(self):
 655                 return self.data.state
 656
 657         def is_broken(self):
 658                 return self.state == "broken"
 659
 660         def obsolete_others(self):
 661                 if not self.type == "release":
 662                         return
 663
 664                 for build in self.backend.builds.get_by_name(self.pkg.name, type="release"):
 665                         # Don't modify ourself.
 666                         if self.id == build.id:
 667                                 continue
 668
 669                         # Don't touch broken builds.
 670                         if build.state in ("obsolete", "broken"):
 671                                 continue
 672
 673                         # Obsolete the build.
 674                         build.update_state("obsolete")
 675
 676         def set_severity(self, severity):
 677                 self._set_attribute("severity", severity)
 678
 679         def get_severity(self):
 680                 return self.data.severity
 681
 682         severity = property(get_severity, set_severity)
 683
 684         @lazy_property
 685         def commit(self):
 686                 if self.pkg and self.pkg.commit:
 687                         return self.pkg.commit
 688
 689         def update_message(self, message):
 690                 self._set_attribute("message", message)
 691
 692         def has_perm(self, user):
 693                 """
 694                         Check, if the given user has the right to perform administrative
 695                         operations on this build.
 696                 """
 697                 if user is None:
 698                         return False
 699
 700                 if user.is_admin():
 701                         return True
 702
 703                 # Check if the user is allowed to manage packages from the critical path.
 704                 if self.critical_path and not user.has_perm("manage_critical_path"):
 705                         return False
 706
 707                 # Search for maintainers...
 708
 709                 # Scratch builds.
 710                 if self.type == "scratch":
 711                         # The owner of a scratch build has the right to do anything with it.
 712                         if self.owner_id == user.id:
 713                                 return True
 714
 715                 # Release builds.
 716                 elif self.type == "release":
 717                         # The maintainer also is allowed to manage the build.
 718                         if self.pkg.maintainer == user:
 719                                 return True
 720
 721                 # Deny permission for all other cases.
 722                 return False
 723
 724         @property
 725         def message(self):
 726                 message = ""
 727
 728                 if self.data.message:
 729                         message = self.data.message
 730
 731                 elif self.commit:
 732                         if self.commit.message:
 733                                 message = "\n".join((self.commit.subject, self.commit.message))
 734                         else:
 735                                 message = self.commit.subject
 736
 737                         prefix = "%s: " % self.pkg.name
 738                         if message.startswith(prefix):
 739                                 message = message[len(prefix):]
 740
 741                 return message
 742
 743         def get_priority(self):
 744                 return self.data.priority
 745
 746         def set_priority(self, priority):
 747                 assert priority in (-2, -1, 0, 1, 2)
 748
 749                 self._set_attribute("priority", priority)
 750
 751         priority = property(get_priority, set_priority)
 752
 753         @property
 754         def path(self):
 755                 path = []
 756                 if self.type == "scratch":
 757                         path.append(BUILD_SCRATCH_DIR)
 758                         path.append(self.uuid)
 759
 760                 elif self.type == "release":
 761                         path.append(BUILD_RELEASE_DIR)
 762                         path.append("%s/%s-%s-%s" % \
 763                                 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
 764
 765                 else:
 766                         raise Exception, "Unknown build type: %s" % self.type
 767
 768                 return os.path.join(*path)
 769
 770         @property
 771         def source_filename(self):
 772                 return os.path.basename(self.pkg.path)
 773
 774         @property
 775         def download_prefix(self):
 776                 return "/".join((self.backend.settings.get("download_baseurl"), "packages"))
 777
 778         @property
 779         def source_download(self):
 780                 return "/".join((self.download_prefix, self.pkg.path))
 781
 782         @property
 783         def source_hash_sha512(self):
 784                 return self.pkg.hash_sha512
 785
 786         @property
 787         def link(self):
 788                 # XXX maybe this should rather live in a uimodule.
 789                 # zlib-1.2.3-2.ip3 [src, i686, blah...]
 790                 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
 791                         (self.state, self.type, self.uuid, self.name)
 792
 793                 s_jobs = []
 794                 for job in self.jobs:
 795                         s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
 796                                 (job.state, job.type, job.uuid, job.arch))
 797
 798                 if s_jobs:
 799                         s += " [%s]" % ", ".join(s_jobs)
 800
 801                 return s
 802
 803         @property
 804         def supported_arches(self):
 805                 return self.pkg.supported_arches
 806
 807         @property
 808         def critical_path(self):
 809                 return self.pkg.critical_path
 810
 811         def get_jobs(self, type=None):
 812                 """
 813                         Returns a list of jobs of this build.
 814                 """
 815                 return self.backend.jobs.get_by_build(self.id, self, type=type)
 816
 817         @lazy_property
 818         def jobs(self):
 819                 """
 820                         Get a list of all build jobs that are in this build.
 821                 """
 822                 return self.get_jobs(type="build")
 823
 824         @property
 825         def test_jobs(self):
 826                 return self.get_jobs(type="test")
 827
 828         @property
 829         def all_jobs_finished(self):
 830                 ret = True
 831
 832                 for job in self.jobs:
 833                         if not job.state == "finished":
 834                                 ret = False
 835                                 break
 836
 837                 return ret
 838
 839         def create_autojobs(self, arches=None, type="build"):
 840                 jobs = []
 841
 842                 # Arches may be passed to this function. If not we use all arches
 843                 # this package supports.
 844                 if arches is None:
 845                         arches = self.supported_arches
 846
 847                 # Create a new job for every given archirecture.
 848                 for arch in self.backend.arches.expand(arches):
 849                         # Don't create jobs for src
 850                         if arch == "src":
 851                                 continue
 852
 853                         job = self.add_job(arch, type=type)
 854                         jobs.append(job)
 855
 856                 # Return all newly created jobs.
 857                 return jobs
 858
 859         def add_job(self, arch, type="build"):
 860                 job = self.backend.jobs.create(self.backend, self, arch, type=type)
 861
 862                 # Add new job to cache.
 863                 self.jobs.append(job)
 864
 865                 return job
 866
 867         ## Update stuff
 868
 869         @property
 870         def update_id(self):
 871                 if not self.type == "release":
 872                         return
 873
 874                 # Generate an update ID if none does exist, yet.
 875                 self.generate_update_id()
 876
 877                 s = [
 878                         "%s" % self.distro.name.replace(" ", "").upper(),
 879                         "%04d" % (self.data.update_year or 0),
 880                         "%04d" % (self.data.update_num or 0),
 881                 ]
 882
 883                 return "-".join(s)
 884
 885         def generate_update_id(self):
 886                 if not self.type == "release":
 887                         return
 888
 889                 if self.data.update_num:
 890                         return
 891
 892                 update = self.db.get("SELECT update_num AS num FROM builds \
 893                         WHERE update_year = EXTRACT(year FROM NOW()) ORDER BY update_num DESC LIMIT 1")
 894
 895                 if update:
 896                         update_num = update.num + 1
 897                 else:
 898                         update_num = 1
 899
 900                 self.db.execute("UPDATE builds SET update_year = EXTRACT(year FROM NOW()), update_num = %s \
 901                         WHERE id = %s", update_num, self.id)
 902
 903         ## Comment stuff
 904
 905         def get_comments(self, limit=10, offset=0):
 906                 query = "SELECT * FROM builds_comments \
 907                         JOIN users ON builds_comments.user_id = users.id \
 908                         WHERE build_id = %s     ORDER BY time_created ASC"
 909
 910                 comments = []
 911                 for comment in self.db.query(query, self.id):
 912                         comment = logs.CommentLogEntry(self.backend, comment)
 913                         comments.append(comment)
 914
 915                 return comments
 916
 917         def add_comment(self, user, text, score):
 918                 # Add the new comment to the database.
 919                 id = self.db.execute("INSERT INTO \
 920                         builds_comments(build_id, user_id, text, credit, time_created) \
 921                         VALUES(%s, %s, %s, %s, NOW())",
 922                         self.id, user.id, text, score)
 923
 924                 # Update the credit cache
 925                 self.score += score
 926
 927                 # Send the new comment to all watchers and stuff.
 928                 self.send_comment_message(id)
 929
 930                 # Return the ID of the newly created comment.
 931                 return id
 932
 933         @lazy_property
 934         def score(self):
 935                 res = self.db.get("SELECT SUM(credit) AS score \
 936                         FROM builds_comments WHERE build_id = %s", self.id)
 937
 938                 return res.score or 0
 939
 940         @property
 941         def credits(self):
 942                 # XXX COMPAT
 943                 return self.score
 944
 945         def get_commenters(self):
 946                 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
 947                         JOIN users ON builds_comments.user_id = users.id \
 948                         WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
 949                         AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
 950
 951                 return [users.User(self.backend, u.id) for u in users]
 952
 953         def send_comment_message(self, comment_id):
 954                 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
 955                         comment_id)
 956
 957                 assert comment
 958                 assert comment.build_id == self.id
 959
 960                 # Get user who wrote the comment.
 961                 user = self.backend.users.get_by_id(comment.user_id)
 962
 963                 format = {
 964                         "build_name" : self.name,
 965                         "user_name"  : user.realname,
 966                 }
 967
 968                 # XXX create beautiful message
 969
 970                 self.backend.messages.send_to_all(self.message_recipients,
 971                         N_("%(user_name)s commented on %(build_name)s"),
 972                         comment.text, format)
 973
 974         ## Logging stuff
 975
 976         def get_log(self, comments=True, repo=True, limit=None):
 977                 entries = []
 978
 979                 # Created entry.
 980                 created_entry = logs.CreatedLogEntry(self.backend, self)
 981                 entries.append(created_entry)
 982
 983                 if comments:
 984                         entries += self.get_comments(limit=limit)
 985
 986                 if repo:
 987                         entries += self.get_repo_moves(limit=limit)
 988
 989                 # Sort all entries in chronological order.
 990                 entries.sort()
 991
 992                 if limit:
 993                         entries = entries[:limit]
 994
 995                 return entries
 996
 997         ## Watchers stuff
 998
 999         def get_watchers(self):
1000                 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
1001                         JOIN users ON builds_watchers.user_id = users.id \
1002                         WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1003                         AND users.activated = 'Y' ORDER BY users.id", self.id)
1004
1005                 return [users.User(self.backend, u.id) for u in query]
1006
1007         def add_watcher(self, user):
1008                 # Don't add a user twice.
1009                 if user in self.get_watchers():
1010                         return
1011
1012                 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1013                         VALUES(%s, %s)", self.id, user.id)
1014
1015         @property
1016         def message_recipients(self):
1017                 ret = []
1018
1019                 for watcher in self.get_watchers():
1020                         ret.append("%s <%s>" % (watcher.realname, watcher.email))
1021
1022                 return ret
1023
1024         @property
1025         def update(self):
1026                 if self._update is None:
1027                         update = self.db.get("SELECT update_id AS id FROM updates_builds \
1028                                 WHERE build_id = %s", self.id)
1029
1030                         if update:
1031                                 self._update = updates.Update(self.backend, update.id)
1032
1033                 return self._update
1034
1035         @lazy_property
1036         def repo(self):
1037                 res = self.db.get("SELECT repo_id FROM repositories_builds \
1038                         WHERE build_id = %s", self.id)
1039
1040                 if res:
1041                         return self.backend.repos.get_by_id(res.repo_id)
1042
1043         def get_repo_moves(self, limit=None):
1044                 query = "SELECT * FROM repositories_history \
1045                         WHERE build_id = %s ORDER BY time ASC"
1046
1047                 actions = []
1048                 for action in self.db.query(query, self.id):
1049                         action = logs.RepositoryLogEntry(self.backend, action)
1050                         actions.append(action)
1051
1052                 return actions
1053
1054         @property
1055         def is_loose(self):
1056                 if self.repo:
1057                         return False
1058
1059                 return True
1060
1061         @property
1062         def repo_time(self):
1063                 repo = self.db.get("SELECT time_added FROM repositories_builds \
1064                         WHERE build_id = %s", self.id)
1065
1066                 if repo:
1067                         return repo.time_added
1068
1069         def get_auto_move(self):
1070                 return self.data.auto_move == "Y"
1071
1072         def set_auto_move(self, state):
1073                 self._set_attribute("auto_move", state)
1074
1075         auto_move = property(get_auto_move, set_auto_move)
1076
1077         @property
1078         def can_move_forward(self):
1079                 if not self.repo:
1080                         return False
1081
1082                 # If there is no next repository, we cannot move anything.
1083                 if not self.repo.next:
1084                         return False
1085
1086                 # If the needed amount of score is reached, we can move forward.
1087                 if self.score >= self.repo.next.score_needed:
1088                         return True
1089
1090                 # If the repository does not require a minimal time,
1091                 # we can move forward immediately.
1092                 if not self.repo.time_min:
1093                         return True
1094
1095                 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1096                         WHERE build_id = %s", self.id)
1097                 duration = query.duration
1098
1099                 if duration >= self.repo.time_min:
1100                         return True
1101
1102                 return False
1103
1104         ## Bugs
1105
1106         def get_bug_ids(self):
1107                 query = self.db.query("SELECT bug_id FROM builds_bugs \
1108                         WHERE build_id = %s", self.id)
1109
1110                 return [b.bug_id for b in query]
1111
1112         def add_bug(self, bug_id, user=None, log=True):
1113                 # Check if this bug is already in the list of bugs.
1114                 if bug_id in self.get_bug_ids():
1115                         return
1116
1117                 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1118                         VALUES(%s, %s)", self.id, bug_id)
1119
1120                 # Log the event.
1121                 if log:
1122                         self.log("bug_added", user=user, bug_id=bug_id)
1123
1124         def rem_bug(self, bug_id, user=None, log=True):
1125                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1126                         bug_id = %s", self.id, bug_id)
1127
1128                 # Log the event.
1129                 if log:
1130                         self.log("bug_removed", user=user, bug_id=bug_id)
1131
1132         def search_for_bugs(self):
1133                 if not self.commit:
1134                         return
1135
1136                 pattern = re.compile(r"(bug\s?|#)(\d+)")
1137
1138                 for txt in (self.commit.subject, self.commit.message):
1139                         for bug in re.finditer(pattern, txt):
1140                                 try:
1141                                         bugid = int(bug.group(2))
1142                                 except ValueError:
1143                                         continue
1144
1145                                 # Check if a bug with the given ID exists in BZ.
1146                                 bug = self.backend.bugzilla.get_bug(bugid)
1147                                 if not bug:
1148                                         continue
1149
1150                                 self.add_bug(bugid)
1151
1152         def get_bugs(self):
1153                 bugs = []
1154                 for bug_id in self.get_bug_ids():
1155                         bug = self.backend.bugzilla.get_bug(bug_id)
1156                         if not bug:
1157                                 continue
1158
1159                         bugs.append(bug)
1160
1161                 return bugs
1162
1163         def _update_bugs_helper(self, repo):
1164                 """
1165                         This function takes a new status and generates messages that
1166                         are appended to all bugs.
1167                 """
1168                 try:
1169                         kwargs = BUG_MESSAGES[repo.type].copy()
1170                 except KeyError:
1171                         return
1172
1173                 baseurl = self.backend.settings.get("baseurl", "")
1174                 args = {
1175                         "build_url"    : "%s/build/%s" % (baseurl, self.uuid),
1176                         "distro_name"  : self.distro.name,
1177                         "package_name" : self.name,
1178                         "repo_name"    : repo.name,
1179                 }
1180                 kwargs["comment"] = kwargs["comment"] % args
1181
1182                 self.update_bugs(**kwargs)
1183
1184         def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1185                 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1186                         VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1187
1188         def update_bugs(self, status, resolution=None, comment=None):
1189                 # Update all bugs linked to this build.
1190                 for bug_id in self.get_bug_ids():
1191                         self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1192
1193
1194 class Jobs(base.Object):
1195         def _get_job(self, query, *args):
1196                 res = self.db.get(query, *args)
1197
1198                 if res:
1199                         return Job(self.backend, res.id, data=res)
1200
1201         def _get_jobs(self, query, *args):
1202                 res = self.db.query(query, *args)
1203
1204                 for row in res:
1205                         yield Job(self.backend, row.id, data=row)
1206
1207         def create(self,  build, arch, type="build"):
1208                 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1209                         VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1210                 job.log("created")
1211
1212                 # Set cache for Build object.
1213                 job.build = build
1214
1215                 # Jobs are by default in state "new" and wait for being checked
1216                 # for dependencies. Packages that do have no build dependencies
1217                 # can directly be forwarded to "pending" state.
1218                 if not job.pkg.requires:
1219                         job.state = "pending"
1220
1221                 return job
1222
1223         def get_by_id(self, id, data=None):
1224                 return Job(self.backend, id, data)
1225
1226         def get_by_uuid(self, uuid):
1227                 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1228
1229                 if job:
1230                         return self.get_by_id(job.id)
1231
1232         def get_by_build(self, build_id, build=None, type=None):
1233                 """
1234                         Get all jobs in the specifies build.
1235                 """
1236                 query = "SELECT * FROM jobs WHERE build_id = %s"
1237                 args = [build_id,]
1238
1239                 if type:
1240                         query += " AND type = %s"
1241                         args.append(type)
1242
1243                 # Get IDs of all builds in this group.
1244                 jobs = []
1245                 for job in self.db.query(query, *args):
1246                         job = Job(self.backend, job.id, job)
1247
1248                         # If the Build object was set, we set it so it won't be retrieved
1249                         # from the database again.
1250                         if build:
1251                                 job._build = build
1252
1253                         jobs.append(job)
1254
1255                 # Return sorted list of jobs.
1256                 return sorted(jobs)
1257
1258         def get_active(self, host_id=None, builder=None, states=None):
1259                 if builder:
1260                         host_id = builder.id
1261
1262                 if states is None:
1263                         states = ["dispatching", "running", "uploading"]
1264
1265                 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1266                 args = states
1267
1268                 if host_id:
1269                         query += " AND builder_id = %s" % host_id
1270
1271                 query += " ORDER BY \
1272                         CASE \
1273                                 WHEN jobs.state = 'running'     THEN 0 \
1274                                 WHEN jobs.state = 'uploading'   THEN 1 \
1275                                 WHEN jobs.state = 'dispatching' THEN 2 \
1276                                 WHEN jobs.state = 'pending'     THEN 3 \
1277                                 WHEN jobs.state = 'new'         THEN 4 \
1278                         END, time_started ASC"
1279
1280                 return [Job(self.backend, j.id, j) for j in self.db.query(query, *args)]
1281
1282         def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
1283                 query = "SELECT * FROM jobs"
1284                 args  = []
1285
1286                 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
1287
1288                 if arch:
1289                         where.append("arch = %s")
1290                         args.append(arch)
1291
1292                 if builder:
1293                         where.append("builder_id = %s")
1294                         args.append(builder.id)
1295
1296                 if date:
1297                         try:
1298                                 year, month, day = date.split("-", 2)
1299                                 date = datetime.date(int(year), int(month), int(day))
1300                         except ValueError:
1301                                 pass
1302                         else:
1303                                 where.append("(time_created::date = %s OR \
1304                                         time_started::date = %s OR time_finished::date = %s)")
1305                                 args += (date, date, date)
1306
1307                 if age:
1308                         where.append("time_finished >= NOW() - '%s'::interval" % age)
1309
1310                 if where:
1311                         query += " WHERE %s" % " AND ".join(where)
1312
1313                 query += " ORDER BY time_finished DESC"
1314
1315                 if limit:
1316                         query += " LIMIT %s"
1317                         args.append(limit)
1318
1319                 return [Job(self.backend, j.id, j) for j in self.db.query(query, *args)]
1320
1321         def get_average_build_time(self):
1322                 """
1323                         Returns the average build time of all finished builds from the
1324                         last 3 months.
1325                 """
1326                 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1327                         FROM jobs WHERE type = 'build' AND state = 'finished' AND \
1328                         time_finished >= NOW() - '3 months'::interval")
1329
1330                 if result:
1331                         return result.average
1332
1333         def count(self, *states):
1334                 query = "SELECT COUNT(*) AS count FROM jobs"
1335                 args  = []
1336
1337                 if states:
1338                         query += " WHERE state IN %s"
1339                         args.append(states)
1340
1341                 jobs = self.db.get(query, *args)
1342                 if jobs:
1343                         return jobs.count
1344
1345         def restart_failed(self, max_tries=9):
1346                 jobs = self._get_jobs("SELECT jobs.* FROM jobs \
1347                         JOIN builds ON builds.id = jobs.build_id \
1348                         WHERE \
1349                                 jobs.type = 'build' AND \
1350                                 jobs.state = 'failed' AND \
1351                                 jobs.tries <= %s AND \
1352                                 NOT builds.state = 'broken' AND \
1353                                 jobs.time_finished < NOW() - '72 hours'::interval \
1354                         ORDER BY \
1355                                 CASE \
1356                                         WHEN jobs.type = 'build' THEN 0 \
1357                                         WHEN jobs.type = 'test'  THEN 1 \
1358                                 END, \
1359                                 builds.priority DESC, jobs.time_created ASC",
1360                         max_tries)
1361
1362                 # Restart the job
1363                 for job in jobs:
1364                         job.set_state("new", log=False)
1365
1366
1367 class Job(base.DataObject):
1368         table = "jobs"
1369
1370         def __str__(self):
1371                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1372
1373         def __eq__(self, other):
1374                 if isinstance(other, self.__class__):
1375                         return self.id == other.id
1376
1377         def __lt__(self, other):
1378                 if isinstance(other, self.__class__):
1379                         if (self.type, other.type) == ("build", "test"):
1380                                 return True
1381
1382                         if self.build == other.build:
1383                                 return arches.priority(self.arch) < arches.priority(other.arch)
1384
1385                         return self.time_created < other.time_created
1386
1387         def __iter__(self):
1388                 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1389                         LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1390                         WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
1391
1392                 return iter(packages)
1393
1394         def __nonzero__(self):
1395                 return True
1396
1397         def __len__(self):
1398                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1399                         WHERE job_id = %s", self.id)
1400
1401                 return res.len
1402
1403         @property
1404         def distro(self):
1405                 return self.build.distro
1406
1407         def delete(self):
1408                 self.__delete_buildroots()
1409                 self.__delete_history()
1410                 self.__delete_packages()
1411                 self.__delete_logfiles()
1412
1413                 # Delete the job itself.
1414                 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
1415
1416         def __delete_buildroots(self):
1417                 """
1418                         Removes all buildroots.
1419                 """
1420                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1421
1422         def __delete_history(self):
1423                 """
1424                         Removes all references in the history to this build job.
1425                 """
1426                 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1427
1428         def __delete_packages(self):
1429                 """
1430                         Deletes all uploaded files from the job.
1431                 """
1432                 for pkg in self.packages:
1433                         pkg.delete()
1434
1435                 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1436
1437         def __delete_logfiles(self):
1438                 for logfile in self.logfiles:
1439                         self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1440
1441         def reset(self, user=None):
1442                 self.__delete_buildroots()
1443                 self.__delete_packages()
1444                 self.__delete_history()
1445                 self.__delete_logfiles()
1446
1447                 self.state = "new"
1448                 self.log("reset", user=user)
1449
1450         ## Logging stuff
1451
1452         def log(self, action, user=None, state=None, builder=None, test_job=None):
1453                 user_id = None
1454                 if user:
1455                         user_id = user.id
1456
1457                 builder_id = None
1458                 if builder:
1459                         builder_id = builder.id
1460
1461                 test_job_id = None
1462                 if test_job:
1463                         test_job_id = test_job.id
1464
1465                 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1466                         time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1467                         self.id, action, state, user_id, builder_id, test_job_id)
1468
1469         def get_log(self, limit=None, offset=None, user=None):
1470                 query = "SELECT * FROM jobs_history"
1471
1472                 conditions = ["job_id = %s",]
1473                 args  = [self.id,]
1474
1475                 if user:
1476                         conditions.append("user_id = %s")
1477                         args.append(user.id)
1478
1479                 if conditions:
1480                         query += " WHERE %s" % " AND ".join(conditions)
1481
1482                 query += " ORDER BY time DESC"
1483
1484                 if limit:
1485                         if offset:
1486                                 query += " LIMIT %s,%s"
1487                                 args  += [offset, limit,]
1488                         else:
1489                                 query += " LIMIT %s"
1490                                 args  += [limit,]
1491
1492                 entries = []
1493                 for entry in self.db.query(query, *args):
1494                         entry = logs.JobLogEntry(self.backend, entry)
1495                         entries.append(entry)
1496
1497                 return entries
1498
1499         @property
1500         def uuid(self):
1501                 return self.data.uuid
1502
1503         @property
1504         def type(self):
1505                 return self.data.type
1506
1507         @property
1508         def build_id(self):
1509                 return self.data.build_id
1510
1511         @lazy_property
1512         def build(self):
1513                 return self.backend.builds.get_by_id(self.build_id)
1514
1515         @property
1516         def related_jobs(self):
1517                 ret = []
1518
1519                 for job in self.build.jobs:
1520                         if job == self:
1521                                 continue
1522
1523                         ret.append(job)
1524
1525                 return ret
1526
1527         @property
1528         def pkg(self):
1529                 return self.build.pkg
1530
1531         @property
1532         def name(self):
1533                 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
1534
1535         @property
1536         def size(self):
1537                 return sum((p.size for p in self.packages))
1538
1539         @lazy_property
1540         def rank(self):
1541                 """
1542                         Returns the rank in the build queue
1543                 """
1544                 if not self.state == "pending":
1545                         return
1546
1547                 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1548
1549                 if res:
1550                         return res.rank
1551
1552         def is_running(self):
1553                 """
1554                         Returns True if job is in a running state.
1555                 """
1556                 return self.state in ("pending", "dispatching", "running", "uploading")
1557
1558         def get_state(self):
1559                 return self.data.state
1560
1561         def set_state(self, state, user=None, log=True):
1562                 # Nothing to do if the state remains.
1563                 if not self.state == state:
1564                         self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
1565
1566                         # Log the event.
1567                         if log and not state == "new":
1568                                 self.log("state_change", state=state, user=user)
1569
1570                         # Update cache.
1571                         if self._data:
1572                                 self._data["state"] = state
1573
1574                 # Always clear the message when the status is changed.
1575                 self.update_message(None)
1576
1577                 # Update some more informations.
1578                 if state == "dispatching":
1579                         # Set start time.
1580                         self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1581                                 WHERE id = %s", self.id)
1582
1583                 elif state == "pending":
1584                         self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1585                                 time_finished = NULL WHERE id = %s", self.id)
1586
1587                 elif state in ("aborted", "dependency_error", "finished", "failed"):
1588                         # Set finish time and reset builder..
1589                         self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
1590
1591                         # Send messages to the user.
1592                         if state == "finished":
1593                                 self.send_finished_message()
1594
1595                         elif state == "failed":
1596                                 # Remove all package files if a job is set to failed state.
1597                                 self.__delete_packages()
1598
1599                                 self.send_failed_message()
1600
1601                 # Automatically update the state of the build (not on test builds).
1602                 if self.type == "build":
1603                         self.build.auto_update_state()
1604
1605         state = property(get_state, set_state)
1606
1607         @property
1608         def message(self):
1609                 return self.data.message
1610
1611         def update_message(self, msg):
1612                 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1613                         msg, self.id)
1614
1615                 if self._data:
1616                         self._data["message"] = msg
1617
1618         def get_builder(self):
1619                 if self.data.builder_id:
1620                         return self.backend.builders.get_by_id(self.data.builder_id)
1621
1622         def set_builder(self, builder, user=None):
1623                 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1624                         builder.id, self.id)
1625
1626                 # Update cache.
1627                 if self._data:
1628                         self._data["builder_id"] = builder.id
1629
1630                 self._builder = builder
1631
1632                 # Log the event.
1633                 if user:
1634                         self.log("builder_assigned", builder=builder, user=user)
1635
1636         builder = lazy_property(get_builder, set_builder)
1637
1638         @property
1639         def arch(self):
1640                 return self.data.arch
1641
1642         @property
1643         def duration(self):
1644                 if not self.time_started:
1645                         return 0
1646
1647                 if self.time_finished:
1648                         delta = self.time_finished - self.time_started
1649                 else:
1650                         delta = datetime.datetime.utcnow() - self.time_started
1651
1652                 return delta.total_seconds()
1653
1654         @property
1655         def time_created(self):
1656                 return self.data.time_created
1657
1658         @property
1659         def time_started(self):
1660                 return self.data.time_started
1661
1662         @property
1663         def time_finished(self):
1664                 return self.data.time_finished
1665
1666         @property
1667         def expected_runtime(self):
1668                 """
1669                         Returns the estimated time and stddev, this job takes to finish.
1670                 """
1671                 # Get the average build time.
1672                 build_times = self.backend.builds.get_build_times_by_arch(self.arch,
1673                         name=self.pkg.name)
1674
1675                 # If there is no statistical data, we cannot estimate anything.
1676                 if not build_times:
1677                         return None, None
1678
1679                 return build_times.average, build_times.stddev
1680
1681         @property
1682         def eta(self):
1683                 expected_runtime, stddev = self.expected_runtime
1684
1685                 if expected_runtime:
1686                         return expected_runtime - int(self.duration), stddev
1687
1688         @property
1689         def tries(self):
1690                 return self.data.tries
1691
1692         def get_pkg_by_uuid(self, uuid):
1693                 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
1694                         JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1695                         WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1696                         self.id, uuid)
1697
1698                 if pkg:
1699                         pkg.job = self
1700                         return pkg
1701
1702         @lazy_property
1703         def logfiles(self):
1704                 logfiles = []
1705
1706                 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1707                         log = logs.LogFile(self.backend, log.id)
1708                         log._job = self
1709
1710                         logfiles.append(log)
1711
1712                 return logfiles
1713
1714         def add_file(self, filename):
1715                 """
1716                         Add the specified file to this job.
1717
1718                         The file is copied to the right directory by this function.
1719                 """
1720                 assert os.path.exists(filename)
1721
1722                 if filename.endswith(".log"):
1723                         self._add_file_log(filename)
1724
1725                 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1726                         # It is not allowed to upload packages on test builds.
1727                         if self.type == "test":
1728                                 return
1729
1730                         self._add_file_package(filename)
1731
1732         def _add_file_log(self, filename):
1733                 """
1734                         Attach a log file to this job.
1735                 """
1736                 target_dirname = os.path.join(self.build.path, "logs")
1737
1738                 if self.type == "test":
1739                         i = 1
1740                         while True:
1741                                 target_filename = os.path.join(target_dirname,
1742                                         "test.%s.%s.%s.log" % (self.arch, i, self.tries))
1743
1744                                 if os.path.exists(target_filename):
1745                                         i += 1
1746                                 else:
1747                                         break
1748                 else:
1749                         target_filename = os.path.join(target_dirname,
1750                                 "build.%s.%s.log" % (self.arch, self.tries))
1751
1752                 # Make sure the target directory exists.
1753                 if not os.path.exists(target_dirname):
1754                         os.makedirs(target_dirname)
1755
1756                 # Calculate a SHA512 hash from that file.
1757                 f = open(filename, "rb")
1758                 h = hashlib.sha512()
1759                 while True:
1760                         buf = f.read(BUFFER_SIZE)
1761                         if not buf:
1762                                 break
1763
1764                         h.update(buf)
1765                 f.close()
1766
1767                 # Copy the file to the final location.
1768                 shutil.copy2(filename, target_filename)
1769
1770                 # Create an entry in the database.
1771                 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1772                         VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1773                         os.path.getsize(target_filename), h.hexdigest())
1774
1775         def _add_file_package(self, filename):
1776                 # Open package (creates entry in the database).
1777                 pkg = packages.Package.open(self.backend, filename)
1778
1779                 # Move package to the build directory.
1780                 pkg.move(os.path.join(self.build.path, self.arch))
1781
1782                 # Attach the package to this job.
1783                 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1784                         self.id, pkg.id)
1785
1786         def get_aborted_state(self):
1787                 return self.data.aborted_state
1788
1789         def set_aborted_state(self, state):
1790                 self._set_attribute("aborted_state", state)
1791
1792         aborted_state = property(get_aborted_state, set_aborted_state)
1793
1794         @property
1795         def message_recipients(self):
1796                 l = []
1797
1798                 # Add all people watching the build.
1799                 l += self.build.message_recipients
1800
1801                 # Add the package maintainer on release builds.
1802                 if self.build.type == "release":
1803                         maint = self.pkg.maintainer
1804
1805                         if isinstance(maint, users.User):
1806                                 l.append("%s <%s>" % (maint.realname, maint.email))
1807                         elif maint:
1808                                 l.append(maint)
1809
1810                         # XXX add committer and commit author.
1811
1812                 # Add the owner of the scratch build on scratch builds.
1813                 elif self.build.type == "scratch" and self.build.user:
1814                         l.append("%s <%s>" % \
1815                                 (self.build.user.realname, self.build.user.email))
1816
1817                 return set(l)
1818
1819         def save_buildroot(self, pkgs):
1820                 rows = []
1821
1822                 for pkg_name, pkg_uuid in pkgs:
1823                         rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1824
1825                 # Cleanup old stuff first (for rebuilding packages).
1826                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1827                         self.id, self.tries)
1828
1829                 self.db.executemany("INSERT INTO \
1830                         jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1831                         VALUES(%s, %s, %s, %s)", rows)
1832
1833         def has_buildroot(self, tries=None):
1834                 if tries is None:
1835                         tries = self.tries
1836
1837                 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
1838                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1839                         self.id, tries)
1840
1841                 if res:
1842                         return res.num
1843
1844                 return 0
1845
1846         def get_buildroot(self, tries=None):
1847                 if tries is None:
1848                         tries = self.tries
1849
1850                 rows = self.db.query("SELECT * FROM jobs_buildroots \
1851                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1852                         ORDER BY pkg_name", self.id, tries)
1853
1854                 pkgs = []
1855                 for row in rows:
1856                         # Search for this package in the packages table.
1857                         pkg = self.backend.packages.get_by_uuid(row.pkg_uuid)
1858                         pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1859
1860                 return pkgs
1861
1862         def send_finished_message(self):
1863                 # Send no finished mails for test jobs.
1864                 if self.type == "test":
1865                         return
1866
1867                 logging.debug("Sending finished message for job %s to %s" % \
1868                         (self.name, ", ".join(self.message_recipients)))
1869
1870                 info = {
1871                         "build_name" : self.name,
1872                         "build_host" : self.builder.name,
1873                         "build_uuid" : self.uuid,
1874                 }
1875
1876                 self.backend.messages.send_to_all(self.message_recipients,
1877                         MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
1878
1879         def send_failed_message(self):
1880                 logging.debug("Sending failed message for job %s to %s" % \
1881                         (self.name, ", ".join(self.message_recipients)))
1882
1883                 build_host = "--"
1884                 if self.builder:
1885                         build_host = self.builder.name
1886
1887                 info = {
1888                         "build_name" : self.name,
1889                         "build_host" : build_host,
1890                         "build_uuid" : self.uuid,
1891                 }
1892
1893                 self.backend.messages.send_to_all(self.message_recipients,
1894                         MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
1895
1896         def set_start_time(self, start_not_before):
1897                 self._set_attribute("start_not_before", start_not_before)
1898
1899         def schedule(self, type, start_time=None, user=None):
1900                 assert type in ("rebuild", "test")
1901
1902                 if type == "rebuild":
1903                         if self.state == "finished":
1904                                 return
1905
1906                         self.set_state("new", user=user, log=False)
1907                         self.set_start_time(start_time)
1908
1909                         # Log the event.
1910                         self.log("schedule_rebuild", user=user)
1911
1912                 elif type == "test":
1913                         if not self.state == "finished":
1914                                 return
1915
1916                         # Create a new job with same build and arch.
1917                         job = self.create(self.backend, self.build, self.arch, type="test")
1918                         job.set_start_time(start_time)
1919
1920                         # Log the event.
1921                         self.log("schedule_test_job", test_job=job, user=user)
1922
1923                         return job
1924
1925         def schedule_test(self, start_not_before=None, user=None):
1926                 # XXX to be removed
1927                 return self.schedule("test", start_time=start_not_before, user=user)
1928
1929         def schedule_rebuild(self, start_not_before=None, user=None):
1930                 # XXX to be removed
1931                 return self.schedule("rebuild", start_time=start_not_before, user=user)
1932
1933         def get_build_repos(self):
1934                 """
1935                         Returns a list of all repositories that should be used when
1936                         building this job.
1937                 """
1938                 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
1939                         self.id)
1940
1941                 if not repo_ids:
1942                         return self.distro.get_build_repos()
1943
1944                 repos = []
1945                 for repo in self.distro.repositories:
1946                         if repo.id in [r.id for r in repo_ids]:
1947                                 repos.append(repo)
1948
1949                 return repos or self.distro.get_build_repos()
1950
1951         def get_repo_config(self):
1952                 """
1953                         Get repository configuration file that is sent to the builder.
1954                 """
1955                 confs = []
1956
1957                 for repo in self.get_build_repos():
1958                         confs.append(repo.get_conf())
1959
1960                 return "\n\n".join(confs)
1961
1962         def get_config(self):
1963                 """
1964                         Get configuration file that is sent to the builder.
1965                 """
1966                 confs = []
1967
1968                 # Add the distribution configuration.
1969                 confs.append(self.distro.get_config())
1970
1971                 # Then add all repositories for this build.
1972                 confs.append(self.get_repo_config())
1973
1974                 return "\n\n".join(confs)
1975
1976         def resolvdep(self):
1977                 config = pakfire.config.Config(files=["general.conf"])
1978                 config.parse(self.get_config())
1979
1980                 # The filename of the source file.
1981                 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
1982                 assert os.path.exists(filename), filename
1983
1984                 # Create a new pakfire instance with the configuration for
1985                 # this build.
1986                 p = pakfire.PakfireServer(config=config, arch=self.arch)
1987
1988                 # Try to solve the build dependencies.
1989                 try:
1990                         solver = p.resolvdep(filename)
1991
1992                 # Catch dependency errors and log the problem string.
1993                 except DependencyError, e:
1994                         self.state = "dependency_error"
1995                         self.update_message(e)
1996
1997                 else:
1998                         # If the build dependencies can be resolved, we set the build in
1999                         # pending state.
2000                         if solver.status is True:
2001                                 if self.state in ("failed",):
2002                                         return
2003
2004                                 self.state = "pending"