src/buildservice/builds.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import hashlib
   5 import logging
   6 import os
   7 import re
   8 import shutil
   9 import uuid
  10
  11 import pakfire
  12 import pakfire.config
  13 import pakfire.packages
  14
  15 from . import arches
  16 from . import base
  17 from . import builders
  18 from . import logs
  19 from . import packages
  20 from . import repository
  21 from . import updates
  22 from . import users
  23
  24 from .constants import *
  25 from .decorators import *
  26
  27 def import_from_package(_pakfire, filename, distro=None, commit=None, type="release",
  28                 arches=None, check_for_duplicates=True, owner=None):
  29
  30         if distro is None:
  31                 distro = commit.source.distro
  32
  33         assert distro
  34
  35         # Open the package file to read some basic information.
  36         pkg = pakfire.packages.open(None, None, filename)
  37
  38         if check_for_duplicates:
  39                 if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
  40                         logging.warning("Duplicate package detected: %s. Skipping." % pkg)
  41                         return
  42
  43         # Open the package and add it to the database.
  44         pkg = packages.Package.open(_pakfire, filename)
  45         logging.debug("Created new package: %s" % pkg)
  46
  47         # Associate the package to the processed commit.
  48         if commit:
  49                 pkg.commit = commit
  50
  51         # Create a new build object from the package which
  52         # is always a release build.
  53         build = Build.create(_pakfire, pkg, type=type, owner=owner, distro=distro)
  54         logging.debug("Created new build job: %s" % build)
  55
  56         # Create all automatic jobs.
  57         build.create_autojobs(arches=arches)
  58
  59         return pkg, build
  60
  61
  62 class Builds(base.Object):
  63         def _get_build(self, query, *args):
  64                 res = self.db.get(query, *args)
  65
  66                 if res:
  67                         return Build(self.backend, res.id, data=res)
  68
  69         def _get_builds(self, query, *args):
  70                 res = self.db.query(query, *args)
  71
  72                 for row in res:
  73                         yield Build(self.backend, row.id, data=row)
  74
  75         def get_by_id(self, id, data=None):
  76                 return Build(self.pakfire, id, data=data)
  77
  78         def get_by_uuid(self, uuid):
  79                 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
  80
  81                 if build:
  82                         return self.get_by_id(build.id)
  83
  84         def get_all(self, limit=50):
  85                 query = "SELECT * FROM builds ORDER BY time_created DESC"
  86
  87                 if limit:
  88                         query += " LIMIT %d" % limit
  89
  90                 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
  91
  92         def get_by_user(self, user, type=None, public=None):
  93                 args = []
  94                 conditions = []
  95
  96                 if not type or type == "scratch":
  97                         # On scratch builds the user id equals the owner id.
  98                         conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
  99                         args.append(user.id)
 100
 101                 elif not type or type == "release":
 102                         pass # TODO
 103
 104                 if public is True:
 105                         conditions.append("public = 'Y'")
 106                 elif public is False:
 107                         conditions.append("public = 'N'")
 108
 109                 query = "SELECT builds.* AS id FROM builds \
 110                         JOIN packages ON builds.pkg_id = packages.id"
 111
 112                 if conditions:
 113                         query += " WHERE %s" % " AND ".join(conditions)
 114
 115                 query += " ORDER BY builds.time_created DESC"
 116
 117                 builds = []
 118                 for build in self.db.query(query, *args):
 119                         build = Build(self.pakfire, build.id, build)
 120                         builds.append(build)
 121
 122                 return builds
 123
 124         def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
 125                 args = [name,]
 126                 conditions = [
 127                         "packages.name = %s",
 128                 ]
 129
 130                 if type:
 131                         conditions.append("builds.type = %s")
 132                         args.append(type)
 133
 134                 or_conditions = []
 135                 if public is True:
 136                         or_conditions.append("public = 'Y'")
 137                 elif public is False:
 138                         or_conditions.append("public = 'N'")
 139
 140                 if user and not user.is_admin():
 141                         or_conditions.append("builds.owner_id = %s")
 142                         args.append(user.id)
 143
 144                 query = "SELECT builds.* AS id FROM builds \
 145                         JOIN packages ON builds.pkg_id = packages.id"
 146
 147                 if or_conditions:
 148                         conditions.append(" OR ".join(or_conditions))
 149
 150                 if conditions:
 151                         query += " WHERE %s" % " AND ".join(conditions)
 152
 153                 if type == "release":
 154                         query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
 155                 elif type == "scratch":
 156                         query += " ORDER BY time_created DESC"
 157
 158                 if limit:
 159                         if offset:
 160                                 query += " LIMIT %s,%s"
 161                                 args.extend([offset, limit])
 162                         else:
 163                                 query += " LIMIT %s"
 164                                 args.append(limit)
 165
 166                 return [Build(self.pakfire, b.id, b) for b in self.db.query(query, *args)]
 167
 168         def get_latest_by_name(self, name, type=None, public=None):
 169                 query = "\
 170                         SELECT * FROM builds \
 171                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 172                         WHERE builds_latest.package_name = %s"
 173                 args = [name,]
 174
 175                 if type:
 176                         query += " AND builds_latest.build_type = %s"
 177                         args.append(type)
 178
 179                 if public is True:
 180                         query += " AND builds.public = %s"
 181                         args.append("Y")
 182                 elif public is False:
 183                         query += " AND builds.public = %s"
 184                         args.append("N")
 185
 186                 # Get the last one only.
 187                 # Prefer release builds over scratch builds.
 188                 query += "\
 189                         ORDER BY \
 190                                 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
 191                                 builds.time_created DESC \
 192                         LIMIT 1"
 193
 194                 res = self.db.get(query, *args)
 195
 196                 if res:
 197                         return Build(self.pakfire, res.id, res)
 198
 199         def get_active_builds(self, name, public=None):
 200                 query = "\
 201                         SELECT * FROM builds \
 202                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 203                         WHERE builds_latest.package_name = %s AND builds.type = %s"
 204                 args = [name, "release"]
 205
 206                 if public is True:
 207                         query += " AND builds.public = %s"
 208                         args.append("Y")
 209                 elif public is False:
 210                         query += " AND builds.public = %s"
 211                         args.append("N")
 212
 213                 builds = []
 214                 for row in self.db.query(query, *args):
 215                         b = Build(self.pakfire, row.id, row)
 216                         builds.append(b)
 217
 218                 # Sort the result. Lastest build first.
 219                 builds.sort(reverse=True)
 220
 221                 return builds
 222
 223         def count(self):
 224                 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
 225                 if builds:
 226                         return builds.count
 227
 228         def get_obsolete(self, repo=None):
 229                 """
 230                         Get all obsoleted builds.
 231
 232                         If repo is True: which are in any repository.
 233                         If repo is some Repository object: which are in this repository.
 234                 """
 235                 args = []
 236
 237                 if repo is None:
 238                         query = "SELECT id FROM builds WHERE state = 'obsolete'"
 239
 240                 else:
 241                         query = "SELECT build_id AS id FROM repositories_builds \
 242                                 JOIN builds ON builds.id = repositories_builds.build_id \
 243                                 WHERE builds.state = 'obsolete'"
 244
 245                         if repo and not repo is True:
 246                                 query += " AND repositories_builds.repo_id = %s"
 247                                 args.append(repo.id)
 248
 249                 res = self.db.query(query, *args)
 250
 251                 builds = []
 252                 for build in res:
 253                         build = Build(self.pakfire, build.id)
 254                         builds.append(build)
 255
 256                 return builds
 257
 258         def get_changelog(self, name, public=None, limit=5, offset=0):
 259                 query = "SELECT builds.* FROM builds \
 260                         JOIN packages ON builds.pkg_id = packages.id \
 261                         WHERE \
 262                                 builds.type = %s \
 263                         AND \
 264                                 packages.name = %s"
 265                 args = ["release", name,]
 266
 267                 if public == True:
 268                         query += " AND builds.public = %s"
 269                         args.append("Y")
 270                 elif public == False:
 271                         query += " AND builds.public = %s"
 272                         args.append("N")
 273
 274                 query += " ORDER BY builds.time_created DESC"
 275
 276                 if limit:
 277                         if offset:
 278                                 query += " LIMIT %s,%s"
 279                                 args += [offset, limit]
 280                         else:
 281                                 query += " LIMIT %s"
 282                                 args.append(limit)
 283
 284                 builds = []
 285                 for b in self.db.query(query, *args):
 286                         b = Build(self.pakfire, b.id, b)
 287                         builds.append(b)
 288
 289                 builds.sort(reverse=True)
 290
 291                 return builds
 292
 293         def get_comments(self, limit=10, offset=None, user=None):
 294                 query = "SELECT * FROM builds_comments \
 295                         JOIN users ON builds_comments.user_id = users.id"
 296                 args = []
 297
 298                 wheres = []
 299                 if user:
 300                         wheres.append("users.id = %s")
 301                         args.append(user.id)
 302
 303                 if wheres:
 304                         query += " WHERE %s" % " AND ".join(wheres)
 305
 306                 # Sort everything.
 307                 query += " ORDER BY time_created DESC"
 308
 309                 # Limits.
 310                 if limit:
 311                         if offset:
 312                                 query += " LIMIT %s,%s"
 313                                 args.append(offset)
 314                         else:
 315                                 query += " LIMIT %s"
 316
 317                         args.append(limit)
 318
 319                 comments = []
 320                 for comment in self.db.query(query, *args):
 321                         comment = logs.CommentLogEntry(self.pakfire, comment)
 322                         comments.append(comment)
 323
 324                 return comments
 325
 326         def get_build_times_summary(self, name=None, job_type=None, arch=None):
 327                 query = "\
 328                         SELECT \
 329                                 builds_times.arch AS arch, \
 330                                 MAX(duration) AS maximum, \
 331                                 MIN(duration) AS minimum, \
 332                                 AVG(duration) AS average, \
 333                                 SUM(duration) AS sum, \
 334                                 STDDEV_POP(duration) AS stddev \
 335                         FROM builds_times \
 336                                 LEFT JOIN builds ON builds_times.build_id = builds.id \
 337                                 LEFT JOIN packages ON builds.pkg_id = packages.id"
 338
 339                 args = []
 340                 conditions = []
 341
 342                 # Filter for name.
 343                 if name:
 344                         conditions.append("packages.name = %s")
 345                         args.append(name)
 346
 347                 # Filter by job types.
 348                 if job_type:
 349                         conditions.append("builds_times.job_type = %s")
 350                         args.append(job_type)
 351
 352                 # Filter by arch.
 353                 if arch:
 354                         conditions.append("builds_times.arch = %s")
 355                         args.append(arch)
 356
 357                 # Add conditions.
 358                 if conditions:
 359                         query += " WHERE %s" % " AND ".join(conditions)
 360
 361                 # Grouping and sorting.
 362                 query += " GROUP BY arch ORDER BY arch DESC"
 363
 364                 return self.db.query(query, *args)
 365
 366         def get_build_times_by_arch(self, arch, **kwargs):
 367                 kwargs.update({
 368                         "arch" : arch,
 369                 })
 370
 371                 build_times = self.get_build_times_summary(**kwargs)
 372                 if build_times:
 373                         return build_times[0]
 374
 375
 376 class Build(base.Object):
 377         def __init__(self, pakfire, id, data=None):
 378                 base.Object.__init__(self, pakfire)
 379
 380                 # ID of this build
 381                 self.id = id
 382
 383                 # Cache data.
 384                 self._data = data
 385                 self._jobs = None
 386                 self._jobs_test = None
 387                 self._depends_on = None
 388                 self._pkg = None
 389                 self._credits = None
 390                 self._owner = None
 391                 self._update = None
 392                 self._repo = None
 393                 self._distro = None
 394
 395         def __repr__(self):
 396                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
 397
 398         def __cmp__(self, other):
 399                 assert self.pkg
 400                 assert other.pkg
 401
 402                 return cmp(self.pkg, other.pkg)
 403
 404         def __iter__(self):
 405                 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
 406                         WHERE build_id = %s", self.id)
 407
 408                 return iter(sorted(jobs))
 409
 410         @classmethod
 411         def create(cls, pakfire, pkg, type="release", owner=None, distro=None, public=True):
 412                 assert type in ("release", "scratch", "test")
 413                 assert distro, "You need to specify the distribution of this build."
 414
 415                 if public:
 416                         public = "Y"
 417                 else:
 418                         public = "N"
 419
 420                 # Check if scratch build has an owner.
 421                 if type == "scratch" and not owner:
 422                         raise Exception, "Scratch builds require an owner"
 423
 424                 # Set the default priority of this build.
 425                 if type == "release":
 426                         priority = 0
 427
 428                 elif type == "scratch":
 429                         priority = 1
 430
 431                 elif type == "test":
 432                         priority = -1
 433
 434                 id = pakfire.db.execute("""
 435                         INSERT INTO builds(uuid, pkg_id, type, distro_id, time_created, public, priority)
 436                         VALUES(%s, %s, %s, %s, NOW(), %s, %s)""", "%s" % uuid.uuid4(), pkg.id,
 437                         type, distro.id, public, priority)
 438
 439                 # Set the owner of this buildgroup.
 440                 if owner:
 441                         pakfire.db.execute("UPDATE builds SET owner_id = %s WHERE id = %s",
 442                                 owner.id, id)
 443
 444                 build = cls(pakfire, id)
 445
 446                 # Log that the build has been created.
 447                 build.log("created", user=owner)
 448
 449                 # Create directory where the files live.
 450                 if not os.path.exists(build.path):
 451                         os.makedirs(build.path)
 452
 453                 # Move package file to the directory of the build.
 454                 source_path = os.path.join(build.path, "src")
 455                 build.pkg.move(source_path)
 456
 457                 # Generate an update id.
 458                 build.generate_update_id()
 459
 460                 # Obsolete all other builds with the same name to track updates.
 461                 build.obsolete_others()
 462
 463                 # Search for possible bug IDs in the commit message.
 464                 build.search_for_bugs()
 465
 466                 return build
 467
 468         def delete(self):
 469                 """
 470                         Deletes this build including all jobs, packages and the source
 471                         package.
 472                 """
 473                 # If the build is in a repository, we need to remove it.
 474                 if self.repo:
 475                         self.repo.rem_build(self)
 476
 477                 for job in self.jobs + self.test_jobs:
 478                         job.delete()
 479
 480                 if self.pkg:
 481                         self.pkg.delete()
 482
 483                 # Delete everything related to this build.
 484                 self.__delete_bugs()
 485                 self.__delete_comments()
 486                 self.__delete_history()
 487                 self.__delete_watchers()
 488
 489                 # Delete the build itself.
 490                 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
 491
 492         def __delete_bugs(self):
 493                 """
 494                         Delete all associated bugs.
 495                 """
 496                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
 497
 498         def __delete_comments(self):
 499                 """
 500                         Delete all comments.
 501                 """
 502                 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
 503
 504         def __delete_history(self):
 505                 """
 506                         Delete the repository history.
 507                 """
 508                 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
 509
 510         def __delete_watchers(self):
 511                 """
 512                         Delete all watchers.
 513                 """
 514                 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
 515
 516         def reset(self):
 517                 """
 518                         Resets the whole build so it can start again (as it has never
 519                         been started).
 520                 """
 521                 for job in self.jobs:
 522                         job.reset()
 523
 524                 #self.__delete_bugs()
 525                 self.__delete_comments()
 526                 self.__delete_history()
 527                 self.__delete_watchers()
 528
 529                 self.state = "building"
 530
 531                 # XXX empty log
 532
 533         @property
 534         def data(self):
 535                 """
 536                         Lazy fetching of data for this object.
 537                 """
 538                 if self._data is None:
 539                         self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id)
 540                         assert self._data
 541
 542                 return self._data
 543
 544         @property
 545         def info(self):
 546                 """
 547                         A set of information that is sent to the XMLRPC client.
 548                 """
 549                 return { "uuid" : self.uuid }
 550
 551         def log(self, action, user=None, bug_id=None):
 552                 user_id = None
 553                 if user:
 554                         user_id = user.id
 555
 556                 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
 557                         VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
 558
 559         @property
 560         def uuid(self):
 561                 """
 562                         The UUID of this build.
 563                 """
 564                 return self.data.uuid
 565
 566         @property
 567         def pkg(self):
 568                 """
 569                         Get package that is to be built in the build.
 570                 """
 571                 if self._pkg is None:
 572                         self._pkg = packages.Package(self.pakfire, self.data.pkg_id)
 573
 574                 return self._pkg
 575
 576         @property
 577         def name(self):
 578                 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
 579
 580         @property
 581         def type(self):
 582                 """
 583                         The type of this build.
 584                 """
 585                 return self.data.type
 586
 587         @property
 588         def owner_id(self):
 589                 """
 590                         The ID of the owner of this build.
 591                 """
 592                 return self.data.owner_id
 593
 594         @property
 595         def owner(self):
 596                 """
 597                         The owner of this build.
 598                 """
 599                 if not self.owner_id:
 600                         return
 601
 602                 if self._owner is None:
 603                         self._owner = self.pakfire.users.get_by_id(self.owner_id)
 604                         assert self._owner
 605
 606                 return self._owner
 607
 608         @property
 609         def distro_id(self):
 610                 return self.data.distro_id
 611
 612         @property
 613         def distro(self):
 614                 if self._distro is None:
 615                         self._distro = self.pakfire.distros.get_by_id(self.distro_id)
 616                         assert self._distro
 617
 618                 return self._distro
 619
 620         @property
 621         def user(self):
 622                 if self.type == "scratch":
 623                         return self.owner
 624
 625         def get_depends_on(self):
 626                 if self.data.depends_on and self._depends_on is None:
 627                         self._depends_on = Build(self.pakfire, self.data.depends_on)
 628
 629                 return self._depends_on
 630
 631         def set_depends_on(self, build):
 632                 self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s",
 633                         build.id, self.id)
 634
 635                 # Update cache.
 636                 self._depends_on = build
 637                 self._data["depends_on"] = build.id
 638
 639         depends_on = property(get_depends_on, set_depends_on)
 640
 641         @property
 642         def created(self):
 643                 return self.data.time_created
 644
 645         @property
 646         def date(self):
 647                 return self.created.date()
 648
 649         @property
 650         def public(self):
 651                 """
 652                         Is this build public?
 653                 """
 654                 return self.data.public == "Y"
 655
 656         @property
 657         def size(self):
 658                 """
 659                         Returns the size on disk of this build.
 660                 """
 661                 s = 0
 662
 663                 # Add the source package.
 664                 if self.pkg:
 665                         s += self.pkg.size
 666
 667                 # Add all jobs.
 668                 s += sum((j.size for j in self.jobs))
 669
 670                 return s
 671
 672         #@property
 673         #def state(self):
 674         #       # Cache all states.
 675         #       states = [j.state for j in self.jobs]
 676         #
 677         #       target_state = "unknown"
 678         #
 679         #       # If at least one job has failed, the whole build has failed.
 680         #       if "failed" in states:
 681         #               target_state = "failed"
 682         #
 683         #       # It at least one of the jobs is still running, the whole
 684         #       # build is in running state.
 685         #       elif "running" in states:
 686         #               target_state = "running"
 687         #
 688         #       # If all jobs are in the finished state, we turn into finished
 689         #       # state as well.
 690         #       elif all([s == "finished" for s in states]):
 691         #               target_state = "finished"
 692         #
 693         #       return target_state
 694
 695         def auto_update_state(self):
 696                 """
 697                         Check if the state of this build can be updated and perform
 698                         the change if possible.
 699                 """
 700                 # Do not change the broken/obsolete state automatically.
 701                 if self.state in ("broken", "obsolete"):
 702                         return
 703
 704                 if self.repo and self.repo.type == "stable":
 705                         self.update_state("stable")
 706                         return
 707
 708                 # If any of the build jobs are finished, the build will be put in testing
 709                 # state.
 710                 for job in self.jobs:
 711                         if job.state == "finished":
 712                                 self.update_state("testing")
 713                                 break
 714
 715         def update_state(self, state, user=None, remove=False):
 716                 assert state in ("stable", "testing", "obsolete", "broken")
 717
 718                 self.db.execute("UPDATE builds SET state = %s WHERE id = %s", state, self.id)
 719
 720                 if self._data:
 721                         self._data["state"] = state
 722
 723                 # In broken state, the removal from the repository is forced and
 724                 # all jobs that are not finished yet will be aborted.
 725                 if state == "broken":
 726                         remove = True
 727
 728                         for job in self.jobs:
 729                                 if job.state in ("new", "pending", "running", "dependency_error"):
 730                                         job.state = "aborted"
 731
 732                 # If this build is in a repository, it will leave it.
 733                 if remove and self.repo:
 734                                 self.repo.rem_build(self)
 735
 736                 # If a release build is now in testing state, we put it into the
 737                 # first repository of the distribution.
 738                 elif self.type == "release" and state == "testing":
 739                         # If the build is not in a repository, yet and if there is
 740                         # a first repository, we put the build there.
 741                         if not self.repo and self.distro.first_repo:
 742                                 self.distro.first_repo.add_build(self, user=user)
 743
 744         @property
 745         def state(self):
 746                 return self.data.state
 747
 748         def is_broken(self):
 749                 return self.state == "broken"
 750
 751         def obsolete_others(self):
 752                 if not self.type == "release":
 753                         return
 754
 755                 for build in self.pakfire.builds.get_by_name(self.pkg.name, type="release"):
 756                         # Don't modify ourself.
 757                         if self.id == build.id:
 758                                 continue
 759
 760                         # Don't touch broken builds.
 761                         if build.state in ("obsolete", "broken"):
 762                                 continue
 763
 764                         # Obsolete the build.
 765                         build.update_state("obsolete")
 766
 767         def set_severity(self, severity):
 768                 self.db.execute("UPDATE builds SET severity = %s WHERE id = %s", state, self.id)
 769
 770                 if self._data:
 771                         self._data["severity"] = severity
 772
 773         def get_severity(self):
 774                 return self.data.severity
 775
 776         severity = property(get_severity, set_severity)
 777
 778         @property
 779         def commit(self):
 780                 if self.pkg and self.pkg.commit:
 781                         return self.pkg.commit
 782
 783         def update_message(self, msg):
 784                 self.db.execute("UPDATE builds SET message = %s WHERE id = %s", msg, self.id)
 785
 786                 if self._data:
 787                         self._data["message"] = msg
 788
 789         def has_perm(self, user):
 790                 """
 791                         Check, if the given user has the right to perform administrative
 792                         operations on this build.
 793                 """
 794                 if user is None:
 795                         return False
 796
 797                 if user.is_admin():
 798                         return True
 799
 800                 # Check if the user is allowed to manage packages from the critical path.
 801                 if self.critical_path and not user.has_perm("manage_critical_path"):
 802                         return False
 803
 804                 # Search for maintainers...
 805
 806                 # Scratch builds.
 807                 if self.type == "scratch":
 808                         # The owner of a scratch build has the right to do anything with it.
 809                         if self.owner_id == user.id:
 810                                 return True
 811
 812                 # Release builds.
 813                 elif self.type == "release":
 814                         # The maintainer also is allowed to manage the build.
 815                         if self.pkg.maintainer == user:
 816                                 return True
 817
 818                 # Deny permission for all other cases.
 819                 return False
 820
 821         @property
 822         def message(self):
 823                 message = ""
 824
 825                 if self.data.message:
 826                         message = self.data.message
 827
 828                 elif self.commit:
 829                         if self.commit.message:
 830                                 message = "\n".join((self.commit.subject, self.commit.message))
 831                         else:
 832                                 message = self.commit.subject
 833
 834                         prefix = "%s: " % self.pkg.name
 835                         if message.startswith(prefix):
 836                                 message = message[len(prefix):]
 837
 838                 return message
 839
 840         def get_priority(self):
 841                 return self.data.priority
 842
 843         def set_priority(self, priority):
 844                 assert priority in (-2, -1, 0, 1, 2)
 845
 846                 self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority,
 847                         self.id)
 848
 849                 if self._data:
 850                         self._data["priority"] = priority
 851
 852         priority = property(get_priority, set_priority)
 853
 854         @property
 855         def path(self):
 856                 path = []
 857                 if self.type == "scratch":
 858                         path.append(BUILD_SCRATCH_DIR)
 859                         path.append(self.uuid)
 860
 861                 elif self.type == "release":
 862                         path.append(BUILD_RELEASE_DIR)
 863                         path.append("%s/%s-%s-%s" % \
 864                                 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
 865
 866                 else:
 867                         raise Exception, "Unknown build type: %s" % self.type
 868
 869                 return os.path.join(*path)
 870
 871         @property
 872         def source_filename(self):
 873                 return os.path.basename(self.pkg.path)
 874
 875         @property
 876         def download_prefix(self):
 877                 return "/".join((self.pakfire.settings.get("download_baseurl"), "packages"))
 878
 879         @property
 880         def source_download(self):
 881                 return "/".join((self.download_prefix, self.pkg.path))
 882
 883         @property
 884         def source_hash_sha512(self):
 885                 return self.pkg.hash_sha512
 886
 887         @property
 888         def link(self):
 889                 # XXX maybe this should rather live in a uimodule.
 890                 # zlib-1.2.3-2.ip3 [src, i686, blah...]
 891                 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
 892                         (self.state, self.type, self.uuid, self.name)
 893
 894                 s_jobs = []
 895                 for job in self.jobs:
 896                         s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
 897                                 (job.state, job.type, job.uuid, job.arch))
 898
 899                 if s_jobs:
 900                         s += " [%s]" % ", ".join(s_jobs)
 901
 902                 return s
 903
 904         @property
 905         def supported_arches(self):
 906                 return self.pkg.supported_arches
 907
 908         @property
 909         def critical_path(self):
 910                 return self.pkg.critical_path
 911
 912         def get_jobs(self, type=None):
 913                 """
 914                         Returns a list of jobs of this build.
 915                 """
 916                 return self.pakfire.jobs.get_by_build(self.id, self, type=type)
 917
 918         @property
 919         def jobs(self):
 920                 """
 921                         Get a list of all build jobs that are in this build.
 922                 """
 923                 if self._jobs is None:
 924                         self._jobs = self.get_jobs(type="build")
 925
 926                 return self._jobs
 927
 928         @property
 929         def test_jobs(self):
 930                 if self._jobs_test is None:
 931                         self._jobs_test = self.get_jobs(type="test")
 932
 933                 return self._jobs_test
 934
 935         @property
 936         def all_jobs_finished(self):
 937                 ret = True
 938
 939                 for job in self.jobs:
 940                         if not job.state == "finished":
 941                                 ret = False
 942                                 break
 943
 944                 return ret
 945
 946         def create_autojobs(self, arches=None, type="build"):
 947                 jobs = []
 948
 949                 # Arches may be passed to this function. If not we use all arches
 950                 # this package supports.
 951                 if arches is None:
 952                         arches = self.supported_arches
 953
 954                 # Create a new job for every given archirecture.
 955                 for arch in self.pakfire.arches.expand(arches):
 956                         # Don't create jobs for src.
 957                         if arch.name == "src":
 958                                 continue
 959
 960                         job = self.add_job(arch, type=type)
 961                         jobs.append(job)
 962
 963                 # Return all newly created jobs.
 964                 return jobs
 965
 966         def add_job(self, arch, type="build"):
 967                 job = Job.create(self.pakfire, self, arch, type=type)
 968
 969                 # Add new job to cache.
 970                 if self._jobs:
 971                         self._jobs.append(job)
 972
 973                 return job
 974
 975         ## Update stuff
 976
 977         @property
 978         def update_id(self):
 979                 if not self.type == "release":
 980                         return
 981
 982                 # Generate an update ID if none does exist, yet.
 983                 self.generate_update_id()
 984
 985                 s = [
 986                         "%s" % self.distro.name.replace(" ", "").upper(),
 987                         "%04d" % (self.data.update_year or 0),
 988                         "%04d" % (self.data.update_num or 0),
 989                 ]
 990
 991                 return "-".join(s)
 992
 993         def generate_update_id(self):
 994                 if not self.type == "release":
 995                         return
 996
 997                 if self.data.update_num:
 998                         return
 999
1000                 update = self.db.get("SELECT update_num AS num FROM builds \
1001                         WHERE update_year = YEAR(NOW()) ORDER BY update_num DESC LIMIT 1")
1002
1003                 if update:
1004                         update_num = update.num + 1
1005                 else:
1006                         update_num = 1
1007
1008                 self.db.execute("UPDATE builds SET update_year = YEAR(NOW()), update_num = %s \
1009                         WHERE id = %s", update_num, self.id)
1010
1011         ## Comment stuff
1012
1013         def get_comments(self, limit=10, offset=0):
1014                 query = "SELECT * FROM builds_comments \
1015                         JOIN users ON builds_comments.user_id = users.id \
1016                         WHERE build_id = %s     ORDER BY time_created ASC"
1017
1018                 comments = []
1019                 for comment in self.db.query(query, self.id):
1020                         comment = logs.CommentLogEntry(self.pakfire, comment)
1021                         comments.append(comment)
1022
1023                 return comments
1024
1025         def add_comment(self, user, text, credit):
1026                 # Add the new comment to the database.
1027                 id = self.db.execute("INSERT INTO \
1028                         builds_comments(build_id, user_id, text, credit, time_created) \
1029                         VALUES(%s, %s, %s, %s, NOW())",
1030                         self.id, user.id, text, credit)
1031
1032                 # Update the credit cache.
1033                 if not self._credits is None:
1034                         self._credits += credit
1035
1036                 # Send the new comment to all watchers and stuff.
1037                 self.send_comment_message(id)
1038
1039                 # Return the ID of the newly created comment.
1040                 return id
1041
1042         @property
1043         def score(self):
1044                 # XXX UPDATE THIS
1045                 if self._credits is None:
1046                         # Get the sum of the credits from the database.
1047                         query = self.db.get(
1048                                 "SELECT SUM(credit) as credits FROM builds_comments WHERE build_id = %s",
1049                                 self.id
1050                         )
1051
1052                         self._credits = query.credits or 0
1053
1054                 return self._credits
1055
1056         @property
1057         def credits(self):
1058                 # XXX COMPAT
1059                 return self.score
1060
1061         def get_commenters(self):
1062                 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
1063                         JOIN users ON builds_comments.user_id = users.id \
1064                         WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
1065                         AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
1066
1067                 return [users.User(self.pakfire, u.id) for u in users]
1068
1069         def send_comment_message(self, comment_id):
1070                 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
1071                         comment_id)
1072
1073                 assert comment
1074                 assert comment.build_id == self.id
1075
1076                 # Get user who wrote the comment.
1077                 user = self.pakfire.users.get_by_id(comment.user_id)
1078
1079                 format = {
1080                         "build_name" : self.name,
1081                         "user_name"  : user.realname,
1082                 }
1083
1084                 # XXX create beautiful message
1085
1086                 self.pakfire.messages.send_to_all(self.message_recipients,
1087                         N_("%(user_name)s commented on %(build_name)s"),
1088                         comment.text, format)
1089
1090         ## Logging stuff
1091
1092         def get_log(self, comments=True, repo=True, limit=None):
1093                 entries = []
1094
1095                 # Created entry.
1096                 created_entry = logs.CreatedLogEntry(self.pakfire, self)
1097                 entries.append(created_entry)
1098
1099                 if comments:
1100                         entries += self.get_comments(limit=limit)
1101
1102                 if repo:
1103                         entries += self.get_repo_moves(limit=limit)
1104
1105                 # Sort all entries in chronological order.
1106                 entries.sort()
1107
1108                 if limit:
1109                         entries = entries[:limit]
1110
1111                 return entries
1112
1113         ## Watchers stuff
1114
1115         def get_watchers(self):
1116                 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
1117                         JOIN users ON builds_watchers.user_id = users.id \
1118                         WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1119                         AND users.activated = 'Y' ORDER BY users.id", self.id)
1120
1121                 return [users.User(self.pakfire, u.id) for u in query]
1122
1123         def add_watcher(self, user):
1124                 # Don't add a user twice.
1125                 if user in self.get_watchers():
1126                         return
1127
1128                 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1129                         VALUES(%s, %s)", self.id, user.id)
1130
1131         @property
1132         def message_recipients(self):
1133                 ret = []
1134
1135                 for watcher in self.get_watchers():
1136                         ret.append("%s <%s>" % (watcher.realname, watcher.email))
1137
1138                 return ret
1139
1140         @property
1141         def update(self):
1142                 if self._update is None:
1143                         update = self.db.get("SELECT update_id AS id FROM updates_builds \
1144                                 WHERE build_id = %s", self.id)
1145
1146                         if update:
1147                                 self._update = updates.Update(self.pakfire, update.id)
1148
1149                 return self._update
1150
1151         @property
1152         def repo(self):
1153                 if self._repo is None:
1154                         repo = self.db.get("SELECT repo_id AS id FROM repositories_builds \
1155                                 WHERE build_id = %s", self.id)
1156
1157                         if repo:
1158                                 self._repo = repository.Repository(self.pakfire, repo.id)
1159
1160                 return self._repo
1161
1162         def get_repo_moves(self, limit=None):
1163                 query = "SELECT * FROM repositories_history \
1164                         WHERE build_id = %s ORDER BY time ASC"
1165
1166                 actions = []
1167                 for action in self.db.query(query, self.id):
1168                         action = logs.RepositoryLogEntry(self.pakfire, action)
1169                         actions.append(action)
1170
1171                 return actions
1172
1173         @property
1174         def is_loose(self):
1175                 if self.repo:
1176                         return False
1177
1178                 return True
1179
1180         @property
1181         def repo_time(self):
1182                 repo = self.db.get("SELECT time_added FROM repositories_builds \
1183                         WHERE build_id = %s", self.id)
1184
1185                 if repo:
1186                         return repo.time_added
1187
1188         def get_auto_move(self):
1189                 return self.data.auto_move == "Y"
1190
1191         def set_auto_move(self, state):
1192                 if state:
1193                         state = "Y"
1194                 else:
1195                         state = "N"
1196
1197                 self.db.execute("UPDATE builds SET auto_move = %s WHERE id = %s", self.id)
1198                 if self._data:
1199                         self._data["auto_move"] = state
1200
1201         auto_move = property(get_auto_move, set_auto_move)
1202
1203         @property
1204         def can_move_forward(self):
1205                 if not self.repo:
1206                         return False
1207
1208                 # If there is no next repository, we cannot move anything.
1209                 if not self.repo.next:
1210                         return False
1211
1212                 # If the needed amount of score is reached, we can move forward.
1213                 if self.score >= self.repo.next.score_needed:
1214                         return True
1215
1216                 # If the repository does not require a minimal time,
1217                 # we can move forward immediately.
1218                 if not self.repo.time_min:
1219                         return True
1220
1221                 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1222                         WHERE build_id = %s", self.id)
1223                 duration = query.duration
1224
1225                 if duration >= self.repo.time_min:
1226                         return True
1227
1228                 return False
1229
1230         ## Bugs
1231
1232         def get_bug_ids(self):
1233                 query = self.db.query("SELECT bug_id FROM builds_bugs \
1234                         WHERE build_id = %s", self.id)
1235
1236                 return [b.bug_id for b in query]
1237
1238         def add_bug(self, bug_id, user=None, log=True):
1239                 # Check if this bug is already in the list of bugs.
1240                 if bug_id in self.get_bug_ids():
1241                         return
1242
1243                 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1244                         VALUES(%s, %s)", self.id, bug_id)
1245
1246                 # Log the event.
1247                 if log:
1248                         self.log("bug_added", user=user, bug_id=bug_id)
1249
1250         def rem_bug(self, bug_id, user=None, log=True):
1251                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1252                         bug_id = %s", self.id, bug_id)
1253
1254                 # Log the event.
1255                 if log:
1256                         self.log("bug_removed", user=user, bug_id=bug_id)
1257
1258         def search_for_bugs(self):
1259                 if not self.commit:
1260                         return
1261
1262                 pattern = re.compile(r"(bug\s?|#)(\d+)")
1263
1264                 for txt in (self.commit.subject, self.commit.message):
1265                         for bug in re.finditer(pattern, txt):
1266                                 try:
1267                                         bugid = int(bug.group(2))
1268                                 except ValueError:
1269                                         continue
1270
1271                                 # Check if a bug with the given ID exists in BZ.
1272                                 bug = self.pakfire.bugzilla.get_bug(bugid)
1273                                 if not bug:
1274                                         continue
1275
1276                                 self.add_bug(bugid)
1277
1278         def get_bugs(self):
1279                 bugs = []
1280                 for bug_id in self.get_bug_ids():
1281                         bug = self.pakfire.bugzilla.get_bug(bug_id)
1282                         if not bug:
1283                                 continue
1284
1285                         bugs.append(bug)
1286
1287                 return bugs
1288
1289         def _update_bugs_helper(self, repo):
1290                 """
1291                         This function takes a new status and generates messages that
1292                         are appended to all bugs.
1293                 """
1294                 try:
1295                         kwargs = BUG_MESSAGES[repo.type].copy()
1296                 except KeyError:
1297                         return
1298
1299                 baseurl = self.pakfire.settings.get("baseurl", "")
1300                 args = {
1301                         "build_url"    : "%s/build/%s" % (baseurl, self.uuid),
1302                         "distro_name"  : self.distro.name,
1303                         "package_name" : self.name,
1304                         "repo_name"    : repo.name,
1305                 }
1306                 kwargs["comment"] = kwargs["comment"] % args
1307
1308                 self.update_bugs(**kwargs)
1309
1310         def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1311                 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1312                         VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1313
1314         def update_bugs(self, status, resolution=None, comment=None):
1315                 # Update all bugs linked to this build.
1316                 for bug_id in self.get_bug_ids():
1317                         self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1318
1319
1320 class Jobs(base.Object):
1321         def _get_job(self, query, *args):
1322                 res = self.db.get(query, *args)
1323
1324                 if res:
1325                         return Job(self.backend, res.id, data=res)
1326
1327         def _get_jobs(self, query, *args):
1328                 res = self.db.query(query, *args)
1329
1330                 for row in res:
1331                         yield Job(self.backend, row.id, data=row)
1332
1333         def create(self,  build, arch, type="build"):
1334                 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1335                         VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1336                 job.log("created")
1337
1338                 # Set cache for Build object.
1339                 job.build = build
1340
1341                 # Jobs are by default in state "new" and wait for being checked
1342                 # for dependencies. Packages that do have no build dependencies
1343                 # can directly be forwarded to "pending" state.
1344                 if not job.pkg.requires:
1345                         job.state = "pending"
1346
1347                 return job
1348
1349         def get_by_id(self, id, data=None):
1350                 return Job(self.pakfire, id, data)
1351
1352         def get_by_uuid(self, uuid):
1353                 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1354
1355                 if job:
1356                         return self.get_by_id(job.id)
1357
1358         def get_by_build(self, build_id, build=None, type=None):
1359                 """
1360                         Get all jobs in the specifies build.
1361                 """
1362                 query = "SELECT * FROM jobs WHERE build_id = %s"
1363                 args = [build_id,]
1364
1365                 if type:
1366                         query += " AND type = %s"
1367                         args.append(type)
1368
1369                 # Get IDs of all builds in this group.
1370                 jobs = []
1371                 for job in self.db.query(query, *args):
1372                         job = Job(self.pakfire, job.id, job)
1373
1374                         # If the Build object was set, we set it so it won't be retrieved
1375                         # from the database again.
1376                         if build:
1377                                 job._build = build
1378
1379                         jobs.append(job)
1380
1381                 # Return sorted list of jobs.
1382                 return sorted(jobs)
1383
1384         def get_active(self, host_id=None, builder=None, states=None):
1385                 if builder:
1386                         host_id = builder.id
1387
1388                 if states is None:
1389                         states = ["dispatching", "running", "uploading"]
1390
1391                 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1392                 args = states
1393
1394                 if host_id:
1395                         query += " AND builder_id = %s" % host_id
1396
1397                 query += " ORDER BY \
1398                         CASE \
1399                                 WHEN jobs.state = 'running'     THEN 0 \
1400                                 WHEN jobs.state = 'uploading'   THEN 1 \
1401                                 WHEN jobs.state = 'dispatching' THEN 2 \
1402                                 WHEN jobs.state = 'pending'     THEN 3 \
1403                                 WHEN jobs.state = 'new'         THEN 4 \
1404                         END, time_started ASC"
1405
1406                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1407
1408         def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
1409                 query = "SELECT * FROM jobs"
1410                 args  = []
1411
1412                 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
1413
1414                 if arch:
1415                         where.append("arch = %s")
1416                         args.append(arch)
1417
1418                 if builder:
1419                         where.append("builder_id = %s")
1420                         args.append(builder.id)
1421
1422                 if date:
1423                         try:
1424                                 year, month, day = date.split("-", 2)
1425                                 date = datetime.date(int(year), int(month), int(day))
1426                         except ValueError:
1427                                 pass
1428                         else:
1429                                 where.append("(DATE(time_created) = %s OR \
1430                                         DATE(time_started) = %s OR DATE(time_finished) = %s)")
1431                                 args += (date, date, date)
1432
1433                 if age:
1434                         where.append("time_finished >= NOW() - '%s'::interval" % age)
1435
1436                 if where:
1437                         query += " WHERE %s" % " AND ".join(where)
1438
1439                 query += " ORDER BY time_finished DESC"
1440
1441                 if limit:
1442                         query += " LIMIT %s"
1443                         args.append(limit)
1444
1445                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1446
1447         def get_average_build_time(self):
1448                 """
1449                         Returns the average build time of all finished builds from the
1450                         last 3 months.
1451                 """
1452                 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1453                         FROM jobs WHERE type = 'build' AND state = 'finished' AND \
1454                         time_finished >= NOW() - '3 months'::interval")
1455
1456                 if result:
1457                         return result.average
1458
1459         def count(self, *states):
1460                 query = "SELECT COUNT(*) AS count FROM jobs"
1461                 args  = []
1462
1463                 if states:
1464                         query += " WHERE state IN %s"
1465                         args.append(states)
1466
1467                 jobs = self.db.get(query, *args)
1468                 if jobs:
1469                         return jobs.count
1470
1471
1472 class Job(base.DataObject):
1473         table = "jobs"
1474
1475         def __str__(self):
1476                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1477
1478         def __eq__(self, other):
1479                 if isinstance(other, self.__class__):
1480                         return self.id == other.id
1481
1482         def __lt__(self, other):
1483                 if isinstance(other, self.__class__):
1484                         if (self.type, other.type) == ("build", "test"):
1485                                 return True
1486
1487                         if self.build == other.build:
1488                                 return arches.priority(self.arch) < arches.priority(other.arch)
1489
1490                         return self.time_created < other.time_created
1491
1492         def __iter__(self):
1493                 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1494                         LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1495                         WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
1496
1497                 return iter(packages)
1498
1499         def __nonzero__(self):
1500                 return True
1501
1502         def __len__(self):
1503                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1504                         WHERE job_id = %s", self.id)
1505
1506                 return res.len
1507
1508         @property
1509         def distro(self):
1510                 return self.build.distro
1511
1512         def delete(self):
1513                 self.__delete_buildroots()
1514                 self.__delete_history()
1515                 self.__delete_packages()
1516                 self.__delete_logfiles()
1517
1518                 # Delete the job itself.
1519                 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
1520
1521         def __delete_buildroots(self):
1522                 """
1523                         Removes all buildroots.
1524                 """
1525                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1526
1527         def __delete_history(self):
1528                 """
1529                         Removes all references in the history to this build job.
1530                 """
1531                 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1532
1533         def __delete_packages(self):
1534                 """
1535                         Deletes all uploaded files from the job.
1536                 """
1537                 for pkg in self.packages:
1538                         pkg.delete()
1539
1540                 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1541
1542         def __delete_logfiles(self):
1543                 for logfile in self.logfiles:
1544                         self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1545
1546         def reset(self, user=None):
1547                 self.__delete_buildroots()
1548                 self.__delete_packages()
1549                 self.__delete_history()
1550                 self.__delete_logfiles()
1551
1552                 self.state = "new"
1553                 self.log("reset", user=user)
1554
1555         ## Logging stuff
1556
1557         def log(self, action, user=None, state=None, builder=None, test_job=None):
1558                 user_id = None
1559                 if user:
1560                         user_id = user.id
1561
1562                 builder_id = None
1563                 if builder:
1564                         builder_id = builder.id
1565
1566                 test_job_id = None
1567                 if test_job:
1568                         test_job_id = test_job.id
1569
1570                 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1571                         time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1572                         self.id, action, state, user_id, builder_id, test_job_id)
1573
1574         def get_log(self, limit=None, offset=None, user=None):
1575                 query = "SELECT * FROM jobs_history"
1576
1577                 conditions = ["job_id = %s",]
1578                 args  = [self.id,]
1579
1580                 if user:
1581                         conditions.append("user_id = %s")
1582                         args.append(user.id)
1583
1584                 if conditions:
1585                         query += " WHERE %s" % " AND ".join(conditions)
1586
1587                 query += " ORDER BY time DESC"
1588
1589                 if limit:
1590                         if offset:
1591                                 query += " LIMIT %s,%s"
1592                                 args  += [offset, limit,]
1593                         else:
1594                                 query += " LIMIT %s"
1595                                 args  += [limit,]
1596
1597                 entries = []
1598                 for entry in self.db.query(query, *args):
1599                         entry = logs.JobLogEntry(self.pakfire, entry)
1600                         entries.append(entry)
1601
1602                 return entries
1603
1604         @property
1605         def uuid(self):
1606                 return self.data.uuid
1607
1608         @property
1609         def type(self):
1610                 return self.data.type
1611
1612         @property
1613         def build_id(self):
1614                 return self.data.build_id
1615
1616         @lazy_property
1617         def build(self):
1618                 return self.pakfire.builds.get_by_id(self.build_id)
1619
1620         @property
1621         def related_jobs(self):
1622                 ret = []
1623
1624                 for job in self.build.jobs:
1625                         if job == self:
1626                                 continue
1627
1628                         ret.append(job)
1629
1630                 return ret
1631
1632         @property
1633         def pkg(self):
1634                 return self.build.pkg
1635
1636         @property
1637         def name(self):
1638                 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
1639
1640         @property
1641         def size(self):
1642                 return sum((p.size for p in self.packages))
1643
1644         @lazy_property
1645         def rank(self):
1646                 """
1647                         Returns the rank in the build queue
1648                 """
1649                 if not self.state == "pending":
1650                         return
1651
1652                 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1653
1654                 if res:
1655                         return res.rank
1656
1657         def is_running(self):
1658                 """
1659                         Returns True if job is in a running state.
1660                 """
1661                 return self.state in ("pending", "dispatching", "running", "uploading")
1662
1663         def get_state(self):
1664                 return self.data.state
1665
1666         def set_state(self, state, user=None, log=True):
1667                 # Nothing to do if the state remains.
1668                 if not self.state == state:
1669                         self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
1670
1671                         # Log the event.
1672                         if log and not state == "new":
1673                                 self.log("state_change", state=state, user=user)
1674
1675                         # Update cache.
1676                         if self._data:
1677                                 self._data["state"] = state
1678
1679                 # Always clear the message when the status is changed.
1680                 self.update_message(None)
1681
1682                 # Update some more informations.
1683                 if state == "dispatching":
1684                         # Set start time.
1685                         self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1686                                 WHERE id = %s", self.id)
1687
1688                 elif state == "pending":
1689                         self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1690                                 time_finished = NULL WHERE id = %s", self.id)
1691
1692                 elif state in ("aborted", "dependency_error", "finished", "failed"):
1693                         # Set finish time and reset builder..
1694                         self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
1695
1696                         # Send messages to the user.
1697                         if state == "finished":
1698                                 self.send_finished_message()
1699
1700                         elif state == "failed":
1701                                 # Remove all package files if a job is set to failed state.
1702                                 self.__delete_packages()
1703
1704                                 self.send_failed_message()
1705
1706                 # Automatically update the state of the build (not on test builds).
1707                 if self.type == "build":
1708                         self.build.auto_update_state()
1709
1710         state = property(get_state, set_state)
1711
1712         @property
1713         def message(self):
1714                 return self.data.message
1715
1716         def update_message(self, msg):
1717                 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1718                         msg, self.id)
1719
1720                 if self._data:
1721                         self._data["message"] = msg
1722
1723         def get_builder(self):
1724                 if self.data.builder_id:
1725                         return self.backend.builders.get_by_id(self.data.builder_id)
1726
1727         def set_builder(self, builder, user=None):
1728                 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1729                         builder.id, self.id)
1730
1731                 # Update cache.
1732                 if self._data:
1733                         self._data["builder_id"] = builder.id
1734
1735                 self._builder = builder
1736
1737                 # Log the event.
1738                 if user:
1739                         self.log("builder_assigned", builder=builder, user=user)
1740
1741         builder = lazy_property(get_builder, set_builder)
1742
1743         @property
1744         def arch(self):
1745                 return self.data.arch
1746
1747         @property
1748         def duration(self):
1749                 if not self.time_started:
1750                         return 0
1751
1752                 if self.time_finished:
1753                         delta = self.time_finished - self.time_started
1754                 else:
1755                         delta = datetime.datetime.utcnow() - self.time_started
1756
1757                 return delta.total_seconds()
1758
1759         @property
1760         def time_created(self):
1761                 return self.data.time_created
1762
1763         @property
1764         def time_started(self):
1765                 return self.data.time_started
1766
1767         @property
1768         def time_finished(self):
1769                 return self.data.time_finished
1770
1771         @property
1772         def expected_runtime(self):
1773                 """
1774                         Returns the estimated time and stddev, this job takes to finish.
1775                 """
1776                 # Get the average build time.
1777                 build_times = self.pakfire.builds.get_build_times_by_arch(self.arch,
1778                         name=self.pkg.name)
1779
1780                 # If there is no statistical data, we cannot estimate anything.
1781                 if not build_times:
1782                         return None, None
1783
1784                 return build_times.average, build_times.stddev
1785
1786         @property
1787         def eta(self):
1788                 expected_runtime, stddev = self.expected_runtime
1789
1790                 if expected_runtime:
1791                         return expected_runtime - int(self.duration), stddev
1792
1793         @property
1794         def tries(self):
1795                 return self.data.tries
1796
1797         def get_pkg_by_uuid(self, uuid):
1798                 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
1799                         JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1800                         WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1801                         self.id, uuid)
1802
1803                 if pkg:
1804                         pkg.job = self
1805                         return pkg
1806
1807         @lazy_property
1808         def logfiles(self):
1809                 logfiles = []
1810
1811                 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1812                         log = logs.LogFile(self.pakfire, log.id)
1813                         log._job = self
1814
1815                         logfiles.append(log)
1816
1817                 return logfiles
1818
1819         def add_file(self, filename):
1820                 """
1821                         Add the specified file to this job.
1822
1823                         The file is copied to the right directory by this function.
1824                 """
1825                 assert os.path.exists(filename)
1826
1827                 if filename.endswith(".log"):
1828                         self._add_file_log(filename)
1829
1830                 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1831                         # It is not allowed to upload packages on test builds.
1832                         if self.type == "test":
1833                                 return
1834
1835                         self._add_file_package(filename)
1836
1837         def _add_file_log(self, filename):
1838                 """
1839                         Attach a log file to this job.
1840                 """
1841                 target_dirname = os.path.join(self.build.path, "logs")
1842
1843                 if self.type == "test":
1844                         i = 1
1845                         while True:
1846                                 target_filename = os.path.join(target_dirname,
1847                                         "test.%s.%s.%s.log" % (self.arch, i, self.tries))
1848
1849                                 if os.path.exists(target_filename):
1850                                         i += 1
1851                                 else:
1852                                         break
1853                 else:
1854                         target_filename = os.path.join(target_dirname,
1855                                 "build.%s.%s.log" % (self.arch, self.tries))
1856
1857                 # Make sure the target directory exists.
1858                 if not os.path.exists(target_dirname):
1859                         os.makedirs(target_dirname)
1860
1861                 # Calculate a SHA512 hash from that file.
1862                 f = open(filename, "rb")
1863                 h = hashlib.sha512()
1864                 while True:
1865                         buf = f.read(BUFFER_SIZE)
1866                         if not buf:
1867                                 break
1868
1869                         h.update(buf)
1870                 f.close()
1871
1872                 # Copy the file to the final location.
1873                 shutil.copy2(filename, target_filename)
1874
1875                 # Create an entry in the database.
1876                 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1877                         VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1878                         os.path.getsize(target_filename), h.hexdigest())
1879
1880         def _add_file_package(self, filename):
1881                 # Open package (creates entry in the database).
1882                 pkg = packages.Package.open(self.pakfire, filename)
1883
1884                 # Move package to the build directory.
1885                 pkg.move(os.path.join(self.build.path, self.arch))
1886
1887                 # Attach the package to this job.
1888                 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1889                         self.id, pkg.id)
1890
1891         def get_aborted_state(self):
1892                 return self.data.aborted_state
1893
1894         def set_aborted_state(self, state):
1895                 self._set_attribute("aborted_state", state)
1896
1897         aborted_state = property(get_aborted_state, set_aborted_state)
1898
1899         @property
1900         def message_recipients(self):
1901                 l = []
1902
1903                 # Add all people watching the build.
1904                 l += self.build.message_recipients
1905
1906                 # Add the package maintainer on release builds.
1907                 if self.build.type == "release":
1908                         maint = self.pkg.maintainer
1909
1910                         if isinstance(maint, users.User):
1911                                 l.append("%s <%s>" % (maint.realname, maint.email))
1912                         elif maint:
1913                                 l.append(maint)
1914
1915                         # XXX add committer and commit author.
1916
1917                 # Add the owner of the scratch build on scratch builds.
1918                 elif self.build.type == "scratch" and self.build.user:
1919                         l.append("%s <%s>" % \
1920                                 (self.build.user.realname, self.build.user.email))
1921
1922                 return set(l)
1923
1924         def save_buildroot(self, pkgs):
1925                 rows = []
1926
1927                 for pkg_name, pkg_uuid in pkgs:
1928                         rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1929
1930                 # Cleanup old stuff first (for rebuilding packages).
1931                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1932                         self.id, self.tries)
1933
1934                 self.db.executemany("INSERT INTO \
1935                         jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1936                         VALUES(%s, %s, %s, %s)", rows)
1937
1938         def has_buildroot(self, tries=None):
1939                 if tries is None:
1940                         tries = self.tries
1941
1942                 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
1943                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1944                         self.id, tries)
1945
1946                 if res:
1947                         return res.num
1948
1949                 return 0
1950
1951         def get_buildroot(self, tries=None):
1952                 if tries is None:
1953                         tries = self.tries
1954
1955                 rows = self.db.query("SELECT * FROM jobs_buildroots \
1956                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1957                         ORDER BY pkg_name", self.id, tries)
1958
1959                 pkgs = []
1960                 for row in rows:
1961                         # Search for this package in the packages table.
1962                         pkg = self.pakfire.packages.get_by_uuid(row.pkg_uuid)
1963                         pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1964
1965                 return pkgs
1966
1967         def send_finished_message(self):
1968                 # Send no finished mails for test jobs.
1969                 if self.type == "test":
1970                         return
1971
1972                 logging.debug("Sending finished message for job %s to %s" % \
1973                         (self.name, ", ".join(self.message_recipients)))
1974
1975                 info = {
1976                         "build_name" : self.name,
1977                         "build_host" : self.builder.name,
1978                         "build_uuid" : self.uuid,
1979                 }
1980
1981                 self.pakfire.messages.send_to_all(self.message_recipients,
1982                         MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
1983
1984         def send_failed_message(self):
1985                 logging.debug("Sending failed message for job %s to %s" % \
1986                         (self.name, ", ".join(self.message_recipients)))
1987
1988                 build_host = "--"
1989                 if self.builder:
1990                         build_host = self.builder.name
1991
1992                 info = {
1993                         "build_name" : self.name,
1994                         "build_host" : build_host,
1995                         "build_uuid" : self.uuid,
1996                 }
1997
1998                 self.pakfire.messages.send_to_all(self.message_recipients,
1999                         MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
2000
2001         def set_start_time(self, start_time):
2002                 if start_time is None:
2003                         return
2004
2005                 self.db.execute("UPDATE jobs SET start_not_before = NOW() + %s \
2006                         WHERE id = %s LIMIT 1", start_time, self.id)
2007
2008         def schedule(self, type, start_time=None, user=None):
2009                 assert type in ("rebuild", "test")
2010
2011                 if type == "rebuild":
2012                         if self.state == "finished":
2013                                 return
2014
2015                         self.set_state("new", user=user, log=False)
2016                         self.set_start_time(start_time)
2017
2018                         # Log the event.
2019                         self.log("schedule_rebuild", user=user)
2020
2021                 elif type == "test":
2022                         if not self.state == "finished":
2023                                 return
2024
2025                         # Create a new job with same build and arch.
2026                         job = self.create(self.pakfire, self.build, self.arch, type="test")
2027                         job.set_start_time(start_time)
2028
2029                         # Log the event.
2030                         self.log("schedule_test_job", test_job=job, user=user)
2031
2032                         return job
2033
2034         def schedule_test(self, start_not_before=None, user=None):
2035                 # XXX to be removed
2036                 return self.schedule("test", start_time=start_not_before, user=user)
2037
2038         def schedule_rebuild(self, start_not_before=None, user=None):
2039                 # XXX to be removed
2040                 return self.schedule("rebuild", start_time=start_not_before, user=user)
2041
2042         def get_build_repos(self):
2043                 """
2044                         Returns a list of all repositories that should be used when
2045                         building this job.
2046                 """
2047                 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
2048                         self.id)
2049
2050                 if not repo_ids:
2051                         return self.distro.get_build_repos()
2052
2053                 repos = []
2054                 for repo in self.distro.repositories:
2055                         if repo.id in [r.id for r in repo_ids]:
2056                                 repos.append(repo)
2057
2058                 return repos or self.distro.get_build_repos()
2059
2060         def get_repo_config(self):
2061                 """
2062                         Get repository configuration file that is sent to the builder.
2063                 """
2064                 confs = []
2065
2066                 for repo in self.get_build_repos():
2067                         confs.append(repo.get_conf())
2068
2069                 return "\n\n".join(confs)
2070
2071         def get_config(self):
2072                 """
2073                         Get configuration file that is sent to the builder.
2074                 """
2075                 confs = []
2076
2077                 # Add the distribution configuration.
2078                 confs.append(self.distro.get_config())
2079
2080                 # Then add all repositories for this build.
2081                 confs.append(self.get_repo_config())
2082
2083                 return "\n\n".join(confs)
2084
2085         def resolvdep(self):
2086                 config = pakfire.config.Config(files=["general.conf"])
2087                 config.parse(self.get_config())
2088
2089                 # The filename of the source file.
2090                 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
2091                 assert os.path.exists(filename), filename
2092
2093                 # Create a new pakfire instance with the configuration for
2094                 # this build.
2095                 p = pakfire.PakfireServer(config=config, arch=self.arch)
2096
2097                 # Try to solve the build dependencies.
2098                 try:
2099                         solver = p.resolvdep(filename)
2100
2101                 # Catch dependency errors and log the problem string.
2102                 except DependencyError, e:
2103                         self.state = "dependency_error"
2104                         self.update_message(e)
2105
2106                 else:
2107                         # If the build dependencies can be resolved, we set the build in
2108                         # pending state.
2109                         if solver.status is True:
2110                                 if self.state in ("failed",):
2111                                         return
2112
2113                                 self.state = "pending"