src/buildservice/builds.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import hashlib
   5 import logging
   6 import os
   7 import re
   8 import shutil
   9 import uuid
  10
  11 import pakfire
  12 import pakfire.config
  13 import pakfire.packages
  14
  15 from . import arches
  16 from . import base
  17 from . import builders
  18 from . import logs
  19 from . import packages
  20 from . import repository
  21 from . import updates
  22 from . import users
  23
  24 log = logging.getLogger("builds")
  25 log.propagate = 1
  26
  27 from .constants import *
  28 from .decorators import *
  29
  30 class Builds(base.Object):
  31         def _get_build(self, query, *args):
  32                 res = self.db.get(query, *args)
  33
  34                 if res:
  35                         return Build(self.backend, res.id, data=res)
  36
  37         def _get_builds(self, query, *args):
  38                 res = self.db.query(query, *args)
  39
  40                 for row in res:
  41                         yield Build(self.backend, row.id, data=row)
  42
  43         def get_by_id(self, id, data=None):
  44                 return Build(self.pakfire, id, data=data)
  45
  46         def get_by_uuid(self, uuid):
  47                 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
  48
  49                 if build:
  50                         return self.get_by_id(build.id)
  51
  52         def get_all(self, limit=50):
  53                 query = "SELECT * FROM builds ORDER BY time_created DESC"
  54
  55                 if limit:
  56                         query += " LIMIT %d" % limit
  57
  58                 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
  59
  60         def get_by_user(self, user, type=None, public=None):
  61                 args = []
  62                 conditions = []
  63
  64                 if not type or type == "scratch":
  65                         # On scratch builds the user id equals the owner id.
  66                         conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
  67                         args.append(user.id)
  68
  69                 elif not type or type == "release":
  70                         pass # TODO
  71
  72                 if public is True:
  73                         conditions.append("public = 'Y'")
  74                 elif public is False:
  75                         conditions.append("public = 'N'")
  76
  77                 query = "SELECT builds.* AS id FROM builds \
  78                         JOIN packages ON builds.pkg_id = packages.id"
  79
  80                 if conditions:
  81                         query += " WHERE %s" % " AND ".join(conditions)
  82
  83                 query += " ORDER BY builds.time_created DESC"
  84
  85                 builds = []
  86                 for build in self.db.query(query, *args):
  87                         build = Build(self.pakfire, build.id, build)
  88                         builds.append(build)
  89
  90                 return builds
  91
  92         def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
  93                 args = [name,]
  94                 conditions = [
  95                         "packages.name = %s",
  96                 ]
  97
  98                 if type:
  99                         conditions.append("builds.type = %s")
 100                         args.append(type)
 101
 102                 or_conditions = []
 103                 if public is True:
 104                         or_conditions.append("public = 'Y'")
 105                 elif public is False:
 106                         or_conditions.append("public = 'N'")
 107
 108                 if user and not user.is_admin():
 109                         or_conditions.append("builds.owner_id = %s")
 110                         args.append(user.id)
 111
 112                 query = "SELECT builds.* AS id FROM builds \
 113                         JOIN packages ON builds.pkg_id = packages.id"
 114
 115                 if or_conditions:
 116                         conditions.append(" OR ".join(or_conditions))
 117
 118                 if conditions:
 119                         query += " WHERE %s" % " AND ".join(conditions)
 120
 121                 if type == "release":
 122                         query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
 123                 elif type == "scratch":
 124                         query += " ORDER BY time_created DESC"
 125
 126                 if limit:
 127                         if offset:
 128                                 query += " LIMIT %s,%s"
 129                                 args.extend([offset, limit])
 130                         else:
 131                                 query += " LIMIT %s"
 132                                 args.append(limit)
 133
 134                 return [Build(self.pakfire, b.id, b) for b in self.db.query(query, *args)]
 135
 136         def get_latest_by_name(self, name, type=None, public=None):
 137                 query = "\
 138                         SELECT * FROM builds \
 139                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 140                         WHERE builds_latest.package_name = %s"
 141                 args = [name,]
 142
 143                 if type:
 144                         query += " AND builds_latest.build_type = %s"
 145                         args.append(type)
 146
 147                 if public is True:
 148                         query += " AND builds.public = %s"
 149                         args.append("Y")
 150                 elif public is False:
 151                         query += " AND builds.public = %s"
 152                         args.append("N")
 153
 154                 # Get the last one only.
 155                 # Prefer release builds over scratch builds.
 156                 query += "\
 157                         ORDER BY \
 158                                 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
 159                                 builds.time_created DESC \
 160                         LIMIT 1"
 161
 162                 res = self.db.get(query, *args)
 163
 164                 if res:
 165                         return Build(self.pakfire, res.id, res)
 166
 167         def get_active_builds(self, name, public=None):
 168                 query = "\
 169                         SELECT * FROM builds \
 170                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 171                         WHERE builds_latest.package_name = %s AND builds.type = %s"
 172                 args = [name, "release"]
 173
 174                 if public is True:
 175                         query += " AND builds.public = %s"
 176                         args.append("Y")
 177                 elif public is False:
 178                         query += " AND builds.public = %s"
 179                         args.append("N")
 180
 181                 builds = []
 182                 for row in self.db.query(query, *args):
 183                         b = Build(self.pakfire, row.id, row)
 184                         builds.append(b)
 185
 186                 # Sort the result. Lastest build first.
 187                 builds.sort(reverse=True)
 188
 189                 return builds
 190
 191         def count(self):
 192                 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
 193                 if builds:
 194                         return builds.count
 195
 196         def get_obsolete(self, repo=None):
 197                 """
 198                         Get all obsoleted builds.
 199
 200                         If repo is True: which are in any repository.
 201                         If repo is some Repository object: which are in this repository.
 202                 """
 203                 args = []
 204
 205                 if repo is None:
 206                         query = "SELECT id FROM builds WHERE state = 'obsolete'"
 207
 208                 else:
 209                         query = "SELECT build_id AS id FROM repositories_builds \
 210                                 JOIN builds ON builds.id = repositories_builds.build_id \
 211                                 WHERE builds.state = 'obsolete'"
 212
 213                         if repo and not repo is True:
 214                                 query += " AND repositories_builds.repo_id = %s"
 215                                 args.append(repo.id)
 216
 217                 res = self.db.query(query, *args)
 218
 219                 builds = []
 220                 for build in res:
 221                         build = Build(self.pakfire, build.id)
 222                         builds.append(build)
 223
 224                 return builds
 225
 226         def create(self, pkg, type="release", owner=None, distro=None):
 227                 assert type in ("release", "scratch", "test")
 228                 assert distro, "You need to specify the distribution of this build."
 229
 230                 # Check if scratch build has an owner.
 231                 if type == "scratch" and not owner:
 232                         raise Exception, "Scratch builds require an owner"
 233
 234                 # Set the default priority of this build.
 235                 if type == "release":
 236                         priority = 0
 237
 238                 elif type == "scratch":
 239                         priority = 1
 240
 241                 elif type == "test":
 242                         priority = -1
 243
 244                 # Create build in database
 245                 build = self._get_build("INSERT INTO builds(uuid, pkg_id, type, distro_id, priority) \
 246                         VALUES(%s, %s, %s, %s, %s) RETURNING *", "%s" % uuid.uuid4(), pkg.id, type, distro.id, priority)
 247
 248                 # Set the owner of this build
 249                 if owner:
 250                         build.owner = owner
 251
 252                 # Log that the build has been created.
 253                 build.log("created", user=owner)
 254
 255                 # Create directory where the files live
 256                 if not os.path.exists(build.path):
 257                         os.makedirs(build.path)
 258
 259                 # Move package file to the directory of the build.
 260                 build.pkg.move(os.path.join(build.path, "src"))
 261
 262                 # Generate an update id.
 263                 build.generate_update_id()
 264
 265                 # Obsolete all other builds with the same name to track updates.
 266                 build.obsolete_others()
 267
 268                 # Search for possible bug IDs in the commit message.
 269                 build.search_for_bugs()
 270
 271                 return build
 272
 273         def create_from_source_package(self, filename, distro, commit=None, type="release",
 274                         arches=None, check_for_duplicates=True, owner=None):
 275                 assert distro
 276
 277                 # Open the package file to read some basic information.
 278                 pkg = pakfire.packages.open(None, None, filename)
 279
 280                 if check_for_duplicates:
 281                         if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
 282                                 log.warning("Duplicate package detected: %s. Skipping." % pkg)
 283                                 return
 284
 285                 # Open the package and add it to the database
 286                 pkg = self.backend.packages.create(filename)
 287
 288                 # Associate the package to the processed commit
 289                 if commit:
 290                         pkg.commit = commit
 291
 292                 # Create a new build object from the package
 293                 build = self.create(pkg, type=type, owner=owner, distro=distro)
 294
 295                 # Create all automatic jobs
 296                 build.create_autojobs(arches=arches)
 297
 298                 return build
 299
 300         def get_changelog(self, name, public=None, limit=5, offset=0):
 301                 query = "SELECT builds.* FROM builds \
 302                         JOIN packages ON builds.pkg_id = packages.id \
 303                         WHERE \
 304                                 builds.type = %s \
 305                         AND \
 306                                 packages.name = %s"
 307                 args = ["release", name,]
 308
 309                 if public == True:
 310                         query += " AND builds.public = %s"
 311                         args.append("Y")
 312                 elif public == False:
 313                         query += " AND builds.public = %s"
 314                         args.append("N")
 315
 316                 query += " ORDER BY builds.time_created DESC"
 317
 318                 if limit:
 319                         if offset:
 320                                 query += " LIMIT %s,%s"
 321                                 args += [offset, limit]
 322                         else:
 323                                 query += " LIMIT %s"
 324                                 args.append(limit)
 325
 326                 builds = []
 327                 for b in self.db.query(query, *args):
 328                         b = Build(self.pakfire, b.id, b)
 329                         builds.append(b)
 330
 331                 builds.sort(reverse=True)
 332
 333                 return builds
 334
 335         def get_comments(self, limit=10, offset=None, user=None):
 336                 query = "SELECT * FROM builds_comments \
 337                         JOIN users ON builds_comments.user_id = users.id"
 338                 args = []
 339
 340                 wheres = []
 341                 if user:
 342                         wheres.append("users.id = %s")
 343                         args.append(user.id)
 344
 345                 if wheres:
 346                         query += " WHERE %s" % " AND ".join(wheres)
 347
 348                 # Sort everything.
 349                 query += " ORDER BY time_created DESC"
 350
 351                 # Limits.
 352                 if limit:
 353                         if offset:
 354                                 query += " LIMIT %s,%s"
 355                                 args.append(offset)
 356                         else:
 357                                 query += " LIMIT %s"
 358
 359                         args.append(limit)
 360
 361                 comments = []
 362                 for comment in self.db.query(query, *args):
 363                         comment = logs.CommentLogEntry(self.pakfire, comment)
 364                         comments.append(comment)
 365
 366                 return comments
 367
 368         def get_build_times_summary(self, name=None, job_type=None, arch=None):
 369                 query = "\
 370                         SELECT \
 371                                 builds_times.arch AS arch, \
 372                                 MAX(duration) AS maximum, \
 373                                 MIN(duration) AS minimum, \
 374                                 AVG(duration) AS average, \
 375                                 SUM(duration) AS sum, \
 376                                 STDDEV_POP(duration) AS stddev \
 377                         FROM builds_times \
 378                                 LEFT JOIN builds ON builds_times.build_id = builds.id \
 379                                 LEFT JOIN packages ON builds.pkg_id = packages.id"
 380
 381                 args = []
 382                 conditions = []
 383
 384                 # Filter for name.
 385                 if name:
 386                         conditions.append("packages.name = %s")
 387                         args.append(name)
 388
 389                 # Filter by job types.
 390                 if job_type:
 391                         conditions.append("builds_times.job_type = %s")
 392                         args.append(job_type)
 393
 394                 # Filter by arch.
 395                 if arch:
 396                         conditions.append("builds_times.arch = %s")
 397                         args.append(arch)
 398
 399                 # Add conditions.
 400                 if conditions:
 401                         query += " WHERE %s" % " AND ".join(conditions)
 402
 403                 # Grouping and sorting.
 404                 query += " GROUP BY arch ORDER BY arch DESC"
 405
 406                 return self.db.query(query, *args)
 407
 408         def get_build_times_by_arch(self, arch, **kwargs):
 409                 kwargs.update({
 410                         "arch" : arch,
 411                 })
 412
 413                 build_times = self.get_build_times_summary(**kwargs)
 414                 if build_times:
 415                         return build_times[0]
 416
 417
 418 class Build(base.Object):
 419         table = "builds"
 420
 421         def __init__(self, pakfire, id, data=None):
 422                 base.Object.__init__(self, pakfire)
 423
 424                 # ID of this build
 425                 self.id = id
 426
 427                 # Cache data.
 428                 self._data = data
 429                 self._jobs = None
 430                 self._jobs_test = None
 431                 self._depends_on = None
 432                 self._pkg = None
 433                 self._credits = None
 434                 self._update = None
 435                 self._repo = None
 436                 self._distro = None
 437
 438         def __repr__(self):
 439                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
 440
 441         def __cmp__(self, other):
 442                 assert self.pkg
 443                 assert other.pkg
 444
 445                 return cmp(self.pkg, other.pkg)
 446
 447         def __iter__(self):
 448                 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
 449                         WHERE build_id = %s", self.id)
 450
 451                 return iter(sorted(jobs))
 452
 453         def delete(self):
 454                 """
 455                         Deletes this build including all jobs, packages and the source
 456                         package.
 457                 """
 458                 # If the build is in a repository, we need to remove it.
 459                 if self.repo:
 460                         self.repo.rem_build(self)
 461
 462                 for job in self.jobs + self.test_jobs:
 463                         job.delete()
 464
 465                 if self.pkg:
 466                         self.pkg.delete()
 467
 468                 # Delete everything related to this build.
 469                 self.__delete_bugs()
 470                 self.__delete_comments()
 471                 self.__delete_history()
 472                 self.__delete_watchers()
 473
 474                 # Delete the build itself.
 475                 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
 476
 477         def __delete_bugs(self):
 478                 """
 479                         Delete all associated bugs.
 480                 """
 481                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
 482
 483         def __delete_comments(self):
 484                 """
 485                         Delete all comments.
 486                 """
 487                 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
 488
 489         def __delete_history(self):
 490                 """
 491                         Delete the repository history.
 492                 """
 493                 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
 494
 495         def __delete_watchers(self):
 496                 """
 497                         Delete all watchers.
 498                 """
 499                 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
 500
 501         def reset(self):
 502                 """
 503                         Resets the whole build so it can start again (as it has never
 504                         been started).
 505                 """
 506                 for job in self.jobs:
 507                         job.reset()
 508
 509                 #self.__delete_bugs()
 510                 self.__delete_comments()
 511                 self.__delete_history()
 512                 self.__delete_watchers()
 513
 514                 self.state = "building"
 515
 516                 # XXX empty log
 517
 518         @property
 519         def data(self):
 520                 """
 521                         Lazy fetching of data for this object.
 522                 """
 523                 if self._data is None:
 524                         self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id)
 525                         assert self._data
 526
 527                 return self._data
 528
 529         @property
 530         def info(self):
 531                 """
 532                         A set of information that is sent to the XMLRPC client.
 533                 """
 534                 return { "uuid" : self.uuid }
 535
 536         def log(self, action, user=None, bug_id=None):
 537                 user_id = None
 538                 if user:
 539                         user_id = user.id
 540
 541                 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
 542                         VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
 543
 544         @property
 545         def uuid(self):
 546                 """
 547                         The UUID of this build.
 548                 """
 549                 return self.data.uuid
 550
 551         @property
 552         def pkg(self):
 553                 """
 554                         Get package that is to be built in the build.
 555                 """
 556                 if self._pkg is None:
 557                         self._pkg = packages.Package(self.pakfire, self.data.pkg_id)
 558
 559                 return self._pkg
 560
 561         @property
 562         def name(self):
 563                 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
 564
 565         @property
 566         def type(self):
 567                 """
 568                         The type of this build.
 569                 """
 570                 return self.data.type
 571
 572         def get_owner(self):
 573                 """
 574                         The owner of this build.
 575                 """
 576                 if self.data.owner_id:
 577                         return self.backend.users.get_by_id(self.data.owner_id)
 578
 579         def set_owner(self, owner):
 580                 if owner:
 581                         self._set_attribute("owner_id", owner.id)
 582                 else:
 583                         self._set_attribute("owner_id", None)
 584
 585         owner = lazy_property(get_owner, set_owner)
 586
 587         @property
 588         def distro_id(self):
 589                 return self.data.distro_id
 590
 591         @property
 592         def distro(self):
 593                 if self._distro is None:
 594                         self._distro = self.pakfire.distros.get_by_id(self.distro_id)
 595                         assert self._distro
 596
 597                 return self._distro
 598
 599         @property
 600         def user(self):
 601                 if self.type == "scratch":
 602                         return self.owner
 603
 604         def get_depends_on(self):
 605                 if self.data.depends_on and self._depends_on is None:
 606                         self._depends_on = Build(self.pakfire, self.data.depends_on)
 607
 608                 return self._depends_on
 609
 610         def set_depends_on(self, build):
 611                 self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s",
 612                         build.id, self.id)
 613
 614                 # Update cache.
 615                 self._depends_on = build
 616                 self._data["depends_on"] = build.id
 617
 618         depends_on = property(get_depends_on, set_depends_on)
 619
 620         @property
 621         def created(self):
 622                 return self.data.time_created
 623
 624         @property
 625         def date(self):
 626                 return self.created.date()
 627
 628         @property
 629         def public(self):
 630                 """
 631                         Is this build public?
 632                 """
 633                 return self.data.public == "Y"
 634
 635         @property
 636         def size(self):
 637                 """
 638                         Returns the size on disk of this build.
 639                 """
 640                 s = 0
 641
 642                 # Add the source package.
 643                 if self.pkg:
 644                         s += self.pkg.size
 645
 646                 # Add all jobs.
 647                 s += sum((j.size for j in self.jobs))
 648
 649                 return s
 650
 651         #@property
 652         #def state(self):
 653         #       # Cache all states.
 654         #       states = [j.state for j in self.jobs]
 655         #
 656         #       target_state = "unknown"
 657         #
 658         #       # If at least one job has failed, the whole build has failed.
 659         #       if "failed" in states:
 660         #               target_state = "failed"
 661         #
 662         #       # It at least one of the jobs is still running, the whole
 663         #       # build is in running state.
 664         #       elif "running" in states:
 665         #               target_state = "running"
 666         #
 667         #       # If all jobs are in the finished state, we turn into finished
 668         #       # state as well.
 669         #       elif all([s == "finished" for s in states]):
 670         #               target_state = "finished"
 671         #
 672         #       return target_state
 673
 674         def auto_update_state(self):
 675                 """
 676                         Check if the state of this build can be updated and perform
 677                         the change if possible.
 678                 """
 679                 # Do not change the broken/obsolete state automatically.
 680                 if self.state in ("broken", "obsolete"):
 681                         return
 682
 683                 if self.repo and self.repo.type == "stable":
 684                         self.update_state("stable")
 685                         return
 686
 687                 # If any of the build jobs are finished, the build will be put in testing
 688                 # state.
 689                 for job in self.jobs:
 690                         if job.state == "finished":
 691                                 self.update_state("testing")
 692                                 break
 693
 694         def update_state(self, state, user=None, remove=False):
 695                 assert state in ("stable", "testing", "obsolete", "broken")
 696
 697                 self.db.execute("UPDATE builds SET state = %s WHERE id = %s", state, self.id)
 698
 699                 if self._data:
 700                         self._data["state"] = state
 701
 702                 # In broken state, the removal from the repository is forced and
 703                 # all jobs that are not finished yet will be aborted.
 704                 if state == "broken":
 705                         remove = True
 706
 707                         for job in self.jobs:
 708                                 if job.state in ("new", "pending", "running", "dependency_error"):
 709                                         job.state = "aborted"
 710
 711                 # If this build is in a repository, it will leave it.
 712                 if remove and self.repo:
 713                                 self.repo.rem_build(self)
 714
 715                 # If a release build is now in testing state, we put it into the
 716                 # first repository of the distribution.
 717                 elif self.type == "release" and state == "testing":
 718                         # If the build is not in a repository, yet and if there is
 719                         # a first repository, we put the build there.
 720                         if not self.repo and self.distro.first_repo:
 721                                 self.distro.first_repo.add_build(self, user=user)
 722
 723         @property
 724         def state(self):
 725                 return self.data.state
 726
 727         def is_broken(self):
 728                 return self.state == "broken"
 729
 730         def obsolete_others(self):
 731                 if not self.type == "release":
 732                         return
 733
 734                 for build in self.pakfire.builds.get_by_name(self.pkg.name, type="release"):
 735                         # Don't modify ourself.
 736                         if self.id == build.id:
 737                                 continue
 738
 739                         # Don't touch broken builds.
 740                         if build.state in ("obsolete", "broken"):
 741                                 continue
 742
 743                         # Obsolete the build.
 744                         build.update_state("obsolete")
 745
 746         def set_severity(self, severity):
 747                 self.db.execute("UPDATE builds SET severity = %s WHERE id = %s", state, self.id)
 748
 749                 if self._data:
 750                         self._data["severity"] = severity
 751
 752         def get_severity(self):
 753                 return self.data.severity
 754
 755         severity = property(get_severity, set_severity)
 756
 757         @property
 758         def commit(self):
 759                 if self.pkg and self.pkg.commit:
 760                         return self.pkg.commit
 761
 762         def update_message(self, msg):
 763                 self.db.execute("UPDATE builds SET message = %s WHERE id = %s", msg, self.id)
 764
 765                 if self._data:
 766                         self._data["message"] = msg
 767
 768         def has_perm(self, user):
 769                 """
 770                         Check, if the given user has the right to perform administrative
 771                         operations on this build.
 772                 """
 773                 if user is None:
 774                         return False
 775
 776                 if user.is_admin():
 777                         return True
 778
 779                 # Check if the user is allowed to manage packages from the critical path.
 780                 if self.critical_path and not user.has_perm("manage_critical_path"):
 781                         return False
 782
 783                 # Search for maintainers...
 784
 785                 # Scratch builds.
 786                 if self.type == "scratch":
 787                         # The owner of a scratch build has the right to do anything with it.
 788                         if self.owner_id == user.id:
 789                                 return True
 790
 791                 # Release builds.
 792                 elif self.type == "release":
 793                         # The maintainer also is allowed to manage the build.
 794                         if self.pkg.maintainer == user:
 795                                 return True
 796
 797                 # Deny permission for all other cases.
 798                 return False
 799
 800         @property
 801         def message(self):
 802                 message = ""
 803
 804                 if self.data.message:
 805                         message = self.data.message
 806
 807                 elif self.commit:
 808                         if self.commit.message:
 809                                 message = "\n".join((self.commit.subject, self.commit.message))
 810                         else:
 811                                 message = self.commit.subject
 812
 813                         prefix = "%s: " % self.pkg.name
 814                         if message.startswith(prefix):
 815                                 message = message[len(prefix):]
 816
 817                 return message
 818
 819         def get_priority(self):
 820                 return self.data.priority
 821
 822         def set_priority(self, priority):
 823                 assert priority in (-2, -1, 0, 1, 2)
 824
 825                 self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority,
 826                         self.id)
 827
 828                 if self._data:
 829                         self._data["priority"] = priority
 830
 831         priority = property(get_priority, set_priority)
 832
 833         @property
 834         def path(self):
 835                 path = []
 836                 if self.type == "scratch":
 837                         path.append(BUILD_SCRATCH_DIR)
 838                         path.append(self.uuid)
 839
 840                 elif self.type == "release":
 841                         path.append(BUILD_RELEASE_DIR)
 842                         path.append("%s/%s-%s-%s" % \
 843                                 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
 844
 845                 else:
 846                         raise Exception, "Unknown build type: %s" % self.type
 847
 848                 return os.path.join(*path)
 849
 850         @property
 851         def source_filename(self):
 852                 return os.path.basename(self.pkg.path)
 853
 854         @property
 855         def download_prefix(self):
 856                 return "/".join((self.pakfire.settings.get("download_baseurl"), "packages"))
 857
 858         @property
 859         def source_download(self):
 860                 return "/".join((self.download_prefix, self.pkg.path))
 861
 862         @property
 863         def source_hash_sha512(self):
 864                 return self.pkg.hash_sha512
 865
 866         @property
 867         def link(self):
 868                 # XXX maybe this should rather live in a uimodule.
 869                 # zlib-1.2.3-2.ip3 [src, i686, blah...]
 870                 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
 871                         (self.state, self.type, self.uuid, self.name)
 872
 873                 s_jobs = []
 874                 for job in self.jobs:
 875                         s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
 876                                 (job.state, job.type, job.uuid, job.arch))
 877
 878                 if s_jobs:
 879                         s += " [%s]" % ", ".join(s_jobs)
 880
 881                 return s
 882
 883         @property
 884         def supported_arches(self):
 885                 return self.pkg.supported_arches
 886
 887         @property
 888         def critical_path(self):
 889                 return self.pkg.critical_path
 890
 891         def get_jobs(self, type=None):
 892                 """
 893                         Returns a list of jobs of this build.
 894                 """
 895                 return self.pakfire.jobs.get_by_build(self.id, self, type=type)
 896
 897         @property
 898         def jobs(self):
 899                 """
 900                         Get a list of all build jobs that are in this build.
 901                 """
 902                 if self._jobs is None:
 903                         self._jobs = self.get_jobs(type="build")
 904
 905                 return self._jobs
 906
 907         @property
 908         def test_jobs(self):
 909                 if self._jobs_test is None:
 910                         self._jobs_test = self.get_jobs(type="test")
 911
 912                 return self._jobs_test
 913
 914         @property
 915         def all_jobs_finished(self):
 916                 ret = True
 917
 918                 for job in self.jobs:
 919                         if not job.state == "finished":
 920                                 ret = False
 921                                 break
 922
 923                 return ret
 924
 925         def create_autojobs(self, arches=None, type="build"):
 926                 jobs = []
 927
 928                 # Arches may be passed to this function. If not we use all arches
 929                 # this package supports.
 930                 if arches is None:
 931                         arches = self.supported_arches
 932
 933                 # Create a new job for every given archirecture.
 934                 for arch in self.pakfire.arches.expand(arches):
 935                         # Don't create jobs for src
 936                         if arch == "src":
 937                                 continue
 938
 939                         job = self.add_job(arch, type=type)
 940                         jobs.append(job)
 941
 942                 # Return all newly created jobs.
 943                 return jobs
 944
 945         def add_job(self, arch, type="build"):
 946                 job = Job.create(self.pakfire, self, arch, type=type)
 947
 948                 # Add new job to cache.
 949                 if self._jobs:
 950                         self._jobs.append(job)
 951
 952                 return job
 953
 954         ## Update stuff
 955
 956         @property
 957         def update_id(self):
 958                 if not self.type == "release":
 959                         return
 960
 961                 # Generate an update ID if none does exist, yet.
 962                 self.generate_update_id()
 963
 964                 s = [
 965                         "%s" % self.distro.name.replace(" ", "").upper(),
 966                         "%04d" % (self.data.update_year or 0),
 967                         "%04d" % (self.data.update_num or 0),
 968                 ]
 969
 970                 return "-".join(s)
 971
 972         def generate_update_id(self):
 973                 if not self.type == "release":
 974                         return
 975
 976                 if self.data.update_num:
 977                         return
 978
 979                 update = self.db.get("SELECT update_num AS num FROM builds \
 980                         WHERE update_year = EXTRACT(year FROM NOW()) ORDER BY update_num DESC LIMIT 1")
 981
 982                 if update:
 983                         update_num = update.num + 1
 984                 else:
 985                         update_num = 1
 986
 987                 self.db.execute("UPDATE builds SET update_year = EXTRACT(year FROM NOW()), update_num = %s \
 988                         WHERE id = %s", update_num, self.id)
 989
 990         ## Comment stuff
 991
 992         def get_comments(self, limit=10, offset=0):
 993                 query = "SELECT * FROM builds_comments \
 994                         JOIN users ON builds_comments.user_id = users.id \
 995                         WHERE build_id = %s     ORDER BY time_created ASC"
 996
 997                 comments = []
 998                 for comment in self.db.query(query, self.id):
 999                         comment = logs.CommentLogEntry(self.pakfire, comment)
1000                         comments.append(comment)
1001
1002                 return comments
1003
1004         def add_comment(self, user, text, credit):
1005                 # Add the new comment to the database.
1006                 id = self.db.execute("INSERT INTO \
1007                         builds_comments(build_id, user_id, text, credit, time_created) \
1008                         VALUES(%s, %s, %s, %s, NOW())",
1009                         self.id, user.id, text, credit)
1010
1011                 # Update the credit cache.
1012                 if not self._credits is None:
1013                         self._credits += credit
1014
1015                 # Send the new comment to all watchers and stuff.
1016                 self.send_comment_message(id)
1017
1018                 # Return the ID of the newly created comment.
1019                 return id
1020
1021         @property
1022         def score(self):
1023                 # XXX UPDATE THIS
1024                 if self._credits is None:
1025                         # Get the sum of the credits from the database.
1026                         query = self.db.get(
1027                                 "SELECT SUM(credit) as credits FROM builds_comments WHERE build_id = %s",
1028                                 self.id
1029                         )
1030
1031                         self._credits = query.credits or 0
1032
1033                 return self._credits
1034
1035         @property
1036         def credits(self):
1037                 # XXX COMPAT
1038                 return self.score
1039
1040         def get_commenters(self):
1041                 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
1042                         JOIN users ON builds_comments.user_id = users.id \
1043                         WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
1044                         AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
1045
1046                 return [users.User(self.pakfire, u.id) for u in users]
1047
1048         def send_comment_message(self, comment_id):
1049                 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
1050                         comment_id)
1051
1052                 assert comment
1053                 assert comment.build_id == self.id
1054
1055                 # Get user who wrote the comment.
1056                 user = self.pakfire.users.get_by_id(comment.user_id)
1057
1058                 format = {
1059                         "build_name" : self.name,
1060                         "user_name"  : user.realname,
1061                 }
1062
1063                 # XXX create beautiful message
1064
1065                 self.pakfire.messages.send_to_all(self.message_recipients,
1066                         N_("%(user_name)s commented on %(build_name)s"),
1067                         comment.text, format)
1068
1069         ## Logging stuff
1070
1071         def get_log(self, comments=True, repo=True, limit=None):
1072                 entries = []
1073
1074                 # Created entry.
1075                 created_entry = logs.CreatedLogEntry(self.pakfire, self)
1076                 entries.append(created_entry)
1077
1078                 if comments:
1079                         entries += self.get_comments(limit=limit)
1080
1081                 if repo:
1082                         entries += self.get_repo_moves(limit=limit)
1083
1084                 # Sort all entries in chronological order.
1085                 entries.sort()
1086
1087                 if limit:
1088                         entries = entries[:limit]
1089
1090                 return entries
1091
1092         ## Watchers stuff
1093
1094         def get_watchers(self):
1095                 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
1096                         JOIN users ON builds_watchers.user_id = users.id \
1097                         WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1098                         AND users.activated = 'Y' ORDER BY users.id", self.id)
1099
1100                 return [users.User(self.pakfire, u.id) for u in query]
1101
1102         def add_watcher(self, user):
1103                 # Don't add a user twice.
1104                 if user in self.get_watchers():
1105                         return
1106
1107                 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1108                         VALUES(%s, %s)", self.id, user.id)
1109
1110         @property
1111         def message_recipients(self):
1112                 ret = []
1113
1114                 for watcher in self.get_watchers():
1115                         ret.append("%s <%s>" % (watcher.realname, watcher.email))
1116
1117                 return ret
1118
1119         @property
1120         def update(self):
1121                 if self._update is None:
1122                         update = self.db.get("SELECT update_id AS id FROM updates_builds \
1123                                 WHERE build_id = %s", self.id)
1124
1125                         if update:
1126                                 self._update = updates.Update(self.pakfire, update.id)
1127
1128                 return self._update
1129
1130         @property
1131         def repo(self):
1132                 if self._repo is None:
1133                         repo = self.db.get("SELECT repo_id AS id FROM repositories_builds \
1134                                 WHERE build_id = %s", self.id)
1135
1136                         if repo:
1137                                 self._repo = repository.Repository(self.pakfire, repo.id)
1138
1139                 return self._repo
1140
1141         def get_repo_moves(self, limit=None):
1142                 query = "SELECT * FROM repositories_history \
1143                         WHERE build_id = %s ORDER BY time ASC"
1144
1145                 actions = []
1146                 for action in self.db.query(query, self.id):
1147                         action = logs.RepositoryLogEntry(self.pakfire, action)
1148                         actions.append(action)
1149
1150                 return actions
1151
1152         @property
1153         def is_loose(self):
1154                 if self.repo:
1155                         return False
1156
1157                 return True
1158
1159         @property
1160         def repo_time(self):
1161                 repo = self.db.get("SELECT time_added FROM repositories_builds \
1162                         WHERE build_id = %s", self.id)
1163
1164                 if repo:
1165                         return repo.time_added
1166
1167         def get_auto_move(self):
1168                 return self.data.auto_move == "Y"
1169
1170         def set_auto_move(self, state):
1171                 if state:
1172                         state = "Y"
1173                 else:
1174                         state = "N"
1175
1176                 self.db.execute("UPDATE builds SET auto_move = %s WHERE id = %s", self.id)
1177                 if self._data:
1178                         self._data["auto_move"] = state
1179
1180         auto_move = property(get_auto_move, set_auto_move)
1181
1182         @property
1183         def can_move_forward(self):
1184                 if not self.repo:
1185                         return False
1186
1187                 # If there is no next repository, we cannot move anything.
1188                 if not self.repo.next:
1189                         return False
1190
1191                 # If the needed amount of score is reached, we can move forward.
1192                 if self.score >= self.repo.next.score_needed:
1193                         return True
1194
1195                 # If the repository does not require a minimal time,
1196                 # we can move forward immediately.
1197                 if not self.repo.time_min:
1198                         return True
1199
1200                 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1201                         WHERE build_id = %s", self.id)
1202                 duration = query.duration
1203
1204                 if duration >= self.repo.time_min:
1205                         return True
1206
1207                 return False
1208
1209         ## Bugs
1210
1211         def get_bug_ids(self):
1212                 query = self.db.query("SELECT bug_id FROM builds_bugs \
1213                         WHERE build_id = %s", self.id)
1214
1215                 return [b.bug_id for b in query]
1216
1217         def add_bug(self, bug_id, user=None, log=True):
1218                 # Check if this bug is already in the list of bugs.
1219                 if bug_id in self.get_bug_ids():
1220                         return
1221
1222                 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1223                         VALUES(%s, %s)", self.id, bug_id)
1224
1225                 # Log the event.
1226                 if log:
1227                         self.log("bug_added", user=user, bug_id=bug_id)
1228
1229         def rem_bug(self, bug_id, user=None, log=True):
1230                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1231                         bug_id = %s", self.id, bug_id)
1232
1233                 # Log the event.
1234                 if log:
1235                         self.log("bug_removed", user=user, bug_id=bug_id)
1236
1237         def search_for_bugs(self):
1238                 if not self.commit:
1239                         return
1240
1241                 pattern = re.compile(r"(bug\s?|#)(\d+)")
1242
1243                 for txt in (self.commit.subject, self.commit.message):
1244                         for bug in re.finditer(pattern, txt):
1245                                 try:
1246                                         bugid = int(bug.group(2))
1247                                 except ValueError:
1248                                         continue
1249
1250                                 # Check if a bug with the given ID exists in BZ.
1251                                 bug = self.pakfire.bugzilla.get_bug(bugid)
1252                                 if not bug:
1253                                         continue
1254
1255                                 self.add_bug(bugid)
1256
1257         def get_bugs(self):
1258                 bugs = []
1259                 for bug_id in self.get_bug_ids():
1260                         bug = self.pakfire.bugzilla.get_bug(bug_id)
1261                         if not bug:
1262                                 continue
1263
1264                         bugs.append(bug)
1265
1266                 return bugs
1267
1268         def _update_bugs_helper(self, repo):
1269                 """
1270                         This function takes a new status and generates messages that
1271                         are appended to all bugs.
1272                 """
1273                 try:
1274                         kwargs = BUG_MESSAGES[repo.type].copy()
1275                 except KeyError:
1276                         return
1277
1278                 baseurl = self.pakfire.settings.get("baseurl", "")
1279                 args = {
1280                         "build_url"    : "%s/build/%s" % (baseurl, self.uuid),
1281                         "distro_name"  : self.distro.name,
1282                         "package_name" : self.name,
1283                         "repo_name"    : repo.name,
1284                 }
1285                 kwargs["comment"] = kwargs["comment"] % args
1286
1287                 self.update_bugs(**kwargs)
1288
1289         def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1290                 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1291                         VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1292
1293         def update_bugs(self, status, resolution=None, comment=None):
1294                 # Update all bugs linked to this build.
1295                 for bug_id in self.get_bug_ids():
1296                         self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1297
1298
1299 class Jobs(base.Object):
1300         def _get_job(self, query, *args):
1301                 res = self.db.get(query, *args)
1302
1303                 if res:
1304                         return Job(self.backend, res.id, data=res)
1305
1306         def _get_jobs(self, query, *args):
1307                 res = self.db.query(query, *args)
1308
1309                 for row in res:
1310                         yield Job(self.backend, row.id, data=row)
1311
1312         def create(self,  build, arch, type="build"):
1313                 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1314                         VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1315                 job.log("created")
1316
1317                 # Set cache for Build object.
1318                 job.build = build
1319
1320                 # Jobs are by default in state "new" and wait for being checked
1321                 # for dependencies. Packages that do have no build dependencies
1322                 # can directly be forwarded to "pending" state.
1323                 if not job.pkg.requires:
1324                         job.state = "pending"
1325
1326                 return job
1327
1328         def get_by_id(self, id, data=None):
1329                 return Job(self.pakfire, id, data)
1330
1331         def get_by_uuid(self, uuid):
1332                 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1333
1334                 if job:
1335                         return self.get_by_id(job.id)
1336
1337         def get_by_build(self, build_id, build=None, type=None):
1338                 """
1339                         Get all jobs in the specifies build.
1340                 """
1341                 query = "SELECT * FROM jobs WHERE build_id = %s"
1342                 args = [build_id,]
1343
1344                 if type:
1345                         query += " AND type = %s"
1346                         args.append(type)
1347
1348                 # Get IDs of all builds in this group.
1349                 jobs = []
1350                 for job in self.db.query(query, *args):
1351                         job = Job(self.pakfire, job.id, job)
1352
1353                         # If the Build object was set, we set it so it won't be retrieved
1354                         # from the database again.
1355                         if build:
1356                                 job._build = build
1357
1358                         jobs.append(job)
1359
1360                 # Return sorted list of jobs.
1361                 return sorted(jobs)
1362
1363         def get_active(self, host_id=None, builder=None, states=None):
1364                 if builder:
1365                         host_id = builder.id
1366
1367                 if states is None:
1368                         states = ["dispatching", "running", "uploading"]
1369
1370                 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1371                 args = states
1372
1373                 if host_id:
1374                         query += " AND builder_id = %s" % host_id
1375
1376                 query += " ORDER BY \
1377                         CASE \
1378                                 WHEN jobs.state = 'running'     THEN 0 \
1379                                 WHEN jobs.state = 'uploading'   THEN 1 \
1380                                 WHEN jobs.state = 'dispatching' THEN 2 \
1381                                 WHEN jobs.state = 'pending'     THEN 3 \
1382                                 WHEN jobs.state = 'new'         THEN 4 \
1383                         END, time_started ASC"
1384
1385                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1386
1387         def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
1388                 query = "SELECT * FROM jobs"
1389                 args  = []
1390
1391                 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
1392
1393                 if arch:
1394                         where.append("arch = %s")
1395                         args.append(arch)
1396
1397                 if builder:
1398                         where.append("builder_id = %s")
1399                         args.append(builder.id)
1400
1401                 if date:
1402                         try:
1403                                 year, month, day = date.split("-", 2)
1404                                 date = datetime.date(int(year), int(month), int(day))
1405                         except ValueError:
1406                                 pass
1407                         else:
1408                                 where.append("(time_created::date = %s OR \
1409                                         time_started::date = %s OR time_finished::date = %s)")
1410                                 args += (date, date, date)
1411
1412                 if age:
1413                         where.append("time_finished >= NOW() - '%s'::interval" % age)
1414
1415                 if where:
1416                         query += " WHERE %s" % " AND ".join(where)
1417
1418                 query += " ORDER BY time_finished DESC"
1419
1420                 if limit:
1421                         query += " LIMIT %s"
1422                         args.append(limit)
1423
1424                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1425
1426         def get_average_build_time(self):
1427                 """
1428                         Returns the average build time of all finished builds from the
1429                         last 3 months.
1430                 """
1431                 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1432                         FROM jobs WHERE type = 'build' AND state = 'finished' AND \
1433                         time_finished >= NOW() - '3 months'::interval")
1434
1435                 if result:
1436                         return result.average
1437
1438         def count(self, *states):
1439                 query = "SELECT COUNT(*) AS count FROM jobs"
1440                 args  = []
1441
1442                 if states:
1443                         query += " WHERE state IN %s"
1444                         args.append(states)
1445
1446                 jobs = self.db.get(query, *args)
1447                 if jobs:
1448                         return jobs.count
1449
1450         def restart_failed(self, max_tries=9):
1451                 jobs = self._get_jobs("SELECT jobs.* FROM jobs \
1452                         JOIN builds ON builds.id = jobs.build_id \
1453                         WHERE \
1454                                 jobs.type = 'build' AND \
1455                                 jobs.state = 'failed' AND \
1456                                 jobs.tries <= %s AND \
1457                                 NOT builds.state = 'broken' AND \
1458                                 jobs.time_finished < NOW() - '72 hours'::interval \
1459                         ORDER BY \
1460                                 CASE \
1461                                         WHEN jobs.type = 'build' THEN 0 \
1462                                         WHEN jobs.type = 'test'  THEN 1 \
1463                                 END, \
1464                                 builds.priority DESC, jobs.time_created ASC",
1465                         max_tries)
1466
1467                 # Restart the job
1468                 for job in jobs:
1469                         job.set_state("new", log=False)
1470
1471
1472 class Job(base.DataObject):
1473         table = "jobs"
1474
1475         def __str__(self):
1476                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1477
1478         def __eq__(self, other):
1479                 if isinstance(other, self.__class__):
1480                         return self.id == other.id
1481
1482         def __lt__(self, other):
1483                 if isinstance(other, self.__class__):
1484                         if (self.type, other.type) == ("build", "test"):
1485                                 return True
1486
1487                         if self.build == other.build:
1488                                 return arches.priority(self.arch) < arches.priority(other.arch)
1489
1490                         return self.time_created < other.time_created
1491
1492         def __iter__(self):
1493                 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1494                         LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1495                         WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
1496
1497                 return iter(packages)
1498
1499         def __nonzero__(self):
1500                 return True
1501
1502         def __len__(self):
1503                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1504                         WHERE job_id = %s", self.id)
1505
1506                 return res.len
1507
1508         @property
1509         def distro(self):
1510                 return self.build.distro
1511
1512         def delete(self):
1513                 self.__delete_buildroots()
1514                 self.__delete_history()
1515                 self.__delete_packages()
1516                 self.__delete_logfiles()
1517
1518                 # Delete the job itself.
1519                 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
1520
1521         def __delete_buildroots(self):
1522                 """
1523                         Removes all buildroots.
1524                 """
1525                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1526
1527         def __delete_history(self):
1528                 """
1529                         Removes all references in the history to this build job.
1530                 """
1531                 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1532
1533         def __delete_packages(self):
1534                 """
1535                         Deletes all uploaded files from the job.
1536                 """
1537                 for pkg in self.packages:
1538                         pkg.delete()
1539
1540                 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1541
1542         def __delete_logfiles(self):
1543                 for logfile in self.logfiles:
1544                         self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1545
1546         def reset(self, user=None):
1547                 self.__delete_buildroots()
1548                 self.__delete_packages()
1549                 self.__delete_history()
1550                 self.__delete_logfiles()
1551
1552                 self.state = "new"
1553                 self.log("reset", user=user)
1554
1555         ## Logging stuff
1556
1557         def log(self, action, user=None, state=None, builder=None, test_job=None):
1558                 user_id = None
1559                 if user:
1560                         user_id = user.id
1561
1562                 builder_id = None
1563                 if builder:
1564                         builder_id = builder.id
1565
1566                 test_job_id = None
1567                 if test_job:
1568                         test_job_id = test_job.id
1569
1570                 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1571                         time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1572                         self.id, action, state, user_id, builder_id, test_job_id)
1573
1574         def get_log(self, limit=None, offset=None, user=None):
1575                 query = "SELECT * FROM jobs_history"
1576
1577                 conditions = ["job_id = %s",]
1578                 args  = [self.id,]
1579
1580                 if user:
1581                         conditions.append("user_id = %s")
1582                         args.append(user.id)
1583
1584                 if conditions:
1585                         query += " WHERE %s" % " AND ".join(conditions)
1586
1587                 query += " ORDER BY time DESC"
1588
1589                 if limit:
1590                         if offset:
1591                                 query += " LIMIT %s,%s"
1592                                 args  += [offset, limit,]
1593                         else:
1594                                 query += " LIMIT %s"
1595                                 args  += [limit,]
1596
1597                 entries = []
1598                 for entry in self.db.query(query, *args):
1599                         entry = logs.JobLogEntry(self.pakfire, entry)
1600                         entries.append(entry)
1601
1602                 return entries
1603
1604         @property
1605         def uuid(self):
1606                 return self.data.uuid
1607
1608         @property
1609         def type(self):
1610                 return self.data.type
1611
1612         @property
1613         def build_id(self):
1614                 return self.data.build_id
1615
1616         @lazy_property
1617         def build(self):
1618                 return self.pakfire.builds.get_by_id(self.build_id)
1619
1620         @property
1621         def related_jobs(self):
1622                 ret = []
1623
1624                 for job in self.build.jobs:
1625                         if job == self:
1626                                 continue
1627
1628                         ret.append(job)
1629
1630                 return ret
1631
1632         @property
1633         def pkg(self):
1634                 return self.build.pkg
1635
1636         @property
1637         def name(self):
1638                 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
1639
1640         @property
1641         def size(self):
1642                 return sum((p.size for p in self.packages))
1643
1644         @lazy_property
1645         def rank(self):
1646                 """
1647                         Returns the rank in the build queue
1648                 """
1649                 if not self.state == "pending":
1650                         return
1651
1652                 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1653
1654                 if res:
1655                         return res.rank
1656
1657         def is_running(self):
1658                 """
1659                         Returns True if job is in a running state.
1660                 """
1661                 return self.state in ("pending", "dispatching", "running", "uploading")
1662
1663         def get_state(self):
1664                 return self.data.state
1665
1666         def set_state(self, state, user=None, log=True):
1667                 # Nothing to do if the state remains.
1668                 if not self.state == state:
1669                         self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
1670
1671                         # Log the event.
1672                         if log and not state == "new":
1673                                 self.log("state_change", state=state, user=user)
1674
1675                         # Update cache.
1676                         if self._data:
1677                                 self._data["state"] = state
1678
1679                 # Always clear the message when the status is changed.
1680                 self.update_message(None)
1681
1682                 # Update some more informations.
1683                 if state == "dispatching":
1684                         # Set start time.
1685                         self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1686                                 WHERE id = %s", self.id)
1687
1688                 elif state == "pending":
1689                         self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1690                                 time_finished = NULL WHERE id = %s", self.id)
1691
1692                 elif state in ("aborted", "dependency_error", "finished", "failed"):
1693                         # Set finish time and reset builder..
1694                         self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
1695
1696                         # Send messages to the user.
1697                         if state == "finished":
1698                                 self.send_finished_message()
1699
1700                         elif state == "failed":
1701                                 # Remove all package files if a job is set to failed state.
1702                                 self.__delete_packages()
1703
1704                                 self.send_failed_message()
1705
1706                 # Automatically update the state of the build (not on test builds).
1707                 if self.type == "build":
1708                         self.build.auto_update_state()
1709
1710         state = property(get_state, set_state)
1711
1712         @property
1713         def message(self):
1714                 return self.data.message
1715
1716         def update_message(self, msg):
1717                 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1718                         msg, self.id)
1719
1720                 if self._data:
1721                         self._data["message"] = msg
1722
1723         def get_builder(self):
1724                 if self.data.builder_id:
1725                         return self.backend.builders.get_by_id(self.data.builder_id)
1726
1727         def set_builder(self, builder, user=None):
1728                 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1729                         builder.id, self.id)
1730
1731                 # Update cache.
1732                 if self._data:
1733                         self._data["builder_id"] = builder.id
1734
1735                 self._builder = builder
1736
1737                 # Log the event.
1738                 if user:
1739                         self.log("builder_assigned", builder=builder, user=user)
1740
1741         builder = lazy_property(get_builder, set_builder)
1742
1743         @property
1744         def arch(self):
1745                 return self.data.arch
1746
1747         @property
1748         def duration(self):
1749                 if not self.time_started:
1750                         return 0
1751
1752                 if self.time_finished:
1753                         delta = self.time_finished - self.time_started
1754                 else:
1755                         delta = datetime.datetime.utcnow() - self.time_started
1756
1757                 return delta.total_seconds()
1758
1759         @property
1760         def time_created(self):
1761                 return self.data.time_created
1762
1763         @property
1764         def time_started(self):
1765                 return self.data.time_started
1766
1767         @property
1768         def time_finished(self):
1769                 return self.data.time_finished
1770
1771         @property
1772         def expected_runtime(self):
1773                 """
1774                         Returns the estimated time and stddev, this job takes to finish.
1775                 """
1776                 # Get the average build time.
1777                 build_times = self.pakfire.builds.get_build_times_by_arch(self.arch,
1778                         name=self.pkg.name)
1779
1780                 # If there is no statistical data, we cannot estimate anything.
1781                 if not build_times:
1782                         return None, None
1783
1784                 return build_times.average, build_times.stddev
1785
1786         @property
1787         def eta(self):
1788                 expected_runtime, stddev = self.expected_runtime
1789
1790                 if expected_runtime:
1791                         return expected_runtime - int(self.duration), stddev
1792
1793         @property
1794         def tries(self):
1795                 return self.data.tries
1796
1797         def get_pkg_by_uuid(self, uuid):
1798                 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
1799                         JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1800                         WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1801                         self.id, uuid)
1802
1803                 if pkg:
1804                         pkg.job = self
1805                         return pkg
1806
1807         @lazy_property
1808         def logfiles(self):
1809                 logfiles = []
1810
1811                 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1812                         log = logs.LogFile(self.pakfire, log.id)
1813                         log._job = self
1814
1815                         logfiles.append(log)
1816
1817                 return logfiles
1818
1819         def add_file(self, filename):
1820                 """
1821                         Add the specified file to this job.
1822
1823                         The file is copied to the right directory by this function.
1824                 """
1825                 assert os.path.exists(filename)
1826
1827                 if filename.endswith(".log"):
1828                         self._add_file_log(filename)
1829
1830                 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1831                         # It is not allowed to upload packages on test builds.
1832                         if self.type == "test":
1833                                 return
1834
1835                         self._add_file_package(filename)
1836
1837         def _add_file_log(self, filename):
1838                 """
1839                         Attach a log file to this job.
1840                 """
1841                 target_dirname = os.path.join(self.build.path, "logs")
1842
1843                 if self.type == "test":
1844                         i = 1
1845                         while True:
1846                                 target_filename = os.path.join(target_dirname,
1847                                         "test.%s.%s.%s.log" % (self.arch, i, self.tries))
1848
1849                                 if os.path.exists(target_filename):
1850                                         i += 1
1851                                 else:
1852                                         break
1853                 else:
1854                         target_filename = os.path.join(target_dirname,
1855                                 "build.%s.%s.log" % (self.arch, self.tries))
1856
1857                 # Make sure the target directory exists.
1858                 if not os.path.exists(target_dirname):
1859                         os.makedirs(target_dirname)
1860
1861                 # Calculate a SHA512 hash from that file.
1862                 f = open(filename, "rb")
1863                 h = hashlib.sha512()
1864                 while True:
1865                         buf = f.read(BUFFER_SIZE)
1866                         if not buf:
1867                                 break
1868
1869                         h.update(buf)
1870                 f.close()
1871
1872                 # Copy the file to the final location.
1873                 shutil.copy2(filename, target_filename)
1874
1875                 # Create an entry in the database.
1876                 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1877                         VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1878                         os.path.getsize(target_filename), h.hexdigest())
1879
1880         def _add_file_package(self, filename):
1881                 # Open package (creates entry in the database).
1882                 pkg = packages.Package.open(self.pakfire, filename)
1883
1884                 # Move package to the build directory.
1885                 pkg.move(os.path.join(self.build.path, self.arch))
1886
1887                 # Attach the package to this job.
1888                 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1889                         self.id, pkg.id)
1890
1891         def get_aborted_state(self):
1892                 return self.data.aborted_state
1893
1894         def set_aborted_state(self, state):
1895                 self._set_attribute("aborted_state", state)
1896
1897         aborted_state = property(get_aborted_state, set_aborted_state)
1898
1899         @property
1900         def message_recipients(self):
1901                 l = []
1902
1903                 # Add all people watching the build.
1904                 l += self.build.message_recipients
1905
1906                 # Add the package maintainer on release builds.
1907                 if self.build.type == "release":
1908                         maint = self.pkg.maintainer
1909
1910                         if isinstance(maint, users.User):
1911                                 l.append("%s <%s>" % (maint.realname, maint.email))
1912                         elif maint:
1913                                 l.append(maint)
1914
1915                         # XXX add committer and commit author.
1916
1917                 # Add the owner of the scratch build on scratch builds.
1918                 elif self.build.type == "scratch" and self.build.user:
1919                         l.append("%s <%s>" % \
1920                                 (self.build.user.realname, self.build.user.email))
1921
1922                 return set(l)
1923
1924         def save_buildroot(self, pkgs):
1925                 rows = []
1926
1927                 for pkg_name, pkg_uuid in pkgs:
1928                         rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1929
1930                 # Cleanup old stuff first (for rebuilding packages).
1931                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1932                         self.id, self.tries)
1933
1934                 self.db.executemany("INSERT INTO \
1935                         jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1936                         VALUES(%s, %s, %s, %s)", rows)
1937
1938         def has_buildroot(self, tries=None):
1939                 if tries is None:
1940                         tries = self.tries
1941
1942                 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
1943                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1944                         self.id, tries)
1945
1946                 if res:
1947                         return res.num
1948
1949                 return 0
1950
1951         def get_buildroot(self, tries=None):
1952                 if tries is None:
1953                         tries = self.tries
1954
1955                 rows = self.db.query("SELECT * FROM jobs_buildroots \
1956                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1957                         ORDER BY pkg_name", self.id, tries)
1958
1959                 pkgs = []
1960                 for row in rows:
1961                         # Search for this package in the packages table.
1962                         pkg = self.pakfire.packages.get_by_uuid(row.pkg_uuid)
1963                         pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1964
1965                 return pkgs
1966
1967         def send_finished_message(self):
1968                 # Send no finished mails for test jobs.
1969                 if self.type == "test":
1970                         return
1971
1972                 logging.debug("Sending finished message for job %s to %s" % \
1973                         (self.name, ", ".join(self.message_recipients)))
1974
1975                 info = {
1976                         "build_name" : self.name,
1977                         "build_host" : self.builder.name,
1978                         "build_uuid" : self.uuid,
1979                 }
1980
1981                 self.pakfire.messages.send_to_all(self.message_recipients,
1982                         MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
1983
1984         def send_failed_message(self):
1985                 logging.debug("Sending failed message for job %s to %s" % \
1986                         (self.name, ", ".join(self.message_recipients)))
1987
1988                 build_host = "--"
1989                 if self.builder:
1990                         build_host = self.builder.name
1991
1992                 info = {
1993                         "build_name" : self.name,
1994                         "build_host" : build_host,
1995                         "build_uuid" : self.uuid,
1996                 }
1997
1998                 self.pakfire.messages.send_to_all(self.message_recipients,
1999                         MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
2000
2001         def set_start_time(self, start_time):
2002                 if start_time is None:
2003                         return
2004
2005                 self.db.execute("UPDATE jobs SET start_not_before = NOW() + %s \
2006                         WHERE id = %s LIMIT 1", start_time, self.id)
2007
2008         def schedule(self, type, start_time=None, user=None):
2009                 assert type in ("rebuild", "test")
2010
2011                 if type == "rebuild":
2012                         if self.state == "finished":
2013                                 return
2014
2015                         self.set_state("new", user=user, log=False)
2016                         self.set_start_time(start_time)
2017
2018                         # Log the event.
2019                         self.log("schedule_rebuild", user=user)
2020
2021                 elif type == "test":
2022                         if not self.state == "finished":
2023                                 return
2024
2025                         # Create a new job with same build and arch.
2026                         job = self.create(self.pakfire, self.build, self.arch, type="test")
2027                         job.set_start_time(start_time)
2028
2029                         # Log the event.
2030                         self.log("schedule_test_job", test_job=job, user=user)
2031
2032                         return job
2033
2034         def schedule_test(self, start_not_before=None, user=None):
2035                 # XXX to be removed
2036                 return self.schedule("test", start_time=start_not_before, user=user)
2037
2038         def schedule_rebuild(self, start_not_before=None, user=None):
2039                 # XXX to be removed
2040                 return self.schedule("rebuild", start_time=start_not_before, user=user)
2041
2042         def get_build_repos(self):
2043                 """
2044                         Returns a list of all repositories that should be used when
2045                         building this job.
2046                 """
2047                 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
2048                         self.id)
2049
2050                 if not repo_ids:
2051                         return self.distro.get_build_repos()
2052
2053                 repos = []
2054                 for repo in self.distro.repositories:
2055                         if repo.id in [r.id for r in repo_ids]:
2056                                 repos.append(repo)
2057
2058                 return repos or self.distro.get_build_repos()
2059
2060         def get_repo_config(self):
2061                 """
2062                         Get repository configuration file that is sent to the builder.
2063                 """
2064                 confs = []
2065
2066                 for repo in self.get_build_repos():
2067                         confs.append(repo.get_conf())
2068
2069                 return "\n\n".join(confs)
2070
2071         def get_config(self):
2072                 """
2073                         Get configuration file that is sent to the builder.
2074                 """
2075                 confs = []
2076
2077                 # Add the distribution configuration.
2078                 confs.append(self.distro.get_config())
2079
2080                 # Then add all repositories for this build.
2081                 confs.append(self.get_repo_config())
2082
2083                 return "\n\n".join(confs)
2084
2085         def resolvdep(self):
2086                 config = pakfire.config.Config(files=["general.conf"])
2087                 config.parse(self.get_config())
2088
2089                 # The filename of the source file.
2090                 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
2091                 assert os.path.exists(filename), filename
2092
2093                 # Create a new pakfire instance with the configuration for
2094                 # this build.
2095                 p = pakfire.PakfireServer(config=config, arch=self.arch)
2096
2097                 # Try to solve the build dependencies.
2098                 try:
2099                         solver = p.resolvdep(filename)
2100
2101                 # Catch dependency errors and log the problem string.
2102                 except DependencyError, e:
2103                         self.state = "dependency_error"
2104                         self.update_message(e)
2105
2106                 else:
2107                         # If the build dependencies can be resolved, we set the build in
2108                         # pending state.
2109                         if solver.status is True:
2110                                 if self.state in ("failed",):
2111                                         return
2112
2113                                 self.state = "pending"