src/buildservice/builds.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import hashlib
   5 import logging
   6 import os
   7 import re
   8 import shutil
   9 import uuid
  10
  11 import pakfire
  12 import pakfire.config
  13 import pakfire.packages
  14
  15 from . import arches
  16 from . import base
  17 from . import builders
  18 from . import logs
  19 from . import packages
  20 from . import repository
  21 from . import updates
  22 from . import users
  23
  24 from .constants import *
  25 from .decorators import *
  26
  27 def import_from_package(_pakfire, filename, distro=None, commit=None, type="release",
  28                 arches=None, check_for_duplicates=True, owner=None):
  29
  30         if distro is None:
  31                 distro = commit.source.distro
  32
  33         assert distro
  34
  35         # Open the package file to read some basic information.
  36         pkg = pakfire.packages.open(None, None, filename)
  37
  38         if check_for_duplicates:
  39                 if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
  40                         logging.warning("Duplicate package detected: %s. Skipping." % pkg)
  41                         return
  42
  43         # Open the package and add it to the database.
  44         pkg = packages.Package.open(_pakfire, filename)
  45         logging.debug("Created new package: %s" % pkg)
  46
  47         # Associate the package to the processed commit.
  48         if commit:
  49                 pkg.commit = commit
  50
  51         # Create a new build object from the package which
  52         # is always a release build.
  53         build = Build.create(_pakfire, pkg, type=type, owner=owner, distro=distro)
  54         logging.debug("Created new build job: %s" % build)
  55
  56         # Create all automatic jobs.
  57         build.create_autojobs(arches=arches)
  58
  59         return pkg, build
  60
  61
  62 class Builds(base.Object):
  63         def _get_build(self, query, *args):
  64                 res = self.db.get(query, *args)
  65
  66                 if res:
  67                         return Build(self.backend, res.id, data=res)
  68
  69         def _get_builds(self, query, *args):
  70                 res = self.db.query(query, *args)
  71
  72                 for row in res:
  73                         yield Build(self.backend, row.id, data=row)
  74
  75         def get_by_id(self, id, data=None):
  76                 return Build(self.pakfire, id, data=data)
  77
  78         def get_by_uuid(self, uuid):
  79                 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
  80
  81                 if build:
  82                         return self.get_by_id(build.id)
  83
  84         def get_all(self, limit=50):
  85                 query = "SELECT * FROM builds ORDER BY time_created DESC"
  86
  87                 if limit:
  88                         query += " LIMIT %d" % limit
  89
  90                 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
  91
  92         def get_by_user(self, user, type=None, public=None):
  93                 args = []
  94                 conditions = []
  95
  96                 if not type or type == "scratch":
  97                         # On scratch builds the user id equals the owner id.
  98                         conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
  99                         args.append(user.id)
 100
 101                 elif not type or type == "release":
 102                         pass # TODO
 103
 104                 if public is True:
 105                         conditions.append("public = 'Y'")
 106                 elif public is False:
 107                         conditions.append("public = 'N'")
 108
 109                 query = "SELECT builds.* AS id FROM builds \
 110                         JOIN packages ON builds.pkg_id = packages.id"
 111
 112                 if conditions:
 113                         query += " WHERE %s" % " AND ".join(conditions)
 114
 115                 query += " ORDER BY builds.time_created DESC"
 116
 117                 builds = []
 118                 for build in self.db.query(query, *args):
 119                         build = Build(self.pakfire, build.id, build)
 120                         builds.append(build)
 121
 122                 return builds
 123
 124         def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
 125                 args = [name,]
 126                 conditions = [
 127                         "packages.name = %s",
 128                 ]
 129
 130                 if type:
 131                         conditions.append("builds.type = %s")
 132                         args.append(type)
 133
 134                 or_conditions = []
 135                 if public is True:
 136                         or_conditions.append("public = 'Y'")
 137                 elif public is False:
 138                         or_conditions.append("public = 'N'")
 139
 140                 if user and not user.is_admin():
 141                         or_conditions.append("builds.owner_id = %s")
 142                         args.append(user.id)
 143
 144                 query = "SELECT builds.* AS id FROM builds \
 145                         JOIN packages ON builds.pkg_id = packages.id"
 146
 147                 if or_conditions:
 148                         conditions.append(" OR ".join(or_conditions))
 149
 150                 if conditions:
 151                         query += " WHERE %s" % " AND ".join(conditions)
 152
 153                 if type == "release":
 154                         query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
 155                 elif type == "scratch":
 156                         query += " ORDER BY time_created DESC"
 157
 158                 if limit:
 159                         if offset:
 160                                 query += " LIMIT %s,%s"
 161                                 args.extend([offset, limit])
 162                         else:
 163                                 query += " LIMIT %s"
 164                                 args.append(limit)
 165
 166                 return [Build(self.pakfire, b.id, b) for b in self.db.query(query, *args)]
 167
 168         def get_latest_by_name(self, name, type=None, public=None):
 169                 query = "\
 170                         SELECT * FROM builds \
 171                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 172                         WHERE builds_latest.package_name = %s"
 173                 args = [name,]
 174
 175                 if type:
 176                         query += " AND builds_latest.build_type = %s"
 177                         args.append(type)
 178
 179                 if public is True:
 180                         query += " AND builds.public = %s"
 181                         args.append("Y")
 182                 elif public is False:
 183                         query += " AND builds.public = %s"
 184                         args.append("N")
 185
 186                 # Get the last one only.
 187                 # Prefer release builds over scratch builds.
 188                 query += "\
 189                         ORDER BY \
 190                                 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
 191                                 builds.time_created DESC \
 192                         LIMIT 1"
 193
 194                 res = self.db.get(query, *args)
 195
 196                 if res:
 197                         return Build(self.pakfire, res.id, res)
 198
 199         def get_active_builds(self, name, public=None):
 200                 query = "\
 201                         SELECT * FROM builds \
 202                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 203                         WHERE builds_latest.package_name = %s AND builds.type = %s"
 204                 args = [name, "release"]
 205
 206                 if public is True:
 207                         query += " AND builds.public = %s"
 208                         args.append("Y")
 209                 elif public is False:
 210                         query += " AND builds.public = %s"
 211                         args.append("N")
 212
 213                 builds = []
 214                 for row in self.db.query(query, *args):
 215                         b = Build(self.pakfire, row.id, row)
 216                         builds.append(b)
 217
 218                 # Sort the result. Lastest build first.
 219                 builds.sort(reverse=True)
 220
 221                 return builds
 222
 223         def count(self):
 224                 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
 225                 if builds:
 226                         return builds.count
 227
 228         def get_obsolete(self, repo=None):
 229                 """
 230                         Get all obsoleted builds.
 231
 232                         If repo is True: which are in any repository.
 233                         If repo is some Repository object: which are in this repository.
 234                 """
 235                 args = []
 236
 237                 if repo is None:
 238                         query = "SELECT id FROM builds WHERE state = 'obsolete'"
 239
 240                 else:
 241                         query = "SELECT build_id AS id FROM repositories_builds \
 242                                 JOIN builds ON builds.id = repositories_builds.build_id \
 243                                 WHERE builds.state = 'obsolete'"
 244
 245                         if repo and not repo is True:
 246                                 query += " AND repositories_builds.repo_id = %s"
 247                                 args.append(repo.id)
 248
 249                 res = self.db.query(query, *args)
 250
 251                 builds = []
 252                 for build in res:
 253                         build = Build(self.pakfire, build.id)
 254                         builds.append(build)
 255
 256                 return builds
 257
 258         def get_changelog(self, name, public=None, limit=5, offset=0):
 259                 query = "SELECT builds.* FROM builds \
 260                         JOIN packages ON builds.pkg_id = packages.id \
 261                         WHERE \
 262                                 builds.type = %s \
 263                         AND \
 264                                 packages.name = %s"
 265                 args = ["release", name,]
 266
 267                 if public == True:
 268                         query += " AND builds.public = %s"
 269                         args.append("Y")
 270                 elif public == False:
 271                         query += " AND builds.public = %s"
 272                         args.append("N")
 273
 274                 query += " ORDER BY builds.time_created DESC"
 275
 276                 if limit:
 277                         if offset:
 278                                 query += " LIMIT %s,%s"
 279                                 args += [offset, limit]
 280                         else:
 281                                 query += " LIMIT %s"
 282                                 args.append(limit)
 283
 284                 builds = []
 285                 for b in self.db.query(query, *args):
 286                         b = Build(self.pakfire, b.id, b)
 287                         builds.append(b)
 288
 289                 builds.sort(reverse=True)
 290
 291                 return builds
 292
 293         def get_comments(self, limit=10, offset=None, user=None):
 294                 query = "SELECT * FROM builds_comments \
 295                         JOIN users ON builds_comments.user_id = users.id"
 296                 args = []
 297
 298                 wheres = []
 299                 if user:
 300                         wheres.append("users.id = %s")
 301                         args.append(user.id)
 302
 303                 if wheres:
 304                         query += " WHERE %s" % " AND ".join(wheres)
 305
 306                 # Sort everything.
 307                 query += " ORDER BY time_created DESC"
 308
 309                 # Limits.
 310                 if limit:
 311                         if offset:
 312                                 query += " LIMIT %s,%s"
 313                                 args.append(offset)
 314                         else:
 315                                 query += " LIMIT %s"
 316
 317                         args.append(limit)
 318
 319                 comments = []
 320                 for comment in self.db.query(query, *args):
 321                         comment = logs.CommentLogEntry(self.pakfire, comment)
 322                         comments.append(comment)
 323
 324                 return comments
 325
 326         def get_build_times_summary(self, name=None, job_type=None, arch=None):
 327                 query = "\
 328                         SELECT \
 329                                 builds_times.arch AS arch, \
 330                                 MAX(duration) AS maximum, \
 331                                 MIN(duration) AS minimum, \
 332                                 AVG(duration) AS average, \
 333                                 SUM(duration) AS sum, \
 334                                 STDDEV_POP(duration) AS stddev \
 335                         FROM builds_times \
 336                                 LEFT JOIN builds ON builds_times.build_id = builds.id \
 337                                 LEFT JOIN packages ON builds.pkg_id = packages.id"
 338
 339                 args = []
 340                 conditions = []
 341
 342                 # Filter for name.
 343                 if name:
 344                         conditions.append("packages.name = %s")
 345                         args.append(name)
 346
 347                 # Filter by job types.
 348                 if job_type:
 349                         conditions.append("builds_times.job_type = %s")
 350                         args.append(job_type)
 351
 352                 # Filter by arch.
 353                 if arch:
 354                         conditions.append("builds_times.arch = %s")
 355                         args.append(arch)
 356
 357                 # Add conditions.
 358                 if conditions:
 359                         query += " WHERE %s" % " AND ".join(conditions)
 360
 361                 # Grouping and sorting.
 362                 query += " GROUP BY arch ORDER BY arch DESC"
 363
 364                 return self.db.query(query, *args)
 365
 366         def get_build_times_by_arch(self, arch, **kwargs):
 367                 kwargs.update({
 368                         "arch" : arch,
 369                 })
 370
 371                 build_times = self.get_build_times_summary(**kwargs)
 372                 if build_times:
 373                         return build_times[0]
 374
 375
 376 class Build(base.Object):
 377         def __init__(self, pakfire, id, data=None):
 378                 base.Object.__init__(self, pakfire)
 379
 380                 # ID of this build
 381                 self.id = id
 382
 383                 # Cache data.
 384                 self._data = data
 385                 self._jobs = None
 386                 self._jobs_test = None
 387                 self._depends_on = None
 388                 self._pkg = None
 389                 self._credits = None
 390                 self._owner = None
 391                 self._update = None
 392                 self._repo = None
 393                 self._distro = None
 394
 395         def __repr__(self):
 396                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
 397
 398         def __cmp__(self, other):
 399                 assert self.pkg
 400                 assert other.pkg
 401
 402                 return cmp(self.pkg, other.pkg)
 403
 404         def __iter__(self):
 405                 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
 406                         WHERE build_id = %s", self.id)
 407
 408                 return iter(sorted(jobs))
 409
 410         @classmethod
 411         def create(cls, pakfire, pkg, type="release", owner=None, distro=None, public=True):
 412                 assert type in ("release", "scratch", "test")
 413                 assert distro, "You need to specify the distribution of this build."
 414
 415                 if public:
 416                         public = "Y"
 417                 else:
 418                         public = "N"
 419
 420                 # Check if scratch build has an owner.
 421                 if type == "scratch" and not owner:
 422                         raise Exception, "Scratch builds require an owner"
 423
 424                 # Set the default priority of this build.
 425                 if type == "release":
 426                         priority = 0
 427
 428                 elif type == "scratch":
 429                         priority = 1
 430
 431                 elif type == "test":
 432                         priority = -1
 433
 434                 id = pakfire.db.execute("""
 435                         INSERT INTO builds(uuid, pkg_id, type, distro_id, time_created, public, priority)
 436                         VALUES(%s, %s, %s, %s, NOW(), %s, %s)""", "%s" % uuid.uuid4(), pkg.id,
 437                         type, distro.id, public, priority)
 438
 439                 # Set the owner of this buildgroup.
 440                 if owner:
 441                         pakfire.db.execute("UPDATE builds SET owner_id = %s WHERE id = %s",
 442                                 owner.id, id)
 443
 444                 build = cls(pakfire, id)
 445
 446                 # Log that the build has been created.
 447                 build.log("created", user=owner)
 448
 449                 # Create directory where the files live.
 450                 if not os.path.exists(build.path):
 451                         os.makedirs(build.path)
 452
 453                 # Move package file to the directory of the build.
 454                 source_path = os.path.join(build.path, "src")
 455                 build.pkg.move(source_path)
 456
 457                 # Generate an update id.
 458                 build.generate_update_id()
 459
 460                 # Obsolete all other builds with the same name to track updates.
 461                 build.obsolete_others()
 462
 463                 # Search for possible bug IDs in the commit message.
 464                 build.search_for_bugs()
 465
 466                 return build
 467
 468         def delete(self):
 469                 """
 470                         Deletes this build including all jobs, packages and the source
 471                         package.
 472                 """
 473                 # If the build is in a repository, we need to remove it.
 474                 if self.repo:
 475                         self.repo.rem_build(self)
 476
 477                 for job in self.jobs + self.test_jobs:
 478                         job.delete()
 479
 480                 if self.pkg:
 481                         self.pkg.delete()
 482
 483                 # Delete everything related to this build.
 484                 self.__delete_bugs()
 485                 self.__delete_comments()
 486                 self.__delete_history()
 487                 self.__delete_watchers()
 488
 489                 # Delete the build itself.
 490                 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
 491
 492         def __delete_bugs(self):
 493                 """
 494                         Delete all associated bugs.
 495                 """
 496                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
 497
 498         def __delete_comments(self):
 499                 """
 500                         Delete all comments.
 501                 """
 502                 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
 503
 504         def __delete_history(self):
 505                 """
 506                         Delete the repository history.
 507                 """
 508                 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
 509
 510         def __delete_watchers(self):
 511                 """
 512                         Delete all watchers.
 513                 """
 514                 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
 515
 516         def reset(self):
 517                 """
 518                         Resets the whole build so it can start again (as it has never
 519                         been started).
 520                 """
 521                 for job in self.jobs:
 522                         job.reset()
 523
 524                 #self.__delete_bugs()
 525                 self.__delete_comments()
 526                 self.__delete_history()
 527                 self.__delete_watchers()
 528
 529                 self.state = "building"
 530
 531                 # XXX empty log
 532
 533         @property
 534         def data(self):
 535                 """
 536                         Lazy fetching of data for this object.
 537                 """
 538                 if self._data is None:
 539                         self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id)
 540                         assert self._data
 541
 542                 return self._data
 543
 544         @property
 545         def info(self):
 546                 """
 547                         A set of information that is sent to the XMLRPC client.
 548                 """
 549                 return { "uuid" : self.uuid }
 550
 551         def log(self, action, user=None, bug_id=None):
 552                 user_id = None
 553                 if user:
 554                         user_id = user.id
 555
 556                 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
 557                         VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
 558
 559         @property
 560         def uuid(self):
 561                 """
 562                         The UUID of this build.
 563                 """
 564                 return self.data.uuid
 565
 566         @property
 567         def pkg(self):
 568                 """
 569                         Get package that is to be built in the build.
 570                 """
 571                 if self._pkg is None:
 572                         self._pkg = packages.Package(self.pakfire, self.data.pkg_id)
 573
 574                 return self._pkg
 575
 576         @property
 577         def name(self):
 578                 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
 579
 580         @property
 581         def type(self):
 582                 """
 583                         The type of this build.
 584                 """
 585                 return self.data.type
 586
 587         @property
 588         def owner_id(self):
 589                 """
 590                         The ID of the owner of this build.
 591                 """
 592                 return self.data.owner_id
 593
 594         @property
 595         def owner(self):
 596                 """
 597                         The owner of this build.
 598                 """
 599                 if not self.owner_id:
 600                         return
 601
 602                 if self._owner is None:
 603                         self._owner = self.pakfire.users.get_by_id(self.owner_id)
 604                         assert self._owner
 605
 606                 return self._owner
 607
 608         @property
 609         def distro_id(self):
 610                 return self.data.distro_id
 611
 612         @property
 613         def distro(self):
 614                 if self._distro is None:
 615                         self._distro = self.pakfire.distros.get_by_id(self.distro_id)
 616                         assert self._distro
 617
 618                 return self._distro
 619
 620         @property
 621         def user(self):
 622                 if self.type == "scratch":
 623                         return self.owner
 624
 625         def get_depends_on(self):
 626                 if self.data.depends_on and self._depends_on is None:
 627                         self._depends_on = Build(self.pakfire, self.data.depends_on)
 628
 629                 return self._depends_on
 630
 631         def set_depends_on(self, build):
 632                 self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s",
 633                         build.id, self.id)
 634
 635                 # Update cache.
 636                 self._depends_on = build
 637                 self._data["depends_on"] = build.id
 638
 639         depends_on = property(get_depends_on, set_depends_on)
 640
 641         @property
 642         def created(self):
 643                 return self.data.time_created
 644
 645         @property
 646         def date(self):
 647                 return self.created.date()
 648
 649         @property
 650         def public(self):
 651                 """
 652                         Is this build public?
 653                 """
 654                 return self.data.public == "Y"
 655
 656         @property
 657         def size(self):
 658                 """
 659                         Returns the size on disk of this build.
 660                 """
 661                 s = 0
 662
 663                 # Add the source package.
 664                 if self.pkg:
 665                         s += self.pkg.size
 666
 667                 # Add all jobs.
 668                 s += sum((j.size for j in self.jobs))
 669
 670                 return s
 671
 672         #@property
 673         #def state(self):
 674         #       # Cache all states.
 675         #       states = [j.state for j in self.jobs]
 676         #
 677         #       target_state = "unknown"
 678         #
 679         #       # If at least one job has failed, the whole build has failed.
 680         #       if "failed" in states:
 681         #               target_state = "failed"
 682         #
 683         #       # It at least one of the jobs is still running, the whole
 684         #       # build is in running state.
 685         #       elif "running" in states:
 686         #               target_state = "running"
 687         #
 688         #       # If all jobs are in the finished state, we turn into finished
 689         #       # state as well.
 690         #       elif all([s == "finished" for s in states]):
 691         #               target_state = "finished"
 692         #
 693         #       return target_state
 694
 695         def auto_update_state(self):
 696                 """
 697                         Check if the state of this build can be updated and perform
 698                         the change if possible.
 699                 """
 700                 # Do not change the broken/obsolete state automatically.
 701                 if self.state in ("broken", "obsolete"):
 702                         return
 703
 704                 if self.repo and self.repo.type == "stable":
 705                         self.update_state("stable")
 706                         return
 707
 708                 # If any of the build jobs are finished, the build will be put in testing
 709                 # state.
 710                 for job in self.jobs:
 711                         if job.state == "finished":
 712                                 self.update_state("testing")
 713                                 break
 714
 715         def update_state(self, state, user=None, remove=False):
 716                 assert state in ("stable", "testing", "obsolete", "broken")
 717
 718                 self.db.execute("UPDATE builds SET state = %s WHERE id = %s", state, self.id)
 719
 720                 if self._data:
 721                         self._data["state"] = state
 722
 723                 # In broken state, the removal from the repository is forced and
 724                 # all jobs that are not finished yet will be aborted.
 725                 if state == "broken":
 726                         remove = True
 727
 728                         for job in self.jobs:
 729                                 if job.state in ("new", "pending", "running", "dependency_error"):
 730                                         job.state = "aborted"
 731
 732                 # If this build is in a repository, it will leave it.
 733                 if remove and self.repo:
 734                                 self.repo.rem_build(self)
 735
 736                 # If a release build is now in testing state, we put it into the
 737                 # first repository of the distribution.
 738                 elif self.type == "release" and state == "testing":
 739                         # If the build is not in a repository, yet and if there is
 740                         # a first repository, we put the build there.
 741                         if not self.repo and self.distro.first_repo:
 742                                 self.distro.first_repo.add_build(self, user=user)
 743
 744         @property
 745         def state(self):
 746                 return self.data.state
 747
 748         def is_broken(self):
 749                 return self.state == "broken"
 750
 751         def obsolete_others(self):
 752                 if not self.type == "release":
 753                         return
 754
 755                 for build in self.pakfire.builds.get_by_name(self.pkg.name, type="release"):
 756                         # Don't modify ourself.
 757                         if self.id == build.id:
 758                                 continue
 759
 760                         # Don't touch broken builds.
 761                         if build.state in ("obsolete", "broken"):
 762                                 continue
 763
 764                         # Obsolete the build.
 765                         build.update_state("obsolete")
 766
 767         def set_severity(self, severity):
 768                 self.db.execute("UPDATE builds SET severity = %s WHERE id = %s", state, self.id)
 769
 770                 if self._data:
 771                         self._data["severity"] = severity
 772
 773         def get_severity(self):
 774                 return self.data.severity
 775
 776         severity = property(get_severity, set_severity)
 777
 778         @property
 779         def commit(self):
 780                 if self.pkg and self.pkg.commit:
 781                         return self.pkg.commit
 782
 783         def update_message(self, msg):
 784                 self.db.execute("UPDATE builds SET message = %s WHERE id = %s", msg, self.id)
 785
 786                 if self._data:
 787                         self._data["message"] = msg
 788
 789         def has_perm(self, user):
 790                 """
 791                         Check, if the given user has the right to perform administrative
 792                         operations on this build.
 793                 """
 794                 if user is None:
 795                         return False
 796
 797                 if user.is_admin():
 798                         return True
 799
 800                 # Check if the user is allowed to manage packages from the critical path.
 801                 if self.critical_path and not user.has_perm("manage_critical_path"):
 802                         return False
 803
 804                 # Search for maintainers...
 805
 806                 # Scratch builds.
 807                 if self.type == "scratch":
 808                         # The owner of a scratch build has the right to do anything with it.
 809                         if self.owner_id == user.id:
 810                                 return True
 811
 812                 # Release builds.
 813                 elif self.type == "release":
 814                         # The maintainer also is allowed to manage the build.
 815                         if self.pkg.maintainer == user:
 816                                 return True
 817
 818                 # Deny permission for all other cases.
 819                 return False
 820
 821         @property
 822         def message(self):
 823                 message = ""
 824
 825                 if self.data.message:
 826                         message = self.data.message
 827
 828                 elif self.commit:
 829                         if self.commit.message:
 830                                 message = "\n".join((self.commit.subject, self.commit.message))
 831                         else:
 832                                 message = self.commit.subject
 833
 834                         prefix = "%s: " % self.pkg.name
 835                         if message.startswith(prefix):
 836                                 message = message[len(prefix):]
 837
 838                 return message
 839
 840         def get_priority(self):
 841                 return self.data.priority
 842
 843         def set_priority(self, priority):
 844                 assert priority in (-2, -1, 0, 1, 2)
 845
 846                 self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority,
 847                         self.id)
 848
 849                 if self._data:
 850                         self._data["priority"] = priority
 851
 852         priority = property(get_priority, set_priority)
 853
 854         @property
 855         def path(self):
 856                 path = []
 857                 if self.type == "scratch":
 858                         path.append(BUILD_SCRATCH_DIR)
 859                         path.append(self.uuid)
 860
 861                 elif self.type == "release":
 862                         path.append(BUILD_RELEASE_DIR)
 863                         path.append("%s/%s-%s-%s" % \
 864                                 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
 865
 866                 else:
 867                         raise Exception, "Unknown build type: %s" % self.type
 868
 869                 return os.path.join(*path)
 870
 871         @property
 872         def source_filename(self):
 873                 return os.path.basename(self.pkg.path)
 874
 875         @property
 876         def download_prefix(self):
 877                 return "/".join((self.pakfire.settings.get("download_baseurl"), "packages"))
 878
 879         @property
 880         def source_download(self):
 881                 return "/".join((self.download_prefix, self.pkg.path))
 882
 883         @property
 884         def source_hash_sha512(self):
 885                 return self.pkg.hash_sha512
 886
 887         @property
 888         def link(self):
 889                 # XXX maybe this should rather live in a uimodule.
 890                 # zlib-1.2.3-2.ip3 [src, i686, blah...]
 891                 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
 892                         (self.state, self.type, self.uuid, self.name)
 893
 894                 s_jobs = []
 895                 for job in self.jobs:
 896                         s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
 897                                 (job.state, job.type, job.uuid, job.arch))
 898
 899                 if s_jobs:
 900                         s += " [%s]" % ", ".join(s_jobs)
 901
 902                 return s
 903
 904         @property
 905         def supported_arches(self):
 906                 return self.pkg.supported_arches
 907
 908         @property
 909         def critical_path(self):
 910                 return self.pkg.critical_path
 911
 912         def get_jobs(self, type=None):
 913                 """
 914                         Returns a list of jobs of this build.
 915                 """
 916                 return self.pakfire.jobs.get_by_build(self.id, self, type=type)
 917
 918         @property
 919         def jobs(self):
 920                 """
 921                         Get a list of all build jobs that are in this build.
 922                 """
 923                 if self._jobs is None:
 924                         self._jobs = self.get_jobs(type="build")
 925
 926                 return self._jobs
 927
 928         @property
 929         def test_jobs(self):
 930                 if self._jobs_test is None:
 931                         self._jobs_test = self.get_jobs(type="test")
 932
 933                 return self._jobs_test
 934
 935         @property
 936         def all_jobs_finished(self):
 937                 ret = True
 938
 939                 for job in self.jobs:
 940                         if not job.state == "finished":
 941                                 ret = False
 942                                 break
 943
 944                 return ret
 945
 946         def create_autojobs(self, arches=None, type="build"):
 947                 jobs = []
 948
 949                 # Arches may be passed to this function. If not we use all arches
 950                 # this package supports.
 951                 if arches is None:
 952                         arches = self.supported_arches
 953
 954                 # Create a new job for every given archirecture.
 955                 for arch in self.pakfire.arches.expand(arches):
 956                         # Don't create jobs for src.
 957                         if arch.name == "src":
 958                                 continue
 959
 960                         job = self.add_job(arch, type=type)
 961                         jobs.append(job)
 962
 963                 # Return all newly created jobs.
 964                 return jobs
 965
 966         def add_job(self, arch, type="build"):
 967                 job = Job.create(self.pakfire, self, arch, type=type)
 968
 969                 # Add new job to cache.
 970                 if self._jobs:
 971                         self._jobs.append(job)
 972
 973                 return job
 974
 975         ## Update stuff
 976
 977         @property
 978         def update_id(self):
 979                 if not self.type == "release":
 980                         return
 981
 982                 # Generate an update ID if none does exist, yet.
 983                 self.generate_update_id()
 984
 985                 s = [
 986                         "%s" % self.distro.name.replace(" ", "").upper(),
 987                         "%04d" % (self.data.update_year or 0),
 988                         "%04d" % (self.data.update_num or 0),
 989                 ]
 990
 991                 return "-".join(s)
 992
 993         def generate_update_id(self):
 994                 if not self.type == "release":
 995                         return
 996
 997                 if self.data.update_num:
 998                         return
 999
1000                 update = self.db.get("SELECT update_num AS num FROM builds \
1001                         WHERE update_year = YEAR(NOW()) ORDER BY update_num DESC LIMIT 1")
1002
1003                 if update:
1004                         update_num = update.num + 1
1005                 else:
1006                         update_num = 1
1007
1008                 self.db.execute("UPDATE builds SET update_year = YEAR(NOW()), update_num = %s \
1009                         WHERE id = %s", update_num, self.id)
1010
1011         ## Comment stuff
1012
1013         def get_comments(self, limit=10, offset=0):
1014                 query = "SELECT * FROM builds_comments \
1015                         JOIN users ON builds_comments.user_id = users.id \
1016                         WHERE build_id = %s     ORDER BY time_created ASC"
1017
1018                 comments = []
1019                 for comment in self.db.query(query, self.id):
1020                         comment = logs.CommentLogEntry(self.pakfire, comment)
1021                         comments.append(comment)
1022
1023                 return comments
1024
1025         def add_comment(self, user, text, credit):
1026                 # Add the new comment to the database.
1027                 id = self.db.execute("INSERT INTO \
1028                         builds_comments(build_id, user_id, text, credit, time_created) \
1029                         VALUES(%s, %s, %s, %s, NOW())",
1030                         self.id, user.id, text, credit)
1031
1032                 # Update the credit cache.
1033                 if not self._credits is None:
1034                         self._credits += credit
1035
1036                 # Send the new comment to all watchers and stuff.
1037                 self.send_comment_message(id)
1038
1039                 # Return the ID of the newly created comment.
1040                 return id
1041
1042         @property
1043         def score(self):
1044                 # XXX UPDATE THIS
1045                 if self._credits is None:
1046                         # Get the sum of the credits from the database.
1047                         query = self.db.get(
1048                                 "SELECT SUM(credit) as credits FROM builds_comments WHERE build_id = %s",
1049                                 self.id
1050                         )
1051
1052                         self._credits = query.credits or 0
1053
1054                 return self._credits
1055
1056         @property
1057         def credits(self):
1058                 # XXX COMPAT
1059                 return self.score
1060
1061         def get_commenters(self):
1062                 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
1063                         JOIN users ON builds_comments.user_id = users.id \
1064                         WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
1065                         AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
1066
1067                 return [users.User(self.pakfire, u.id) for u in users]
1068
1069         def send_comment_message(self, comment_id):
1070                 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
1071                         comment_id)
1072
1073                 assert comment
1074                 assert comment.build_id == self.id
1075
1076                 # Get user who wrote the comment.
1077                 user = self.pakfire.users.get_by_id(comment.user_id)
1078
1079                 format = {
1080                         "build_name" : self.name,
1081                         "user_name"  : user.realname,
1082                 }
1083
1084                 # XXX create beautiful message
1085
1086                 self.pakfire.messages.send_to_all(self.message_recipients,
1087                         N_("%(user_name)s commented on %(build_name)s"),
1088                         comment.text, format)
1089
1090         ## Logging stuff
1091
1092         def get_log(self, comments=True, repo=True, limit=None):
1093                 entries = []
1094
1095                 # Created entry.
1096                 created_entry = logs.CreatedLogEntry(self.pakfire, self)
1097                 entries.append(created_entry)
1098
1099                 if comments:
1100                         entries += self.get_comments(limit=limit)
1101
1102                 if repo:
1103                         entries += self.get_repo_moves(limit=limit)
1104
1105                 # Sort all entries in chronological order.
1106                 entries.sort()
1107
1108                 if limit:
1109                         entries = entries[:limit]
1110
1111                 return entries
1112
1113         ## Watchers stuff
1114
1115         def get_watchers(self):
1116                 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
1117                         JOIN users ON builds_watchers.user_id = users.id \
1118                         WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1119                         AND users.activated = 'Y' ORDER BY users.id", self.id)
1120
1121                 return [users.User(self.pakfire, u.id) for u in query]
1122
1123         def add_watcher(self, user):
1124                 # Don't add a user twice.
1125                 if user in self.get_watchers():
1126                         return
1127
1128                 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1129                         VALUES(%s, %s)", self.id, user.id)
1130
1131         @property
1132         def message_recipients(self):
1133                 ret = []
1134
1135                 for watcher in self.get_watchers():
1136                         ret.append("%s <%s>" % (watcher.realname, watcher.email))
1137
1138                 return ret
1139
1140         @property
1141         def update(self):
1142                 if self._update is None:
1143                         update = self.db.get("SELECT update_id AS id FROM updates_builds \
1144                                 WHERE build_id = %s", self.id)
1145
1146                         if update:
1147                                 self._update = updates.Update(self.pakfire, update.id)
1148
1149                 return self._update
1150
1151         @property
1152         def repo(self):
1153                 if self._repo is None:
1154                         repo = self.db.get("SELECT repo_id AS id FROM repositories_builds \
1155                                 WHERE build_id = %s", self.id)
1156
1157                         if repo:
1158                                 self._repo = repository.Repository(self.pakfire, repo.id)
1159
1160                 return self._repo
1161
1162         def get_repo_moves(self, limit=None):
1163                 query = "SELECT * FROM repositories_history \
1164                         WHERE build_id = %s ORDER BY time ASC"
1165
1166                 actions = []
1167                 for action in self.db.query(query, self.id):
1168                         action = logs.RepositoryLogEntry(self.pakfire, action)
1169                         actions.append(action)
1170
1171                 return actions
1172
1173         @property
1174         def is_loose(self):
1175                 if self.repo:
1176                         return False
1177
1178                 return True
1179
1180         @property
1181         def repo_time(self):
1182                 repo = self.db.get("SELECT time_added FROM repositories_builds \
1183                         WHERE build_id = %s", self.id)
1184
1185                 if repo:
1186                         return repo.time_added
1187
1188         def get_auto_move(self):
1189                 return self.data.auto_move == "Y"
1190
1191         def set_auto_move(self, state):
1192                 if state:
1193                         state = "Y"
1194                 else:
1195                         state = "N"
1196
1197                 self.db.execute("UPDATE builds SET auto_move = %s WHERE id = %s", self.id)
1198                 if self._data:
1199                         self._data["auto_move"] = state
1200
1201         auto_move = property(get_auto_move, set_auto_move)
1202
1203         @property
1204         def can_move_forward(self):
1205                 if not self.repo:
1206                         return False
1207
1208                 # If there is no next repository, we cannot move anything.
1209                 if not self.repo.next:
1210                         return False
1211
1212                 # If the needed amount of score is reached, we can move forward.
1213                 if self.score >= self.repo.next.score_needed:
1214                         return True
1215
1216                 # If the repository does not require a minimal time,
1217                 # we can move forward immediately.
1218                 if not self.repo.time_min:
1219                         return True
1220
1221                 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1222                         WHERE build_id = %s", self.id)
1223                 duration = query.duration
1224
1225                 if duration >= self.repo.time_min:
1226                         return True
1227
1228                 return False
1229
1230         ## Bugs
1231
1232         def get_bug_ids(self):
1233                 query = self.db.query("SELECT bug_id FROM builds_bugs \
1234                         WHERE build_id = %s", self.id)
1235
1236                 return [b.bug_id for b in query]
1237
1238         def add_bug(self, bug_id, user=None, log=True):
1239                 # Check if this bug is already in the list of bugs.
1240                 if bug_id in self.get_bug_ids():
1241                         return
1242
1243                 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1244                         VALUES(%s, %s)", self.id, bug_id)
1245
1246                 # Log the event.
1247                 if log:
1248                         self.log("bug_added", user=user, bug_id=bug_id)
1249
1250         def rem_bug(self, bug_id, user=None, log=True):
1251                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1252                         bug_id = %s", self.id, bug_id)
1253
1254                 # Log the event.
1255                 if log:
1256                         self.log("bug_removed", user=user, bug_id=bug_id)
1257
1258         def search_for_bugs(self):
1259                 if not self.commit:
1260                         return
1261
1262                 pattern = re.compile(r"(bug\s?|#)(\d+)")
1263
1264                 for txt in (self.commit.subject, self.commit.message):
1265                         for bug in re.finditer(pattern, txt):
1266                                 try:
1267                                         bugid = int(bug.group(2))
1268                                 except ValueError:
1269                                         continue
1270
1271                                 # Check if a bug with the given ID exists in BZ.
1272                                 bug = self.pakfire.bugzilla.get_bug(bugid)
1273                                 if not bug:
1274                                         continue
1275
1276                                 self.add_bug(bugid)
1277
1278         def get_bugs(self):
1279                 bugs = []
1280                 for bug_id in self.get_bug_ids():
1281                         bug = self.pakfire.bugzilla.get_bug(bug_id)
1282                         if not bug:
1283                                 continue
1284
1285                         bugs.append(bug)
1286
1287                 return bugs
1288
1289         def _update_bugs_helper(self, repo):
1290                 """
1291                         This function takes a new status and generates messages that
1292                         are appended to all bugs.
1293                 """
1294                 try:
1295                         kwargs = BUG_MESSAGES[repo.type].copy()
1296                 except KeyError:
1297                         return
1298
1299                 baseurl = self.pakfire.settings.get("baseurl", "")
1300                 args = {
1301                         "build_url"    : "%s/build/%s" % (baseurl, self.uuid),
1302                         "distro_name"  : self.distro.name,
1303                         "package_name" : self.name,
1304                         "repo_name"    : repo.name,
1305                 }
1306                 kwargs["comment"] = kwargs["comment"] % args
1307
1308                 self.update_bugs(**kwargs)
1309
1310         def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1311                 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1312                         VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1313
1314         def update_bugs(self, status, resolution=None, comment=None):
1315                 # Update all bugs linked to this build.
1316                 for bug_id in self.get_bug_ids():
1317                         self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1318
1319
1320 class Jobs(base.Object):
1321         def _get_job(self, query, *args):
1322                 res = self.db.get(query, *args)
1323
1324                 if res:
1325                         return Job(self.backend, res.id, data=res)
1326
1327         def _get_jobs(self, query, *args):
1328                 res = self.db.query(query, *args)
1329
1330                 for row in res:
1331                         yield Job(self.backend, row.id, data=row)
1332
1333         def create(self,  build, arch, type="build"):
1334                 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1335                         VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1336                 job.log("created")
1337
1338                 # Set cache for Build object.
1339                 job.build = build
1340
1341                 # Jobs are by default in state "new" and wait for being checked
1342                 # for dependencies. Packages that do have no build dependencies
1343                 # can directly be forwarded to "pending" state.
1344                 if not job.pkg.requires:
1345                         job.state = "pending"
1346
1347                 return job
1348
1349         def get_by_id(self, id, data=None):
1350                 return Job(self.pakfire, id, data)
1351
1352         def get_by_uuid(self, uuid):
1353                 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1354
1355                 if job:
1356                         return self.get_by_id(job.id)
1357
1358         def get_by_build(self, build_id, build=None, type=None):
1359                 """
1360                         Get all jobs in the specifies build.
1361                 """
1362                 query = "SELECT * FROM jobs WHERE build_id = %s"
1363                 args = [build_id,]
1364
1365                 if type:
1366                         query += " AND type = %s"
1367                         args.append(type)
1368
1369                 # Get IDs of all builds in this group.
1370                 jobs = []
1371                 for job in self.db.query(query, *args):
1372                         job = Job(self.pakfire, job.id, job)
1373
1374                         # If the Build object was set, we set it so it won't be retrieved
1375                         # from the database again.
1376                         if build:
1377                                 job._build = build
1378
1379                         jobs.append(job)
1380
1381                 # Return sorted list of jobs.
1382                 return sorted(jobs)
1383
1384         def get_active(self, host_id=None, builder=None, states=None):
1385                 if builder:
1386                         host_id = builder.id
1387
1388                 if states is None:
1389                         states = ["dispatching", "running", "uploading"]
1390
1391                 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1392                 args = states
1393
1394                 if host_id:
1395                         query += " AND builder_id = %s" % host_id
1396
1397                 query += " ORDER BY \
1398                         CASE \
1399                                 WHEN jobs.state = 'running'     THEN 0 \
1400                                 WHEN jobs.state = 'uploading'   THEN 1 \
1401                                 WHEN jobs.state = 'dispatching' THEN 2 \
1402                                 WHEN jobs.state = 'pending'     THEN 3 \
1403                                 WHEN jobs.state = 'new'         THEN 4 \
1404                         END, time_started ASC"
1405
1406                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1407
1408         def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
1409                 query = "SELECT * FROM jobs"
1410                 args  = []
1411
1412                 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
1413
1414                 if arch:
1415                         where.append("arch = %s")
1416                         args.append(arch)
1417
1418                 if builder:
1419                         where.append("builder_id = %s")
1420                         args.append(builder.id)
1421
1422                 if date:
1423                         try:
1424                                 year, month, day = date.split("-", 2)
1425                                 date = datetime.date(int(year), int(month), int(day))
1426                         except ValueError:
1427                                 pass
1428                         else:
1429                                 where.append("(DATE(time_created) = %s OR \
1430                                         DATE(time_started) = %s OR DATE(time_finished) = %s)")
1431                                 args += (date, date, date)
1432
1433                 if age:
1434                         where.append("time_finished >= NOW() - '%s'::interval" % age)
1435
1436                 if where:
1437                         query += " WHERE %s" % " AND ".join(where)
1438
1439                 query += " ORDER BY time_finished DESC"
1440
1441                 if limit:
1442                         query += " LIMIT %s"
1443                         args.append(limit)
1444
1445                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1446
1447         def get_average_build_time(self):
1448                 """
1449                         Returns the average build time of all finished builds from the
1450                         last 3 months.
1451                 """
1452                 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1453                         FROM jobs WHERE type = 'build' AND state = 'finished' AND \
1454                         time_finished >= NOW() - '3 months'::interval")
1455
1456                 if result:
1457                         return result.average
1458
1459         def count(self, *states):
1460                 query = "SELECT COUNT(*) AS count FROM jobs"
1461                 args  = []
1462
1463                 if states:
1464                         query += " WHERE state IN %s"
1465                         args.append(states)
1466
1467                 jobs = self.db.get(query, *args)
1468                 if jobs:
1469                         return jobs.count
1470
1471         def restart_failed(self, max_tries=9):
1472                 jobs = self._get_jobs("SELECT jobs.* FROM jobs \
1473                         JOIN builds ON builds.id = jobs.build_id \
1474                         WHERE \
1475                                 jobs.type = 'build' AND \
1476                                 jobs.state = 'failed' AND \
1477                                 jobs.tries <= %s AND \
1478                                 NOT builds.state = 'broken' AND \
1479                                 jobs.time_finished < NOW() - '72 hours'::interval \
1480                         ORDER BY \
1481                                 CASE \
1482                                         WHEN jobs.type = 'build' THEN 0 \
1483                                         WHEN jobs.type = 'test'  THEN 1 \
1484                                 END, \
1485                                 builds.priority DESC, jobs.time_created ASC",
1486                         max_tries)
1487
1488                 # Restart the job
1489                 for job in jobs:
1490                         job.set_state("new", log=False)
1491
1492
1493 class Job(base.DataObject):
1494         table = "jobs"
1495
1496         def __str__(self):
1497                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1498
1499         def __eq__(self, other):
1500                 if isinstance(other, self.__class__):
1501                         return self.id == other.id
1502
1503         def __lt__(self, other):
1504                 if isinstance(other, self.__class__):
1505                         if (self.type, other.type) == ("build", "test"):
1506                                 return True
1507
1508                         if self.build == other.build:
1509                                 return arches.priority(self.arch) < arches.priority(other.arch)
1510
1511                         return self.time_created < other.time_created
1512
1513         def __iter__(self):
1514                 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1515                         LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1516                         WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
1517
1518                 return iter(packages)
1519
1520         def __nonzero__(self):
1521                 return True
1522
1523         def __len__(self):
1524                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1525                         WHERE job_id = %s", self.id)
1526
1527                 return res.len
1528
1529         @property
1530         def distro(self):
1531                 return self.build.distro
1532
1533         def delete(self):
1534                 self.__delete_buildroots()
1535                 self.__delete_history()
1536                 self.__delete_packages()
1537                 self.__delete_logfiles()
1538
1539                 # Delete the job itself.
1540                 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
1541
1542         def __delete_buildroots(self):
1543                 """
1544                         Removes all buildroots.
1545                 """
1546                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1547
1548         def __delete_history(self):
1549                 """
1550                         Removes all references in the history to this build job.
1551                 """
1552                 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1553
1554         def __delete_packages(self):
1555                 """
1556                         Deletes all uploaded files from the job.
1557                 """
1558                 for pkg in self.packages:
1559                         pkg.delete()
1560
1561                 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1562
1563         def __delete_logfiles(self):
1564                 for logfile in self.logfiles:
1565                         self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1566
1567         def reset(self, user=None):
1568                 self.__delete_buildroots()
1569                 self.__delete_packages()
1570                 self.__delete_history()
1571                 self.__delete_logfiles()
1572
1573                 self.state = "new"
1574                 self.log("reset", user=user)
1575
1576         ## Logging stuff
1577
1578         def log(self, action, user=None, state=None, builder=None, test_job=None):
1579                 user_id = None
1580                 if user:
1581                         user_id = user.id
1582
1583                 builder_id = None
1584                 if builder:
1585                         builder_id = builder.id
1586
1587                 test_job_id = None
1588                 if test_job:
1589                         test_job_id = test_job.id
1590
1591                 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1592                         time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1593                         self.id, action, state, user_id, builder_id, test_job_id)
1594
1595         def get_log(self, limit=None, offset=None, user=None):
1596                 query = "SELECT * FROM jobs_history"
1597
1598                 conditions = ["job_id = %s",]
1599                 args  = [self.id,]
1600
1601                 if user:
1602                         conditions.append("user_id = %s")
1603                         args.append(user.id)
1604
1605                 if conditions:
1606                         query += " WHERE %s" % " AND ".join(conditions)
1607
1608                 query += " ORDER BY time DESC"
1609
1610                 if limit:
1611                         if offset:
1612                                 query += " LIMIT %s,%s"
1613                                 args  += [offset, limit,]
1614                         else:
1615                                 query += " LIMIT %s"
1616                                 args  += [limit,]
1617
1618                 entries = []
1619                 for entry in self.db.query(query, *args):
1620                         entry = logs.JobLogEntry(self.pakfire, entry)
1621                         entries.append(entry)
1622
1623                 return entries
1624
1625         @property
1626         def uuid(self):
1627                 return self.data.uuid
1628
1629         @property
1630         def type(self):
1631                 return self.data.type
1632
1633         @property
1634         def build_id(self):
1635                 return self.data.build_id
1636
1637         @lazy_property
1638         def build(self):
1639                 return self.pakfire.builds.get_by_id(self.build_id)
1640
1641         @property
1642         def related_jobs(self):
1643                 ret = []
1644
1645                 for job in self.build.jobs:
1646                         if job == self:
1647                                 continue
1648
1649                         ret.append(job)
1650
1651                 return ret
1652
1653         @property
1654         def pkg(self):
1655                 return self.build.pkg
1656
1657         @property
1658         def name(self):
1659                 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
1660
1661         @property
1662         def size(self):
1663                 return sum((p.size for p in self.packages))
1664
1665         @lazy_property
1666         def rank(self):
1667                 """
1668                         Returns the rank in the build queue
1669                 """
1670                 if not self.state == "pending":
1671                         return
1672
1673                 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1674
1675                 if res:
1676                         return res.rank
1677
1678         def is_running(self):
1679                 """
1680                         Returns True if job is in a running state.
1681                 """
1682                 return self.state in ("pending", "dispatching", "running", "uploading")
1683
1684         def get_state(self):
1685                 return self.data.state
1686
1687         def set_state(self, state, user=None, log=True):
1688                 # Nothing to do if the state remains.
1689                 if not self.state == state:
1690                         self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
1691
1692                         # Log the event.
1693                         if log and not state == "new":
1694                                 self.log("state_change", state=state, user=user)
1695
1696                         # Update cache.
1697                         if self._data:
1698                                 self._data["state"] = state
1699
1700                 # Always clear the message when the status is changed.
1701                 self.update_message(None)
1702
1703                 # Update some more informations.
1704                 if state == "dispatching":
1705                         # Set start time.
1706                         self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1707                                 WHERE id = %s", self.id)
1708
1709                 elif state == "pending":
1710                         self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1711                                 time_finished = NULL WHERE id = %s", self.id)
1712
1713                 elif state in ("aborted", "dependency_error", "finished", "failed"):
1714                         # Set finish time and reset builder..
1715                         self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
1716
1717                         # Send messages to the user.
1718                         if state == "finished":
1719                                 self.send_finished_message()
1720
1721                         elif state == "failed":
1722                                 # Remove all package files if a job is set to failed state.
1723                                 self.__delete_packages()
1724
1725                                 self.send_failed_message()
1726
1727                 # Automatically update the state of the build (not on test builds).
1728                 if self.type == "build":
1729                         self.build.auto_update_state()
1730
1731         state = property(get_state, set_state)
1732
1733         @property
1734         def message(self):
1735                 return self.data.message
1736
1737         def update_message(self, msg):
1738                 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1739                         msg, self.id)
1740
1741                 if self._data:
1742                         self._data["message"] = msg
1743
1744         def get_builder(self):
1745                 if self.data.builder_id:
1746                         return self.backend.builders.get_by_id(self.data.builder_id)
1747
1748         def set_builder(self, builder, user=None):
1749                 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1750                         builder.id, self.id)
1751
1752                 # Update cache.
1753                 if self._data:
1754                         self._data["builder_id"] = builder.id
1755
1756                 self._builder = builder
1757
1758                 # Log the event.
1759                 if user:
1760                         self.log("builder_assigned", builder=builder, user=user)
1761
1762         builder = lazy_property(get_builder, set_builder)
1763
1764         @property
1765         def arch(self):
1766                 return self.data.arch
1767
1768         @property
1769         def duration(self):
1770                 if not self.time_started:
1771                         return 0
1772
1773                 if self.time_finished:
1774                         delta = self.time_finished - self.time_started
1775                 else:
1776                         delta = datetime.datetime.utcnow() - self.time_started
1777
1778                 return delta.total_seconds()
1779
1780         @property
1781         def time_created(self):
1782                 return self.data.time_created
1783
1784         @property
1785         def time_started(self):
1786                 return self.data.time_started
1787
1788         @property
1789         def time_finished(self):
1790                 return self.data.time_finished
1791
1792         @property
1793         def expected_runtime(self):
1794                 """
1795                         Returns the estimated time and stddev, this job takes to finish.
1796                 """
1797                 # Get the average build time.
1798                 build_times = self.pakfire.builds.get_build_times_by_arch(self.arch,
1799                         name=self.pkg.name)
1800
1801                 # If there is no statistical data, we cannot estimate anything.
1802                 if not build_times:
1803                         return None, None
1804
1805                 return build_times.average, build_times.stddev
1806
1807         @property
1808         def eta(self):
1809                 expected_runtime, stddev = self.expected_runtime
1810
1811                 if expected_runtime:
1812                         return expected_runtime - int(self.duration), stddev
1813
1814         @property
1815         def tries(self):
1816                 return self.data.tries
1817
1818         def get_pkg_by_uuid(self, uuid):
1819                 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
1820                         JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1821                         WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1822                         self.id, uuid)
1823
1824                 if pkg:
1825                         pkg.job = self
1826                         return pkg
1827
1828         @lazy_property
1829         def logfiles(self):
1830                 logfiles = []
1831
1832                 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1833                         log = logs.LogFile(self.pakfire, log.id)
1834                         log._job = self
1835
1836                         logfiles.append(log)
1837
1838                 return logfiles
1839
1840         def add_file(self, filename):
1841                 """
1842                         Add the specified file to this job.
1843
1844                         The file is copied to the right directory by this function.
1845                 """
1846                 assert os.path.exists(filename)
1847
1848                 if filename.endswith(".log"):
1849                         self._add_file_log(filename)
1850
1851                 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1852                         # It is not allowed to upload packages on test builds.
1853                         if self.type == "test":
1854                                 return
1855
1856                         self._add_file_package(filename)
1857
1858         def _add_file_log(self, filename):
1859                 """
1860                         Attach a log file to this job.
1861                 """
1862                 target_dirname = os.path.join(self.build.path, "logs")
1863
1864                 if self.type == "test":
1865                         i = 1
1866                         while True:
1867                                 target_filename = os.path.join(target_dirname,
1868                                         "test.%s.%s.%s.log" % (self.arch, i, self.tries))
1869
1870                                 if os.path.exists(target_filename):
1871                                         i += 1
1872                                 else:
1873                                         break
1874                 else:
1875                         target_filename = os.path.join(target_dirname,
1876                                 "build.%s.%s.log" % (self.arch, self.tries))
1877
1878                 # Make sure the target directory exists.
1879                 if not os.path.exists(target_dirname):
1880                         os.makedirs(target_dirname)
1881
1882                 # Calculate a SHA512 hash from that file.
1883                 f = open(filename, "rb")
1884                 h = hashlib.sha512()
1885                 while True:
1886                         buf = f.read(BUFFER_SIZE)
1887                         if not buf:
1888                                 break
1889
1890                         h.update(buf)
1891                 f.close()
1892
1893                 # Copy the file to the final location.
1894                 shutil.copy2(filename, target_filename)
1895
1896                 # Create an entry in the database.
1897                 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1898                         VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1899                         os.path.getsize(target_filename), h.hexdigest())
1900
1901         def _add_file_package(self, filename):
1902                 # Open package (creates entry in the database).
1903                 pkg = packages.Package.open(self.pakfire, filename)
1904
1905                 # Move package to the build directory.
1906                 pkg.move(os.path.join(self.build.path, self.arch))
1907
1908                 # Attach the package to this job.
1909                 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1910                         self.id, pkg.id)
1911
1912         def get_aborted_state(self):
1913                 return self.data.aborted_state
1914
1915         def set_aborted_state(self, state):
1916                 self._set_attribute("aborted_state", state)
1917
1918         aborted_state = property(get_aborted_state, set_aborted_state)
1919
1920         @property
1921         def message_recipients(self):
1922                 l = []
1923
1924                 # Add all people watching the build.
1925                 l += self.build.message_recipients
1926
1927                 # Add the package maintainer on release builds.
1928                 if self.build.type == "release":
1929                         maint = self.pkg.maintainer
1930
1931                         if isinstance(maint, users.User):
1932                                 l.append("%s <%s>" % (maint.realname, maint.email))
1933                         elif maint:
1934                                 l.append(maint)
1935
1936                         # XXX add committer and commit author.
1937
1938                 # Add the owner of the scratch build on scratch builds.
1939                 elif self.build.type == "scratch" and self.build.user:
1940                         l.append("%s <%s>" % \
1941                                 (self.build.user.realname, self.build.user.email))
1942
1943                 return set(l)
1944
1945         def save_buildroot(self, pkgs):
1946                 rows = []
1947
1948                 for pkg_name, pkg_uuid in pkgs:
1949                         rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1950
1951                 # Cleanup old stuff first (for rebuilding packages).
1952                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1953                         self.id, self.tries)
1954
1955                 self.db.executemany("INSERT INTO \
1956                         jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1957                         VALUES(%s, %s, %s, %s)", rows)
1958
1959         def has_buildroot(self, tries=None):
1960                 if tries is None:
1961                         tries = self.tries
1962
1963                 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
1964                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1965                         self.id, tries)
1966
1967                 if res:
1968                         return res.num
1969
1970                 return 0
1971
1972         def get_buildroot(self, tries=None):
1973                 if tries is None:
1974                         tries = self.tries
1975
1976                 rows = self.db.query("SELECT * FROM jobs_buildroots \
1977                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1978                         ORDER BY pkg_name", self.id, tries)
1979
1980                 pkgs = []
1981                 for row in rows:
1982                         # Search for this package in the packages table.
1983                         pkg = self.pakfire.packages.get_by_uuid(row.pkg_uuid)
1984                         pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1985
1986                 return pkgs
1987
1988         def send_finished_message(self):
1989                 # Send no finished mails for test jobs.
1990                 if self.type == "test":
1991                         return
1992
1993                 logging.debug("Sending finished message for job %s to %s" % \
1994                         (self.name, ", ".join(self.message_recipients)))
1995
1996                 info = {
1997                         "build_name" : self.name,
1998                         "build_host" : self.builder.name,
1999                         "build_uuid" : self.uuid,
2000                 }
2001
2002                 self.pakfire.messages.send_to_all(self.message_recipients,
2003                         MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
2004
2005         def send_failed_message(self):
2006                 logging.debug("Sending failed message for job %s to %s" % \
2007                         (self.name, ", ".join(self.message_recipients)))
2008
2009                 build_host = "--"
2010                 if self.builder:
2011                         build_host = self.builder.name
2012
2013                 info = {
2014                         "build_name" : self.name,
2015                         "build_host" : build_host,
2016                         "build_uuid" : self.uuid,
2017                 }
2018
2019                 self.pakfire.messages.send_to_all(self.message_recipients,
2020                         MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
2021
2022         def set_start_time(self, start_time):
2023                 if start_time is None:
2024                         return
2025
2026                 self.db.execute("UPDATE jobs SET start_not_before = NOW() + %s \
2027                         WHERE id = %s LIMIT 1", start_time, self.id)
2028
2029         def schedule(self, type, start_time=None, user=None):
2030                 assert type in ("rebuild", "test")
2031
2032                 if type == "rebuild":
2033                         if self.state == "finished":
2034                                 return
2035
2036                         self.set_state("new", user=user, log=False)
2037                         self.set_start_time(start_time)
2038
2039                         # Log the event.
2040                         self.log("schedule_rebuild", user=user)
2041
2042                 elif type == "test":
2043                         if not self.state == "finished":
2044                                 return
2045
2046                         # Create a new job with same build and arch.
2047                         job = self.create(self.pakfire, self.build, self.arch, type="test")
2048                         job.set_start_time(start_time)
2049
2050                         # Log the event.
2051                         self.log("schedule_test_job", test_job=job, user=user)
2052
2053                         return job
2054
2055         def schedule_test(self, start_not_before=None, user=None):
2056                 # XXX to be removed
2057                 return self.schedule("test", start_time=start_not_before, user=user)
2058
2059         def schedule_rebuild(self, start_not_before=None, user=None):
2060                 # XXX to be removed
2061                 return self.schedule("rebuild", start_time=start_not_before, user=user)
2062
2063         def get_build_repos(self):
2064                 """
2065                         Returns a list of all repositories that should be used when
2066                         building this job.
2067                 """
2068                 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
2069                         self.id)
2070
2071                 if not repo_ids:
2072                         return self.distro.get_build_repos()
2073
2074                 repos = []
2075                 for repo in self.distro.repositories:
2076                         if repo.id in [r.id for r in repo_ids]:
2077                                 repos.append(repo)
2078
2079                 return repos or self.distro.get_build_repos()
2080
2081         def get_repo_config(self):
2082                 """
2083                         Get repository configuration file that is sent to the builder.
2084                 """
2085                 confs = []
2086
2087                 for repo in self.get_build_repos():
2088                         confs.append(repo.get_conf())
2089
2090                 return "\n\n".join(confs)
2091
2092         def get_config(self):
2093                 """
2094                         Get configuration file that is sent to the builder.
2095                 """
2096                 confs = []
2097
2098                 # Add the distribution configuration.
2099                 confs.append(self.distro.get_config())
2100
2101                 # Then add all repositories for this build.
2102                 confs.append(self.get_repo_config())
2103
2104                 return "\n\n".join(confs)
2105
2106         def resolvdep(self):
2107                 config = pakfire.config.Config(files=["general.conf"])
2108                 config.parse(self.get_config())
2109
2110                 # The filename of the source file.
2111                 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
2112                 assert os.path.exists(filename), filename
2113
2114                 # Create a new pakfire instance with the configuration for
2115                 # this build.
2116                 p = pakfire.PakfireServer(config=config, arch=self.arch)
2117
2118                 # Try to solve the build dependencies.
2119                 try:
2120                         solver = p.resolvdep(filename)
2121
2122                 # Catch dependency errors and log the problem string.
2123                 except DependencyError, e:
2124                         self.state = "dependency_error"
2125                         self.update_message(e)
2126
2127                 else:
2128                         # If the build dependencies can be resolved, we set the build in
2129                         # pending state.
2130                         if solver.status is True:
2131                                 if self.state in ("failed",):
2132                                         return
2133
2134                                 self.state = "pending"