src/buildservice/builds.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import hashlib
   5 import logging
   6 import os
   7 import re
   8 import shutil
   9 import uuid
  10
  11 import pakfire
  12 import pakfire.config
  13 import pakfire.packages
  14
  15 from . import arches
  16 from . import base
  17 from . import builders
  18 from . import logs
  19 from . import packages
  20 from . import repository
  21 from . import updates
  22 from . import users
  23
  24 from .constants import *
  25 from .decorators import *
  26
  27 def import_from_package(_pakfire, filename, distro=None, commit=None, type="release",
  28                 arches=None, check_for_duplicates=True, owner=None):
  29
  30         if distro is None:
  31                 distro = commit.source.distro
  32
  33         assert distro
  34
  35         # Open the package file to read some basic information.
  36         pkg = pakfire.packages.open(None, None, filename)
  37
  38         if check_for_duplicates:
  39                 if distro.has_package(pkg.name, pkg.epoch, pkg.version, pkg.release):
  40                         logging.warning("Duplicate package detected: %s. Skipping." % pkg)
  41                         return
  42
  43         # Open the package and add it to the database.
  44         pkg = packages.Package.open(_pakfire, filename)
  45         logging.debug("Created new package: %s" % pkg)
  46
  47         # Associate the package to the processed commit.
  48         if commit:
  49                 pkg.commit = commit
  50
  51         # Create a new build object from the package which
  52         # is always a release build.
  53         build = Build.create(_pakfire, pkg, type=type, owner=owner, distro=distro)
  54         logging.debug("Created new build job: %s" % build)
  55
  56         # Create all automatic jobs.
  57         build.create_autojobs(arches=arches)
  58
  59         return pkg, build
  60
  61
  62 class Builds(base.Object):
  63         def _get_build(self, query, *args):
  64                 res = self.db.get(query, *args)
  65
  66                 if res:
  67                         return Build(self.backend, res.id, data=res)
  68
  69         def _get_builds(self, query, *args):
  70                 res = self.db.query(query, *args)
  71
  72                 for row in res:
  73                         yield Build(self.backend, row.id, data=row)
  74
  75         def get_by_id(self, id, data=None):
  76                 return Build(self.pakfire, id, data=data)
  77
  78         def get_by_uuid(self, uuid):
  79                 build = self.db.get("SELECT id FROM builds WHERE uuid = %s LIMIT 1", uuid)
  80
  81                 if build:
  82                         return self.get_by_id(build.id)
  83
  84         def get_all(self, limit=50):
  85                 query = "SELECT * FROM builds ORDER BY time_created DESC"
  86
  87                 if limit:
  88                         query += " LIMIT %d" % limit
  89
  90                 return [self.get_by_id(b.id, b) for b in self.db.query(query)]
  91
  92         def get_by_user(self, user, type=None, public=None):
  93                 args = []
  94                 conditions = []
  95
  96                 if not type or type == "scratch":
  97                         # On scratch builds the user id equals the owner id.
  98                         conditions.append("(builds.type = 'scratch' AND owner_id = %s)")
  99                         args.append(user.id)
 100
 101                 elif not type or type == "release":
 102                         pass # TODO
 103
 104                 if public is True:
 105                         conditions.append("public = 'Y'")
 106                 elif public is False:
 107                         conditions.append("public = 'N'")
 108
 109                 query = "SELECT builds.* AS id FROM builds \
 110                         JOIN packages ON builds.pkg_id = packages.id"
 111
 112                 if conditions:
 113                         query += " WHERE %s" % " AND ".join(conditions)
 114
 115                 query += " ORDER BY builds.time_created DESC"
 116
 117                 builds = []
 118                 for build in self.db.query(query, *args):
 119                         build = Build(self.pakfire, build.id, build)
 120                         builds.append(build)
 121
 122                 return builds
 123
 124         def get_by_name(self, name, type=None, public=None, user=None, limit=None, offset=None):
 125                 args = [name,]
 126                 conditions = [
 127                         "packages.name = %s",
 128                 ]
 129
 130                 if type:
 131                         conditions.append("builds.type = %s")
 132                         args.append(type)
 133
 134                 or_conditions = []
 135                 if public is True:
 136                         or_conditions.append("public = 'Y'")
 137                 elif public is False:
 138                         or_conditions.append("public = 'N'")
 139
 140                 if user and not user.is_admin():
 141                         or_conditions.append("builds.owner_id = %s")
 142                         args.append(user.id)
 143
 144                 query = "SELECT builds.* AS id FROM builds \
 145                         JOIN packages ON builds.pkg_id = packages.id"
 146
 147                 if or_conditions:
 148                         conditions.append(" OR ".join(or_conditions))
 149
 150                 if conditions:
 151                         query += " WHERE %s" % " AND ".join(conditions)
 152
 153                 if type == "release":
 154                         query += " ORDER BY packages.name,packages.epoch,packages.version,packages.release,id ASC"
 155                 elif type == "scratch":
 156                         query += " ORDER BY time_created DESC"
 157
 158                 if limit:
 159                         if offset:
 160                                 query += " LIMIT %s,%s"
 161                                 args.extend([offset, limit])
 162                         else:
 163                                 query += " LIMIT %s"
 164                                 args.append(limit)
 165
 166                 return [Build(self.pakfire, b.id, b) for b in self.db.query(query, *args)]
 167
 168         def get_latest_by_name(self, name, type=None, public=None):
 169                 query = "\
 170                         SELECT * FROM builds \
 171                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 172                         WHERE builds_latest.package_name = %s"
 173                 args = [name,]
 174
 175                 if type:
 176                         query += " AND builds_latest.build_type = %s"
 177                         args.append(type)
 178
 179                 if public is True:
 180                         query += " AND builds.public = %s"
 181                         args.append("Y")
 182                 elif public is False:
 183                         query += " AND builds.public = %s"
 184                         args.append("N")
 185
 186                 # Get the last one only.
 187                 # Prefer release builds over scratch builds.
 188                 query += "\
 189                         ORDER BY \
 190                                 CASE builds.type WHEN 'release' THEN 0 ELSE 1 END, \
 191                                 builds.time_created DESC \
 192                         LIMIT 1"
 193
 194                 res = self.db.get(query, *args)
 195
 196                 if res:
 197                         return Build(self.pakfire, res.id, res)
 198
 199         def get_active_builds(self, name, public=None):
 200                 query = "\
 201                         SELECT * FROM builds \
 202                                 LEFT JOIN builds_latest ON builds.id = builds_latest.build_id \
 203                         WHERE builds_latest.package_name = %s AND builds.type = %s"
 204                 args = [name, "release"]
 205
 206                 if public is True:
 207                         query += " AND builds.public = %s"
 208                         args.append("Y")
 209                 elif public is False:
 210                         query += " AND builds.public = %s"
 211                         args.append("N")
 212
 213                 builds = []
 214                 for row in self.db.query(query, *args):
 215                         b = Build(self.pakfire, row.id, row)
 216                         builds.append(b)
 217
 218                 # Sort the result. Lastest build first.
 219                 builds.sort(reverse=True)
 220
 221                 return builds
 222
 223         def count(self):
 224                 builds = self.db.get("SELECT COUNT(*) AS count FROM builds")
 225                 if builds:
 226                         return builds.count
 227
 228         def needs_test(self, threshold, arch, limit=None, randomize=False):
 229                 query = "SELECT id FROM builds \
 230                         WHERE NOT EXISTS \
 231                                 (SELECT * FROM jobs WHERE \
 232                                         jobs.build_id = builds.id AND \
 233                                         jobs.arch = %s AND \
 234                                         (jobs.state != 'finished' OR \
 235                                         jobs.time_finished >= %s) \
 236                                 ) \
 237                         AND EXISTS \
 238                                 (SELECT * FROM jobs WHERE \
 239                                         jobs.build_id = builds.id AND \
 240                                         jobs.arch = %s AND \
 241                                         jobs.type = 'build' AND \
 242                                         jobs.state = 'finished' AND \
 243                                         jobs.time_finished < %s \
 244                                 ) \
 245                         AND builds.type = 'release' \
 246                         AND (builds.state = 'stable' OR builds.state = 'testing')"
 247                 args  = [arch, threshold, arch, threshold]
 248
 249                 if randomize:
 250                         query += " ORDER BY RAND()"
 251
 252                 if limit:
 253                         query += " LIMIT %s"
 254                         args.append(limit)
 255
 256                 return [Build(self.pakfire, b.id) for b in self.db.query(query, *args)]
 257
 258         def get_obsolete(self, repo=None):
 259                 """
 260                         Get all obsoleted builds.
 261
 262                         If repo is True: which are in any repository.
 263                         If repo is some Repository object: which are in this repository.
 264                 """
 265                 args = []
 266
 267                 if repo is None:
 268                         query = "SELECT id FROM builds WHERE state = 'obsolete'"
 269
 270                 else:
 271                         query = "SELECT build_id AS id FROM repositories_builds \
 272                                 JOIN builds ON builds.id = repositories_builds.build_id \
 273                                 WHERE builds.state = 'obsolete'"
 274
 275                         if repo and not repo is True:
 276                                 query += " AND repositories_builds.repo_id = %s"
 277                                 args.append(repo.id)
 278
 279                 res = self.db.query(query, *args)
 280
 281                 builds = []
 282                 for build in res:
 283                         build = Build(self.pakfire, build.id)
 284                         builds.append(build)
 285
 286                 return builds
 287
 288         def get_changelog(self, name, public=None, limit=5, offset=0):
 289                 query = "SELECT builds.* FROM builds \
 290                         JOIN packages ON builds.pkg_id = packages.id \
 291                         WHERE \
 292                                 builds.type = %s \
 293                         AND \
 294                                 packages.name = %s"
 295                 args = ["release", name,]
 296
 297                 if public == True:
 298                         query += " AND builds.public = %s"
 299                         args.append("Y")
 300                 elif public == False:
 301                         query += " AND builds.public = %s"
 302                         args.append("N")
 303
 304                 query += " ORDER BY builds.time_created DESC"
 305
 306                 if limit:
 307                         if offset:
 308                                 query += " LIMIT %s,%s"
 309                                 args += [offset, limit]
 310                         else:
 311                                 query += " LIMIT %s"
 312                                 args.append(limit)
 313
 314                 builds = []
 315                 for b in self.db.query(query, *args):
 316                         b = Build(self.pakfire, b.id, b)
 317                         builds.append(b)
 318
 319                 builds.sort(reverse=True)
 320
 321                 return builds
 322
 323         def get_comments(self, limit=10, offset=None, user=None):
 324                 query = "SELECT * FROM builds_comments \
 325                         JOIN users ON builds_comments.user_id = users.id"
 326                 args = []
 327
 328                 wheres = []
 329                 if user:
 330                         wheres.append("users.id = %s")
 331                         args.append(user.id)
 332
 333                 if wheres:
 334                         query += " WHERE %s" % " AND ".join(wheres)
 335
 336                 # Sort everything.
 337                 query += " ORDER BY time_created DESC"
 338
 339                 # Limits.
 340                 if limit:
 341                         if offset:
 342                                 query += " LIMIT %s,%s"
 343                                 args.append(offset)
 344                         else:
 345                                 query += " LIMIT %s"
 346
 347                         args.append(limit)
 348
 349                 comments = []
 350                 for comment in self.db.query(query, *args):
 351                         comment = logs.CommentLogEntry(self.pakfire, comment)
 352                         comments.append(comment)
 353
 354                 return comments
 355
 356         def get_build_times_summary(self, name=None, job_type=None, arch=None):
 357                 query = "\
 358                         SELECT \
 359                                 builds_times.arch AS arch, \
 360                                 MAX(duration) AS maximum, \
 361                                 MIN(duration) AS minimum, \
 362                                 AVG(duration) AS average, \
 363                                 SUM(duration) AS sum, \
 364                                 STDDEV_POP(duration) AS stddev \
 365                         FROM builds_times \
 366                                 LEFT JOIN builds ON builds_times.build_id = builds.id \
 367                                 LEFT JOIN packages ON builds.pkg_id = packages.id"
 368
 369                 args = []
 370                 conditions = []
 371
 372                 # Filter for name.
 373                 if name:
 374                         conditions.append("packages.name = %s")
 375                         args.append(name)
 376
 377                 # Filter by job types.
 378                 if job_type:
 379                         conditions.append("builds_times.job_type = %s")
 380                         args.append(job_type)
 381
 382                 # Filter by arch.
 383                 if arch:
 384                         conditions.append("builds_times.arch = %s")
 385                         args.append(arch)
 386
 387                 # Add conditions.
 388                 if conditions:
 389                         query += " WHERE %s" % " AND ".join(conditions)
 390
 391                 # Grouping and sorting.
 392                 query += " GROUP BY arch ORDER BY arch DESC"
 393
 394                 return self.db.query(query, *args)
 395
 396         def get_build_times_by_arch(self, arch, **kwargs):
 397                 kwargs.update({
 398                         "arch" : arch,
 399                 })
 400
 401                 build_times = self.get_build_times_summary(**kwargs)
 402                 if build_times:
 403                         return build_times[0]
 404
 405
 406 class Build(base.Object):
 407         def __init__(self, pakfire, id, data=None):
 408                 base.Object.__init__(self, pakfire)
 409
 410                 # ID of this build
 411                 self.id = id
 412
 413                 # Cache data.
 414                 self._data = data
 415                 self._jobs = None
 416                 self._jobs_test = None
 417                 self._depends_on = None
 418                 self._pkg = None
 419                 self._credits = None
 420                 self._owner = None
 421                 self._update = None
 422                 self._repo = None
 423                 self._distro = None
 424
 425         def __repr__(self):
 426                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.pkg)
 427
 428         def __cmp__(self, other):
 429                 assert self.pkg
 430                 assert other.pkg
 431
 432                 return cmp(self.pkg, other.pkg)
 433
 434         def __iter__(self):
 435                 jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \
 436                         WHERE build_id = %s", self.id)
 437
 438                 return iter(sorted(jobs))
 439
 440         @classmethod
 441         def create(cls, pakfire, pkg, type="release", owner=None, distro=None, public=True):
 442                 assert type in ("release", "scratch", "test")
 443                 assert distro, "You need to specify the distribution of this build."
 444
 445                 if public:
 446                         public = "Y"
 447                 else:
 448                         public = "N"
 449
 450                 # Check if scratch build has an owner.
 451                 if type == "scratch" and not owner:
 452                         raise Exception, "Scratch builds require an owner"
 453
 454                 # Set the default priority of this build.
 455                 if type == "release":
 456                         priority = 0
 457
 458                 elif type == "scratch":
 459                         priority = 1
 460
 461                 elif type == "test":
 462                         priority = -1
 463
 464                 id = pakfire.db.execute("""
 465                         INSERT INTO builds(uuid, pkg_id, type, distro_id, time_created, public, priority)
 466                         VALUES(%s, %s, %s, %s, NOW(), %s, %s)""", "%s" % uuid.uuid4(), pkg.id,
 467                         type, distro.id, public, priority)
 468
 469                 # Set the owner of this buildgroup.
 470                 if owner:
 471                         pakfire.db.execute("UPDATE builds SET owner_id = %s WHERE id = %s",
 472                                 owner.id, id)
 473
 474                 build = cls(pakfire, id)
 475
 476                 # Log that the build has been created.
 477                 build.log("created", user=owner)
 478
 479                 # Create directory where the files live.
 480                 if not os.path.exists(build.path):
 481                         os.makedirs(build.path)
 482
 483                 # Move package file to the directory of the build.
 484                 source_path = os.path.join(build.path, "src")
 485                 build.pkg.move(source_path)
 486
 487                 # Generate an update id.
 488                 build.generate_update_id()
 489
 490                 # Obsolete all other builds with the same name to track updates.
 491                 build.obsolete_others()
 492
 493                 # Search for possible bug IDs in the commit message.
 494                 build.search_for_bugs()
 495
 496                 return build
 497
 498         def delete(self):
 499                 """
 500                         Deletes this build including all jobs, packages and the source
 501                         package.
 502                 """
 503                 # If the build is in a repository, we need to remove it.
 504                 if self.repo:
 505                         self.repo.rem_build(self)
 506
 507                 for job in self.jobs + self.test_jobs:
 508                         job.delete()
 509
 510                 if self.pkg:
 511                         self.pkg.delete()
 512
 513                 # Delete everything related to this build.
 514                 self.__delete_bugs()
 515                 self.__delete_comments()
 516                 self.__delete_history()
 517                 self.__delete_watchers()
 518
 519                 # Delete the build itself.
 520                 self.db.execute("DELETE FROM builds WHERE id = %s", self.id)
 521
 522         def __delete_bugs(self):
 523                 """
 524                         Delete all associated bugs.
 525                 """
 526                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s", self.id)
 527
 528         def __delete_comments(self):
 529                 """
 530                         Delete all comments.
 531                 """
 532                 self.db.execute("DELETE FROM builds_comments WHERE build_id = %s", self.id)
 533
 534         def __delete_history(self):
 535                 """
 536                         Delete the repository history.
 537                 """
 538                 self.db.execute("DELETE FROM repositories_history WHERE build_id = %s", self.id)
 539
 540         def __delete_watchers(self):
 541                 """
 542                         Delete all watchers.
 543                 """
 544                 self.db.execute("DELETE FROM builds_watchers WHERE build_id = %s", self.id)
 545
 546         def reset(self):
 547                 """
 548                         Resets the whole build so it can start again (as it has never
 549                         been started).
 550                 """
 551                 for job in self.jobs:
 552                         job.reset()
 553
 554                 #self.__delete_bugs()
 555                 self.__delete_comments()
 556                 self.__delete_history()
 557                 self.__delete_watchers()
 558
 559                 self.state = "building"
 560
 561                 # XXX empty log
 562
 563         @property
 564         def data(self):
 565                 """
 566                         Lazy fetching of data for this object.
 567                 """
 568                 if self._data is None:
 569                         self._data = self.db.get("SELECT * FROM builds WHERE id = %s", self.id)
 570                         assert self._data
 571
 572                 return self._data
 573
 574         @property
 575         def info(self):
 576                 """
 577                         A set of information that is sent to the XMLRPC client.
 578                 """
 579                 return { "uuid" : self.uuid }
 580
 581         def log(self, action, user=None, bug_id=None):
 582                 user_id = None
 583                 if user:
 584                         user_id = user.id
 585
 586                 self.db.execute("INSERT INTO builds_history(build_id, action, user_id, time, bug_id) \
 587                         VALUES(%s, %s, %s, NOW(), %s)", self.id, action, user_id, bug_id)
 588
 589         @property
 590         def uuid(self):
 591                 """
 592                         The UUID of this build.
 593                 """
 594                 return self.data.uuid
 595
 596         @property
 597         def pkg(self):
 598                 """
 599                         Get package that is to be built in the build.
 600                 """
 601                 if self._pkg is None:
 602                         self._pkg = packages.Package(self.pakfire, self.data.pkg_id)
 603
 604                 return self._pkg
 605
 606         @property
 607         def name(self):
 608                 return "%s-%s" % (self.pkg.name, self.pkg.friendly_version)
 609
 610         @property
 611         def type(self):
 612                 """
 613                         The type of this build.
 614                 """
 615                 return self.data.type
 616
 617         @property
 618         def owner_id(self):
 619                 """
 620                         The ID of the owner of this build.
 621                 """
 622                 return self.data.owner_id
 623
 624         @property
 625         def owner(self):
 626                 """
 627                         The owner of this build.
 628                 """
 629                 if not self.owner_id:
 630                         return
 631
 632                 if self._owner is None:
 633                         self._owner = self.pakfire.users.get_by_id(self.owner_id)
 634                         assert self._owner
 635
 636                 return self._owner
 637
 638         @property
 639         def distro_id(self):
 640                 return self.data.distro_id
 641
 642         @property
 643         def distro(self):
 644                 if self._distro is None:
 645                         self._distro = self.pakfire.distros.get_by_id(self.distro_id)
 646                         assert self._distro
 647
 648                 return self._distro
 649
 650         @property
 651         def user(self):
 652                 if self.type == "scratch":
 653                         return self.owner
 654
 655         def get_depends_on(self):
 656                 if self.data.depends_on and self._depends_on is None:
 657                         self._depends_on = Build(self.pakfire, self.data.depends_on)
 658
 659                 return self._depends_on
 660
 661         def set_depends_on(self, build):
 662                 self.db.execute("UPDATE builds SET depends_on = %s WHERE id = %s",
 663                         build.id, self.id)
 664
 665                 # Update cache.
 666                 self._depends_on = build
 667                 self._data["depends_on"] = build.id
 668
 669         depends_on = property(get_depends_on, set_depends_on)
 670
 671         @property
 672         def created(self):
 673                 return self.data.time_created
 674
 675         @property
 676         def date(self):
 677                 return self.created.date()
 678
 679         @property
 680         def public(self):
 681                 """
 682                         Is this build public?
 683                 """
 684                 return self.data.public == "Y"
 685
 686         @property
 687         def size(self):
 688                 """
 689                         Returns the size on disk of this build.
 690                 """
 691                 s = 0
 692
 693                 # Add the source package.
 694                 if self.pkg:
 695                         s += self.pkg.size
 696
 697                 # Add all jobs.
 698                 s += sum((j.size for j in self.jobs))
 699
 700                 return s
 701
 702         #@property
 703         #def state(self):
 704         #       # Cache all states.
 705         #       states = [j.state for j in self.jobs]
 706         #
 707         #       target_state = "unknown"
 708         #
 709         #       # If at least one job has failed, the whole build has failed.
 710         #       if "failed" in states:
 711         #               target_state = "failed"
 712         #
 713         #       # It at least one of the jobs is still running, the whole
 714         #       # build is in running state.
 715         #       elif "running" in states:
 716         #               target_state = "running"
 717         #
 718         #       # If all jobs are in the finished state, we turn into finished
 719         #       # state as well.
 720         #       elif all([s == "finished" for s in states]):
 721         #               target_state = "finished"
 722         #
 723         #       return target_state
 724
 725         def auto_update_state(self):
 726                 """
 727                         Check if the state of this build can be updated and perform
 728                         the change if possible.
 729                 """
 730                 # Do not change the broken/obsolete state automatically.
 731                 if self.state in ("broken", "obsolete"):
 732                         return
 733
 734                 if self.repo and self.repo.type == "stable":
 735                         self.update_state("stable")
 736                         return
 737
 738                 # If any of the build jobs are finished, the build will be put in testing
 739                 # state.
 740                 for job in self.jobs:
 741                         if job.state == "finished":
 742                                 self.update_state("testing")
 743                                 break
 744
 745         def update_state(self, state, user=None, remove=False):
 746                 assert state in ("stable", "testing", "obsolete", "broken")
 747
 748                 self.db.execute("UPDATE builds SET state = %s WHERE id = %s", state, self.id)
 749
 750                 if self._data:
 751                         self._data["state"] = state
 752
 753                 # In broken state, the removal from the repository is forced and
 754                 # all jobs that are not finished yet will be aborted.
 755                 if state == "broken":
 756                         remove = True
 757
 758                         for job in self.jobs:
 759                                 if job.state in ("new", "pending", "running", "dependency_error"):
 760                                         job.state = "aborted"
 761
 762                 # If this build is in a repository, it will leave it.
 763                 if remove and self.repo:
 764                                 self.repo.rem_build(self)
 765
 766                 # If a release build is now in testing state, we put it into the
 767                 # first repository of the distribution.
 768                 elif self.type == "release" and state == "testing":
 769                         # If the build is not in a repository, yet and if there is
 770                         # a first repository, we put the build there.
 771                         if not self.repo and self.distro.first_repo:
 772                                 self.distro.first_repo.add_build(self, user=user)
 773
 774         @property
 775         def state(self):
 776                 return self.data.state
 777
 778         def is_broken(self):
 779                 return self.state == "broken"
 780
 781         def obsolete_others(self):
 782                 if not self.type == "release":
 783                         return
 784
 785                 for build in self.pakfire.builds.get_by_name(self.pkg.name, type="release"):
 786                         # Don't modify ourself.
 787                         if self.id == build.id:
 788                                 continue
 789
 790                         # Don't touch broken builds.
 791                         if build.state in ("obsolete", "broken"):
 792                                 continue
 793
 794                         # Obsolete the build.
 795                         build.update_state("obsolete")
 796
 797         def set_severity(self, severity):
 798                 self.db.execute("UPDATE builds SET severity = %s WHERE id = %s", state, self.id)
 799
 800                 if self._data:
 801                         self._data["severity"] = severity
 802
 803         def get_severity(self):
 804                 return self.data.severity
 805
 806         severity = property(get_severity, set_severity)
 807
 808         @property
 809         def commit(self):
 810                 if self.pkg and self.pkg.commit:
 811                         return self.pkg.commit
 812
 813         def update_message(self, msg):
 814                 self.db.execute("UPDATE builds SET message = %s WHERE id = %s", msg, self.id)
 815
 816                 if self._data:
 817                         self._data["message"] = msg
 818
 819         def has_perm(self, user):
 820                 """
 821                         Check, if the given user has the right to perform administrative
 822                         operations on this build.
 823                 """
 824                 if user is None:
 825                         return False
 826
 827                 if user.is_admin():
 828                         return True
 829
 830                 # Check if the user is allowed to manage packages from the critical path.
 831                 if self.critical_path and not user.has_perm("manage_critical_path"):
 832                         return False
 833
 834                 # Search for maintainers...
 835
 836                 # Scratch builds.
 837                 if self.type == "scratch":
 838                         # The owner of a scratch build has the right to do anything with it.
 839                         if self.owner_id == user.id:
 840                                 return True
 841
 842                 # Release builds.
 843                 elif self.type == "release":
 844                         # The maintainer also is allowed to manage the build.
 845                         if self.pkg.maintainer == user:
 846                                 return True
 847
 848                 # Deny permission for all other cases.
 849                 return False
 850
 851         @property
 852         def message(self):
 853                 message = ""
 854
 855                 if self.data.message:
 856                         message = self.data.message
 857
 858                 elif self.commit:
 859                         if self.commit.message:
 860                                 message = "\n".join((self.commit.subject, self.commit.message))
 861                         else:
 862                                 message = self.commit.subject
 863
 864                         prefix = "%s: " % self.pkg.name
 865                         if message.startswith(prefix):
 866                                 message = message[len(prefix):]
 867
 868                 return message
 869
 870         def get_priority(self):
 871                 return self.data.priority
 872
 873         def set_priority(self, priority):
 874                 assert priority in (-2, -1, 0, 1, 2)
 875
 876                 self.db.execute("UPDATE builds SET priority = %s WHERE id = %s", priority,
 877                         self.id)
 878
 879                 if self._data:
 880                         self._data["priority"] = priority
 881
 882         priority = property(get_priority, set_priority)
 883
 884         @property
 885         def path(self):
 886                 path = []
 887                 if self.type == "scratch":
 888                         path.append(BUILD_SCRATCH_DIR)
 889                         path.append(self.uuid)
 890
 891                 elif self.type == "release":
 892                         path.append(BUILD_RELEASE_DIR)
 893                         path.append("%s/%s-%s-%s" % \
 894                                 (self.pkg.name, self.pkg.epoch, self.pkg.version, self.pkg.release))
 895
 896                 else:
 897                         raise Exception, "Unknown build type: %s" % self.type
 898
 899                 return os.path.join(*path)
 900
 901         @property
 902         def source_filename(self):
 903                 return os.path.basename(self.pkg.path)
 904
 905         @property
 906         def download_prefix(self):
 907                 return "/".join((self.pakfire.settings.get("download_baseurl"), "packages"))
 908
 909         @property
 910         def source_download(self):
 911                 return "/".join((self.download_prefix, self.pkg.path))
 912
 913         @property
 914         def source_hash_sha512(self):
 915                 return self.pkg.hash_sha512
 916
 917         @property
 918         def link(self):
 919                 # XXX maybe this should rather live in a uimodule.
 920                 # zlib-1.2.3-2.ip3 [src, i686, blah...]
 921                 s = """<a class="state_%s %s" href="/build/%s">%s</a>""" % \
 922                         (self.state, self.type, self.uuid, self.name)
 923
 924                 s_jobs = []
 925                 for job in self.jobs:
 926                         s_jobs.append("""<a class="state_%s %s" href="/job/%s">%s</a>""" % \
 927                                 (job.state, job.type, job.uuid, job.arch))
 928
 929                 if s_jobs:
 930                         s += " [%s]" % ", ".join(s_jobs)
 931
 932                 return s
 933
 934         @property
 935         def supported_arches(self):
 936                 return self.pkg.supported_arches
 937
 938         @property
 939         def critical_path(self):
 940                 return self.pkg.critical_path
 941
 942         def get_jobs(self, type=None):
 943                 """
 944                         Returns a list of jobs of this build.
 945                 """
 946                 return self.pakfire.jobs.get_by_build(self.id, self, type=type)
 947
 948         @property
 949         def jobs(self):
 950                 """
 951                         Get a list of all build jobs that are in this build.
 952                 """
 953                 if self._jobs is None:
 954                         self._jobs = self.get_jobs(type="build")
 955
 956                 return self._jobs
 957
 958         @property
 959         def test_jobs(self):
 960                 if self._jobs_test is None:
 961                         self._jobs_test = self.get_jobs(type="test")
 962
 963                 return self._jobs_test
 964
 965         @property
 966         def all_jobs_finished(self):
 967                 ret = True
 968
 969                 for job in self.jobs:
 970                         if not job.state == "finished":
 971                                 ret = False
 972                                 break
 973
 974                 return ret
 975
 976         def create_autojobs(self, arches=None, type="build"):
 977                 jobs = []
 978
 979                 # Arches may be passed to this function. If not we use all arches
 980                 # this package supports.
 981                 if arches is None:
 982                         arches = self.supported_arches
 983
 984                 # Create a new job for every given archirecture.
 985                 for arch in self.pakfire.arches.expand(arches):
 986                         # Don't create jobs for src.
 987                         if arch.name == "src":
 988                                 continue
 989
 990                         job = self.add_job(arch, type=type)
 991                         jobs.append(job)
 992
 993                 # Return all newly created jobs.
 994                 return jobs
 995
 996         def add_job(self, arch, type="build"):
 997                 job = Job.create(self.pakfire, self, arch, type=type)
 998
 999                 # Add new job to cache.
1000                 if self._jobs:
1001                         self._jobs.append(job)
1002
1003                 return job
1004
1005         ## Update stuff
1006
1007         @property
1008         def update_id(self):
1009                 if not self.type == "release":
1010                         return
1011
1012                 # Generate an update ID if none does exist, yet.
1013                 self.generate_update_id()
1014
1015                 s = [
1016                         "%s" % self.distro.name.replace(" ", "").upper(),
1017                         "%04d" % (self.data.update_year or 0),
1018                         "%04d" % (self.data.update_num or 0),
1019                 ]
1020
1021                 return "-".join(s)
1022
1023         def generate_update_id(self):
1024                 if not self.type == "release":
1025                         return
1026
1027                 if self.data.update_num:
1028                         return
1029
1030                 update = self.db.get("SELECT update_num AS num FROM builds \
1031                         WHERE update_year = YEAR(NOW()) ORDER BY update_num DESC LIMIT 1")
1032
1033                 if update:
1034                         update_num = update.num + 1
1035                 else:
1036                         update_num = 1
1037
1038                 self.db.execute("UPDATE builds SET update_year = YEAR(NOW()), update_num = %s \
1039                         WHERE id = %s", update_num, self.id)
1040
1041         ## Comment stuff
1042
1043         def get_comments(self, limit=10, offset=0):
1044                 query = "SELECT * FROM builds_comments \
1045                         JOIN users ON builds_comments.user_id = users.id \
1046                         WHERE build_id = %s     ORDER BY time_created ASC"
1047
1048                 comments = []
1049                 for comment in self.db.query(query, self.id):
1050                         comment = logs.CommentLogEntry(self.pakfire, comment)
1051                         comments.append(comment)
1052
1053                 return comments
1054
1055         def add_comment(self, user, text, credit):
1056                 # Add the new comment to the database.
1057                 id = self.db.execute("INSERT INTO \
1058                         builds_comments(build_id, user_id, text, credit, time_created) \
1059                         VALUES(%s, %s, %s, %s, NOW())",
1060                         self.id, user.id, text, credit)
1061
1062                 # Update the credit cache.
1063                 if not self._credits is None:
1064                         self._credits += credit
1065
1066                 # Send the new comment to all watchers and stuff.
1067                 self.send_comment_message(id)
1068
1069                 # Return the ID of the newly created comment.
1070                 return id
1071
1072         @property
1073         def score(self):
1074                 # XXX UPDATE THIS
1075                 if self._credits is None:
1076                         # Get the sum of the credits from the database.
1077                         query = self.db.get(
1078                                 "SELECT SUM(credit) as credits FROM builds_comments WHERE build_id = %s",
1079                                 self.id
1080                         )
1081
1082                         self._credits = query.credits or 0
1083
1084                 return self._credits
1085
1086         @property
1087         def credits(self):
1088                 # XXX COMPAT
1089                 return self.score
1090
1091         def get_commenters(self):
1092                 users = self.db.query("SELECT DISTINCT users.id AS id FROM builds_comments \
1093                         JOIN users ON builds_comments.user_id = users.id \
1094                         WHERE builds_comments.build_id = %s AND NOT users.deleted = 'Y' \
1095                         AND NOT users.activated = 'Y' ORDER BY users.id", self.id)
1096
1097                 return [users.User(self.pakfire, u.id) for u in users]
1098
1099         def send_comment_message(self, comment_id):
1100                 comment = self.db.get("SELECT * FROM builds_comments WHERE id = %s",
1101                         comment_id)
1102
1103                 assert comment
1104                 assert comment.build_id == self.id
1105
1106                 # Get user who wrote the comment.
1107                 user = self.pakfire.users.get_by_id(comment.user_id)
1108
1109                 format = {
1110                         "build_name" : self.name,
1111                         "user_name"  : user.realname,
1112                 }
1113
1114                 # XXX create beautiful message
1115
1116                 self.pakfire.messages.send_to_all(self.message_recipients,
1117                         N_("%(user_name)s commented on %(build_name)s"),
1118                         comment.text, format)
1119
1120         ## Logging stuff
1121
1122         def get_log(self, comments=True, repo=True, limit=None):
1123                 entries = []
1124
1125                 # Created entry.
1126                 created_entry = logs.CreatedLogEntry(self.pakfire, self)
1127                 entries.append(created_entry)
1128
1129                 if comments:
1130                         entries += self.get_comments(limit=limit)
1131
1132                 if repo:
1133                         entries += self.get_repo_moves(limit=limit)
1134
1135                 # Sort all entries in chronological order.
1136                 entries.sort()
1137
1138                 if limit:
1139                         entries = entries[:limit]
1140
1141                 return entries
1142
1143         ## Watchers stuff
1144
1145         def get_watchers(self):
1146                 query = self.db.query("SELECT DISTINCT users.id AS id FROM builds_watchers \
1147                         JOIN users ON builds_watchers.user_id = users.id \
1148                         WHERE builds_watchers.build_id = %s AND NOT users.deleted = 'Y' \
1149                         AND users.activated = 'Y' ORDER BY users.id", self.id)
1150
1151                 return [users.User(self.pakfire, u.id) for u in query]
1152
1153         def add_watcher(self, user):
1154                 # Don't add a user twice.
1155                 if user in self.get_watchers():
1156                         return
1157
1158                 self.db.execute("INSERT INTO builds_watchers(build_id, user_id) \
1159                         VALUES(%s, %s)", self.id, user.id)
1160
1161         @property
1162         def message_recipients(self):
1163                 ret = []
1164
1165                 for watcher in self.get_watchers():
1166                         ret.append("%s <%s>" % (watcher.realname, watcher.email))
1167
1168                 return ret
1169
1170         @property
1171         def update(self):
1172                 if self._update is None:
1173                         update = self.db.get("SELECT update_id AS id FROM updates_builds \
1174                                 WHERE build_id = %s", self.id)
1175
1176                         if update:
1177                                 self._update = updates.Update(self.pakfire, update.id)
1178
1179                 return self._update
1180
1181         @property
1182         def repo(self):
1183                 if self._repo is None:
1184                         repo = self.db.get("SELECT repo_id AS id FROM repositories_builds \
1185                                 WHERE build_id = %s", self.id)
1186
1187                         if repo:
1188                                 self._repo = repository.Repository(self.pakfire, repo.id)
1189
1190                 return self._repo
1191
1192         def get_repo_moves(self, limit=None):
1193                 query = "SELECT * FROM repositories_history \
1194                         WHERE build_id = %s ORDER BY time ASC"
1195
1196                 actions = []
1197                 for action in self.db.query(query, self.id):
1198                         action = logs.RepositoryLogEntry(self.pakfire, action)
1199                         actions.append(action)
1200
1201                 return actions
1202
1203         @property
1204         def is_loose(self):
1205                 if self.repo:
1206                         return False
1207
1208                 return True
1209
1210         @property
1211         def repo_time(self):
1212                 repo = self.db.get("SELECT time_added FROM repositories_builds \
1213                         WHERE build_id = %s", self.id)
1214
1215                 if repo:
1216                         return repo.time_added
1217
1218         def get_auto_move(self):
1219                 return self.data.auto_move == "Y"
1220
1221         def set_auto_move(self, state):
1222                 if state:
1223                         state = "Y"
1224                 else:
1225                         state = "N"
1226
1227                 self.db.execute("UPDATE builds SET auto_move = %s WHERE id = %s", self.id)
1228                 if self._data:
1229                         self._data["auto_move"] = state
1230
1231         auto_move = property(get_auto_move, set_auto_move)
1232
1233         @property
1234         def can_move_forward(self):
1235                 if not self.repo:
1236                         return False
1237
1238                 # If there is no next repository, we cannot move anything.
1239                 if not self.repo.next:
1240                         return False
1241
1242                 # If the needed amount of score is reached, we can move forward.
1243                 if self.score >= self.repo.next.score_needed:
1244                         return True
1245
1246                 # If the repository does not require a minimal time,
1247                 # we can move forward immediately.
1248                 if not self.repo.time_min:
1249                         return True
1250
1251                 query = self.db.get("SELECT NOW() - time_added AS duration FROM repositories_builds \
1252                         WHERE build_id = %s", self.id)
1253                 duration = query.duration
1254
1255                 if duration >= self.repo.time_min:
1256                         return True
1257
1258                 return False
1259
1260         ## Bugs
1261
1262         def get_bug_ids(self):
1263                 query = self.db.query("SELECT bug_id FROM builds_bugs \
1264                         WHERE build_id = %s", self.id)
1265
1266                 return [b.bug_id for b in query]
1267
1268         def add_bug(self, bug_id, user=None, log=True):
1269                 # Check if this bug is already in the list of bugs.
1270                 if bug_id in self.get_bug_ids():
1271                         return
1272
1273                 self.db.execute("INSERT INTO builds_bugs(build_id, bug_id) \
1274                         VALUES(%s, %s)", self.id, bug_id)
1275
1276                 # Log the event.
1277                 if log:
1278                         self.log("bug_added", user=user, bug_id=bug_id)
1279
1280         def rem_bug(self, bug_id, user=None, log=True):
1281                 self.db.execute("DELETE FROM builds_bugs WHERE build_id = %s AND \
1282                         bug_id = %s", self.id, bug_id)
1283
1284                 # Log the event.
1285                 if log:
1286                         self.log("bug_removed", user=user, bug_id=bug_id)
1287
1288         def search_for_bugs(self):
1289                 if not self.commit:
1290                         return
1291
1292                 pattern = re.compile(r"(bug\s?|#)(\d+)")
1293
1294                 for txt in (self.commit.subject, self.commit.message):
1295                         for bug in re.finditer(pattern, txt):
1296                                 try:
1297                                         bugid = int(bug.group(2))
1298                                 except ValueError:
1299                                         continue
1300
1301                                 # Check if a bug with the given ID exists in BZ.
1302                                 bug = self.pakfire.bugzilla.get_bug(bugid)
1303                                 if not bug:
1304                                         continue
1305
1306                                 self.add_bug(bugid)
1307
1308         def get_bugs(self):
1309                 bugs = []
1310                 for bug_id in self.get_bug_ids():
1311                         bug = self.pakfire.bugzilla.get_bug(bug_id)
1312                         if not bug:
1313                                 continue
1314
1315                         bugs.append(bug)
1316
1317                 return bugs
1318
1319         def _update_bugs_helper(self, repo):
1320                 """
1321                         This function takes a new status and generates messages that
1322                         are appended to all bugs.
1323                 """
1324                 try:
1325                         kwargs = BUG_MESSAGES[repo.type].copy()
1326                 except KeyError:
1327                         return
1328
1329                 baseurl = self.pakfire.settings.get("baseurl", "")
1330                 args = {
1331                         "build_url"    : "%s/build/%s" % (baseurl, self.uuid),
1332                         "distro_name"  : self.distro.name,
1333                         "package_name" : self.name,
1334                         "repo_name"    : repo.name,
1335                 }
1336                 kwargs["comment"] = kwargs["comment"] % args
1337
1338                 self.update_bugs(**kwargs)
1339
1340         def _update_bug(self, bug_id, status=None, resolution=None, comment=None):
1341                 self.db.execute("INSERT INTO builds_bugs_updates(bug_id, status, resolution, comment, time) \
1342                         VALUES(%s, %s, %s, %s, NOW())", bug_id, status, resolution, comment)
1343
1344         def update_bugs(self, status, resolution=None, comment=None):
1345                 # Update all bugs linked to this build.
1346                 for bug_id in self.get_bug_ids():
1347                         self._update_bug(bug_id, status=status, resolution=resolution, comment=comment)
1348
1349
1350 class Jobs(base.Object):
1351         def _get_job(self, query, *args):
1352                 res = self.db.get(query, *args)
1353
1354                 if res:
1355                         return Job(self.backend, res.id, data=res)
1356
1357         def _get_jobs(self, query, *args):
1358                 res = self.db.query(query, *args)
1359
1360                 for row in res:
1361                         yield Job(self.backend, row.id, data=row)
1362
1363         def create(self,  build, arch, type="build"):
1364                 job = self._get_job("INSERT INTO jobs(uuid, type, build_id, arch, time_created) \
1365                         VALUES(%s, %s, %s, %s, NOW()) RETURNING *", "%s" % uuid.uuid4(), type, build.id, arch)
1366                 job.log("created")
1367
1368                 # Set cache for Build object.
1369                 job.build = build
1370
1371                 # Jobs are by default in state "new" and wait for being checked
1372                 # for dependencies. Packages that do have no build dependencies
1373                 # can directly be forwarded to "pending" state.
1374                 if not job.pkg.requires:
1375                         job.state = "pending"
1376
1377                 return job
1378
1379         def get_by_id(self, id, data=None):
1380                 return Job(self.pakfire, id, data)
1381
1382         def get_by_uuid(self, uuid):
1383                 job = self.db.get("SELECT id FROM jobs WHERE uuid = %s", uuid)
1384
1385                 if job:
1386                         return self.get_by_id(job.id)
1387
1388         def get_by_build(self, build_id, build=None, type=None):
1389                 """
1390                         Get all jobs in the specifies build.
1391                 """
1392                 query = "SELECT * FROM jobs WHERE build_id = %s"
1393                 args = [build_id,]
1394
1395                 if type:
1396                         query += " AND type = %s"
1397                         args.append(type)
1398
1399                 # Get IDs of all builds in this group.
1400                 jobs = []
1401                 for job in self.db.query(query, *args):
1402                         job = Job(self.pakfire, job.id, job)
1403
1404                         # If the Build object was set, we set it so it won't be retrieved
1405                         # from the database again.
1406                         if build:
1407                                 job._build = build
1408
1409                         jobs.append(job)
1410
1411                 # Return sorted list of jobs.
1412                 return sorted(jobs)
1413
1414         def get_active(self, host_id=None, builder=None, states=None):
1415                 if builder:
1416                         host_id = builder.id
1417
1418                 if states is None:
1419                         states = ["dispatching", "running", "uploading"]
1420
1421                 query = "SELECT * FROM jobs WHERE state IN (%s)" % ", ".join(["%s"] * len(states))
1422                 args = states
1423
1424                 if host_id:
1425                         query += " AND builder_id = %s" % host_id
1426
1427                 query += " ORDER BY \
1428                         CASE \
1429                                 WHEN jobs.state = 'running'     THEN 0 \
1430                                 WHEN jobs.state = 'uploading'   THEN 1 \
1431                                 WHEN jobs.state = 'dispatching' THEN 2 \
1432                                 WHEN jobs.state = 'pending'     THEN 3 \
1433                                 WHEN jobs.state = 'new'         THEN 4 \
1434                         END, time_started ASC"
1435
1436                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1437
1438         def get_latest(self, arch=None, builder=None, limit=None, age=None, date=None):
1439                 query = "SELECT * FROM jobs"
1440                 args  = []
1441
1442                 where = ["(state = 'finished' OR state = 'failed' OR state = 'aborted')"]
1443
1444                 if arch:
1445                         where.append("arch = %s")
1446                         args.append(arch)
1447
1448                 if builder:
1449                         where.append("builder_id = %s")
1450                         args.append(builder.id)
1451
1452                 if date:
1453                         try:
1454                                 year, month, day = date.split("-", 2)
1455                                 date = datetime.date(int(year), int(month), int(day))
1456                         except ValueError:
1457                                 pass
1458                         else:
1459                                 where.append("(DATE(time_created) = %s OR \
1460                                         DATE(time_started) = %s OR DATE(time_finished) = %s)")
1461                                 args += (date, date, date)
1462
1463                 if age:
1464                         where.append("time_finished >= NOW() - '%s'::interval" % age)
1465
1466                 if where:
1467                         query += " WHERE %s" % " AND ".join(where)
1468
1469                 query += " ORDER BY time_finished DESC"
1470
1471                 if limit:
1472                         query += " LIMIT %s"
1473                         args.append(limit)
1474
1475                 return [Job(self.pakfire, j.id, j) for j in self.db.query(query, *args)]
1476
1477         def get_average_build_time(self):
1478                 """
1479                         Returns the average build time of all finished builds from the
1480                         last 3 months.
1481                 """
1482                 result = self.db.get("SELECT AVG(time_finished - time_started) as average \
1483                         FROM jobs WHERE type = 'build' AND state = 'finished' AND \
1484                         time_finished >= NOW() - '3 months'::interval")
1485
1486                 if result:
1487                         return result.average
1488
1489         def count(self, *states):
1490                 query = "SELECT COUNT(*) AS count FROM jobs"
1491                 args  = []
1492
1493                 if states:
1494                         query += " WHERE state IN %s"
1495                         args.append(states)
1496
1497                 jobs = self.db.get(query, *args)
1498                 if jobs:
1499                         return jobs.count
1500
1501
1502 class Job(base.DataObject):
1503         table = "jobs"
1504
1505         def __str__(self):
1506                 return "<%s id=%s %s>" % (self.__class__.__name__, self.id, self.name)
1507
1508         def __eq__(self, other):
1509                 if isinstance(other, self.__class__):
1510                         return self.id == other.id
1511
1512         def __lt__(self, other):
1513                 if isinstance(other, self.__class__):
1514                         if (self.type, other.type) == ("build", "test"):
1515                                 return True
1516
1517                         if self.build == other.build:
1518                                 return arches.priority(self.arch) < arches.priority(other.arch)
1519
1520                         return self.time_created < other.time_created
1521
1522         def __iter__(self):
1523                 packages = self.backend.packages._get_packages("SELECT packages.* FROM jobs_packages \
1524                         LEFT JOIN packages ON jobs_packages.pkg_id = packages.id \
1525                         WHERE jobs_packages.job_id = %s ORDER BY packages.name", self.id)
1526
1527                 return iter(packages)
1528
1529         def __nonzero__(self):
1530                 return True
1531
1532         def __len__(self):
1533                 res = self.db.get("SELECT COUNT(*) AS len FROM jobs_packages \
1534                         WHERE job_id = %s", self.id)
1535
1536                 return res.len
1537
1538         @property
1539         def distro(self):
1540                 return self.build.distro
1541
1542         def delete(self):
1543                 self.__delete_buildroots()
1544                 self.__delete_history()
1545                 self.__delete_packages()
1546                 self.__delete_logfiles()
1547
1548                 # Delete the job itself.
1549                 self.db.execute("DELETE FROM jobs WHERE id = %s", self.id)
1550
1551         def __delete_buildroots(self):
1552                 """
1553                         Removes all buildroots.
1554                 """
1555                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s", self.id)
1556
1557         def __delete_history(self):
1558                 """
1559                         Removes all references in the history to this build job.
1560                 """
1561                 self.db.execute("DELETE FROM jobs_history WHERE job_id = %s", self.id)
1562
1563         def __delete_packages(self):
1564                 """
1565                         Deletes all uploaded files from the job.
1566                 """
1567                 for pkg in self.packages:
1568                         pkg.delete()
1569
1570                 self.db.execute("DELETE FROM jobs_packages WHERE job_id = %s", self.id)
1571
1572         def __delete_logfiles(self):
1573                 for logfile in self.logfiles:
1574                         self.db.execute("INSERT INTO queue_delete(path) VALUES(%s)", logfile.path)
1575
1576         def reset(self, user=None):
1577                 self.__delete_buildroots()
1578                 self.__delete_packages()
1579                 self.__delete_history()
1580                 self.__delete_logfiles()
1581
1582                 self.state = "new"
1583                 self.log("reset", user=user)
1584
1585         ## Logging stuff
1586
1587         def log(self, action, user=None, state=None, builder=None, test_job=None):
1588                 user_id = None
1589                 if user:
1590                         user_id = user.id
1591
1592                 builder_id = None
1593                 if builder:
1594                         builder_id = builder.id
1595
1596                 test_job_id = None
1597                 if test_job:
1598                         test_job_id = test_job.id
1599
1600                 self.db.execute("INSERT INTO jobs_history(job_id, action, state, user_id, \
1601                         time, builder_id, test_job_id) VALUES(%s, %s, %s, %s, NOW(), %s, %s)",
1602                         self.id, action, state, user_id, builder_id, test_job_id)
1603
1604         def get_log(self, limit=None, offset=None, user=None):
1605                 query = "SELECT * FROM jobs_history"
1606
1607                 conditions = ["job_id = %s",]
1608                 args  = [self.id,]
1609
1610                 if user:
1611                         conditions.append("user_id = %s")
1612                         args.append(user.id)
1613
1614                 if conditions:
1615                         query += " WHERE %s" % " AND ".join(conditions)
1616
1617                 query += " ORDER BY time DESC"
1618
1619                 if limit:
1620                         if offset:
1621                                 query += " LIMIT %s,%s"
1622                                 args  += [offset, limit,]
1623                         else:
1624                                 query += " LIMIT %s"
1625                                 args  += [limit,]
1626
1627                 entries = []
1628                 for entry in self.db.query(query, *args):
1629                         entry = logs.JobLogEntry(self.pakfire, entry)
1630                         entries.append(entry)
1631
1632                 return entries
1633
1634         @property
1635         def uuid(self):
1636                 return self.data.uuid
1637
1638         @property
1639         def type(self):
1640                 return self.data.type
1641
1642         @property
1643         def build_id(self):
1644                 return self.data.build_id
1645
1646         @lazy_property
1647         def build(self):
1648                 return self.pakfire.builds.get_by_id(self.build_id)
1649
1650         @property
1651         def related_jobs(self):
1652                 ret = []
1653
1654                 for job in self.build.jobs:
1655                         if job == self:
1656                                 continue
1657
1658                         ret.append(job)
1659
1660                 return ret
1661
1662         @property
1663         def pkg(self):
1664                 return self.build.pkg
1665
1666         @property
1667         def name(self):
1668                 return "%s-%s.%s" % (self.pkg.name, self.pkg.friendly_version, self.arch)
1669
1670         @property
1671         def size(self):
1672                 return sum((p.size for p in self.packages))
1673
1674         @lazy_property
1675         def rank(self):
1676                 """
1677                         Returns the rank in the build queue
1678                 """
1679                 if not self.state == "pending":
1680                         return
1681
1682                 res = self.db.get("SELECT rank FROM jobs_queue WHERE job_id = %s", self.id)
1683
1684                 if res:
1685                         return res.rank
1686
1687         def is_running(self):
1688                 """
1689                         Returns True if job is in a running state.
1690                 """
1691                 return self.state in ("pending", "dispatching", "running", "uploading")
1692
1693         def get_state(self):
1694                 return self.data.state
1695
1696         def set_state(self, state, user=None, log=True):
1697                 # Nothing to do if the state remains.
1698                 if not self.state == state:
1699                         self.db.execute("UPDATE jobs SET state = %s WHERE id = %s", state, self.id)
1700
1701                         # Log the event.
1702                         if log and not state == "new":
1703                                 self.log("state_change", state=state, user=user)
1704
1705                         # Update cache.
1706                         if self._data:
1707                                 self._data["state"] = state
1708
1709                 # Always clear the message when the status is changed.
1710                 self.update_message(None)
1711
1712                 # Update some more informations.
1713                 if state == "dispatching":
1714                         # Set start time.
1715                         self.db.execute("UPDATE jobs SET time_started = NOW(), time_finished = NULL \
1716                                 WHERE id = %s", self.id)
1717
1718                 elif state == "pending":
1719                         self.db.execute("UPDATE jobs SET tries = tries + 1, time_started = NULL, \
1720                                 time_finished = NULL WHERE id = %s", self.id)
1721
1722                 elif state in ("aborted", "dependency_error", "finished", "failed"):
1723                         # Set finish time and reset builder..
1724                         self.db.execute("UPDATE jobs SET time_finished = NOW() WHERE id = %s", self.id)
1725
1726                         # Send messages to the user.
1727                         if state == "finished":
1728                                 self.send_finished_message()
1729
1730                         elif state == "failed":
1731                                 # Remove all package files if a job is set to failed state.
1732                                 self.__delete_packages()
1733
1734                                 self.send_failed_message()
1735
1736                 # Automatically update the state of the build (not on test builds).
1737                 if self.type == "build":
1738                         self.build.auto_update_state()
1739
1740         state = property(get_state, set_state)
1741
1742         @property
1743         def message(self):
1744                 return self.data.message
1745
1746         def update_message(self, msg):
1747                 self.db.execute("UPDATE jobs SET message = %s WHERE id = %s",
1748                         msg, self.id)
1749
1750                 if self._data:
1751                         self._data["message"] = msg
1752
1753         def get_builder(self):
1754                 if self.data.builder_id:
1755                         return self.backend.builders.get_by_id(self.data.builder_id)
1756
1757         def set_builder(self, builder, user=None):
1758                 self.db.execute("UPDATE jobs SET builder_id = %s WHERE id = %s",
1759                         builder.id, self.id)
1760
1761                 # Update cache.
1762                 if self._data:
1763                         self._data["builder_id"] = builder.id
1764
1765                 self._builder = builder
1766
1767                 # Log the event.
1768                 if user:
1769                         self.log("builder_assigned", builder=builder, user=user)
1770
1771         builder = lazy_property(get_builder, set_builder)
1772
1773         @property
1774         def arch(self):
1775                 return self.data.arch
1776
1777         @property
1778         def duration(self):
1779                 if not self.time_started:
1780                         return 0
1781
1782                 if self.time_finished:
1783                         delta = self.time_finished - self.time_started
1784                 else:
1785                         delta = datetime.datetime.utcnow() - self.time_started
1786
1787                 return delta.total_seconds()
1788
1789         @property
1790         def time_created(self):
1791                 return self.data.time_created
1792
1793         @property
1794         def time_started(self):
1795                 return self.data.time_started
1796
1797         @property
1798         def time_finished(self):
1799                 return self.data.time_finished
1800
1801         @property
1802         def expected_runtime(self):
1803                 """
1804                         Returns the estimated time and stddev, this job takes to finish.
1805                 """
1806                 # Get the average build time.
1807                 build_times = self.pakfire.builds.get_build_times_by_arch(self.arch,
1808                         name=self.pkg.name)
1809
1810                 # If there is no statistical data, we cannot estimate anything.
1811                 if not build_times:
1812                         return None, None
1813
1814                 return build_times.average, build_times.stddev
1815
1816         @property
1817         def eta(self):
1818                 expected_runtime, stddev = self.expected_runtime
1819
1820                 if expected_runtime:
1821                         return expected_runtime - int(self.duration), stddev
1822
1823         @property
1824         def tries(self):
1825                 return self.data.tries
1826
1827         def get_pkg_by_uuid(self, uuid):
1828                 pkg = self.backend.packages._get_package("SELECT packages.id FROM packages \
1829                         JOIN jobs_packages ON jobs_packages.pkg_id = packages.id \
1830                         WHERE jobs_packages.job_id = %s AND packages.uuid = %s",
1831                         self.id, uuid)
1832
1833                 if pkg:
1834                         pkg.job = self
1835                         return pkg
1836
1837         @lazy_property
1838         def logfiles(self):
1839                 logfiles = []
1840
1841                 for log in self.db.query("SELECT id FROM logfiles WHERE job_id = %s", self.id):
1842                         log = logs.LogFile(self.pakfire, log.id)
1843                         log._job = self
1844
1845                         logfiles.append(log)
1846
1847                 return logfiles
1848
1849         def add_file(self, filename):
1850                 """
1851                         Add the specified file to this job.
1852
1853                         The file is copied to the right directory by this function.
1854                 """
1855                 assert os.path.exists(filename)
1856
1857                 if filename.endswith(".log"):
1858                         self._add_file_log(filename)
1859
1860                 elif filename.endswith(".%s" % PACKAGE_EXTENSION):
1861                         # It is not allowed to upload packages on test builds.
1862                         if self.type == "test":
1863                                 return
1864
1865                         self._add_file_package(filename)
1866
1867         def _add_file_log(self, filename):
1868                 """
1869                         Attach a log file to this job.
1870                 """
1871                 target_dirname = os.path.join(self.build.path, "logs")
1872
1873                 if self.type == "test":
1874                         i = 1
1875                         while True:
1876                                 target_filename = os.path.join(target_dirname,
1877                                         "test.%s.%s.%s.log" % (self.arch, i, self.tries))
1878
1879                                 if os.path.exists(target_filename):
1880                                         i += 1
1881                                 else:
1882                                         break
1883                 else:
1884                         target_filename = os.path.join(target_dirname,
1885                                 "build.%s.%s.log" % (self.arch, self.tries))
1886
1887                 # Make sure the target directory exists.
1888                 if not os.path.exists(target_dirname):
1889                         os.makedirs(target_dirname)
1890
1891                 # Calculate a SHA512 hash from that file.
1892                 f = open(filename, "rb")
1893                 h = hashlib.sha512()
1894                 while True:
1895                         buf = f.read(BUFFER_SIZE)
1896                         if not buf:
1897                                 break
1898
1899                         h.update(buf)
1900                 f.close()
1901
1902                 # Copy the file to the final location.
1903                 shutil.copy2(filename, target_filename)
1904
1905                 # Create an entry in the database.
1906                 self.db.execute("INSERT INTO logfiles(job_id, path, filesize, hash_sha512) \
1907                         VALUES(%s, %s, %s, %s)", self.id, os.path.relpath(target_filename, PACKAGES_DIR),
1908                         os.path.getsize(target_filename), h.hexdigest())
1909
1910         def _add_file_package(self, filename):
1911                 # Open package (creates entry in the database).
1912                 pkg = packages.Package.open(self.pakfire, filename)
1913
1914                 # Move package to the build directory.
1915                 pkg.move(os.path.join(self.build.path, self.arch))
1916
1917                 # Attach the package to this job.
1918                 self.db.execute("INSERT INTO jobs_packages(job_id, pkg_id) VALUES(%s, %s)",
1919                         self.id, pkg.id)
1920
1921         def get_aborted_state(self):
1922                 return self.data.aborted_state
1923
1924         def set_aborted_state(self, state):
1925                 self._set_attribute("aborted_state", state)
1926
1927         aborted_state = property(get_aborted_state, set_aborted_state)
1928
1929         @property
1930         def message_recipients(self):
1931                 l = []
1932
1933                 # Add all people watching the build.
1934                 l += self.build.message_recipients
1935
1936                 # Add the package maintainer on release builds.
1937                 if self.build.type == "release":
1938                         maint = self.pkg.maintainer
1939
1940                         if isinstance(maint, users.User):
1941                                 l.append("%s <%s>" % (maint.realname, maint.email))
1942                         elif maint:
1943                                 l.append(maint)
1944
1945                         # XXX add committer and commit author.
1946
1947                 # Add the owner of the scratch build on scratch builds.
1948                 elif self.build.type == "scratch" and self.build.user:
1949                         l.append("%s <%s>" % \
1950                                 (self.build.user.realname, self.build.user.email))
1951
1952                 return set(l)
1953
1954         def save_buildroot(self, pkgs):
1955                 rows = []
1956
1957                 for pkg_name, pkg_uuid in pkgs:
1958                         rows.append((self.id, self.tries, pkg_uuid, pkg_name))
1959
1960                 # Cleanup old stuff first (for rebuilding packages).
1961                 self.db.execute("DELETE FROM jobs_buildroots WHERE job_id = %s AND tries = %s",
1962                         self.id, self.tries)
1963
1964                 self.db.executemany("INSERT INTO \
1965                         jobs_buildroots(job_id, tries, pkg_uuid, pkg_name) \
1966                         VALUES(%s, %s, %s, %s)", rows)
1967
1968         def has_buildroot(self, tries=None):
1969                 if tries is None:
1970                         tries = self.tries
1971
1972                 res = self.db.get("SELECT COUNT(*) AS num FROM jobs_buildroots \
1973                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s",
1974                         self.id, tries)
1975
1976                 if res:
1977                         return res.num
1978
1979                 return 0
1980
1981         def get_buildroot(self, tries=None):
1982                 if tries is None:
1983                         tries = self.tries
1984
1985                 rows = self.db.query("SELECT * FROM jobs_buildroots \
1986                         WHERE jobs_buildroots.job_id = %s AND jobs_buildroots.tries = %s \
1987                         ORDER BY pkg_name", self.id, tries)
1988
1989                 pkgs = []
1990                 for row in rows:
1991                         # Search for this package in the packages table.
1992                         pkg = self.pakfire.packages.get_by_uuid(row.pkg_uuid)
1993                         pkgs.append((row.pkg_name, row.pkg_uuid, pkg))
1994
1995                 return pkgs
1996
1997         def send_finished_message(self):
1998                 # Send no finished mails for test jobs.
1999                 if self.type == "test":
2000                         return
2001
2002                 logging.debug("Sending finished message for job %s to %s" % \
2003                         (self.name, ", ".join(self.message_recipients)))
2004
2005                 info = {
2006                         "build_name" : self.name,
2007                         "build_host" : self.builder.name,
2008                         "build_uuid" : self.uuid,
2009                 }
2010
2011                 self.pakfire.messages.send_to_all(self.message_recipients,
2012                         MSG_BUILD_FINISHED_SUBJECT, MSG_BUILD_FINISHED, info)
2013
2014         def send_failed_message(self):
2015                 logging.debug("Sending failed message for job %s to %s" % \
2016                         (self.name, ", ".join(self.message_recipients)))
2017
2018                 build_host = "--"
2019                 if self.builder:
2020                         build_host = self.builder.name
2021
2022                 info = {
2023                         "build_name" : self.name,
2024                         "build_host" : build_host,
2025                         "build_uuid" : self.uuid,
2026                 }
2027
2028                 self.pakfire.messages.send_to_all(self.message_recipients,
2029                         MSG_BUILD_FAILED_SUBJECT, MSG_BUILD_FAILED, info)
2030
2031         def set_start_time(self, start_time):
2032                 if start_time is None:
2033                         return
2034
2035                 self.db.execute("UPDATE jobs SET start_not_before = NOW() + %s \
2036                         WHERE id = %s LIMIT 1", start_time, self.id)
2037
2038         def schedule(self, type, start_time=None, user=None):
2039                 assert type in ("rebuild", "test")
2040
2041                 if type == "rebuild":
2042                         if self.state == "finished":
2043                                 return
2044
2045                         self.set_state("new", user=user, log=False)
2046                         self.set_start_time(start_time)
2047
2048                         # Log the event.
2049                         self.log("schedule_rebuild", user=user)
2050
2051                 elif type == "test":
2052                         if not self.state == "finished":
2053                                 return
2054
2055                         # Create a new job with same build and arch.
2056                         job = self.create(self.pakfire, self.build, self.arch, type="test")
2057                         job.set_start_time(start_time)
2058
2059                         # Log the event.
2060                         self.log("schedule_test_job", test_job=job, user=user)
2061
2062                         return job
2063
2064         def schedule_test(self, start_not_before=None, user=None):
2065                 # XXX to be removed
2066                 return self.schedule("test", start_time=start_not_before, user=user)
2067
2068         def schedule_rebuild(self, start_not_before=None, user=None):
2069                 # XXX to be removed
2070                 return self.schedule("rebuild", start_time=start_not_before, user=user)
2071
2072         def get_build_repos(self):
2073                 """
2074                         Returns a list of all repositories that should be used when
2075                         building this job.
2076                 """
2077                 repo_ids = self.db.query("SELECT repo_id FROM jobs_repos WHERE job_id = %s",
2078                         self.id)
2079
2080                 if not repo_ids:
2081                         return self.distro.get_build_repos()
2082
2083                 repos = []
2084                 for repo in self.distro.repositories:
2085                         if repo.id in [r.id for r in repo_ids]:
2086                                 repos.append(repo)
2087
2088                 return repos or self.distro.get_build_repos()
2089
2090         def get_repo_config(self):
2091                 """
2092                         Get repository configuration file that is sent to the builder.
2093                 """
2094                 confs = []
2095
2096                 for repo in self.get_build_repos():
2097                         confs.append(repo.get_conf())
2098
2099                 return "\n\n".join(confs)
2100
2101         def get_config(self):
2102                 """
2103                         Get configuration file that is sent to the builder.
2104                 """
2105                 confs = []
2106
2107                 # Add the distribution configuration.
2108                 confs.append(self.distro.get_config())
2109
2110                 # Then add all repositories for this build.
2111                 confs.append(self.get_repo_config())
2112
2113                 return "\n\n".join(confs)
2114
2115         def resolvdep(self):
2116                 config = pakfire.config.Config(files=["general.conf"])
2117                 config.parse(self.get_config())
2118
2119                 # The filename of the source file.
2120                 filename = os.path.join(PACKAGES_DIR, self.build.pkg.path)
2121                 assert os.path.exists(filename), filename
2122
2123                 # Create a new pakfire instance with the configuration for
2124                 # this build.
2125                 p = pakfire.PakfireServer(config=config, arch=self.arch)
2126
2127                 # Try to solve the build dependencies.
2128                 try:
2129                         solver = p.resolvdep(filename)
2130
2131                 # Catch dependency errors and log the problem string.
2132                 except DependencyError, e:
2133                         self.state = "dependency_error"
2134                         self.update_message(e)
2135
2136                 else:
2137                         # If the build dependencies can be resolved, we set the build in
2138                         # pending state.
2139                         if solver.status is True:
2140                                 if self.state in ("failed",):
2141                                         return
2142
2143                                 self.state = "pending"