src/hub/handlers.py

   1 #!/usr/bin/python
   2
   3 import base64
   4 import hashlib
   5 import json
   6 import logging
   7 import time
   8 import tornado.web
   9
  10 from .. import builds
  11 from .. import builders
  12 from .. import uploads
  13 from .. import users
  14
  15 class LongPollMixin(object):
  16         def initialize(self):
  17                 self._start_time = time.time()
  18
  19         def add_timeout(self, timeout, callback):
  20                 deadline = time.time() + timeout
  21
  22                 return self.application.ioloop.add_timeout(deadline, callback)
  23
  24         def on_connection_close(self):
  25                 logging.debug("Connection closed unexpectedly")
  26
  27         def connection_closed(self):
  28                 return self.request.connection.stream.closed()
  29
  30         @property
  31         def runtime(self):
  32                 return time.time() - self._start_time
  33
  34
  35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
  36         @property
  37         def backend(self):
  38                 """
  39                         Shortcut handler to pakfire instance
  40                 """
  41                 return self.application.backend
  42
  43         def get_basic_auth_credentials(self):
  44                 """
  45                         This handles HTTP Basic authentication.
  46                 """
  47                 auth_header = self.request.headers.get("Authorization", None)
  48
  49                 # If no authentication information was provided, we stop here.
  50                 if not auth_header:
  51                         return None, None
  52
  53                 # No basic auth? We cannot handle that.
  54                 if not auth_header.startswith("Basic "):
  55                         raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
  56
  57                 try:
  58                         # Decode the authentication information.
  59                         auth_header = base64.decodestring(auth_header[6:])
  60
  61                         name, password = auth_header.split(":", 1)
  62                 except:
  63                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
  64
  65                 return name, password
  66
  67         def get_current_user(self):
  68                 name, password = self.get_basic_auth_credentials()
  69                 if name is None:
  70                         return
  71
  72                 builder = self.backend.builders.auth(name, password)
  73                 if builder:
  74                         return builder
  75
  76                 user = self.backend.users.auth(name, password)
  77                 if user:
  78                         return user
  79
  80         @property
  81         def builder(self):
  82                 if isinstance(self.current_user, builders.Builder):
  83                         return self.current_user
  84
  85         @property
  86         def user(self):
  87                 if isinstance(self.current_user, users.User):
  88                         return self.current_user
  89
  90         def get_argument_int(self, *args, **kwargs):
  91                 arg = self.get_argument(*args, **kwargs)
  92
  93                 try:
  94                         return int(arg)
  95                 except (TypeError, ValueError):
  96                         return None
  97
  98         def get_argument_float(self, *args, **kwargs):
  99                 arg = self.get_argument(*args, **kwargs)
 100
 101                 try:
 102                         return float(arg)
 103                 except (TypeError, ValueError):
 104                         return None
 105
 106         def get_argument_json(self, *args, **kwargs):
 107                 arg = self.get_argument(*args, **kwargs)
 108
 109                 if arg:
 110                         return json.loads(arg)
 111
 112
 113 class NoopHandler(BaseHandler):
 114         def get(self):
 115                 if self.builder:
 116                         self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
 117                 elif self.user:
 118                         self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
 119                 else:
 120                         self.write("Welcome to the Pakfire hub!")
 121
 122
 123 class ErrorTestHandler(BaseHandler):
 124         def get(self, error_code=200):
 125                 """
 126                         For testing a client.
 127
 128                         This just returns a HTTP response with the given code.
 129                 """
 130                 try:
 131                         error_code = int(error_code)
 132                 except ValueError:
 133                         error_code = 200
 134
 135                 raise tornado.web.HTTPError(error_code)
 136
 137
 138 class StatsBuildsTypesHandler(BaseHandler):
 139         def get(self):
 140                 ret = {}
 141
 142                 types = self.backend.builds.get_types_stats()
 143                 for type, count in types.items():
 144                         ret["builds_type_%s" % type] = count
 145
 146                 self.write(ret)
 147
 148
 149 class StatsJobsHandler(BaseHandler):
 150         def get(self):
 151                 ret = {}
 152
 153                 self.write(ret)
 154
 155
 156 class StatsJobsDurationsHandler(BaseHandler):
 157         def get(self):
 158                 durations = self.backend.jobs.get_build_durations()
 159
 160                 ret = {}
 161                 for platform, d in durations.items():
 162                         ret.update({
 163                                 "jobs_durations_%s_minimum" % platform : d.get("minimum", None),
 164                                 "jobs_durations_%s_maximum" % platform : d.get("maximum", None),
 165                                 "jobs_durations_%s_average" % platform : d.get("average", None),
 166                                 "jobs_durations_%s_stddev"  % platform : d.get("stddev", None),
 167                         })
 168
 169                 self.write(ret)
 170
 171 class StatsJobsStatesHandler(BaseHandler):
 172         def get(self):
 173                 ret = {}
 174
 175                 states = self.backend.jobs.get_state_stats()
 176                 for state, count in states.items():
 177                         ret["jobs_state_%s" % state] = count
 178
 179                 self.write(ret)
 180
 181
 182 class StatsJobsQueueHandler(BaseHandler):
 183         def get(self):
 184                 ret = {}
 185
 186                 # Queue length(s).
 187                 ret["job_queue_length"] = self.backend.jobs.get_queue_length()
 188                 for state in ("new", "pending"):
 189                         ret["job_queue_length_%s" % state] = self.backend.jobs.get_queue_length(state)
 190
 191                 # Average waiting time.
 192                 ret["job_queue_avg_wait_time"] = self.backend.jobs.get_avg_wait_time()
 193
 194                 self.write(ret)
 195
 196
 197 # Uploads
 198
 199 class UploadsCreateHandler(BaseHandler):
 200         """
 201                 Create a new upload object in the database and return a unique ID
 202                 to the uploader.
 203         """
 204
 205         @tornado.web.authenticated
 206         def get(self):
 207                 # XXX Check permissions
 208
 209                 filename = self.get_argument("filename")
 210                 filesize = self.get_argument_int("filesize")
 211                 filehash = self.get_argument("hash")
 212
 213                 upload = uploads.Upload.create(self.backend, filename, filesize,
 214                         filehash, user=self.user, builder=self.builder)
 215
 216                 self.finish(upload.uuid)
 217
 218
 219 class UploadsSendChunkHandler(BaseHandler):
 220         @tornado.web.authenticated
 221         def post(self, upload_id):
 222                 upload = self.backend.uploads.get_by_uuid(upload_id)
 223                 if not upload:
 224                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 225
 226                 if not upload.builder == self.builder:
 227                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 228
 229                 chksum = self.get_argument("chksum")
 230                 data = self.get_argument("data")
 231
 232                 # Decode data.
 233                 data = base64.b64decode(data)
 234
 235                 # Calculate hash and compare.
 236                 h = hashlib.new("sha512")
 237                 h.update(data)
 238
 239                 if not chksum == h.hexdigest():
 240                         raise tornado.web.HTTPError(400, "Checksum mismatch")
 241
 242                 # Append the data to file.
 243                 upload.append(data)
 244
 245
 246 class UploadsFinishedHandler(BaseHandler):
 247         @tornado.web.authenticated
 248         def get(self, upload_id):
 249                 upload = self.backend.uploads.get_by_uuid(upload_id)
 250                 if not upload:
 251                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 252
 253                 if not upload.builder == self.builder:
 254                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 255
 256                 # Validate the uploaded data to its hash.
 257                 ret = upload.validate()
 258
 259                 # If the validation was successfull, we mark the upload
 260                 # as finished and send True to the client.
 261                 if ret:
 262                         upload.finished()
 263                         self.finish("OK")
 264
 265                         return
 266
 267                 # In case the download was corrupted or incomplete, we delete it
 268                 # and tell the client to start over.
 269                 upload.remove()
 270
 271                 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
 272
 273
 274 class UploadsDestroyHandler(BaseHandler):
 275         @tornado.web.authenticated
 276         def get(self, upload_id):
 277                 upload = self.backend.uploads.get_by_uuid(upload_id)
 278                 if not upload:
 279                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 280
 281                 if not upload.builder == self.builder:
 282                         raise tornado.web.HTTPError(403, "Removing an other host's file.")
 283
 284                 # Remove the upload from the database and trash the data.
 285                 upload.remove()
 286
 287
 288 # Builds
 289
 290 class BuildsCreateHandler(BaseHandler):
 291         @tornado.web.authenticated
 292         def get(self):
 293                 # Get the upload ID of the package file.
 294                 upload_id = self.get_argument("upload_id")
 295
 296                 # Get the identifier of the distribution we build for.
 297                 distro_ident = self.get_argument("distro")
 298
 299                 # Get a list of arches to build for.
 300                 arches = self.get_argument("arches", None)
 301                 if arches == "":
 302                         arches = None
 303
 304                 # Process build type.
 305                 build_type = self.get_argument("build_type")
 306                 if build_type == "release":
 307                         check_for_duplicates = True
 308                 elif build_type == "scratch":
 309                         check_for_duplicates = False
 310                 else:
 311                         raise tornado.web.HTTPError(400, "Invalid build type")
 312
 313                 ## Check if the user has permission to create a build.
 314                 # Users only have the permission to create scratch builds.
 315                 if self.user and not build_type == "scratch":
 316                         raise tornado.web.HTTPError(403, "Users are only allowed to upload scratch builds")
 317
 318                 # Get previously uploaded file to create this build from.
 319                 upload = self.backend.uploads.get_by_uuid(upload_id)
 320                 if not upload:
 321                         raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
 322
 323                 # Check if the uploaded file belongs to this user/builder.
 324                 if self.user and not upload.user == self.user:
 325                         raise tornado.web.HTTPError(400, "Upload does not belong to this user.")
 326
 327                 elif self.builder and not upload.builder == self.builder:
 328                         raise tornado.web.HTTPError(400, "Upload does not belong to this builder.")
 329
 330                 # Get distribution this package should be built for.
 331                 distro = self.backend.distros.get_by_ident(distro_ident)
 332                 if not distro:
 333                         distro = self.backend.distros.get_default()
 334
 335                 # Open the package that was uploaded earlier and add it to
 336                 # the database. Create a new build object from the uploaded package.
 337                 args = {
 338                         "arches"               : arches,
 339                         "check_for_duplicates" : check_for_duplicates,
 340                         "distro"               : distro,
 341                         "type"                 : build_type,
 342                 }
 343                 if self.user:
 344                         args["owner"] = self.user
 345
 346                 try:
 347                         pkg, build = builds.import_from_package(self.backend, upload.path, **args)
 348
 349                 except:
 350                         # Raise any exception.
 351                         raise
 352
 353                 else:
 354                         # Creating the build will move the file to the build directory,
 355                         # so we can safely remove the uploaded file.
 356                         upload.remove()
 357
 358                 # Send the build ID back to the user.
 359                 self.finish(build.uuid)
 360
 361
 362 class BuildsGetHandler(BaseHandler):
 363         def get(self, build_uuid):
 364                 build = self.backend.builds.get_by_uuid(build_uuid)
 365                 if not build:
 366                         raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
 367
 368                 ret = {
 369                         "distro"       : build.distro.identifier,
 370                         "jobs"         : [j.uuid for j in build.jobs],
 371                         "name"         : build.name,
 372                         "package"      : build.pkg.uuid,
 373                         "priority"     : build.priority,
 374                         "score"        : build.credits,
 375                         "severity"     : build.severity,
 376                         "state"        : build.state,
 377                         "sup_arches"   : build.supported_arches,
 378                         "time_created" : build.created.isoformat(),
 379                         "type"         : build.type,
 380                         "uuid"         : build.uuid,
 381                 }
 382
 383                 # If the build is in a repository, update that bit.
 384                 if build.repo:
 385                         ret["repo"] = build.repo.identifier
 386
 387                 self.finish(ret)
 388
 389
 390 # Jobs
 391
 392 class JobsBaseHandler(BaseHandler):
 393         def job2json(self, job):
 394                 ret = {
 395                         "arch"         : job.arch.name,
 396                         "build"        : job.build.uuid,
 397                         "duration"     : job.duration,
 398                         "name"         : job.name,
 399                         "packages"     : [p.uuid for p in job.packages],
 400                         "state"        : job.state,
 401                         "time_created" : job.time_created.isoformat(),
 402                         "type"         : job.type,
 403                         "uuid"         : job.uuid,
 404                 }
 405
 406                 if job.builder:
 407                         ret["builder"] = job.builder.hostname
 408
 409                 if job.time_started:
 410                         ret["time_started"] = job.time_started.isoformat()
 411
 412                 if job.time_finished:
 413                         ret["time_finished"] = job.time_finished.isoformat()
 414
 415                 return ret
 416
 417
 418 class JobsGetActiveHandler(JobsBaseHandler):
 419         def get(self):
 420                 # Get list of all active jobs.
 421                 jobs = self.backend.jobs.get_active()
 422
 423                 args = {
 424                         "jobs" : [self.job2json(j) for j in jobs],
 425                 }
 426
 427                 self.finish(args)
 428
 429
 430 class JobsGetLatestHandler(JobsBaseHandler):
 431         def get(self):
 432                 limit = self.get_argument_int("limit", 5)
 433
 434                 # Get the latest jobs.
 435                 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
 436
 437                 args = {
 438                         "jobs" : [self.job2json(j) for j in jobs],
 439                 }
 440
 441                 self.finish(args)
 442
 443
 444 class JobsGetQueueHandler(JobsBaseHandler):
 445         def get(self):
 446                 limit = self.get_argument_int("limit", 5)
 447
 448                 # Get the job queue.
 449                 jobs = self.backend.jobs.get_next(limit=limit)
 450
 451                 args = {
 452                         "jobs" : [self.job2json(j) for j in jobs],
 453                 }
 454
 455                 self.finish(args)
 456
 457
 458 class JobsGetHandler(JobsBaseHandler):
 459         def get(self, job_uuid):
 460                 job = self.backend.jobs.get_by_uuid(job_uuid)
 461                 if not job:
 462                         raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
 463
 464                 # Check if user is allowed to view this job.
 465                 if job.build.public == False:
 466                         if not self.user:
 467                                 raise tornado.web.HTTPError(401)
 468
 469                         # Check if an authenticated user has permission to see this build.
 470                         if not job.build.has_perm(self.user):
 471                                 raise tornado.web.HTTPError(403)
 472
 473                 ret = self.job2json(job)
 474                 self.finish(ret)
 475
 476
 477 # Packages
 478
 479 class PackagesGetHandler(BaseHandler):
 480         def get(self, package_uuid):
 481                 pkg = self.backend.packages.get_by_uuid(package_uuid)
 482                 if not pkg:
 483                         raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
 484
 485                 ret = {
 486                         "arch"             : pkg.arch.name,
 487                         "build_id"         : pkg.build_id,
 488                         "build_host"       : pkg.build_host,
 489                         "build_time"       : pkg.build_time.isoformat(),
 490                         "description"      : pkg.description,
 491                         "epoch"            : pkg.epoch,
 492                         "filesize"         : pkg.filesize,
 493                         "friendly_name"    : pkg.friendly_name,
 494                         "friendly_version" : pkg.friendly_version,
 495                         "groups"           : pkg.groups,
 496                         "hash_sha512"      : pkg.hash_sha512,
 497                         "license"          : pkg.license,
 498                         "name"             : pkg.name,
 499                         "release"          : pkg.release,
 500                         "size"             : pkg.size,
 501                         "summary"          : pkg.summary,
 502                         "type"             : pkg.type,
 503                         "url"              : pkg.url,
 504                         "uuid"             : pkg.uuid,
 505                         "version"          : pkg.version,
 506
 507                         # Dependencies.
 508                         "prerequires"      : pkg.prerequires,
 509                         "requires"         : pkg.requires,
 510                         "provides"         : pkg.provides,
 511                         "obsoletes"        : pkg.obsoletes,
 512                         "conflicts"        : pkg.conflicts,
 513                 }
 514
 515                 if pkg.type == "source":
 516                         ret["supported_arches"] = pkg.supported_arches
 517
 518                 if isinstance(pkg.maintainer, users.User):
 519                         ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
 520                 elif pkg.maintainer:
 521                         ret["maintainer"] = pkg.maintainer
 522
 523                 if pkg.distro:
 524                         ret["distro"] = pkg.distro.identifier
 525
 526                 self.finish(ret)
 527
 528
 529 # Builders
 530
 531 class BuildersBaseHandler(BaseHandler):
 532         def prepare(self):
 533                 # The request must come from an authenticated buider.
 534                 if not self.builder:
 535                         raise tornado.web.HTTPError(403)
 536
 537
 538 class BuildersInfoHandler(BuildersBaseHandler):
 539         @tornado.web.authenticated
 540         def post(self):
 541                 args = {
 542                         # CPU info
 543                         "cpu_model"    : self.get_argument("cpu_model", None),
 544                         "cpu_count"    : self.get_argument("cpu_count", None),
 545                         "cpu_arch"     : self.get_argument("cpu_arch", None),
 546                         "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
 547
 548                         # Pakfire
 549                         "pakfire_version" : self.get_argument("pakfire_version", None),
 550                         "host_key"     : self.get_argument("host_key", None),
 551
 552                         # OS
 553                         "os_name"      : self.get_argument("os_name", None),
 554                 }
 555                 self.builder.update_info(**args)
 556
 557
 558 class BuildersKeepaliveHandler(BuildersBaseHandler):
 559         @tornado.web.authenticated
 560         def post(self):
 561                 args = {
 562                         # Load average
 563                         "loadavg1"   : self.get_argument_float("loadavg1", None),
 564                         "loadavg5"   : self.get_argument_float("loadavg5", None),
 565                         "loadavg15"  : self.get_argument_float("loadavg15", None),
 566
 567                         # Memory
 568                         "mem_total"  : self.get_argument_int("mem_total", None),
 569                         "mem_free"   : self.get_argument_int("mem_free", None),
 570
 571                         # swap
 572                         "swap_total" : self.get_argument_int("swap_total", None),
 573                         "swap_free"  : self.get_argument_int("swap_free", None),
 574
 575                         # Disk space
 576                         "space_free" : self.get_argument_int("space_free", None),
 577                 }
 578                 self.builder.update_keepalive(**args)
 579
 580                 self.finish("OK")
 581
 582
 583 class BuildersJobsQueueHandler(BuildersBaseHandler):
 584         @tornado.web.asynchronous
 585         @tornado.web.authenticated
 586         def get(self):
 587                 self.callback()
 588
 589         def callback(self):
 590                 # Break if the connection has been closed in the mean time.
 591                 if self.connection_closed():
 592                         logging.warning("Connection closed")
 593                         return
 594
 595                 # Check if there is a job for us.
 596                 job = self.builder.get_next_job()
 597
 598                 # Got no job, wait and try again.
 599                 if not job:
 600                         # Check if we have been running for too long.
 601                         if self.runtime >= self.max_runtime:
 602                                 logging.debug("Exceeded max. runtime. Finishing request.")
 603                                 return self.finish()
 604
 605                         # Try again in a jiffy.
 606                         self.add_timeout(self.heartbeat, self.callback)
 607                         return
 608
 609                 try:
 610                         # Set job to dispatching state.
 611                         job.state = "dispatching"
 612
 613                         # Set our build host.
 614                         job.builder = self.builder
 615
 616                         ret = {
 617                                 "id"                 : job.uuid,
 618                                 "arch"               : job.arch.name,
 619                                 "source_url"         : job.build.source_download,
 620                                 "source_hash_sha512" : job.build.source_hash_sha512,
 621                                 "type"               : job.type,
 622                                 "config"             : job.get_config(),
 623                         }
 624
 625                         # Send build information to the builder.
 626                         self.finish(ret)
 627                 except:
 628                         # If anything went wrong, we reset the state.
 629                         job.state = "pending"
 630                         raise
 631
 632         @property
 633         def heartbeat(self):
 634                 return 15 # 15 seconds
 635
 636         @property
 637         def max_runtime(self):
 638                 timeout = self.get_argument_int("timeout", None)
 639                 if timeout:
 640                         return timeout - self.heartbeat
 641
 642                 return 300 # 5 min
 643
 644
 645 class BuildersJobsStateHandler(BuildersBaseHandler):
 646         @tornado.web.authenticated
 647         def post(self, job_uuid, state):
 648                 job = self.backend.jobs.get_by_uuid(job_uuid)
 649                 if not job:
 650                         raise tornado.web.HTTPError(404, "Invalid job id.")
 651
 652                 if not job.builder == self.builder:
 653                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 654
 655                 # Save information to database.
 656                 job.state = state
 657
 658                 message = self.get_argument("message", None)
 659                 job.update_message(message)
 660
 661                 self.finish("OK")
 662
 663
 664 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
 665         @tornado.web.authenticated
 666         def post(self, job_uuid):
 667                 job = self.backend.jobs.get_by_uuid(job_uuid)
 668                 if not job:
 669                         raise tornado.web.HTTPError(404, "Invalid job id.")
 670
 671                 if not job.builder == self.builder:
 672                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 673
 674                 # Get buildroot.
 675                 buildroot = self.get_argument_json("buildroot", None)
 676                 if buildroot:
 677                         job.save_buildroot(buildroot)
 678
 679                 self.finish("OK")
 680
 681
 682 class BuildersJobsAddFileHandler(BuildersBaseHandler):
 683         @tornado.web.authenticated
 684         def post(self, job_uuid, upload_id):
 685                 type = self.get_argument("type")
 686                 assert type in ("package", "log")
 687
 688                 # Fetch job we are working on and check if it is actually ours.
 689                 job = self.backend.jobs.get_by_uuid(job_uuid)
 690                 if not job:
 691                         raise tornado.web.HTTPError(404, "Invalid job id.")
 692
 693                 if not job.builder == self.builder:
 694                         raise tornado.web.HTTPError(403, "Altering another builder's job.")
 695
 696                 # Fetch uploaded file object and check we uploaded it ourself.
 697                 upload = self.backend.uploads.get_by_uuid(upload_id)
 698                 if not upload:
 699                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 700
 701                 if not upload.builder == self.builder:
 702                         raise tornado.web.HTTPError(403, "Using an other host's file.")
 703
 704                 # Remove all files that have to be deleted, first.
 705                 self.backend.cleanup_files()
 706
 707                 try:
 708                         job.add_file(upload.path)
 709
 710                 finally:
 711                         # Finally, remove the uploaded file.
 712                         upload.remove()
 713
 714                 self.finish("OK")