src/hub/handlers.py

   1 #!/usr/bin/python
   2
   3 import base64
   4 import hashlib
   5 import json
   6 import logging
   7 import time
   8 import tornado.web
   9
  10 from .. import builds
  11 from .. import builders
  12 from .. import uploads
  13 from .. import users
  14
  15 class LongPollMixin(object):
  16         def initialize(self):
  17                 self._start_time = time.time()
  18
  19         def add_timeout(self, timeout, callback):
  20                 deadline = time.time() + timeout
  21
  22                 return self.application.ioloop.add_timeout(deadline, callback)
  23
  24         def on_connection_close(self):
  25                 logging.debug("Connection closed unexpectedly")
  26
  27         def connection_closed(self):
  28                 return self.request.connection.stream.closed()
  29
  30         @property
  31         def runtime(self):
  32                 return time.time() - self._start_time
  33
  34
  35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
  36         @property
  37         def backend(self):
  38                 """
  39                         Shortcut handler to pakfire instance
  40                 """
  41                 return self.application.backend
  42
  43         @property
  44         def db(self):
  45                 return self.backend.db
  46
  47         def get_basic_auth_credentials(self):
  48                 """
  49                         This handles HTTP Basic authentication.
  50                 """
  51                 auth_header = self.request.headers.get("Authorization", None)
  52
  53                 # If no authentication information was provided, we stop here.
  54                 if not auth_header:
  55                         return None, None
  56
  57                 # No basic auth? We cannot handle that.
  58                 if not auth_header.startswith("Basic "):
  59                         raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
  60
  61                 try:
  62                         # Decode the authentication information.
  63                         auth_header = base64.decodestring(auth_header[6:])
  64
  65                         name, password = auth_header.split(":", 1)
  66                 except:
  67                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
  68
  69                 return name, password
  70
  71         def get_current_user(self):
  72                 name, password = self.get_basic_auth_credentials()
  73                 if name is None:
  74                         return
  75
  76                 builder = self.backend.builders.auth(name, password)
  77                 if builder:
  78                         return builder
  79
  80                 user = self.backend.users.auth(name, password)
  81                 if user:
  82                         return user
  83
  84         @property
  85         def builder(self):
  86                 if isinstance(self.current_user, builders.Builder):
  87                         return self.current_user
  88
  89         @property
  90         def user(self):
  91                 if isinstance(self.current_user, users.User):
  92                         return self.current_user
  93
  94         def get_argument_int(self, *args, **kwargs):
  95                 arg = self.get_argument(*args, **kwargs)
  96
  97                 try:
  98                         return int(arg)
  99                 except (TypeError, ValueError):
 100                         return None
 101
 102         def get_argument_float(self, *args, **kwargs):
 103                 arg = self.get_argument(*args, **kwargs)
 104
 105                 try:
 106                         return float(arg)
 107                 except (TypeError, ValueError):
 108                         return None
 109
 110         def get_argument_json(self, *args, **kwargs):
 111                 arg = self.get_argument(*args, **kwargs)
 112
 113                 if arg:
 114                         return json.loads(arg)
 115
 116
 117 class NoopHandler(BaseHandler):
 118         def get(self):
 119                 if self.builder:
 120                         self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
 121                 elif self.user:
 122                         self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
 123                 else:
 124                         self.write("Welcome to the Pakfire hub!")
 125
 126
 127 class ErrorTestHandler(BaseHandler):
 128         def get(self, error_code=200):
 129                 """
 130                         For testing a client.
 131
 132                         This just returns a HTTP response with the given code.
 133                 """
 134                 try:
 135                         error_code = int(error_code)
 136                 except ValueError:
 137                         error_code = 200
 138
 139                 raise tornado.web.HTTPError(error_code)
 140
 141
 142 # Uploads
 143
 144 class UploadsCreateHandler(BaseHandler):
 145         """
 146                 Create a new upload object in the database and return a unique ID
 147                 to the uploader.
 148         """
 149
 150         @tornado.web.authenticated
 151         def get(self):
 152                 # XXX Check permissions
 153
 154                 filename = self.get_argument("filename")
 155                 filesize = self.get_argument_int("filesize")
 156                 filehash = self.get_argument("hash", None)
 157
 158                 with self.db.transaction():
 159                         upload = self.backend.uploads.create(filename, filesize,
 160                                 filehash, user=self.user, builder=self.builder)
 161
 162                         self.finish(upload.uuid)
 163
 164
 165 @tornado.web.stream_request_body
 166 class UploadsStreamHandler(BaseHandler):
 167         @tornado.web.authenticated
 168         def prepare(self):
 169                 # Received file size
 170                 self.size = 0
 171
 172                 upload_uuid = self.get_argument("id")
 173
 174                 # Fetch upload object from database
 175                 self.upload = self.backend.uploads.get_by_uuid(upload_uuid)
 176                 if not self.upload:
 177                         raise tornado.web.HTTPError(404)
 178
 179         def data_received(self, data):
 180                 logging.debug("Received chunk of %s bytes" % len(data))
 181                 self.size += len(data)
 182
 183                 # Write the received chunk to disk
 184                 with self.db.transaction():
 185                         self.upload.append(data)
 186
 187         def put(self):
 188                 logging.info("Received entire file (%s bytes)" % self.size)
 189
 190                 with self.db.transaction():
 191                         self.upload.finished()
 192
 193                 self.finish("OK")
 194
 195
 196 class UploadsSendChunkHandler(BaseHandler):
 197         @tornado.web.authenticated
 198         def post(self, upload_id):
 199                 upload = self.backend.uploads.get_by_uuid(upload_id)
 200                 if not upload:
 201                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 202
 203                 if not upload.builder == self.builder:
 204                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 205
 206                 chksum = self.get_argument("chksum")
 207                 data = self.get_argument("data")
 208
 209                 # Decode data.
 210                 data = base64.b64decode(data)
 211
 212                 # Calculate hash and compare.
 213                 h = hashlib.new("sha512")
 214                 h.update(data)
 215
 216                 if not chksum == h.hexdigest():
 217                         raise tornado.web.HTTPError(400, "Checksum mismatch")
 218
 219                 # Append the data to file.
 220                 with self.db.transaction():
 221                         upload.append(data)
 222
 223
 224 class UploadsFinishedHandler(BaseHandler):
 225         @tornado.web.authenticated
 226         def get(self, upload_id):
 227                 upload = self.backend.uploads.get_by_uuid(upload_id)
 228                 if not upload:
 229                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 230
 231                 if not upload.builder == self.builder:
 232                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 233
 234                 # Validate the uploaded data to its hash.
 235                 ret = upload.validate()
 236
 237                 # If the validation was successfull, we mark the upload
 238                 # as finished and send True to the client.
 239                 if ret:
 240                         upload.finished()
 241                         self.finish("OK")
 242
 243                         return
 244
 245                 # In case the download was corrupted or incomplete, we delete it
 246                 # and tell the client to start over.
 247                 with self.db.transaction():
 248                         upload.remove()
 249
 250                 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
 251
 252
 253 class UploadsDestroyHandler(BaseHandler):
 254         @tornado.web.authenticated
 255         def get(self, upload_id):
 256                 upload = self.backend.uploads.get_by_uuid(upload_id)
 257                 if not upload:
 258                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 259
 260                 if not upload.builder == self.builder:
 261                         raise tornado.web.HTTPError(403, "Removing an other host's file.")
 262
 263                 # Remove the upload from the database and trash the data.
 264                 with self.db.transaction():
 265                         upload.remove()
 266
 267
 268 # Builds
 269
 270 class BuildsCreateHandler(BaseHandler):
 271         @tornado.web.authenticated
 272         def get(self):
 273                 # Get the upload ID of the package file.
 274                 upload_id = self.get_argument("upload_id")
 275
 276                 # Get the identifier of the distribution we build for.
 277                 distro_ident = self.get_argument("distro")
 278
 279                 # Get a list of arches to build for.
 280                 arches = self.get_argument("arches", None)
 281                 if arches == "":
 282                         arches = None
 283
 284                 # Get previously uploaded file to create this build from.
 285                 upload = self.backend.uploads.get_by_uuid(upload_id)
 286                 if not upload:
 287                         raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
 288
 289                 # Check if the uploaded file belongs to this user/builder.
 290                 if self.user and not upload.user == self.user:
 291                         raise tornado.web.HTTPError(400, "Upload does not belong to this user")
 292
 293                 elif self.builder and not upload.builder == self.builder:
 294                         raise tornado.web.HTTPError(400, "Upload does not belong to this builder")
 295
 296                 # Get distribution this package should be built for.
 297                 distro = self.backend.distros.get_by_ident(distro_ident)
 298                 if not distro:
 299                         distro = self.backend.distros.get_default()
 300
 301                 # Open the package that was uploaded earlier and add it to
 302                 # the database. Create a new build object from the uploaded package.
 303                 try:
 304                         build = self.backend.builds.create_from_source_package(upload.path, distro=distro,
 305                                 type="scratch", arches=arches, owner=self.user)
 306
 307                 except:
 308                         # Raise any exception.
 309                         raise
 310
 311                 else:
 312                         # Creating the build will move the file to the build directory,
 313                         # so we can safely remove the uploaded file.
 314                         upload.remove()
 315
 316                 # Send the build ID back to the user.
 317                 self.finish(build.uuid)
 318
 319
 320 class BuildsGetHandler(BaseHandler):
 321         def get(self, build_uuid):
 322                 build = self.backend.builds.get_by_uuid(build_uuid)
 323                 if not build:
 324                         raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
 325
 326                 ret = {
 327                         "distro"       : build.distro.identifier,
 328                         "jobs"         : [j.uuid for j in build.jobs],
 329                         "name"         : build.name,
 330                         "package"      : build.pkg.uuid,
 331                         "priority"     : build.priority,
 332                         "score"        : build.score,
 333                         "severity"     : build.severity,
 334                         "state"        : build.state,
 335                         "sup_arches"   : build.supported_arches,
 336                         "time_created" : build.created.isoformat(),
 337                         "type"         : build.type,
 338                         "uuid"         : build.uuid,
 339                 }
 340
 341                 # If the build is in a repository, update that bit.
 342                 if build.repo:
 343                         ret["repo"] = build.repo.identifier
 344
 345                 self.finish(ret)
 346
 347
 348 # Jobs
 349
 350 class JobsBaseHandler(BaseHandler):
 351         def job2json(self, job):
 352                 ret = {
 353                         "arch"         : job.arch,
 354                         "build"        : job.build.uuid,
 355                         "duration"     : job.duration,
 356                         "name"         : job.name,
 357                         "packages"     : [p.uuid for p in job.packages],
 358                         "state"        : job.state,
 359                         "time_created" : job.time_created.isoformat(),
 360                         "type"         : "test" if job.test else "release",
 361                         "uuid"         : job.uuid,
 362                 }
 363
 364                 if job.builder:
 365                         ret["builder"] = job.builder.hostname
 366
 367                 if job.time_started:
 368                         ret["time_started"] = job.time_started.isoformat()
 369
 370                 if job.time_finished:
 371                         ret["time_finished"] = job.time_finished.isoformat()
 372
 373                 return ret
 374
 375
 376 class JobsGetActiveHandler(JobsBaseHandler):
 377         def get(self):
 378                 # Get list of all active jobs.
 379                 jobs = self.backend.jobs.get_active()
 380
 381                 args = {
 382                         "jobs" : [self.job2json(j) for j in jobs],
 383                 }
 384
 385                 self.finish(args)
 386
 387
 388 class JobsGetLatestHandler(JobsBaseHandler):
 389         def get(self):
 390                 limit = self.get_argument_int("limit", 5)
 391
 392                 # Get the latest jobs.
 393                 jobs = self.backend.jobs.get_recently_ended(limit=limit)
 394
 395                 args = {
 396                         "jobs" : [self.job2json(j) for j in jobs],
 397                 }
 398
 399                 self.finish(args)
 400
 401
 402 class JobsGetQueueHandler(JobsBaseHandler):
 403         def get(self):
 404                 limit = self.get_argument_int("limit", 5)
 405
 406                 # Get the job queue.
 407                 jobs = []
 408                 for job in self.backend.jobqueue:
 409                         jobs.append(job)
 410
 411                         limit -= 1
 412                         if not limit: break
 413
 414                 args = {
 415                         "jobs" : [self.job2json(j) for j in jobs],
 416                 }
 417
 418                 self.finish(args)
 419
 420
 421 class JobsGetHandler(JobsBaseHandler):
 422         def get(self, job_uuid):
 423                 job = self.backend.jobs.get_by_uuid(job_uuid)
 424                 if not job:
 425                         raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
 426
 427                 ret = self.job2json(job)
 428                 self.finish(ret)
 429
 430
 431 # Packages
 432
 433 class PackagesGetHandler(BaseHandler):
 434         def get(self, package_uuid):
 435                 pkg = self.backend.packages.get_by_uuid(package_uuid)
 436                 if not pkg:
 437                         raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
 438
 439                 ret = {
 440                         "arch"             : pkg.arch,
 441                         "build_id"         : pkg.build_id,
 442                         "build_host"       : pkg.build_host,
 443                         "build_time"       : pkg.build_time.isoformat(),
 444                         "description"      : pkg.description,
 445                         "epoch"            : pkg.epoch,
 446                         "filesize"         : pkg.filesize,
 447                         "friendly_name"    : pkg.friendly_name,
 448                         "friendly_version" : pkg.friendly_version,
 449                         "groups"           : pkg.groups,
 450                         "hash_sha512"      : pkg.hash_sha512,
 451                         "license"          : pkg.license,
 452                         "name"             : pkg.name,
 453                         "release"          : pkg.release,
 454                         "size"             : pkg.size,
 455                         "summary"          : pkg.summary,
 456                         "type"             : pkg.type,
 457                         "url"              : pkg.url,
 458                         "uuid"             : pkg.uuid,
 459                         "version"          : pkg.version,
 460
 461                         # Dependencies.
 462                         "prerequires"      : pkg.prerequires,
 463                         "requires"         : pkg.requires,
 464                         "provides"         : pkg.provides,
 465                         "obsoletes"        : pkg.obsoletes,
 466                         "conflicts"        : pkg.conflicts,
 467                 }
 468
 469                 if pkg.type == "source":
 470                         ret["supported_arches"] = pkg.supported_arches
 471
 472                 if isinstance(pkg.maintainer, users.User):
 473                         ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
 474                 elif pkg.maintainer:
 475                         ret["maintainer"] = pkg.maintainer
 476
 477                 if pkg.distro:
 478                         ret["distro"] = pkg.distro.identifier
 479
 480                 self.finish(ret)
 481
 482
 483 # Builders
 484
 485 class BuildersBaseHandler(BaseHandler):
 486         def prepare(self):
 487                 # The request must come from an authenticated buider.
 488                 if not self.builder:
 489                         raise tornado.web.HTTPError(403)
 490
 491
 492 class BuildersInfoHandler(BuildersBaseHandler):
 493         @tornado.web.authenticated
 494         def post(self):
 495                 args = {
 496                         # CPU info
 497                         "cpu_model"    : self.get_argument("cpu_model", None),
 498                         "cpu_count"    : self.get_argument("cpu_count", None),
 499                         "cpu_arch"     : self.get_argument("cpu_arch", None),
 500                         "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
 501
 502                         # Pakfire
 503                         "pakfire_version" : self.get_argument("pakfire_version", None),
 504                         "host_key"     : self.get_argument("host_key", None),
 505
 506                         # OS
 507                         "os_name"      : self.get_argument("os_name", None),
 508                 }
 509                 self.builder.update_info(**args)
 510
 511
 512 class BuildersKeepaliveHandler(BuildersBaseHandler):
 513         @tornado.web.authenticated
 514         def post(self):
 515                 args = {
 516                         # Load average
 517                         "loadavg1"   : self.get_argument_float("loadavg1", None),
 518                         "loadavg5"   : self.get_argument_float("loadavg5", None),
 519                         "loadavg15"  : self.get_argument_float("loadavg15", None),
 520
 521                         # Memory
 522                         "mem_total"  : self.get_argument_int("mem_total", None),
 523                         "mem_free"   : self.get_argument_int("mem_free", None),
 524
 525                         # swap
 526                         "swap_total" : self.get_argument_int("swap_total", None),
 527                         "swap_free"  : self.get_argument_int("swap_free", None),
 528
 529                         # Disk space
 530                         "space_free" : self.get_argument_int("space_free", None),
 531                 }
 532                 self.builder.update_keepalive(**args)
 533
 534                 self.finish("OK")
 535
 536
 537 class BuildersGetNextJobHandler(BuildersBaseHandler):
 538         @tornado.web.authenticated
 539         def get(self):
 540                 # XXX Set keepalive
 541
 542                 # If the builder is disabled, we don't need to do anything
 543                 # but will ask it to return after 5 min
 544                 if not self.builder.enabled:
 545                         self.set_header("Retry-After", "300")
 546                         return
 547
 548                 # If the builder has too many jobs running,
 549                 # we will tell it to return after 1 min
 550                 if self.builder.too_many_jobs:
 551                         self.set_header("Retry-After", "60")
 552                         return
 553
 554                 # Okay, we are ready for the next job
 555                 job = self.builder.get_next_job()
 556
 557                 # If we got no job, we will ask the builder
 558                 # to return after 30 seconds
 559                 if not job:
 560                         self.set_header("Retry-After", "30")
 561                         return
 562
 563                 # If we got a job, we will serialise it
 564                 # and send it to the builder
 565                 with self.db.transaction():
 566                         job.start(builder=self.builder)
 567
 568                         ret = {
 569                                 "id"                 : job.uuid,
 570                                 "arch"               : job.arch,
 571                                 "source_url"         : job.build.source_download,
 572                                 "source_hash_sha512" : job.build.source_hash_sha512,
 573                                 "type"               : "test" if job.test else "release",
 574                                 "config"             : job.get_config(),
 575                         }
 576                         self.finish(ret)
 577
 578
 579 class BuildersJobsQueueHandler(BuildersBaseHandler):
 580         @tornado.web.asynchronous
 581         @tornado.web.authenticated
 582         def get(self):
 583                 self.callback()
 584
 585         def callback(self):
 586                 # Break if the connection has been closed in the mean time.
 587                 if self.connection_closed():
 588                         logging.warning("Connection closed")
 589                         return
 590
 591                 with self.db.transaction():
 592                         # Check if there is a job for us.
 593                         job = self.builder.get_next_job()
 594
 595                         # Got no job, wait and try again.
 596                         if not job:
 597                                 return self.add_timeout(self.heartbeat, self.callback)
 598
 599                         # We got a job!
 600                         job.start(builder=self.builder)
 601
 602                         ret = {
 603                                 "id"                 : job.uuid,
 604                                 "arch"               : job.arch,
 605                                 "source_url"         : job.build.source_download,
 606                                 "source_hash_sha512" : job.build.source_hash_sha512,
 607                                 "type"               : "test" if job.test else "release",
 608                                 "config"             : job.get_config(),
 609                         }
 610
 611                         # Send build information to the builder.
 612                         self.finish(ret)
 613
 614         @property
 615         def heartbeat(self):
 616                 return 15 # 15 seconds
 617
 618         @property
 619         def max_runtime(self):
 620                 timeout = self.get_argument_int("timeout", None)
 621                 if timeout:
 622                         return timeout - self.heartbeat
 623
 624                 return 300 # 5 min
 625
 626
 627 class BuildersJobsStateHandler(BuildersBaseHandler):
 628         @tornado.web.authenticated
 629         def post(self, job_uuid, state):
 630                 job = self.backend.jobs.get_by_uuid(job_uuid)
 631                 if not job:
 632                         raise tornado.web.HTTPError(404, "Invalid job id.")
 633
 634                 if not job.builder == self.builder:
 635                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 636
 637                 message = self.get_argument("message", None)
 638
 639                 # Save information to database.
 640                 with self.db.transaction():
 641                         if state == "running":
 642                                 job.running()
 643                         elif state == "failed":
 644                                 job.failed(message)
 645                         elif state == "finished":
 646                                 job.finished()
 647                         else:
 648                                 job.state = state
 649
 650                 self.finish("OK")
 651
 652
 653 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
 654         @tornado.web.authenticated
 655         def post(self, job_uuid):
 656                 job = self.backend.jobs.get_by_uuid(job_uuid)
 657                 if not job:
 658                         raise tornado.web.HTTPError(404, "Invalid job id.")
 659
 660                 if not job.builder == self.builder:
 661                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 662
 663                 # Get buildroot.
 664                 buildroot = self.get_argument_json("buildroot", None)
 665                 if buildroot:
 666                         job.save_buildroot(buildroot)
 667
 668                 self.finish("OK")
 669
 670
 671 class BuildersJobsAddFileHandler(BuildersBaseHandler):
 672         @tornado.web.authenticated
 673         def post(self, job_uuid, upload_id):
 674                 type = self.get_argument("type")
 675                 assert type in ("package", "log")
 676
 677                 # Fetch job we are working on and check if it is actually ours.
 678                 job = self.backend.jobs.get_by_uuid(job_uuid)
 679                 if not job:
 680                         raise tornado.web.HTTPError(404, "Invalid job id.")
 681
 682                 if not job.builder == self.builder:
 683                         raise tornado.web.HTTPError(403, "Altering another builder's job.")
 684
 685                 # Fetch uploaded file object and check we uploaded it ourself.
 686                 upload = self.backend.uploads.get_by_uuid(upload_id)
 687                 if not upload:
 688                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 689
 690                 if not upload.builder == self.builder:
 691                         raise tornado.web.HTTPError(403, "Using an other host's file.")
 692
 693                 try:
 694                         job.add_file(upload.path)
 695
 696                 finally:
 697                         # Finally, remove the uploaded file.
 698                         upload.remove()
 699
 700                 self.finish("OK")