src/hub/handlers.py

   1 #!/usr/bin/python
   2
   3 import base64
   4 import hashlib
   5 import json
   6 import logging
   7 import time
   8 import tornado.web
   9
  10 from .. import builds
  11 from .. import builders
  12 from .. import uploads
  13 from .. import users
  14
  15 class LongPollMixin(object):
  16         def initialize(self):
  17                 self._start_time = time.time()
  18
  19         def add_timeout(self, timeout, callback):
  20                 deadline = time.time() + timeout
  21
  22                 return self.application.ioloop.add_timeout(deadline, callback)
  23
  24         def on_connection_close(self):
  25                 logging.debug("Connection closed unexpectedly")
  26
  27         def connection_closed(self):
  28                 return self.request.connection.stream.closed()
  29
  30         @property
  31         def runtime(self):
  32                 return time.time() - self._start_time
  33
  34
  35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
  36         @property
  37         def backend(self):
  38                 """
  39                         Shortcut handler to pakfire instance
  40                 """
  41                 return self.application.backend
  42
  43         def get_basic_auth_credentials(self):
  44                 """
  45                         This handles HTTP Basic authentication.
  46                 """
  47                 auth_header = self.request.headers.get("Authorization", None)
  48
  49                 # If no authentication information was provided, we stop here.
  50                 if not auth_header:
  51                         return None, None
  52
  53                 # No basic auth? We cannot handle that.
  54                 if not auth_header.startswith("Basic "):
  55                         raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
  56
  57                 try:
  58                         # Decode the authentication information.
  59                         auth_header = base64.decodestring(auth_header[6:])
  60
  61                         name, password = auth_header.split(":", 1)
  62                 except:
  63                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
  64
  65                 return name, password
  66
  67         def get_current_user(self):
  68                 name, password = self.get_basic_auth_credentials()
  69                 if name is None:
  70                         return
  71
  72                 builder = self.backend.builders.auth(name, password)
  73                 if builder:
  74                         return builder
  75
  76                 user = self.backend.users.auth(name, password)
  77                 if user:
  78                         return user
  79
  80         @property
  81         def builder(self):
  82                 if isinstance(self.current_user, builders.Builder):
  83                         return self.current_user
  84
  85         @property
  86         def user(self):
  87                 if isinstance(self.current_user, users.User):
  88                         return self.current_user
  89
  90         def get_argument_int(self, *args, **kwargs):
  91                 arg = self.get_argument(*args, **kwargs)
  92
  93                 try:
  94                         return int(arg)
  95                 except (TypeError, ValueError):
  96                         return None
  97
  98         def get_argument_float(self, *args, **kwargs):
  99                 arg = self.get_argument(*args, **kwargs)
 100
 101                 try:
 102                         return float(arg)
 103                 except (TypeError, ValueError):
 104                         return None
 105
 106         def get_argument_json(self, *args, **kwargs):
 107                 arg = self.get_argument(*args, **kwargs)
 108
 109                 if arg:
 110                         return json.loads(arg)
 111
 112
 113 class NoopHandler(BaseHandler):
 114         def get(self):
 115                 if self.builder:
 116                         self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
 117                 elif self.user:
 118                         self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
 119                 else:
 120                         self.write("Welcome to the Pakfire hub!")
 121
 122
 123 class ErrorTestHandler(BaseHandler):
 124         def get(self, error_code=200):
 125                 """
 126                         For testing a client.
 127
 128                         This just returns a HTTP response with the given code.
 129                 """
 130                 try:
 131                         error_code = int(error_code)
 132                 except ValueError:
 133                         error_code = 200
 134
 135                 raise tornado.web.HTTPError(error_code)
 136
 137
 138 # Uploads
 139
 140 class UploadsCreateHandler(BaseHandler):
 141         """
 142                 Create a new upload object in the database and return a unique ID
 143                 to the uploader.
 144         """
 145
 146         @tornado.web.authenticated
 147         def get(self):
 148                 # XXX Check permissions
 149
 150                 filename = self.get_argument("filename")
 151                 filesize = self.get_argument_int("filesize")
 152                 filehash = self.get_argument("hash")
 153
 154                 upload = uploads.Upload.create(self.backend, filename, filesize,
 155                         filehash, user=self.user, builder=self.builder)
 156
 157                 self.finish(upload.uuid)
 158
 159
 160 class UploadsSendChunkHandler(BaseHandler):
 161         @tornado.web.authenticated
 162         def post(self, upload_id):
 163                 upload = self.backend.uploads.get_by_uuid(upload_id)
 164                 if not upload:
 165                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 166
 167                 if not upload.builder == self.builder:
 168                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 169
 170                 chksum = self.get_argument("chksum")
 171                 data = self.get_argument("data")
 172
 173                 # Decode data.
 174                 data = base64.b64decode(data)
 175
 176                 # Calculate hash and compare.
 177                 h = hashlib.new("sha512")
 178                 h.update(data)
 179
 180                 if not chksum == h.hexdigest():
 181                         raise tornado.web.HTTPError(400, "Checksum mismatch")
 182
 183                 # Append the data to file.
 184                 upload.append(data)
 185
 186
 187 class UploadsFinishedHandler(BaseHandler):
 188         @tornado.web.authenticated
 189         def get(self, upload_id):
 190                 upload = self.backend.uploads.get_by_uuid(upload_id)
 191                 if not upload:
 192                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 193
 194                 if not upload.builder == self.builder:
 195                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 196
 197                 # Validate the uploaded data to its hash.
 198                 ret = upload.validate()
 199
 200                 # If the validation was successfull, we mark the upload
 201                 # as finished and send True to the client.
 202                 if ret:
 203                         upload.finished()
 204                         self.finish("OK")
 205
 206                         return
 207
 208                 # In case the download was corrupted or incomplete, we delete it
 209                 # and tell the client to start over.
 210                 upload.remove()
 211
 212                 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
 213
 214
 215 class UploadsDestroyHandler(BaseHandler):
 216         @tornado.web.authenticated
 217         def get(self, upload_id):
 218                 upload = self.backend.uploads.get_by_uuid(upload_id)
 219                 if not upload:
 220                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 221
 222                 if not upload.builder == self.builder:
 223                         raise tornado.web.HTTPError(403, "Removing an other host's file.")
 224
 225                 # Remove the upload from the database and trash the data.
 226                 upload.remove()
 227
 228
 229 # Builds
 230
 231 class BuildsCreateHandler(BaseHandler):
 232         @tornado.web.authenticated
 233         def get(self):
 234                 # Get the upload ID of the package file.
 235                 upload_id = self.get_argument("upload_id")
 236
 237                 # Get the identifier of the distribution we build for.
 238                 distro_ident = self.get_argument("distro")
 239
 240                 # Get a list of arches to build for.
 241                 arches = self.get_argument("arches", None)
 242                 if arches == "":
 243                         arches = None
 244
 245                 # Process build type.
 246                 build_type = self.get_argument("build_type")
 247                 if build_type == "release":
 248                         check_for_duplicates = True
 249                 elif build_type == "scratch":
 250                         check_for_duplicates = False
 251                 else:
 252                         raise tornado.web.HTTPError(400, "Invalid build type")
 253
 254                 ## Check if the user has permission to create a build.
 255                 # Users only have the permission to create scratch builds.
 256                 if self.user and not build_type == "scratch":
 257                         raise tornado.web.HTTPError(403, "Users are only allowed to upload scratch builds")
 258
 259                 # Get previously uploaded file to create this build from.
 260                 upload = self.backend.uploads.get_by_uuid(upload_id)
 261                 if not upload:
 262                         raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
 263
 264                 # Check if the uploaded file belongs to this user/builder.
 265                 if self.user and not upload.user == self.user:
 266                         raise tornado.web.HTTPError(400, "Upload does not belong to this user.")
 267
 268                 elif self.builder and not upload.builder == self.builder:
 269                         raise tornado.web.HTTPError(400, "Upload does not belong to this builder.")
 270
 271                 # Get distribution this package should be built for.
 272                 distro = self.backend.distros.get_by_ident(distro_ident)
 273                 if not distro:
 274                         distro = self.backend.distros.get_default()
 275
 276                 # Open the package that was uploaded earlier and add it to
 277                 # the database. Create a new build object from the uploaded package.
 278                 args = {
 279                         "arches"               : arches,
 280                         "check_for_duplicates" : check_for_duplicates,
 281                         "distro"               : distro,
 282                         "type"                 : build_type,
 283                 }
 284                 if self.user:
 285                         args["owner"] = self.user
 286
 287                 try:
 288                         pkg, build = builds.import_from_package(self.backend, upload.path, **args)
 289
 290                 except:
 291                         # Raise any exception.
 292                         raise
 293
 294                 else:
 295                         # Creating the build will move the file to the build directory,
 296                         # so we can safely remove the uploaded file.
 297                         upload.remove()
 298
 299                 # Send the build ID back to the user.
 300                 self.finish(build.uuid)
 301
 302
 303 class BuildsGetHandler(BaseHandler):
 304         def get(self, build_uuid):
 305                 build = self.backend.builds.get_by_uuid(build_uuid)
 306                 if not build:
 307                         raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
 308
 309                 ret = {
 310                         "distro"       : build.distro.identifier,
 311                         "jobs"         : [j.uuid for j in build.jobs],
 312                         "name"         : build.name,
 313                         "package"      : build.pkg.uuid,
 314                         "priority"     : build.priority,
 315                         "score"        : build.credits,
 316                         "severity"     : build.severity,
 317                         "state"        : build.state,
 318                         "sup_arches"   : build.supported_arches,
 319                         "time_created" : build.created.isoformat(),
 320                         "type"         : build.type,
 321                         "uuid"         : build.uuid,
 322                 }
 323
 324                 # If the build is in a repository, update that bit.
 325                 if build.repo:
 326                         ret["repo"] = build.repo.identifier
 327
 328                 self.finish(ret)
 329
 330
 331 # Jobs
 332
 333 class JobsBaseHandler(BaseHandler):
 334         def job2json(self, job):
 335                 ret = {
 336                         "arch"         : job.arch,
 337                         "build"        : job.build.uuid,
 338                         "duration"     : job.duration,
 339                         "name"         : job.name,
 340                         "packages"     : [p.uuid for p in job.packages],
 341                         "state"        : job.state,
 342                         "time_created" : job.time_created.isoformat(),
 343                         "type"         : "test" if job.test else "release",
 344                         "uuid"         : job.uuid,
 345                 }
 346
 347                 if job.builder:
 348                         ret["builder"] = job.builder.hostname
 349
 350                 if job.time_started:
 351                         ret["time_started"] = job.time_started.isoformat()
 352
 353                 if job.time_finished:
 354                         ret["time_finished"] = job.time_finished.isoformat()
 355
 356                 return ret
 357
 358
 359 class JobsGetActiveHandler(JobsBaseHandler):
 360         def get(self):
 361                 # Get list of all active jobs.
 362                 jobs = self.backend.jobs.get_active()
 363
 364                 args = {
 365                         "jobs" : [self.job2json(j) for j in jobs],
 366                 }
 367
 368                 self.finish(args)
 369
 370
 371 class JobsGetLatestHandler(JobsBaseHandler):
 372         def get(self):
 373                 limit = self.get_argument_int("limit", 5)
 374
 375                 # Get the latest jobs.
 376                 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
 377
 378                 args = {
 379                         "jobs" : [self.job2json(j) for j in jobs],
 380                 }
 381
 382                 self.finish(args)
 383
 384
 385 class JobsGetQueueHandler(JobsBaseHandler):
 386         def get(self):
 387                 limit = self.get_argument_int("limit", 5)
 388
 389                 # Get the job queue.
 390                 jobs = []
 391                 for job in self.backend.jobqueue:
 392                         jobs.append(job)
 393
 394                         limit -= 1
 395                         if not limit: break
 396
 397                 args = {
 398                         "jobs" : [self.job2json(j) for j in jobs],
 399                 }
 400
 401                 self.finish(args)
 402
 403
 404 class JobsGetHandler(JobsBaseHandler):
 405         def get(self, job_uuid):
 406                 job = self.backend.jobs.get_by_uuid(job_uuid)
 407                 if not job:
 408                         raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
 409
 410                 ret = self.job2json(job)
 411                 self.finish(ret)
 412
 413
 414 # Packages
 415
 416 class PackagesGetHandler(BaseHandler):
 417         def get(self, package_uuid):
 418                 pkg = self.backend.packages.get_by_uuid(package_uuid)
 419                 if not pkg:
 420                         raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
 421
 422                 ret = {
 423                         "arch"             : pkg.arch,
 424                         "build_id"         : pkg.build_id,
 425                         "build_host"       : pkg.build_host,
 426                         "build_time"       : pkg.build_time.isoformat(),
 427                         "description"      : pkg.description,
 428                         "epoch"            : pkg.epoch,
 429                         "filesize"         : pkg.filesize,
 430                         "friendly_name"    : pkg.friendly_name,
 431                         "friendly_version" : pkg.friendly_version,
 432                         "groups"           : pkg.groups,
 433                         "hash_sha512"      : pkg.hash_sha512,
 434                         "license"          : pkg.license,
 435                         "name"             : pkg.name,
 436                         "release"          : pkg.release,
 437                         "size"             : pkg.size,
 438                         "summary"          : pkg.summary,
 439                         "type"             : pkg.type,
 440                         "url"              : pkg.url,
 441                         "uuid"             : pkg.uuid,
 442                         "version"          : pkg.version,
 443
 444                         # Dependencies.
 445                         "prerequires"      : pkg.prerequires,
 446                         "requires"         : pkg.requires,
 447                         "provides"         : pkg.provides,
 448                         "obsoletes"        : pkg.obsoletes,
 449                         "conflicts"        : pkg.conflicts,
 450                 }
 451
 452                 if pkg.type == "source":
 453                         ret["supported_arches"] = pkg.supported_arches
 454
 455                 if isinstance(pkg.maintainer, users.User):
 456                         ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
 457                 elif pkg.maintainer:
 458                         ret["maintainer"] = pkg.maintainer
 459
 460                 if pkg.distro:
 461                         ret["distro"] = pkg.distro.identifier
 462
 463                 self.finish(ret)
 464
 465
 466 # Builders
 467
 468 class BuildersBaseHandler(BaseHandler):
 469         def prepare(self):
 470                 # The request must come from an authenticated buider.
 471                 if not self.builder:
 472                         raise tornado.web.HTTPError(403)
 473
 474
 475 class BuildersInfoHandler(BuildersBaseHandler):
 476         @tornado.web.authenticated
 477         def post(self):
 478                 args = {
 479                         # CPU info
 480                         "cpu_model"    : self.get_argument("cpu_model", None),
 481                         "cpu_count"    : self.get_argument("cpu_count", None),
 482                         "cpu_arch"     : self.get_argument("cpu_arch", None),
 483                         "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
 484
 485                         # Pakfire
 486                         "pakfire_version" : self.get_argument("pakfire_version", None),
 487                         "host_key"     : self.get_argument("host_key", None),
 488
 489                         # OS
 490                         "os_name"      : self.get_argument("os_name", None),
 491                 }
 492                 self.builder.update_info(**args)
 493
 494
 495 class BuildersKeepaliveHandler(BuildersBaseHandler):
 496         @tornado.web.authenticated
 497         def post(self):
 498                 args = {
 499                         # Load average
 500                         "loadavg1"   : self.get_argument_float("loadavg1", None),
 501                         "loadavg5"   : self.get_argument_float("loadavg5", None),
 502                         "loadavg15"  : self.get_argument_float("loadavg15", None),
 503
 504                         # Memory
 505                         "mem_total"  : self.get_argument_int("mem_total", None),
 506                         "mem_free"   : self.get_argument_int("mem_free", None),
 507
 508                         # swap
 509                         "swap_total" : self.get_argument_int("swap_total", None),
 510                         "swap_free"  : self.get_argument_int("swap_free", None),
 511
 512                         # Disk space
 513                         "space_free" : self.get_argument_int("space_free", None),
 514                 }
 515                 self.builder.update_keepalive(**args)
 516
 517                 self.finish("OK")
 518
 519
 520 class BuildersJobsQueueHandler(BuildersBaseHandler):
 521         @tornado.web.asynchronous
 522         @tornado.web.authenticated
 523         def get(self):
 524                 self.callback()
 525
 526         def callback(self):
 527                 # Break if the connection has been closed in the mean time.
 528                 if self.connection_closed():
 529                         logging.warning("Connection closed")
 530                         return
 531
 532                 # Check if there is a job for us.
 533                 job = self.builder.get_next_job()
 534
 535                 # Got no job, wait and try again.
 536                 if not job:
 537                         # Check if we have been running for too long.
 538                         if self.runtime >= self.max_runtime:
 539                                 logging.debug("Exceeded max. runtime. Finishing request.")
 540                                 return self.finish()
 541
 542                         # Try again in a jiffy.
 543                         self.add_timeout(self.heartbeat, self.callback)
 544                         return
 545
 546                 try:
 547                         # Set job to dispatching state.
 548                         job.state = "dispatching"
 549
 550                         # Set our build host.
 551                         job.builder = self.builder
 552
 553                         ret = {
 554                                 "id"                 : job.uuid,
 555                                 "arch"               : job.arch,
 556                                 "source_url"         : job.build.source_download,
 557                                 "source_hash_sha512" : job.build.source_hash_sha512,
 558                                 "type"               : "test" if job.test else "release",
 559                                 "config"             : job.get_config(),
 560                         }
 561
 562                         # Send build information to the builder.
 563                         self.finish(ret)
 564                 except:
 565                         # If anything went wrong, we reset the state.
 566                         job.state = "pending"
 567                         raise
 568
 569         @property
 570         def heartbeat(self):
 571                 return 15 # 15 seconds
 572
 573         @property
 574         def max_runtime(self):
 575                 timeout = self.get_argument_int("timeout", None)
 576                 if timeout:
 577                         return timeout - self.heartbeat
 578
 579                 return 300 # 5 min
 580
 581
 582 class BuildersJobsStateHandler(BuildersBaseHandler):
 583         @tornado.web.authenticated
 584         def post(self, job_uuid, state):
 585                 job = self.backend.jobs.get_by_uuid(job_uuid)
 586                 if not job:
 587                         raise tornado.web.HTTPError(404, "Invalid job id.")
 588
 589                 if not job.builder == self.builder:
 590                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 591
 592                 # Save information to database.
 593                 job.state = state
 594
 595                 message = self.get_argument("message", None)
 596                 job.update_message(message)
 597
 598                 self.finish("OK")
 599
 600
 601 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
 602         @tornado.web.authenticated
 603         def post(self, job_uuid):
 604                 job = self.backend.jobs.get_by_uuid(job_uuid)
 605                 if not job:
 606                         raise tornado.web.HTTPError(404, "Invalid job id.")
 607
 608                 if not job.builder == self.builder:
 609                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 610
 611                 # Get buildroot.
 612                 buildroot = self.get_argument_json("buildroot", None)
 613                 if buildroot:
 614                         job.save_buildroot(buildroot)
 615
 616                 self.finish("OK")
 617
 618
 619 class BuildersJobsAddFileHandler(BuildersBaseHandler):
 620         @tornado.web.authenticated
 621         def post(self, job_uuid, upload_id):
 622                 type = self.get_argument("type")
 623                 assert type in ("package", "log")
 624
 625                 # Fetch job we are working on and check if it is actually ours.
 626                 job = self.backend.jobs.get_by_uuid(job_uuid)
 627                 if not job:
 628                         raise tornado.web.HTTPError(404, "Invalid job id.")
 629
 630                 if not job.builder == self.builder:
 631                         raise tornado.web.HTTPError(403, "Altering another builder's job.")
 632
 633                 # Fetch uploaded file object and check we uploaded it ourself.
 634                 upload = self.backend.uploads.get_by_uuid(upload_id)
 635                 if not upload:
 636                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 637
 638                 if not upload.builder == self.builder:
 639                         raise tornado.web.HTTPError(403, "Using an other host's file.")
 640
 641                 # Remove all files that have to be deleted, first.
 642                 self.backend.cleanup_files()
 643
 644                 try:
 645                         job.add_file(upload.path)
 646
 647                 finally:
 648                         # Finally, remove the uploaded file.
 649                         upload.remove()
 650
 651                 self.finish("OK")