src/hub/handlers.py

   1 #!/usr/bin/python
   2
   3 import base64
   4 import hashlib
   5 import json
   6 import logging
   7 import time
   8 import tornado.web
   9
  10 from .. import builds
  11 from .. import builders
  12 from .. import uploads
  13 from .. import users
  14
  15 class LongPollMixin(object):
  16         def initialize(self):
  17                 self._start_time = time.time()
  18
  19         def add_timeout(self, timeout, callback):
  20                 deadline = time.time() + timeout
  21
  22                 return self.application.ioloop.add_timeout(deadline, callback)
  23
  24         def on_connection_close(self):
  25                 logging.debug("Connection closed unexpectedly")
  26
  27         def connection_closed(self):
  28                 return self.request.connection.stream.closed()
  29
  30         @property
  31         def runtime(self):
  32                 return time.time() - self._start_time
  33
  34
  35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
  36         @property
  37         def backend(self):
  38                 """
  39                         Shortcut handler to pakfire instance
  40                 """
  41                 return self.application.backend
  42
  43         @property
  44         def db(self):
  45                 return self.backend.db
  46
  47         def get_basic_auth_credentials(self):
  48                 """
  49                         This handles HTTP Basic authentication.
  50                 """
  51                 auth_header = self.request.headers.get("Authorization", None)
  52
  53                 # If no authentication information was provided, we stop here.
  54                 if not auth_header:
  55                         return None, None
  56
  57                 # No basic auth? We cannot handle that.
  58                 if not auth_header.startswith("Basic "):
  59                         raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
  60
  61                 try:
  62                         # Decode the authentication information.
  63                         auth_header = base64.decodestring(auth_header[6:])
  64
  65                         name, password = auth_header.split(":", 1)
  66                 except:
  67                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
  68
  69                 return name, password
  70
  71         def get_current_user(self):
  72                 name, password = self.get_basic_auth_credentials()
  73                 if name is None:
  74                         return
  75
  76                 builder = self.backend.builders.auth(name, password)
  77                 if builder:
  78                         return builder
  79
  80                 user = self.backend.users.auth(name, password)
  81                 if user:
  82                         return user
  83
  84         @property
  85         def builder(self):
  86                 if isinstance(self.current_user, builders.Builder):
  87                         return self.current_user
  88
  89         @property
  90         def user(self):
  91                 if isinstance(self.current_user, users.User):
  92                         return self.current_user
  93
  94         def get_argument_int(self, *args, **kwargs):
  95                 arg = self.get_argument(*args, **kwargs)
  96
  97                 try:
  98                         return int(arg)
  99                 except (TypeError, ValueError):
 100                         return None
 101
 102         def get_argument_float(self, *args, **kwargs):
 103                 arg = self.get_argument(*args, **kwargs)
 104
 105                 try:
 106                         return float(arg)
 107                 except (TypeError, ValueError):
 108                         return None
 109
 110         def get_argument_json(self, *args, **kwargs):
 111                 arg = self.get_argument(*args, **kwargs)
 112
 113                 if arg:
 114                         return json.loads(arg)
 115
 116
 117 class NoopHandler(BaseHandler):
 118         def get(self):
 119                 if self.builder:
 120                         self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
 121                 elif self.user:
 122                         self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
 123                 else:
 124                         self.write("Welcome to the Pakfire hub!")
 125
 126
 127 class ErrorTestHandler(BaseHandler):
 128         def get(self, error_code=200):
 129                 """
 130                         For testing a client.
 131
 132                         This just returns a HTTP response with the given code.
 133                 """
 134                 try:
 135                         error_code = int(error_code)
 136                 except ValueError:
 137                         error_code = 200
 138
 139                 raise tornado.web.HTTPError(error_code)
 140
 141
 142 # Uploads
 143
 144 class UploadsCreateHandler(BaseHandler):
 145         """
 146                 Create a new upload object in the database and return a unique ID
 147                 to the uploader.
 148         """
 149
 150         @tornado.web.authenticated
 151         def get(self):
 152                 # XXX Check permissions
 153
 154                 filename = self.get_argument("filename")
 155                 filesize = self.get_argument_int("filesize")
 156                 filehash = self.get_argument("hash")
 157
 158                 with self.db.transaction():
 159                         upload = self.backend.uploads.create(filename, filesize,
 160                                 filehash, user=self.user, builder=self.builder)
 161
 162                         self.finish(upload.uuid)
 163
 164
 165 class UploadsSendChunkHandler(BaseHandler):
 166         @tornado.web.authenticated
 167         def post(self, upload_id):
 168                 upload = self.backend.uploads.get_by_uuid(upload_id)
 169                 if not upload:
 170                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 171
 172                 if not upload.builder == self.builder:
 173                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 174
 175                 chksum = self.get_argument("chksum")
 176                 data = self.get_argument("data")
 177
 178                 # Decode data.
 179                 data = base64.b64decode(data)
 180
 181                 # Calculate hash and compare.
 182                 h = hashlib.new("sha512")
 183                 h.update(data)
 184
 185                 if not chksum == h.hexdigest():
 186                         raise tornado.web.HTTPError(400, "Checksum mismatch")
 187
 188                 # Append the data to file.
 189                 with self.db.transaction():
 190                         upload.append(data)
 191
 192
 193 class UploadsFinishedHandler(BaseHandler):
 194         @tornado.web.authenticated
 195         def get(self, upload_id):
 196                 upload = self.backend.uploads.get_by_uuid(upload_id)
 197                 if not upload:
 198                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 199
 200                 if not upload.builder == self.builder:
 201                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 202
 203                 # Validate the uploaded data to its hash.
 204                 ret = upload.validate()
 205
 206                 # If the validation was successfull, we mark the upload
 207                 # as finished and send True to the client.
 208                 if ret:
 209                         upload.finished()
 210                         self.finish("OK")
 211
 212                         return
 213
 214                 # In case the download was corrupted or incomplete, we delete it
 215                 # and tell the client to start over.
 216                 with self.db.transaction():
 217                         upload.remove()
 218
 219                 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
 220
 221
 222 class UploadsDestroyHandler(BaseHandler):
 223         @tornado.web.authenticated
 224         def get(self, upload_id):
 225                 upload = self.backend.uploads.get_by_uuid(upload_id)
 226                 if not upload:
 227                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 228
 229                 if not upload.builder == self.builder:
 230                         raise tornado.web.HTTPError(403, "Removing an other host's file.")
 231
 232                 # Remove the upload from the database and trash the data.
 233                 with self.db.transaction():
 234                         upload.remove()
 235
 236
 237 # Builds
 238
 239 class BuildsCreateHandler(BaseHandler):
 240         @tornado.web.authenticated
 241         def get(self):
 242                 # Get the upload ID of the package file.
 243                 upload_id = self.get_argument("upload_id")
 244
 245                 # Get the identifier of the distribution we build for.
 246                 distro_ident = self.get_argument("distro")
 247
 248                 # Get a list of arches to build for.
 249                 arches = self.get_argument("arches", None)
 250                 if arches == "":
 251                         arches = None
 252
 253                 # Get previously uploaded file to create this build from.
 254                 upload = self.backend.uploads.get_by_uuid(upload_id)
 255                 if not upload:
 256                         raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
 257
 258                 # Check if the uploaded file belongs to this user/builder.
 259                 if self.user and not upload.user == self.user:
 260                         raise tornado.web.HTTPError(400, "Upload does not belong to this user")
 261
 262                 elif self.builder and not upload.builder == self.builder:
 263                         raise tornado.web.HTTPError(400, "Upload does not belong to this builder")
 264
 265                 # Get distribution this package should be built for.
 266                 distro = self.backend.distros.get_by_ident(distro_ident)
 267                 if not distro:
 268                         distro = self.backend.distros.get_default()
 269
 270                 # Open the package that was uploaded earlier and add it to
 271                 # the database. Create a new build object from the uploaded package.
 272                 try:
 273                         build = self.backend.builds.create_from_source_package(upload.path, distro=distro,
 274                                 type="scratch", arches=arches, owner=self.user)
 275
 276                 except:
 277                         # Raise any exception.
 278                         raise
 279
 280                 else:
 281                         # Creating the build will move the file to the build directory,
 282                         # so we can safely remove the uploaded file.
 283                         upload.remove()
 284
 285                 # Send the build ID back to the user.
 286                 self.finish(build.uuid)
 287
 288
 289 class BuildsGetHandler(BaseHandler):
 290         def get(self, build_uuid):
 291                 build = self.backend.builds.get_by_uuid(build_uuid)
 292                 if not build:
 293                         raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
 294
 295                 ret = {
 296                         "distro"       : build.distro.identifier,
 297                         "jobs"         : [j.uuid for j in build.jobs],
 298                         "name"         : build.name,
 299                         "package"      : build.pkg.uuid,
 300                         "priority"     : build.priority,
 301                         "score"        : build.score,
 302                         "severity"     : build.severity,
 303                         "state"        : build.state,
 304                         "sup_arches"   : build.supported_arches,
 305                         "time_created" : build.created.isoformat(),
 306                         "type"         : build.type,
 307                         "uuid"         : build.uuid,
 308                 }
 309
 310                 # If the build is in a repository, update that bit.
 311                 if build.repo:
 312                         ret["repo"] = build.repo.identifier
 313
 314                 self.finish(ret)
 315
 316
 317 # Jobs
 318
 319 class JobsBaseHandler(BaseHandler):
 320         def job2json(self, job):
 321                 ret = {
 322                         "arch"         : job.arch,
 323                         "build"        : job.build.uuid,
 324                         "duration"     : job.duration,
 325                         "name"         : job.name,
 326                         "packages"     : [p.uuid for p in job.packages],
 327                         "state"        : job.state,
 328                         "time_created" : job.time_created.isoformat(),
 329                         "type"         : "test" if job.test else "release",
 330                         "uuid"         : job.uuid,
 331                 }
 332
 333                 if job.builder:
 334                         ret["builder"] = job.builder.hostname
 335
 336                 if job.time_started:
 337                         ret["time_started"] = job.time_started.isoformat()
 338
 339                 if job.time_finished:
 340                         ret["time_finished"] = job.time_finished.isoformat()
 341
 342                 return ret
 343
 344
 345 class JobsGetActiveHandler(JobsBaseHandler):
 346         def get(self):
 347                 # Get list of all active jobs.
 348                 jobs = self.backend.jobs.get_active()
 349
 350                 args = {
 351                         "jobs" : [self.job2json(j) for j in jobs],
 352                 }
 353
 354                 self.finish(args)
 355
 356
 357 class JobsGetLatestHandler(JobsBaseHandler):
 358         def get(self):
 359                 limit = self.get_argument_int("limit", 5)
 360
 361                 # Get the latest jobs.
 362                 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
 363
 364                 args = {
 365                         "jobs" : [self.job2json(j) for j in jobs],
 366                 }
 367
 368                 self.finish(args)
 369
 370
 371 class JobsGetQueueHandler(JobsBaseHandler):
 372         def get(self):
 373                 limit = self.get_argument_int("limit", 5)
 374
 375                 # Get the job queue.
 376                 jobs = []
 377                 for job in self.backend.jobqueue:
 378                         jobs.append(job)
 379
 380                         limit -= 1
 381                         if not limit: break
 382
 383                 args = {
 384                         "jobs" : [self.job2json(j) for j in jobs],
 385                 }
 386
 387                 self.finish(args)
 388
 389
 390 class JobsGetHandler(JobsBaseHandler):
 391         def get(self, job_uuid):
 392                 job = self.backend.jobs.get_by_uuid(job_uuid)
 393                 if not job:
 394                         raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
 395
 396                 ret = self.job2json(job)
 397                 self.finish(ret)
 398
 399
 400 # Packages
 401
 402 class PackagesGetHandler(BaseHandler):
 403         def get(self, package_uuid):
 404                 pkg = self.backend.packages.get_by_uuid(package_uuid)
 405                 if not pkg:
 406                         raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
 407
 408                 ret = {
 409                         "arch"             : pkg.arch,
 410                         "build_id"         : pkg.build_id,
 411                         "build_host"       : pkg.build_host,
 412                         "build_time"       : pkg.build_time.isoformat(),
 413                         "description"      : pkg.description,
 414                         "epoch"            : pkg.epoch,
 415                         "filesize"         : pkg.filesize,
 416                         "friendly_name"    : pkg.friendly_name,
 417                         "friendly_version" : pkg.friendly_version,
 418                         "groups"           : pkg.groups,
 419                         "hash_sha512"      : pkg.hash_sha512,
 420                         "license"          : pkg.license,
 421                         "name"             : pkg.name,
 422                         "release"          : pkg.release,
 423                         "size"             : pkg.size,
 424                         "summary"          : pkg.summary,
 425                         "type"             : pkg.type,
 426                         "url"              : pkg.url,
 427                         "uuid"             : pkg.uuid,
 428                         "version"          : pkg.version,
 429
 430                         # Dependencies.
 431                         "prerequires"      : pkg.prerequires,
 432                         "requires"         : pkg.requires,
 433                         "provides"         : pkg.provides,
 434                         "obsoletes"        : pkg.obsoletes,
 435                         "conflicts"        : pkg.conflicts,
 436                 }
 437
 438                 if pkg.type == "source":
 439                         ret["supported_arches"] = pkg.supported_arches
 440
 441                 if isinstance(pkg.maintainer, users.User):
 442                         ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
 443                 elif pkg.maintainer:
 444                         ret["maintainer"] = pkg.maintainer
 445
 446                 if pkg.distro:
 447                         ret["distro"] = pkg.distro.identifier
 448
 449                 self.finish(ret)
 450
 451
 452 # Builders
 453
 454 class BuildersBaseHandler(BaseHandler):
 455         def prepare(self):
 456                 # The request must come from an authenticated buider.
 457                 if not self.builder:
 458                         raise tornado.web.HTTPError(403)
 459
 460
 461 class BuildersInfoHandler(BuildersBaseHandler):
 462         @tornado.web.authenticated
 463         def post(self):
 464                 args = {
 465                         # CPU info
 466                         "cpu_model"    : self.get_argument("cpu_model", None),
 467                         "cpu_count"    : self.get_argument("cpu_count", None),
 468                         "cpu_arch"     : self.get_argument("cpu_arch", None),
 469                         "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
 470
 471                         # Pakfire
 472                         "pakfire_version" : self.get_argument("pakfire_version", None),
 473                         "host_key"     : self.get_argument("host_key", None),
 474
 475                         # OS
 476                         "os_name"      : self.get_argument("os_name", None),
 477                 }
 478                 self.builder.update_info(**args)
 479
 480
 481 class BuildersKeepaliveHandler(BuildersBaseHandler):
 482         @tornado.web.authenticated
 483         def post(self):
 484                 args = {
 485                         # Load average
 486                         "loadavg1"   : self.get_argument_float("loadavg1", None),
 487                         "loadavg5"   : self.get_argument_float("loadavg5", None),
 488                         "loadavg15"  : self.get_argument_float("loadavg15", None),
 489
 490                         # Memory
 491                         "mem_total"  : self.get_argument_int("mem_total", None),
 492                         "mem_free"   : self.get_argument_int("mem_free", None),
 493
 494                         # swap
 495                         "swap_total" : self.get_argument_int("swap_total", None),
 496                         "swap_free"  : self.get_argument_int("swap_free", None),
 497
 498                         # Disk space
 499                         "space_free" : self.get_argument_int("space_free", None),
 500                 }
 501                 self.builder.update_keepalive(**args)
 502
 503                 self.finish("OK")
 504
 505
 506 class BuildersJobsQueueHandler(BuildersBaseHandler):
 507         @tornado.web.asynchronous
 508         @tornado.web.authenticated
 509         def get(self):
 510                 self.callback()
 511
 512         def callback(self):
 513                 # Break if the connection has been closed in the mean time.
 514                 if self.connection_closed():
 515                         logging.warning("Connection closed")
 516                         return
 517
 518                 # Check if there is a job for us.
 519                 job = self.builder.get_next_job()
 520
 521                 # Got no job, wait and try again.
 522                 if not job:
 523                         # Check if we have been running for too long.
 524                         if self.runtime >= self.max_runtime:
 525                                 logging.debug("Exceeded max. runtime. Finishing request.")
 526                                 return self.finish()
 527
 528                         # Try again in a jiffy.
 529                         self.add_timeout(self.heartbeat, self.callback)
 530                         return
 531
 532                 try:
 533                         # Set job to dispatching state.
 534                         job.state = "dispatching"
 535
 536                         # Set our build host.
 537                         job.builder = self.builder
 538
 539                         ret = {
 540                                 "id"                 : job.uuid,
 541                                 "arch"               : job.arch,
 542                                 "source_url"         : job.build.source_download,
 543                                 "source_hash_sha512" : job.build.source_hash_sha512,
 544                                 "type"               : "test" if job.test else "release",
 545                                 "config"             : job.get_config(),
 546                         }
 547
 548                         # Send build information to the builder.
 549                         self.finish(ret)
 550                 except:
 551                         # If anything went wrong, we reset the state.
 552                         job.state = "pending"
 553                         raise
 554
 555         @property
 556         def heartbeat(self):
 557                 return 15 # 15 seconds
 558
 559         @property
 560         def max_runtime(self):
 561                 timeout = self.get_argument_int("timeout", None)
 562                 if timeout:
 563                         return timeout - self.heartbeat
 564
 565                 return 300 # 5 min
 566
 567
 568 class BuildersJobsStateHandler(BuildersBaseHandler):
 569         @tornado.web.authenticated
 570         def post(self, job_uuid, state):
 571                 job = self.backend.jobs.get_by_uuid(job_uuid)
 572                 if not job:
 573                         raise tornado.web.HTTPError(404, "Invalid job id.")
 574
 575                 if not job.builder == self.builder:
 576                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 577
 578                 # Save information to database.
 579                 job.state = state
 580
 581                 message = self.get_argument("message", None)
 582                 job.update_message(message)
 583
 584                 self.finish("OK")
 585
 586
 587 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
 588         @tornado.web.authenticated
 589         def post(self, job_uuid):
 590                 job = self.backend.jobs.get_by_uuid(job_uuid)
 591                 if not job:
 592                         raise tornado.web.HTTPError(404, "Invalid job id.")
 593
 594                 if not job.builder == self.builder:
 595                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 596
 597                 # Get buildroot.
 598                 buildroot = self.get_argument_json("buildroot", None)
 599                 if buildroot:
 600                         job.save_buildroot(buildroot)
 601
 602                 self.finish("OK")
 603
 604
 605 class BuildersJobsAddFileHandler(BuildersBaseHandler):
 606         @tornado.web.authenticated
 607         def post(self, job_uuid, upload_id):
 608                 type = self.get_argument("type")
 609                 assert type in ("package", "log")
 610
 611                 # Fetch job we are working on and check if it is actually ours.
 612                 job = self.backend.jobs.get_by_uuid(job_uuid)
 613                 if not job:
 614                         raise tornado.web.HTTPError(404, "Invalid job id.")
 615
 616                 if not job.builder == self.builder:
 617                         raise tornado.web.HTTPError(403, "Altering another builder's job.")
 618
 619                 # Fetch uploaded file object and check we uploaded it ourself.
 620                 upload = self.backend.uploads.get_by_uuid(upload_id)
 621                 if not upload:
 622                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 623
 624                 if not upload.builder == self.builder:
 625                         raise tornado.web.HTTPError(403, "Using an other host's file.")
 626
 627                 # Remove all files that have to be deleted, first.
 628                 self.backend.cleanup_files()
 629
 630                 try:
 631                         job.add_file(upload.path)
 632
 633                 finally:
 634                         # Finally, remove the uploaded file.
 635                         upload.remove()
 636
 637                 self.finish("OK")