src/hub/handlers.py

   1 #!/usr/bin/python
   2
   3 import base64
   4 import hashlib
   5 import json
   6 import logging
   7 import time
   8 import tornado.web
   9
  10 from .. import builds
  11 from .. import builders
  12 from .. import uploads
  13 from .. import users
  14
  15 class LongPollMixin(object):
  16         def initialize(self):
  17                 self._start_time = time.time()
  18
  19         def add_timeout(self, timeout, callback):
  20                 deadline = time.time() + timeout
  21
  22                 return self.application.ioloop.add_timeout(deadline, callback)
  23
  24         def on_connection_close(self):
  25                 logging.debug("Connection closed unexpectedly")
  26
  27         def connection_closed(self):
  28                 return self.request.connection.stream.closed()
  29
  30         @property
  31         def runtime(self):
  32                 return time.time() - self._start_time
  33
  34
  35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
  36         @property
  37         def backend(self):
  38                 """
  39                         Shortcut handler to pakfire instance
  40                 """
  41                 return self.application.backend
  42
  43         def get_basic_auth_credentials(self):
  44                 """
  45                         This handles HTTP Basic authentication.
  46                 """
  47                 auth_header = self.request.headers.get("Authorization", None)
  48
  49                 # If no authentication information was provided, we stop here.
  50                 if not auth_header:
  51                         return None, None
  52
  53                 # No basic auth? We cannot handle that.
  54                 if not auth_header.startswith("Basic "):
  55                         raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
  56
  57                 try:
  58                         # Decode the authentication information.
  59                         auth_header = base64.decodestring(auth_header[6:])
  60
  61                         name, password = auth_header.split(":", 1)
  62                 except:
  63                         raise tornado.web.HTTPError(400, "Authorization data was malformed")
  64
  65                 return name, password
  66
  67         def get_current_user(self):
  68                 name, password = self.get_basic_auth_credentials()
  69                 if name is None:
  70                         return
  71
  72                 builder = self.backend.builders.auth(name, password)
  73                 if builder:
  74                         return builder
  75
  76                 user = self.backend.users.auth(name, password)
  77                 if user:
  78                         return user
  79
  80         @property
  81         def builder(self):
  82                 if isinstance(self.current_user, builders.Builder):
  83                         return self.current_user
  84
  85         @property
  86         def user(self):
  87                 if isinstance(self.current_user, users.User):
  88                         return self.current_user
  89
  90         def get_argument_int(self, *args, **kwargs):
  91                 arg = self.get_argument(*args, **kwargs)
  92
  93                 try:
  94                         return int(arg)
  95                 except (TypeError, ValueError):
  96                         return None
  97
  98         def get_argument_float(self, *args, **kwargs):
  99                 arg = self.get_argument(*args, **kwargs)
 100
 101                 try:
 102                         return float(arg)
 103                 except (TypeError, ValueError):
 104                         return None
 105
 106         def get_argument_json(self, *args, **kwargs):
 107                 arg = self.get_argument(*args, **kwargs)
 108
 109                 if arg:
 110                         return json.loads(arg)
 111
 112
 113 class NoopHandler(BaseHandler):
 114         def get(self):
 115                 if self.builder:
 116                         self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
 117                 elif self.user:
 118                         self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
 119                 else:
 120                         self.write("Welcome to the Pakfire hub!")
 121
 122
 123 class ErrorTestHandler(BaseHandler):
 124         def get(self, error_code=200):
 125                 """
 126                         For testing a client.
 127
 128                         This just returns a HTTP response with the given code.
 129                 """
 130                 try:
 131                         error_code = int(error_code)
 132                 except ValueError:
 133                         error_code = 200
 134
 135                 raise tornado.web.HTTPError(error_code)
 136
 137
 138 # Uploads
 139
 140 class UploadsCreateHandler(BaseHandler):
 141         """
 142                 Create a new upload object in the database and return a unique ID
 143                 to the uploader.
 144         """
 145
 146         @tornado.web.authenticated
 147         def get(self):
 148                 # XXX Check permissions
 149
 150                 filename = self.get_argument("filename")
 151                 filesize = self.get_argument_int("filesize")
 152                 filehash = self.get_argument("hash")
 153
 154                 with self.db.transaction():
 155                         upload = self.backend.uploads.create(filename, filesize,
 156                                 filehash, user=self.user, builder=self.builder)
 157
 158                         self.finish(upload.uuid)
 159
 160
 161 class UploadsSendChunkHandler(BaseHandler):
 162         @tornado.web.authenticated
 163         def post(self, upload_id):
 164                 upload = self.backend.uploads.get_by_uuid(upload_id)
 165                 if not upload:
 166                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 167
 168                 if not upload.builder == self.builder:
 169                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 170
 171                 chksum = self.get_argument("chksum")
 172                 data = self.get_argument("data")
 173
 174                 # Decode data.
 175                 data = base64.b64decode(data)
 176
 177                 # Calculate hash and compare.
 178                 h = hashlib.new("sha512")
 179                 h.update(data)
 180
 181                 if not chksum == h.hexdigest():
 182                         raise tornado.web.HTTPError(400, "Checksum mismatch")
 183
 184                 # Append the data to file.
 185                 with self.db.transaction():
 186                         upload.append(data)
 187
 188
 189 class UploadsFinishedHandler(BaseHandler):
 190         @tornado.web.authenticated
 191         def get(self, upload_id):
 192                 upload = self.backend.uploads.get_by_uuid(upload_id)
 193                 if not upload:
 194                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 195
 196                 if not upload.builder == self.builder:
 197                         raise tornado.web.HTTPError(403, "Uploading an other host's file.")
 198
 199                 # Validate the uploaded data to its hash.
 200                 ret = upload.validate()
 201
 202                 # If the validation was successfull, we mark the upload
 203                 # as finished and send True to the client.
 204                 if ret:
 205                         upload.finished()
 206                         self.finish("OK")
 207
 208                         return
 209
 210                 # In case the download was corrupted or incomplete, we delete it
 211                 # and tell the client to start over.
 212                 with self.db.transaction():
 213                         upload.remove()
 214
 215                 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
 216
 217
 218 class UploadsDestroyHandler(BaseHandler):
 219         @tornado.web.authenticated
 220         def get(self, upload_id):
 221                 upload = self.backend.uploads.get_by_uuid(upload_id)
 222                 if not upload:
 223                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 224
 225                 if not upload.builder == self.builder:
 226                         raise tornado.web.HTTPError(403, "Removing an other host's file.")
 227
 228                 # Remove the upload from the database and trash the data.
 229                 with self.db.transaction():
 230                         upload.remove()
 231
 232
 233 # Builds
 234
 235 class BuildsCreateHandler(BaseHandler):
 236         @tornado.web.authenticated
 237         def get(self):
 238                 # Get the upload ID of the package file.
 239                 upload_id = self.get_argument("upload_id")
 240
 241                 # Get the identifier of the distribution we build for.
 242                 distro_ident = self.get_argument("distro")
 243
 244                 # Get a list of arches to build for.
 245                 arches = self.get_argument("arches", None)
 246                 if arches == "":
 247                         arches = None
 248
 249                 # Process build type.
 250                 build_type = self.get_argument("build_type")
 251                 if build_type == "release":
 252                         check_for_duplicates = True
 253                 elif build_type == "scratch":
 254                         check_for_duplicates = False
 255                 else:
 256                         raise tornado.web.HTTPError(400, "Invalid build type")
 257
 258                 ## Check if the user has permission to create a build.
 259                 # Users only have the permission to create scratch builds.
 260                 if self.user and not build_type == "scratch":
 261                         raise tornado.web.HTTPError(403, "Users are only allowed to upload scratch builds")
 262
 263                 # Get previously uploaded file to create this build from.
 264                 upload = self.backend.uploads.get_by_uuid(upload_id)
 265                 if not upload:
 266                         raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
 267
 268                 # Check if the uploaded file belongs to this user/builder.
 269                 if self.user and not upload.user == self.user:
 270                         raise tornado.web.HTTPError(400, "Upload does not belong to this user.")
 271
 272                 elif self.builder and not upload.builder == self.builder:
 273                         raise tornado.web.HTTPError(400, "Upload does not belong to this builder.")
 274
 275                 # Get distribution this package should be built for.
 276                 distro = self.backend.distros.get_by_ident(distro_ident)
 277                 if not distro:
 278                         distro = self.backend.distros.get_default()
 279
 280                 # Open the package that was uploaded earlier and add it to
 281                 # the database. Create a new build object from the uploaded package.
 282                 args = {
 283                         "arches"               : arches,
 284                         "check_for_duplicates" : check_for_duplicates,
 285                         "distro"               : distro,
 286                         "type"                 : build_type,
 287                 }
 288                 if self.user:
 289                         args["owner"] = self.user
 290
 291                 try:
 292                         pkg, build = builds.import_from_package(self.backend, upload.path, **args)
 293
 294                 except:
 295                         # Raise any exception.
 296                         raise
 297
 298                 else:
 299                         # Creating the build will move the file to the build directory,
 300                         # so we can safely remove the uploaded file.
 301                         upload.remove()
 302
 303                 # Send the build ID back to the user.
 304                 self.finish(build.uuid)
 305
 306
 307 class BuildsGetHandler(BaseHandler):
 308         def get(self, build_uuid):
 309                 build = self.backend.builds.get_by_uuid(build_uuid)
 310                 if not build:
 311                         raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
 312
 313                 ret = {
 314                         "distro"       : build.distro.identifier,
 315                         "jobs"         : [j.uuid for j in build.jobs],
 316                         "name"         : build.name,
 317                         "package"      : build.pkg.uuid,
 318                         "priority"     : build.priority,
 319                         "score"        : build.score,
 320                         "severity"     : build.severity,
 321                         "state"        : build.state,
 322                         "sup_arches"   : build.supported_arches,
 323                         "time_created" : build.created.isoformat(),
 324                         "type"         : build.type,
 325                         "uuid"         : build.uuid,
 326                 }
 327
 328                 # If the build is in a repository, update that bit.
 329                 if build.repo:
 330                         ret["repo"] = build.repo.identifier
 331
 332                 self.finish(ret)
 333
 334
 335 # Jobs
 336
 337 class JobsBaseHandler(BaseHandler):
 338         def job2json(self, job):
 339                 ret = {
 340                         "arch"         : job.arch,
 341                         "build"        : job.build.uuid,
 342                         "duration"     : job.duration,
 343                         "name"         : job.name,
 344                         "packages"     : [p.uuid for p in job.packages],
 345                         "state"        : job.state,
 346                         "time_created" : job.time_created.isoformat(),
 347                         "type"         : "test" if job.test else "release",
 348                         "uuid"         : job.uuid,
 349                 }
 350
 351                 if job.builder:
 352                         ret["builder"] = job.builder.hostname
 353
 354                 if job.time_started:
 355                         ret["time_started"] = job.time_started.isoformat()
 356
 357                 if job.time_finished:
 358                         ret["time_finished"] = job.time_finished.isoformat()
 359
 360                 return ret
 361
 362
 363 class JobsGetActiveHandler(JobsBaseHandler):
 364         def get(self):
 365                 # Get list of all active jobs.
 366                 jobs = self.backend.jobs.get_active()
 367
 368                 args = {
 369                         "jobs" : [self.job2json(j) for j in jobs],
 370                 }
 371
 372                 self.finish(args)
 373
 374
 375 class JobsGetLatestHandler(JobsBaseHandler):
 376         def get(self):
 377                 limit = self.get_argument_int("limit", 5)
 378
 379                 # Get the latest jobs.
 380                 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
 381
 382                 args = {
 383                         "jobs" : [self.job2json(j) for j in jobs],
 384                 }
 385
 386                 self.finish(args)
 387
 388
 389 class JobsGetQueueHandler(JobsBaseHandler):
 390         def get(self):
 391                 limit = self.get_argument_int("limit", 5)
 392
 393                 # Get the job queue.
 394                 jobs = []
 395                 for job in self.backend.jobqueue:
 396                         jobs.append(job)
 397
 398                         limit -= 1
 399                         if not limit: break
 400
 401                 args = {
 402                         "jobs" : [self.job2json(j) for j in jobs],
 403                 }
 404
 405                 self.finish(args)
 406
 407
 408 class JobsGetHandler(JobsBaseHandler):
 409         def get(self, job_uuid):
 410                 job = self.backend.jobs.get_by_uuid(job_uuid)
 411                 if not job:
 412                         raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
 413
 414                 ret = self.job2json(job)
 415                 self.finish(ret)
 416
 417
 418 # Packages
 419
 420 class PackagesGetHandler(BaseHandler):
 421         def get(self, package_uuid):
 422                 pkg = self.backend.packages.get_by_uuid(package_uuid)
 423                 if not pkg:
 424                         raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
 425
 426                 ret = {
 427                         "arch"             : pkg.arch,
 428                         "build_id"         : pkg.build_id,
 429                         "build_host"       : pkg.build_host,
 430                         "build_time"       : pkg.build_time.isoformat(),
 431                         "description"      : pkg.description,
 432                         "epoch"            : pkg.epoch,
 433                         "filesize"         : pkg.filesize,
 434                         "friendly_name"    : pkg.friendly_name,
 435                         "friendly_version" : pkg.friendly_version,
 436                         "groups"           : pkg.groups,
 437                         "hash_sha512"      : pkg.hash_sha512,
 438                         "license"          : pkg.license,
 439                         "name"             : pkg.name,
 440                         "release"          : pkg.release,
 441                         "size"             : pkg.size,
 442                         "summary"          : pkg.summary,
 443                         "type"             : pkg.type,
 444                         "url"              : pkg.url,
 445                         "uuid"             : pkg.uuid,
 446                         "version"          : pkg.version,
 447
 448                         # Dependencies.
 449                         "prerequires"      : pkg.prerequires,
 450                         "requires"         : pkg.requires,
 451                         "provides"         : pkg.provides,
 452                         "obsoletes"        : pkg.obsoletes,
 453                         "conflicts"        : pkg.conflicts,
 454                 }
 455
 456                 if pkg.type == "source":
 457                         ret["supported_arches"] = pkg.supported_arches
 458
 459                 if isinstance(pkg.maintainer, users.User):
 460                         ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
 461                 elif pkg.maintainer:
 462                         ret["maintainer"] = pkg.maintainer
 463
 464                 if pkg.distro:
 465                         ret["distro"] = pkg.distro.identifier
 466
 467                 self.finish(ret)
 468
 469
 470 # Builders
 471
 472 class BuildersBaseHandler(BaseHandler):
 473         def prepare(self):
 474                 # The request must come from an authenticated buider.
 475                 if not self.builder:
 476                         raise tornado.web.HTTPError(403)
 477
 478
 479 class BuildersInfoHandler(BuildersBaseHandler):
 480         @tornado.web.authenticated
 481         def post(self):
 482                 args = {
 483                         # CPU info
 484                         "cpu_model"    : self.get_argument("cpu_model", None),
 485                         "cpu_count"    : self.get_argument("cpu_count", None),
 486                         "cpu_arch"     : self.get_argument("cpu_arch", None),
 487                         "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
 488
 489                         # Pakfire
 490                         "pakfire_version" : self.get_argument("pakfire_version", None),
 491                         "host_key"     : self.get_argument("host_key", None),
 492
 493                         # OS
 494                         "os_name"      : self.get_argument("os_name", None),
 495                 }
 496                 self.builder.update_info(**args)
 497
 498
 499 class BuildersKeepaliveHandler(BuildersBaseHandler):
 500         @tornado.web.authenticated
 501         def post(self):
 502                 args = {
 503                         # Load average
 504                         "loadavg1"   : self.get_argument_float("loadavg1", None),
 505                         "loadavg5"   : self.get_argument_float("loadavg5", None),
 506                         "loadavg15"  : self.get_argument_float("loadavg15", None),
 507
 508                         # Memory
 509                         "mem_total"  : self.get_argument_int("mem_total", None),
 510                         "mem_free"   : self.get_argument_int("mem_free", None),
 511
 512                         # swap
 513                         "swap_total" : self.get_argument_int("swap_total", None),
 514                         "swap_free"  : self.get_argument_int("swap_free", None),
 515
 516                         # Disk space
 517                         "space_free" : self.get_argument_int("space_free", None),
 518                 }
 519                 self.builder.update_keepalive(**args)
 520
 521                 self.finish("OK")
 522
 523
 524 class BuildersJobsQueueHandler(BuildersBaseHandler):
 525         @tornado.web.asynchronous
 526         @tornado.web.authenticated
 527         def get(self):
 528                 self.callback()
 529
 530         def callback(self):
 531                 # Break if the connection has been closed in the mean time.
 532                 if self.connection_closed():
 533                         logging.warning("Connection closed")
 534                         return
 535
 536                 # Check if there is a job for us.
 537                 job = self.builder.get_next_job()
 538
 539                 # Got no job, wait and try again.
 540                 if not job:
 541                         # Check if we have been running for too long.
 542                         if self.runtime >= self.max_runtime:
 543                                 logging.debug("Exceeded max. runtime. Finishing request.")
 544                                 return self.finish()
 545
 546                         # Try again in a jiffy.
 547                         self.add_timeout(self.heartbeat, self.callback)
 548                         return
 549
 550                 try:
 551                         # Set job to dispatching state.
 552                         job.state = "dispatching"
 553
 554                         # Set our build host.
 555                         job.builder = self.builder
 556
 557                         ret = {
 558                                 "id"                 : job.uuid,
 559                                 "arch"               : job.arch,
 560                                 "source_url"         : job.build.source_download,
 561                                 "source_hash_sha512" : job.build.source_hash_sha512,
 562                                 "type"               : "test" if job.test else "release",
 563                                 "config"             : job.get_config(),
 564                         }
 565
 566                         # Send build information to the builder.
 567                         self.finish(ret)
 568                 except:
 569                         # If anything went wrong, we reset the state.
 570                         job.state = "pending"
 571                         raise
 572
 573         @property
 574         def heartbeat(self):
 575                 return 15 # 15 seconds
 576
 577         @property
 578         def max_runtime(self):
 579                 timeout = self.get_argument_int("timeout", None)
 580                 if timeout:
 581                         return timeout - self.heartbeat
 582
 583                 return 300 # 5 min
 584
 585
 586 class BuildersJobsStateHandler(BuildersBaseHandler):
 587         @tornado.web.authenticated
 588         def post(self, job_uuid, state):
 589                 job = self.backend.jobs.get_by_uuid(job_uuid)
 590                 if not job:
 591                         raise tornado.web.HTTPError(404, "Invalid job id.")
 592
 593                 if not job.builder == self.builder:
 594                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 595
 596                 # Save information to database.
 597                 job.state = state
 598
 599                 message = self.get_argument("message", None)
 600                 job.update_message(message)
 601
 602                 self.finish("OK")
 603
 604
 605 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
 606         @tornado.web.authenticated
 607         def post(self, job_uuid):
 608                 job = self.backend.jobs.get_by_uuid(job_uuid)
 609                 if not job:
 610                         raise tornado.web.HTTPError(404, "Invalid job id.")
 611
 612                 if not job.builder == self.builder:
 613                         raise tornado.web.HTTPError(403, "Altering another builder's build.")
 614
 615                 # Get buildroot.
 616                 buildroot = self.get_argument_json("buildroot", None)
 617                 if buildroot:
 618                         job.save_buildroot(buildroot)
 619
 620                 self.finish("OK")
 621
 622
 623 class BuildersJobsAddFileHandler(BuildersBaseHandler):
 624         @tornado.web.authenticated
 625         def post(self, job_uuid, upload_id):
 626                 type = self.get_argument("type")
 627                 assert type in ("package", "log")
 628
 629                 # Fetch job we are working on and check if it is actually ours.
 630                 job = self.backend.jobs.get_by_uuid(job_uuid)
 631                 if not job:
 632                         raise tornado.web.HTTPError(404, "Invalid job id.")
 633
 634                 if not job.builder == self.builder:
 635                         raise tornado.web.HTTPError(403, "Altering another builder's job.")
 636
 637                 # Fetch uploaded file object and check we uploaded it ourself.
 638                 upload = self.backend.uploads.get_by_uuid(upload_id)
 639                 if not upload:
 640                         raise tornado.web.HTTPError(404, "Invalid upload id.")
 641
 642                 if not upload.builder == self.builder:
 643                         raise tornado.web.HTTPError(403, "Using an other host's file.")
 644
 645                 # Remove all files that have to be deleted, first.
 646                 self.backend.cleanup_files()
 647
 648                 try:
 649                         job.add_file(upload.path)
 650
 651                 finally:
 652                         # Finally, remove the uploaded file.
 653                         upload.remove()
 654
 655                 self.finish("OK")