]> git.ipfire.org Git - people/jschlag/pbs.git/blob - src/hub/handlers.py
c51d9ad2c6bd4a4f24a81121d6fa4dc396891237
[people/jschlag/pbs.git] / src / hub / handlers.py
1 #!/usr/bin/python
2
3 import base64
4 import hashlib
5 import json
6 import logging
7 import time
8 import tornado.web
9
10 from .. import builds
11 from .. import builders
12 from .. import uploads
13 from .. import users
14
15 class LongPollMixin(object):
16 def initialize(self):
17 self._start_time = time.time()
18
19 def add_timeout(self, timeout, callback):
20 deadline = time.time() + timeout
21
22 return self.application.ioloop.add_timeout(deadline, callback)
23
24 def on_connection_close(self):
25 logging.debug("Connection closed unexpectedly")
26
27 def connection_closed(self):
28 return self.request.connection.stream.closed()
29
30 @property
31 def runtime(self):
32 return time.time() - self._start_time
33
34
35 class BaseHandler(LongPollMixin, tornado.web.RequestHandler):
36 @property
37 def backend(self):
38 """
39 Shortcut handler to pakfire instance
40 """
41 return self.application.backend
42
43 def get_basic_auth_credentials(self):
44 """
45 This handles HTTP Basic authentication.
46 """
47 auth_header = self.request.headers.get("Authorization", None)
48
49 # If no authentication information was provided, we stop here.
50 if not auth_header:
51 return None, None
52
53 # No basic auth? We cannot handle that.
54 if not auth_header.startswith("Basic "):
55 raise tornado.web.HTTPError(400, "Can only handle Basic auth.")
56
57 try:
58 # Decode the authentication information.
59 auth_header = base64.decodestring(auth_header[6:])
60
61 name, password = auth_header.split(":", 1)
62 except:
63 raise tornado.web.HTTPError(400, "Authorization data was malformed")
64
65 return name, password
66
67 def get_current_user(self):
68 name, password = self.get_basic_auth_credentials()
69 if name is None:
70 return
71
72 builder = self.backend.builders.auth(name, password)
73 if builder:
74 return builder
75
76 user = self.backend.users.auth(name, password)
77 if user:
78 return user
79
80 @property
81 def builder(self):
82 if isinstance(self.current_user, builders.Builder):
83 return self.current_user
84
85 @property
86 def user(self):
87 if isinstance(self.current_user, users.User):
88 return self.current_user
89
90 def get_argument_int(self, *args, **kwargs):
91 arg = self.get_argument(*args, **kwargs)
92
93 try:
94 return int(arg)
95 except (TypeError, ValueError):
96 return None
97
98 def get_argument_float(self, *args, **kwargs):
99 arg = self.get_argument(*args, **kwargs)
100
101 try:
102 return float(arg)
103 except (TypeError, ValueError):
104 return None
105
106 def get_argument_json(self, *args, **kwargs):
107 arg = self.get_argument(*args, **kwargs)
108
109 if arg:
110 return json.loads(arg)
111
112
113 class NoopHandler(BaseHandler):
114 def get(self):
115 if self.builder:
116 self.write("Welcome to the Pakfire hub, %s!" % self.builder.hostname)
117 elif self.user:
118 self.write("Welcome to the Pakfire hub, %s!" % self.user.name)
119 else:
120 self.write("Welcome to the Pakfire hub!")
121
122
123 class ErrorTestHandler(BaseHandler):
124 def get(self, error_code=200):
125 """
126 For testing a client.
127
128 This just returns a HTTP response with the given code.
129 """
130 try:
131 error_code = int(error_code)
132 except ValueError:
133 error_code = 200
134
135 raise tornado.web.HTTPError(error_code)
136
137
138 # Uploads
139
140 class UploadsCreateHandler(BaseHandler):
141 """
142 Create a new upload object in the database and return a unique ID
143 to the uploader.
144 """
145
146 @tornado.web.authenticated
147 def get(self):
148 # XXX Check permissions
149
150 filename = self.get_argument("filename")
151 filesize = self.get_argument_int("filesize")
152 filehash = self.get_argument("hash")
153
154 upload = uploads.Upload.create(self.backend, filename, filesize,
155 filehash, user=self.user, builder=self.builder)
156
157 self.finish(upload.uuid)
158
159
160 class UploadsSendChunkHandler(BaseHandler):
161 @tornado.web.authenticated
162 def post(self, upload_id):
163 upload = self.backend.uploads.get_by_uuid(upload_id)
164 if not upload:
165 raise tornado.web.HTTPError(404, "Invalid upload id.")
166
167 if not upload.builder == self.builder:
168 raise tornado.web.HTTPError(403, "Uploading an other host's file.")
169
170 chksum = self.get_argument("chksum")
171 data = self.get_argument("data")
172
173 # Decode data.
174 data = base64.b64decode(data)
175
176 # Calculate hash and compare.
177 h = hashlib.new("sha512")
178 h.update(data)
179
180 if not chksum == h.hexdigest():
181 raise tornado.web.HTTPError(400, "Checksum mismatch")
182
183 # Append the data to file.
184 upload.append(data)
185
186
187 class UploadsFinishedHandler(BaseHandler):
188 @tornado.web.authenticated
189 def get(self, upload_id):
190 upload = self.backend.uploads.get_by_uuid(upload_id)
191 if not upload:
192 raise tornado.web.HTTPError(404, "Invalid upload id.")
193
194 if not upload.builder == self.builder:
195 raise tornado.web.HTTPError(403, "Uploading an other host's file.")
196
197 # Validate the uploaded data to its hash.
198 ret = upload.validate()
199
200 # If the validation was successfull, we mark the upload
201 # as finished and send True to the client.
202 if ret:
203 upload.finished()
204 self.finish("OK")
205
206 return
207
208 # In case the download was corrupted or incomplete, we delete it
209 # and tell the client to start over.
210 upload.remove()
211
212 self.finish("ERROR: CORRUPTED OR INCOMPLETE FILE")
213
214
215 class UploadsDestroyHandler(BaseHandler):
216 @tornado.web.authenticated
217 def get(self, upload_id):
218 upload = self.backend.uploads.get_by_uuid(upload_id)
219 if not upload:
220 raise tornado.web.HTTPError(404, "Invalid upload id.")
221
222 if not upload.builder == self.builder:
223 raise tornado.web.HTTPError(403, "Removing an other host's file.")
224
225 # Remove the upload from the database and trash the data.
226 upload.remove()
227
228
229 # Builds
230
231 class BuildsCreateHandler(BaseHandler):
232 @tornado.web.authenticated
233 def get(self):
234 # Get the upload ID of the package file.
235 upload_id = self.get_argument("upload_id")
236
237 # Get the identifier of the distribution we build for.
238 distro_ident = self.get_argument("distro")
239
240 # Get a list of arches to build for.
241 arches = self.get_argument("arches", None)
242 if arches == "":
243 arches = None
244
245 # Process build type.
246 build_type = self.get_argument("build_type")
247 if build_type == "release":
248 check_for_duplicates = True
249 elif build_type == "scratch":
250 check_for_duplicates = False
251 else:
252 raise tornado.web.HTTPError(400, "Invalid build type")
253
254 ## Check if the user has permission to create a build.
255 # Users only have the permission to create scratch builds.
256 if self.user and not build_type == "scratch":
257 raise tornado.web.HTTPError(403, "Users are only allowed to upload scratch builds")
258
259 # Get previously uploaded file to create this build from.
260 upload = self.backend.uploads.get_by_uuid(upload_id)
261 if not upload:
262 raise tornado.web.HTTPError(400, "Upload does not exist: %s" % upload_id)
263
264 # Check if the uploaded file belongs to this user/builder.
265 if self.user and not upload.user == self.user:
266 raise tornado.web.HTTPError(400, "Upload does not belong to this user.")
267
268 elif self.builder and not upload.builder == self.builder:
269 raise tornado.web.HTTPError(400, "Upload does not belong to this builder.")
270
271 # Get distribution this package should be built for.
272 distro = self.backend.distros.get_by_ident(distro_ident)
273 if not distro:
274 distro = self.backend.distros.get_default()
275
276 # Open the package that was uploaded earlier and add it to
277 # the database. Create a new build object from the uploaded package.
278 args = {
279 "arches" : arches,
280 "check_for_duplicates" : check_for_duplicates,
281 "distro" : distro,
282 "type" : build_type,
283 }
284 if self.user:
285 args["owner"] = self.user
286
287 try:
288 pkg, build = builds.import_from_package(self.backend, upload.path, **args)
289
290 except:
291 # Raise any exception.
292 raise
293
294 else:
295 # Creating the build will move the file to the build directory,
296 # so we can safely remove the uploaded file.
297 upload.remove()
298
299 # Send the build ID back to the user.
300 self.finish(build.uuid)
301
302
303 class BuildsGetHandler(BaseHandler):
304 def get(self, build_uuid):
305 build = self.backend.builds.get_by_uuid(build_uuid)
306 if not build:
307 raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
308
309 ret = {
310 "distro" : build.distro.identifier,
311 "jobs" : [j.uuid for j in build.jobs],
312 "name" : build.name,
313 "package" : build.pkg.uuid,
314 "priority" : build.priority,
315 "score" : build.score,
316 "severity" : build.severity,
317 "state" : build.state,
318 "sup_arches" : build.supported_arches,
319 "time_created" : build.created.isoformat(),
320 "type" : build.type,
321 "uuid" : build.uuid,
322 }
323
324 # If the build is in a repository, update that bit.
325 if build.repo:
326 ret["repo"] = build.repo.identifier
327
328 self.finish(ret)
329
330
331 # Jobs
332
333 class JobsBaseHandler(BaseHandler):
334 def job2json(self, job):
335 ret = {
336 "arch" : job.arch,
337 "build" : job.build.uuid,
338 "duration" : job.duration,
339 "name" : job.name,
340 "packages" : [p.uuid for p in job.packages],
341 "state" : job.state,
342 "time_created" : job.time_created.isoformat(),
343 "type" : "test" if job.test else "release",
344 "uuid" : job.uuid,
345 }
346
347 if job.builder:
348 ret["builder"] = job.builder.hostname
349
350 if job.time_started:
351 ret["time_started"] = job.time_started.isoformat()
352
353 if job.time_finished:
354 ret["time_finished"] = job.time_finished.isoformat()
355
356 return ret
357
358
359 class JobsGetActiveHandler(JobsBaseHandler):
360 def get(self):
361 # Get list of all active jobs.
362 jobs = self.backend.jobs.get_active()
363
364 args = {
365 "jobs" : [self.job2json(j) for j in jobs],
366 }
367
368 self.finish(args)
369
370
371 class JobsGetLatestHandler(JobsBaseHandler):
372 def get(self):
373 limit = self.get_argument_int("limit", 5)
374
375 # Get the latest jobs.
376 jobs = self.backend.jobs.get_latest(age="24 HOUR", limit=limit)
377
378 args = {
379 "jobs" : [self.job2json(j) for j in jobs],
380 }
381
382 self.finish(args)
383
384
385 class JobsGetQueueHandler(JobsBaseHandler):
386 def get(self):
387 limit = self.get_argument_int("limit", 5)
388
389 # Get the job queue.
390 jobs = []
391 for job in self.backend.jobqueue:
392 jobs.append(job)
393
394 limit -= 1
395 if not limit: break
396
397 args = {
398 "jobs" : [self.job2json(j) for j in jobs],
399 }
400
401 self.finish(args)
402
403
404 class JobsGetHandler(JobsBaseHandler):
405 def get(self, job_uuid):
406 job = self.backend.jobs.get_by_uuid(job_uuid)
407 if not job:
408 raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
409
410 ret = self.job2json(job)
411 self.finish(ret)
412
413
414 # Packages
415
416 class PackagesGetHandler(BaseHandler):
417 def get(self, package_uuid):
418 pkg = self.backend.packages.get_by_uuid(package_uuid)
419 if not pkg:
420 raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
421
422 ret = {
423 "arch" : pkg.arch,
424 "build_id" : pkg.build_id,
425 "build_host" : pkg.build_host,
426 "build_time" : pkg.build_time.isoformat(),
427 "description" : pkg.description,
428 "epoch" : pkg.epoch,
429 "filesize" : pkg.filesize,
430 "friendly_name" : pkg.friendly_name,
431 "friendly_version" : pkg.friendly_version,
432 "groups" : pkg.groups,
433 "hash_sha512" : pkg.hash_sha512,
434 "license" : pkg.license,
435 "name" : pkg.name,
436 "release" : pkg.release,
437 "size" : pkg.size,
438 "summary" : pkg.summary,
439 "type" : pkg.type,
440 "url" : pkg.url,
441 "uuid" : pkg.uuid,
442 "version" : pkg.version,
443
444 # Dependencies.
445 "prerequires" : pkg.prerequires,
446 "requires" : pkg.requires,
447 "provides" : pkg.provides,
448 "obsoletes" : pkg.obsoletes,
449 "conflicts" : pkg.conflicts,
450 }
451
452 if pkg.type == "source":
453 ret["supported_arches"] = pkg.supported_arches
454
455 if isinstance(pkg.maintainer, users.User):
456 ret["maintainer"] = "%s <%s>" % (pkg.maintainer.realname, pkg.maintainer.email)
457 elif pkg.maintainer:
458 ret["maintainer"] = pkg.maintainer
459
460 if pkg.distro:
461 ret["distro"] = pkg.distro.identifier
462
463 self.finish(ret)
464
465
466 # Builders
467
468 class BuildersBaseHandler(BaseHandler):
469 def prepare(self):
470 # The request must come from an authenticated buider.
471 if not self.builder:
472 raise tornado.web.HTTPError(403)
473
474
475 class BuildersInfoHandler(BuildersBaseHandler):
476 @tornado.web.authenticated
477 def post(self):
478 args = {
479 # CPU info
480 "cpu_model" : self.get_argument("cpu_model", None),
481 "cpu_count" : self.get_argument("cpu_count", None),
482 "cpu_arch" : self.get_argument("cpu_arch", None),
483 "cpu_bogomips" : self.get_argument("cpu_bogomips", None),
484
485 # Pakfire
486 "pakfire_version" : self.get_argument("pakfire_version", None),
487 "host_key" : self.get_argument("host_key", None),
488
489 # OS
490 "os_name" : self.get_argument("os_name", None),
491 }
492 self.builder.update_info(**args)
493
494
495 class BuildersKeepaliveHandler(BuildersBaseHandler):
496 @tornado.web.authenticated
497 def post(self):
498 args = {
499 # Load average
500 "loadavg1" : self.get_argument_float("loadavg1", None),
501 "loadavg5" : self.get_argument_float("loadavg5", None),
502 "loadavg15" : self.get_argument_float("loadavg15", None),
503
504 # Memory
505 "mem_total" : self.get_argument_int("mem_total", None),
506 "mem_free" : self.get_argument_int("mem_free", None),
507
508 # swap
509 "swap_total" : self.get_argument_int("swap_total", None),
510 "swap_free" : self.get_argument_int("swap_free", None),
511
512 # Disk space
513 "space_free" : self.get_argument_int("space_free", None),
514 }
515 self.builder.update_keepalive(**args)
516
517 self.finish("OK")
518
519
520 class BuildersJobsQueueHandler(BuildersBaseHandler):
521 @tornado.web.asynchronous
522 @tornado.web.authenticated
523 def get(self):
524 self.callback()
525
526 def callback(self):
527 # Break if the connection has been closed in the mean time.
528 if self.connection_closed():
529 logging.warning("Connection closed")
530 return
531
532 # Check if there is a job for us.
533 job = self.builder.get_next_job()
534
535 # Got no job, wait and try again.
536 if not job:
537 # Check if we have been running for too long.
538 if self.runtime >= self.max_runtime:
539 logging.debug("Exceeded max. runtime. Finishing request.")
540 return self.finish()
541
542 # Try again in a jiffy.
543 self.add_timeout(self.heartbeat, self.callback)
544 return
545
546 try:
547 # Set job to dispatching state.
548 job.state = "dispatching"
549
550 # Set our build host.
551 job.builder = self.builder
552
553 ret = {
554 "id" : job.uuid,
555 "arch" : job.arch,
556 "source_url" : job.build.source_download,
557 "source_hash_sha512" : job.build.source_hash_sha512,
558 "type" : "test" if job.test else "release",
559 "config" : job.get_config(),
560 }
561
562 # Send build information to the builder.
563 self.finish(ret)
564 except:
565 # If anything went wrong, we reset the state.
566 job.state = "pending"
567 raise
568
569 @property
570 def heartbeat(self):
571 return 15 # 15 seconds
572
573 @property
574 def max_runtime(self):
575 timeout = self.get_argument_int("timeout", None)
576 if timeout:
577 return timeout - self.heartbeat
578
579 return 300 # 5 min
580
581
582 class BuildersJobsStateHandler(BuildersBaseHandler):
583 @tornado.web.authenticated
584 def post(self, job_uuid, state):
585 job = self.backend.jobs.get_by_uuid(job_uuid)
586 if not job:
587 raise tornado.web.HTTPError(404, "Invalid job id.")
588
589 if not job.builder == self.builder:
590 raise tornado.web.HTTPError(403, "Altering another builder's build.")
591
592 # Save information to database.
593 job.state = state
594
595 message = self.get_argument("message", None)
596 job.update_message(message)
597
598 self.finish("OK")
599
600
601 class BuildersJobsBuildrootHandler(BuildersBaseHandler):
602 @tornado.web.authenticated
603 def post(self, job_uuid):
604 job = self.backend.jobs.get_by_uuid(job_uuid)
605 if not job:
606 raise tornado.web.HTTPError(404, "Invalid job id.")
607
608 if not job.builder == self.builder:
609 raise tornado.web.HTTPError(403, "Altering another builder's build.")
610
611 # Get buildroot.
612 buildroot = self.get_argument_json("buildroot", None)
613 if buildroot:
614 job.save_buildroot(buildroot)
615
616 self.finish("OK")
617
618
619 class BuildersJobsAddFileHandler(BuildersBaseHandler):
620 @tornado.web.authenticated
621 def post(self, job_uuid, upload_id):
622 type = self.get_argument("type")
623 assert type in ("package", "log")
624
625 # Fetch job we are working on and check if it is actually ours.
626 job = self.backend.jobs.get_by_uuid(job_uuid)
627 if not job:
628 raise tornado.web.HTTPError(404, "Invalid job id.")
629
630 if not job.builder == self.builder:
631 raise tornado.web.HTTPError(403, "Altering another builder's job.")
632
633 # Fetch uploaded file object and check we uploaded it ourself.
634 upload = self.backend.uploads.get_by_uuid(upload_id)
635 if not upload:
636 raise tornado.web.HTTPError(404, "Invalid upload id.")
637
638 if not upload.builder == self.builder:
639 raise tornado.web.HTTPError(403, "Using an other host's file.")
640
641 # Remove all files that have to be deleted, first.
642 self.backend.cleanup_files()
643
644 try:
645 job.add_file(upload.path)
646
647 finally:
648 # Finally, remove the uploaded file.
649 upload.remove()
650
651 self.finish("OK")